From 3e92607fa7f7813c55eaec44d4c0b37864fed9e4 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 13 Sep 2026 11:50:40 -0600 Subject: [PATCH 1/2] build: remove unused dependencies and unify reqwest --- Cargo.lock | 683 ++++----------------------------------- Cargo.toml | 1 + control_plane/Cargo.toml | 4 +- data_plane/Cargo.toml | 8 +- 4 files changed, 72 insertions(+), 624 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4edd38bd..6000e376 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,15 +73,6 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - [[package]] name = "anstream" version = "1.0.0" @@ -464,7 +455,7 @@ dependencies = [ "asap-aware-mapping", "asap-types", "base64 0.21.7", - "clap 4.6.6", + "clap", "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser)", "serde", "serde_json", @@ -541,17 +532,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.5.1" @@ -569,10 +549,10 @@ dependencies = [ "base64 0.22.1", "bytes", "futures-util", - "http 1.5.0", - "http-body 1.1.0", + "http", + "http-body", "http-body-util", - "hyper 1.11.1", + "hyper", "hyper-util", "itoa", "matchit", @@ -586,7 +566,7 @@ dependencies = [ "serde_path_to_error", "serde_urlencoded", "sha1", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", "tokio-tungstenite 0.24.0", "tower 0.5.3", @@ -604,13 +584,13 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.5.0", - "http-body 1.1.0", + "http", + "http-body", "http-body-util", "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.2", + "sync_wrapper", "tower-layer", "tower-service", "tracing", @@ -821,7 +801,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -861,21 +841,6 @@ dependencies = [ "half", ] -[[package]] -name = "clap" -version = "2.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" -dependencies = [ - "ansi_term", - "atty", - "bitflags 1.3.2", - "strsim 0.8.0", - "textwrap", - "unicode-width 0.1.14", - "vec_map", -] - [[package]] name = "clap" version = "4.6.6" @@ -895,7 +860,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.1", + "strsim", ] [[package]] @@ -904,7 +869,7 @@ version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "syn 3.0.5", @@ -929,7 +894,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ "unicode-segmentation", - "unicode-width 0.2.2", + "unicode-width", ] [[package]] @@ -978,14 +943,13 @@ dependencies = [ "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser)", "prost", "prost-build", - "reqwest 0.12.28", + "reqwest", "serde", "serde_json", "serde_yaml", "sha2", "thiserror 1.0.69", "tokio", - "tokio-stream", "tokio-tungstenite 0.21.0", "toml", "tonic", @@ -993,17 +957,6 @@ dependencies = [ "tower 0.4.13", "tracing", "tracing-subscriber", - "zstd", -] - -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", ] [[package]] @@ -1012,12 +965,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core_detect" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" - [[package]] name = "cpufeatures" version = "0.2.17" @@ -1054,7 +1001,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.6.6", + "clap", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -1205,7 +1152,7 @@ dependencies = [ "bincode", "chrono", "chrono-tz", - "clap 4.6.6", + "clap", "control_plane", "crc32fast", "criterion", @@ -1223,15 +1170,13 @@ dependencies = [ "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser)", "prost", "regex", - "reqwest 0.11.27", + "reqwest", "rmp-serde", - "rusqlite", "serde", "serde_json", "serde_yaml", "sha2", "snap", - "structopt", "tempfile", "thiserror 1.0.69", "tokio", @@ -1242,8 +1187,6 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", - "urlencoding", - "uuid", "xxhash-rust", ] @@ -1686,21 +1629,6 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" -[[package]] -name = "encoding_rs" -version = "0.8.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a" -dependencies = [ - "cfg-if", - "core_detect", - "multiversion", - "multiversion_no_op", - "rustversion", - "scopeguard", - "simdutf8", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -1717,18 +1645,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - [[package]] name = "fastrand" version = "2.5.0" @@ -1913,7 +1829,7 @@ version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" dependencies = [ - "unicode-width 0.2.2", + "unicode-width", ] [[package]] @@ -1961,25 +1877,6 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" -[[package]] -name = "h2" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.14.2", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.19" @@ -1991,7 +1888,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.5.0", + "http", "indexmap 2.14.2", "slab", "tokio", @@ -2039,39 +1936,12 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -[[package]] -name = "hashlink" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" -dependencies = [ - "hashbrown 0.14.5", -] - -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.5.3" @@ -2084,17 +1954,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.5.0" @@ -2105,17 +1964,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.1.0" @@ -2123,7 +1971,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", - "http 1.5.0", + "http", ] [[package]] @@ -2134,8 +1982,8 @@ checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", - "http 1.5.0", - "http-body 1.1.0", + "http", + "http-body", "pin-project-lite", ] @@ -2157,30 +2005,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.27", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.10", - "tokio", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "1.11.1" @@ -2191,9 +2015,9 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.19", - "http 1.5.0", - "http-body 1.1.0", + "h2", + "http", + "http-body", "httparse", "httpdate", "itoa", @@ -2203,34 +2027,20 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http 0.2.12", - "hyper 0.14.32", - "rustls 0.21.12", - "tokio", - "tokio-rustls 0.24.1", -] - [[package]] name = "hyper-rustls" version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.5.0", - "hyper 1.11.1", + "http", + "hyper", "hyper-util", - "rustls 0.23.44", + "rustls", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls", "tower-service", - "webpki-roots 1.0.9", + "webpki-roots", ] [[package]] @@ -2239,7 +2049,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.11.1", + "hyper", "hyper-util", "pin-project-lite", "tokio", @@ -2256,9 +2066,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.5.0", - "http-body 1.1.0", - "hyper 1.11.1", + "http", + "http-body", + "hyper", "ipnet", "libc", "percent-encoding", @@ -2447,7 +2257,7 @@ version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ - "hermit-abi 0.5.3", + "hermit-abi", "libc", "windows-sys 0.61.2", ] @@ -2587,17 +2397,6 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" -[[package]] -name = "libsqlite3-sys" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -2801,33 +2600,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" -[[package]] -name = "multiversion" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c" -dependencies = [ - "multiversion-macros", -] - -[[package]] -name = "multiversion-macros" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83" -dependencies = [ - "proc-macro2", - "quote", - "rustversion", - "syn 3.0.5", -] - -[[package]] -name = "multiversion_no_op" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2922,7 +2694,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "hermit-abi 0.5.3", + "hermit-abi", "libc", ] @@ -3202,30 +2974,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro2" version = "1.0.107" @@ -3317,7 +3065,7 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.5.0", + "heck", "itertools 0.14.0", "log", "multimap", @@ -3435,7 +3183,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.44", + "rustls", "socket2 0.6.5", "thiserror 2.0.20", "tokio", @@ -3456,7 +3204,7 @@ dependencies = [ "rand_pcg", "ring", "rustc-hash", - "rustls 0.23.44", + "rustls", "rustls-pki-types", "slab", "thiserror 2.0.20", @@ -3643,47 +3391,6 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" -[[package]] -name = "reqwest" -version = "0.11.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" -dependencies = [ - "base64 0.21.7", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2 0.3.27", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.32", - "hyper-rustls 0.24.2", - "ipnet", - "js-sys", - "log", - "mime", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls 0.21.12", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper 0.1.2", - "system-configuration", - "tokio", - "tokio-rustls 0.24.1", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "webpki-roots 0.25.4", - "winreg", -] - [[package]] name = "reqwest" version = "0.12.28" @@ -3693,25 +3400,25 @@ dependencies = [ "base64 0.22.1", "bytes", "futures-core", - "http 1.5.0", - "http-body 1.1.0", + "http", + "http-body", "http-body-util", - "hyper 1.11.1", - "hyper-rustls 0.27.9", + "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.44", + "rustls", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls", "tower 0.5.3", "tower-http", "tower-service", @@ -3719,7 +3426,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 1.0.9", + "webpki-roots", ] [[package]] @@ -3755,20 +3462,6 @@ dependencies = [ "serde", ] -[[package]] -name = "rusqlite" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" -dependencies = [ - "bitflags 2.13.2", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", -] - [[package]] name = "rustc-hash" version = "2.1.3" @@ -3810,18 +3503,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", -] - [[package]] name = "rustls" version = "0.23.44" @@ -3831,20 +3512,11 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.15", + "rustls-webpki", "subtle", "zeroize", ] -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - [[package]] name = "rustls-pki-types" version = "1.15.1" @@ -3855,16 +3527,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "rustls-webpki" version = "0.103.15" @@ -3903,16 +3565,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "semver" version = "1.0.28" @@ -4085,12 +3737,6 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" -[[package]] -name = "simdutf8" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" - [[package]] name = "siphasher" version = "1.0.3" @@ -4124,7 +3770,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "syn 2.0.119", @@ -4201,42 +3847,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "structopt" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" -dependencies = [ - "clap 2.34.0", - "lazy_static", - "structopt-derive", -] - -[[package]] -name = "structopt-derive" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" -dependencies = [ - "heck 0.3.3", - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "strum" version = "0.26.3" @@ -4252,7 +3868,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "rustversion", @@ -4271,17 +3887,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.119" @@ -4304,12 +3909,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4330,27 +3929,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "tagptr" version = "0.2.0" @@ -4370,15 +3948,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width 0.1.14", -] - [[package]] name = "thiserror" version = "1.0.69" @@ -4543,23 +4112,13 @@ dependencies = [ "syn 3.0.5", ] -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.12", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" dependencies = [ - "rustls 0.23.44", + "rustls", "tokio", ] @@ -4665,11 +4224,11 @@ dependencies = [ "base64 0.22.1", "bytes", "flate2", - "h2 0.4.19", - "http 1.5.0", - "http-body 1.1.0", + "h2", + "http", + "http-body", "http-body-util", - "hyper 1.11.1", + "hyper", "hyper-timeout", "hyper-util", "percent-encoding", @@ -4727,7 +4286,7 @@ dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", "tower-layer", "tower-service", @@ -4743,8 +4302,8 @@ dependencies = [ "bitflags 2.13.2", "bytes", "futures-util", - "http 1.5.0", - "http-body 1.1.0", + "http", + "http-body", "pin-project-lite", "tower 0.5.3", "tower-layer", @@ -4854,7 +4413,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.5.0", + "http", "httparse", "log", "rand 0.8.8", @@ -4873,7 +4432,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.5.0", + "http", "httparse", "log", "rand 0.8.8", @@ -4919,12 +4478,6 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "unicode-width" version = "0.2.2" @@ -4955,12 +4508,6 @@ dependencies = [ "serde", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf-8" version = "0.7.6" @@ -4996,18 +4543,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - [[package]] name = "vergen" version = "8.3.2" @@ -5154,12 +4689,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.25.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" - [[package]] name = "webpki-roots" version = "1.0.9" @@ -5259,22 +4788,13 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -5283,7 +4803,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -5295,67 +4815,34 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -5368,48 +4855,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -5425,16 +4888,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/Cargo.toml b/Cargo.toml index 3e61c62d..94de6139 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ chrono = { version = "0.4", features = ["serde"] } promql-parser = { git = "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/ProjectASAP/promql-parser" } tokio = { version = "1.0", features = ["full"] } arc-swap = "1.7" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } # Internal crates asap_types = { path = "crates/asap_types" } diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index e87a6c64..ac83339b 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -19,7 +19,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" serde_yaml = "0.9" -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +reqwest.workspace = true anyhow = "1" thiserror = "1" tracing = "0.1" @@ -28,11 +28,9 @@ chrono = { version = "0.4", features = ["serde"] } promql-parser.workspace = true prost = "0.13" bytes = "1" -zstd = "0.13" parking_lot = "0.12" prometheus = { version = "0.13", default-features = false, features = ["process"] } tonic = { version = "0.12", features = ["gzip"] } -tokio-stream = { version = "0.1", features = ["net"] } asap_types.workspace = true # Planner's IR, replacement search, and frontends share the workspace pin. diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 8a0aff0a..60d4549a 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -36,12 +36,8 @@ arc-swap.workspace = true # Crate-specific (keep version pinned here) form_urlencoded = "1.2" axum = "0.7" -rusqlite = { version = "0.31", features = ["bundled"] } bincode = "1.3" dashmap = "5.5" -uuid = { version = "1.0", features = ["v4"] } -structopt = "0.3" -urlencoding = "2.1" flate2 = "1.0" async-trait = "0.1" xxhash-rust = { version = "0.8", features = ["xxh32", "xxh64"] } @@ -61,11 +57,11 @@ asap_otel_proto.workspace = true # `asap-gzip` arm's agent. control_plane already pulls tonic with this # feature for its RuntimeSamples gRPC. tonic = { version = "0.12", features = ["gzip"] } -tokio-stream = "0.1" +tokio-stream = { version = "0.1", features = ["net"] } regex = "1" prometheus = "0.13" lazy_static = "1.4" -reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"] } +reqwest.workspace = true tracing-appender = "0.2" asap_sketchlib = { git = "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/ProjectASAP/asap_sketchlib", branch = "main" } # Generic MessagePack codec for decoding the heap-bearing CountSketch From dea89329e25aca2c84a4d78638c53f3bfcf9e6e9 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 13 Sep 2026 11:50:50 -0600 Subject: [PATCH 2/2] refactor: require installed query plans in tests and remove unused APIs --- .../examples/offline_frequency_plan.rs | 2 +- .../examples/offline_planner_replay.rs | 2 +- control_plane/src/asap_tier_implement.rs | 2 +- control_plane/src/backend_client.rs | 44 +- control_plane/src/clickhouse.rs | 1 + control_plane/src/emit/mod.rs | 2 +- control_plane/src/emit/monitor.rs | 2 + control_plane/src/lib.rs | 1 - control_plane/src/main.rs | 25 +- control_plane/src/opamp/mod.rs | 8 - .../src/physical/colored_dag/allocator.rs | 14 +- control_plane/src/physical/colored_dag/dag.rs | 8 +- .../src/physical/colored_dag/emitter.rs | 14 +- .../src/physical/colored_dag/tests.rs | 10 +- control_plane/src/physical/compiler.rs | 13 +- .../src/physical/deployment_cost/delta.rs | 2 +- .../src/physical/deployment_cost/mod.rs | 17 +- .../src/physical/deployment_cost/pareto.rs | 6 +- .../deployment_cost/sketch_capability.rs | 7 +- control_plane/src/physical/erp.rs | 4 +- control_plane/src/physical/plan_cache.rs | 3 +- .../src/physical/post_asap/cost_model.rs | 106 +- .../src/physical/post_asap/deployment_expr.rs | 8 +- control_plane/src/physical/post_asap/lower.rs | 14 +- control_plane/src/physical/post_asap/tests.rs | 2 +- .../src/physical/runtime_capability.rs | 4 +- control_plane/src/physical/sketch_catalog.rs | 14 +- control_plane/src/physical/stage_split.rs | 2 +- control_plane/src/physical/workload_cost.rs | 6 +- .../src/physical/workload_planner.rs | 17 +- control_plane/src/pipeline.rs | 6 +- control_plane/src/planner_selection.rs | 20 +- control_plane/src/query_parser/mod.rs | 4 +- control_plane/src/query_plan.rs | 79 - .../src/query_plan/clickhouse_exact.rs | 3 +- control_plane/src/query_plan/logical.rs | 3 +- control_plane/src/query_planning.rs | 1 + control_plane/src/replan.rs | 3 +- control_plane/src/store/workload.rs | 2 +- control_plane/src/types.rs | 256 +++- control_plane/src/types_v2.rs | 303 ---- control_plane/src/workload.rs | 7 +- control_plane/tests/o11y_exact_fallback.rs | 6 +- control_plane/tests/offline_evidence.rs | 5 +- crates/asap_types/src/aggregation_config.rs | 14 +- crates/asap_types/src/aggregation_type.rs | 23 - crates/asap_types/src/grouping_projection.rs | 1 + crates/asap_types/src/policy_registry.rs | 1 + crates/asap_types/src/query_plan.rs | 40 - crates/asap_types/src/summary_catalog.rs | 1 + .../control_plane_client/config_fetcher.rs | 154 -- .../src/drivers/control_plane_client/mod.rs | 2 - .../src/drivers/ingest/series_resolver.rs | 8 - data_plane/src/drivers/query/servers/http.rs | 1 + .../src/drivers/query/servers/metrics.rs | 18 - data_plane/src/monitor/coordinator.rs | 1 + data_plane/src/monitor/server.rs | 1 + .../precompute_engine/accumulator_factory.rs | 1 + .../coordination_checkpoint.rs | 1 + data_plane/src/precompute_engine/group_key.rs | 1 + .../precompute_engine/maintenance_runtime.rs | 1 + data_plane/src/precompute_engine/metrics.rs | 2 + .../multisource_coordinator.rs | 26 +- .../operators/edge_runtime_adapter.rs | 1 + .../multiple_increase_accumulator.rs | 4 - .../operators/multiple_min_max_accumulator.rs | 9 +- .../operators/multiple_sum_accumulator.rs | 4 - .../operators/sketch_envelope_accumulator.rs | 5 - .../src/precompute_engine/output_sink.rs | 8 +- .../src/precompute_engine/series_buffer.rs | 2 + .../src/precompute_engine/series_router.rs | 23 - .../src/precompute_engine/window_manager.rs | 1 + data_plane/src/precompute_engine/worker.rs | 2 + .../asap_clickhouse_query_engine/execution.rs | 21 - .../query_engines/asap_query_engine/engine.rs | 1311 ++++------------- .../asap_query_engine/live_serve.rs | 298 +--- .../query_engines/asap_query_engine/mod.rs | 4 +- .../asap_query_engine/physical_dag.rs | 1 + .../asap_query_engine/post_asap_planner.rs | 709 --------- .../asap_query_engine/post_asap_readout.rs | 239 ++- .../asap_query_engine/summary_executor.rs | 2 +- .../asap_query_engine/test_plan.rs | 150 ++ data_plane/src/query_engines/query_result.rs | 11 +- .../routing/backend_storage_routing.rs | 1 + .../thanos_query_engine/forward.rs | 6 +- .../storage_engines/sketch_db/backfill/mod.rs | 1 + .../sketch_db/backfill/prometheus_reader.rs | 18 - .../sketch_db/backfill/raw_sample_reader.rs | 1 + .../sketch_db/index/epoch_columnar.rs | 22 - .../storage_engines/sketch_db/index/mod.rs | 66 +- .../sketch_db/lifecycle/eviction.rs | 8 - .../sketch_db/persistence/cache.rs | 5 - .../sketch_db/persistence/flusher.rs | 65 - .../sketch_db/persistence/manifest.rs | 1 + .../sketch_db/query/delta_apply.rs | 86 +- .../types/hot_reload_config.rs | 6 +- .../types/key_by_label_values.rs | 1 + .../types/precomputed_output.rs | 12 - .../tests/all_sketches_process_oracle_e2e.rs | 27 +- ...e2e_controller_plans_and_backend_serves.rs | 17 +- 100 files changed, 976 insertions(+), 3527 deletions(-) delete mode 100644 control_plane/src/types_v2.rs delete mode 100644 data_plane/src/drivers/control_plane_client/config_fetcher.rs delete mode 100644 data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs create mode 100644 data_plane/src/query_engines/asap_query_engine/test_plan.rs diff --git a/control_plane/examples/offline_frequency_plan.rs b/control_plane/examples/offline_frequency_plan.rs index b30d5de1..218571bf 100644 --- a/control_plane/examples/offline_frequency_plan.rs +++ b/control_plane/examples/offline_frequency_plan.rs @@ -8,7 +8,7 @@ use asap_aware_mapping::empirical_comparison::{ use control_plane::{ physical::post_asap::{bind_query_expr_with_cost_model, cost_model::ControlPlaneCostModel}, planner_selection::frequency, - types_v2::AccuracyTarget, + types::AccuracyTarget, }; use planner_types::pre_asap::{AggIntent, Column, DataType, QueryExpr, Reduction, Schema, Source}; use serde_json::json; diff --git a/control_plane/examples/offline_planner_replay.rs b/control_plane/examples/offline_planner_replay.rs index f0e31758..7a703ce1 100644 --- a/control_plane/examples/offline_planner_replay.rs +++ b/control_plane/examples/offline_planner_replay.rs @@ -11,7 +11,7 @@ use control_plane::{ PostAsapPlan, }, query_parser::parse_query_expr_canonical, - types_v2::AccuracyTarget, + types::AccuracyTarget, }; use planner_types::post_asap::{SummaryExpr, SummaryFamilyType, SummaryNode}; use serde_json::{json, Value}; diff --git a/control_plane/src/asap_tier_implement.rs b/control_plane/src/asap_tier_implement.rs index 48853ab8..858c8e3c 100644 --- a/control_plane/src/asap_tier_implement.rs +++ b/control_plane/src/asap_tier_implement.rs @@ -77,7 +77,7 @@ use asap_aware_mapping::DefaultCostModel; use planner_types::post_asap::SummaryNode; use crate::query_parser::parse_query_expr_canonical; -use crate::types_v2::AccuracyTarget; +use crate::types::AccuracyTarget; use planner_types::pre_asap::QueryExpr; /// Fixed accuracy target for this L1 call site (L1 adoption, diff --git a/control_plane/src/backend_client.rs b/control_plane/src/backend_client.rs index 13e2179c..3ff47856 100644 --- a/control_plane/src/backend_client.rs +++ b/control_plane/src/backend_client.rs @@ -17,7 +17,9 @@ use std::time::Duration; use anyhow::{Context, Result}; use reqwest::Client; -use tracing::{debug, warn}; +use tracing::debug; +#[cfg(test)] +use tracing::warn; /// Classification of an HTTP push failure used by the retry layer in /// [`crate::emit::backend_push`]. Transient errors are safe to retry @@ -167,6 +169,7 @@ impl BackendClient { } } + #[cfg(test)] /// Post streaming configuration as `application/json`. The backend accepts /// both JSON and YAML; a non-2xx response is an error for the caller to log. pub async fn post_streaming_config_json(&self, json: String) -> Result<()> { @@ -197,6 +200,7 @@ impl BackendClient { } } + #[cfg(test)] /// Typed sibling of [`Self::post_streaming_config_json`] for the /// retry layer. Returns the same `Ok(())` on 2xx, but on failure /// classifies the underlying cause as [`BackendPostError::Transient`] @@ -239,41 +243,7 @@ impl BackendClient { } } - /// Typed sibling of [`Self::post_storage_routing_json`] for the - /// retry layer. Identical contract to - /// [`Self::post_streaming_config_json_typed`]. - pub async fn post_storage_routing_json_typed( - &self, - json: String, - ) -> std::result::Result<(), BackendPostError> { - let url = derive_storage_routing_url(&self.endpoint); - debug!( - endpoint = %url, - json_bytes = json.len(), - "posting storage-routing JSON to ASAPQuery-backend (typed)" - ); - let resp = self - .http - .post(&url) - .header("content-type", "application/json") - .body(json) - .send() - .await - .map_err(classify_reqwest_error)?; - - let status = resp.status(); - if status.is_success() { - Ok(()) - } else { - let body = resp.text().await.unwrap_or_default(); - Err(classify_http_status( - status, - body, - "storage-routing JSON POST", - )) - } - } - + #[cfg(test)] /// Post backend storage-routing JSON. Derive the URL by replacing the /// `/api/v1/streaming-config` suffix with `/api/v1/storage_routing`; URLs /// without that suffix are used verbatim. @@ -409,6 +379,7 @@ fn derive_physical_plan_url(endpoint: &str) -> String { /// end with `/api/v1/streaming-config` (or `/api/v1/streaming_config` — /// either spelling is supported) pass through unchanged so tests can /// inject a mock-server URL directly. +#[cfg(test)] fn derive_storage_routing_url(endpoint: &str) -> String { const STREAMING_PATH_DASH: &str = "/api/v1/streaming-config"; const STREAMING_PATH_UNDERSCORE: &str = "/api/v1/streaming_config"; @@ -422,6 +393,7 @@ fn derive_storage_routing_url(endpoint: &str) -> String { endpoint.to_string() } +#[cfg(test)] /// Fire-and-forget convenience helper used by the replanner. Logs /// errors at WARN and never propagates them — the replanner should /// never fail an entire replan because the backend was temporarily diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index 4abd29e6..944cdb1e 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -72,6 +72,7 @@ pub async fn plan_clickhouse_sql( }) } +#[cfg(test)] pub async fn canonicalize_clickhouse_sql( sql: &str, catalog: &SqlCatalog, diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index edbcf379..bd270e4c 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -1131,7 +1131,7 @@ mod runtime_tests { time_window: Duration::from_secs(60), repeat_every: None, accuracy_sla: 0.01, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: override_family, exact_required: false, diff --git a/control_plane/src/emit/monitor.rs b/control_plane/src/emit/monitor.rs index 1e35bbd3..4bd65e6e 100644 --- a/control_plane/src/emit/monitor.rs +++ b/control_plane/src/emit/monitor.rs @@ -17,6 +17,7 @@ //! `ASAPCollector/docs/continuous-monitoring-tumbling-cost-analysis.md`. pub use asap_types::MonitorFunctional as Functional; +#[cfg(test)] use serde_yaml::{Mapping, Value}; /// One monitored standing-query intent: "alert when the global Σ of `metric`'s @@ -53,6 +54,7 @@ pub fn agg_id_for_metric(metric: &str) -> u64 { h } +#[cfg(test)] /// Render the edge `threshold:` YAML mapping for the fused asap_edge processor's /// per-metric entry. Omits `key`/`coeffs` when not applicable to the functional. pub fn edge_threshold_block(intent: &MonitorIntent) -> Value { diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index aa0658e8..b4f41302 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -35,7 +35,6 @@ pub mod runtime_samples; pub mod sketch_selection; pub mod store; pub mod types; -pub mod types_v2; pub mod workload; /// PromQL → ASAPPlanner's canonical post-ASAP plan via diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 08790ef8..91913ac3 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -14,7 +14,6 @@ use control_plane::replan; use control_plane::runtime_samples; use control_plane::store; use control_plane::types; -use control_plane::types_v2; use control_plane::workload; use axum::{ @@ -520,7 +519,7 @@ struct PhysicalPlanQueryRequest { window_secs: u64, #[serde(default)] group_by: Vec, - accuracy: types_v2::AccuracyTarget, + accuracy: types::AccuracyTarget, lifecycle: physical::compiler::LifecyclePlanningInput, window_implementations: Vec, #[serde(default)] @@ -577,7 +576,7 @@ impl PhysicalQueryFrontend { fn parse( self, query: &str, - accuracy: types_v2::AccuracyTarget, + accuracy: types::AccuracyTarget, ) -> Result { match self { Self::PromQl => parse_query_expr_canonical(query, accuracy) @@ -2384,7 +2383,7 @@ mod api_tests { /// HTTP and stored replan inputs share the resolved typed target, including delta. #[tokio::test] async fn plan_preserves_typed_accuracy_requirements() { - use control_plane::types_v2::AccuracyTarget; + use control_plane::types::AccuracyTarget; for target in [ AccuracyTarget::Epsilon(0.05), AccuracyTarget::EpsilonDelta { @@ -2514,7 +2513,7 @@ mod api_tests { time_window: std::time::Duration::from_secs(300), repeat_every: None, accuracy_sla: 0.01, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, @@ -2777,8 +2776,8 @@ mod api_tests { accuracy: None, dollars: None, deployment_model: None, - shape: types_v2::QueryShape::default(), - data: types_v2::DataShape::default(), + shape: types::QueryShape::default(), + data: types::DataShape::default(), }; let wl = analyzer.analyze(spec).unwrap(); let wc = types::WorkloadCharacteristics::default(); @@ -2922,8 +2921,8 @@ mod api_tests { accuracy: None, dollars: None, deployment_model: None, - shape: types_v2::QueryShape::default(), - data: types_v2::DataShape::default(), + shape: types::QueryShape::default(), + data: types::DataShape::default(), }; let wl = analyzer.analyze(spec).unwrap(); let wc = types::WorkloadCharacteristics::default(); @@ -3187,8 +3186,8 @@ mod api_tests { accuracy: None, dollars: None, deployment_model: None, - shape: types_v2::QueryShape::default(), - data: types_v2::DataShape::default(), + shape: types::QueryShape::default(), + data: types::DataShape::default(), }; let wl = analyzer.analyze(spec).expect("analyze"); let wc = types::WorkloadCharacteristics::default(); @@ -3422,8 +3421,8 @@ mod api_tests { accuracy: None, dollars: None, deployment_model: None, - shape: types_v2::QueryShape::default(), - data: types_v2::DataShape::default(), + shape: types::QueryShape::default(), + data: types::DataShape::default(), }; let wl = analyzer.analyze(spec).expect("analyze"); let wc = types::WorkloadCharacteristics::default(); diff --git a/control_plane/src/opamp/mod.rs b/control_plane/src/opamp/mod.rs index 91e99d7f..cb9318bc 100644 --- a/control_plane/src/opamp/mod.rs +++ b/control_plane/src/opamp/mod.rs @@ -245,14 +245,6 @@ impl OpampServer { } } - /// Broadcasts a config to every connected agent regardless of role. - pub async fn push_all(&self, cfg: RemoteConfig) { - let ids: Vec = self.agents.read().await.keys().cloned().collect(); - for id in ids { - self.push(&id, cfg.clone()).await; - } - } - /// Broadcasts a config only to agents matching the given role. pub async fn push_to_role(&self, role: AgentRole, cfg: RemoteConfig) { let ids: Vec = self diff --git a/control_plane/src/physical/colored_dag/allocator.rs b/control_plane/src/physical/colored_dag/allocator.rs index 7574aa22..a74699cb 100644 --- a/control_plane/src/physical/colored_dag/allocator.rs +++ b/control_plane/src/physical/colored_dag/allocator.rs @@ -45,7 +45,6 @@ use crate::physical::colored_dag::dag::{ColoredDag, ColoredNode, NodeId}; use crate::physical::colored_dag::stage_id::{StageId, Topology}; use crate::physical::post_asap::deployment_expr::PostAsapPlan; use crate::physical::post_asap::PhysicalExpr; -use crate::types_v2::BindingName; /// Errors surfaced by [`StageAllocator::allocate`]. #[derive(Debug, thiserror::Error, PartialEq)] @@ -343,17 +342,6 @@ impl ThreeStageWalker { } } -// Convenience helper used by tests / external callers that only need a -// stage lookup keyed by binding name. -pub(crate) fn binding_stage(dag: &ColoredDag, name: &BindingName) -> Option { - dag.nodes.iter().find_map(|n| match &n.expr { - PhysicalExpr::Committed(PostAsapPlan::LetBinding { name: n2, .. }) if n2 == name => { - Some(n.stage) - } - _ => None, - }) -} - // ── Tests ───────────────────────────────────────────────────────────────────── #[cfg(test)] @@ -417,7 +405,7 @@ mod tests { measures: vec![planner_types::pre_asap::AggIntent::Quantile { col: None, q: 0.99, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), }], output_names: Vec::new(), having: None, diff --git a/control_plane/src/physical/colored_dag/dag.rs b/control_plane/src/physical/colored_dag/dag.rs index 7a04017d..63cb5870 100644 --- a/control_plane/src/physical/colored_dag/dag.rs +++ b/control_plane/src/physical/colored_dag/dag.rs @@ -79,11 +79,6 @@ impl ColoredDag { self.nodes.first() } - /// All nodes painted with `stage`. - pub fn nodes_in_stage(&self, stage: StageId) -> impl Iterator { - self.nodes.iter().filter(move |n| n.stage == stage) - } - /// Set of `StageId`s actually present in this colouring (subset of /// `topology.stages()`). pub fn occupied_stages(&self) -> Vec { @@ -96,6 +91,7 @@ impl ColoredDag { seen } + #[cfg(test)] /// Edges crossing stage boundaries. They describe the required transport hops. pub fn cut_edges(&self) -> Vec<(NodeId, NodeId)> { self.edges @@ -175,7 +171,7 @@ mod tests { measures: vec![planner_types::pre_asap::AggIntent::Quantile { col: None, q: 0.99, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), }], output_names: Vec::new(), having: None, diff --git a/control_plane/src/physical/colored_dag/emitter.rs b/control_plane/src/physical/colored_dag/emitter.rs index 9d0de45b..7bcfc3f5 100644 --- a/control_plane/src/physical/colored_dag/emitter.rs +++ b/control_plane/src/physical/colored_dag/emitter.rs @@ -13,7 +13,6 @@ use serde::{Deserialize, Serialize}; use crate::physical::colored_dag::dag::ColoredDag; use crate::physical::colored_dag::stage_id::{StageId, Topology}; use crate::physical::post_asap::deployment_expr::{PhysicalExpr, PostAsapPlan}; -use crate::types_v2::BindingName; use planner_types::post_asap::{ ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, @@ -45,10 +44,10 @@ enum NodeKind<'a> { }, SketchMerge, LetBinding { - name: &'a BindingName, + name: &'a String, }, Ref { - name: &'a BindingName, + name: &'a String, }, RawAtEdgeSketchAtBackend { family: &'a SketchAlgorithm, @@ -447,14 +446,7 @@ pub enum ColdFormat { Intchunk, } -impl ColdFormat { - /// `true` for the default ([`ColdFormat::Fragment`]). Drives the - /// `skip_serializing_if` on [`EdgeStageConfig::cold_format`] so an unset - /// format leaves the serialized config byte-identical to today. - pub fn is_default(&self) -> bool { - matches!(self, ColdFormat::Fragment) - } -} +impl ColdFormat {} /// Named default for [`EdgeStageConfig::cold_ship_endpoint`]. /// diff --git a/control_plane/src/physical/colored_dag/tests.rs b/control_plane/src/physical/colored_dag/tests.rs index 23e8860d..7794dae8 100644 --- a/control_plane/src/physical/colored_dag/tests.rs +++ b/control_plane/src/physical/colored_dag/tests.rs @@ -19,7 +19,7 @@ use crate::physical::colored_dag::allocator::StageAllocator; use crate::physical::colored_dag::emitter::{EmitError, Emitter, StageConfig, ThreeStageEmitter}; use crate::physical::colored_dag::stage_id::{StageId, Topology}; use crate::physical::post_asap::deployment_expr::{PhysicalExpr, PostAsapPlan}; -use crate::types_v2::{AccuracyTarget, BindingName}; +use crate::types::AccuracyTarget; use planner_types::pre_asap::{Column, DataType}; use planner_types::pre_asap::{ColumnRef, QueryExpr, Reduction, Schema, Source}; @@ -276,10 +276,10 @@ fn allocator_let_binding_color_propagates() { logical_l4(windowed_scan()), ); let bind = PhysicalExpr::Committed(PostAsapPlan::LetBinding { - name: BindingName::new("kll_state"), + name: String::from("kll_state"), expr: Rc::new(PostAsapPlan::Summary(inner_agg)), child: Rc::new(PostAsapPlan::Ref { - name: BindingName::new("kll_state"), + name: String::from("kll_state"), }), }); let dag = StageAllocator @@ -306,10 +306,10 @@ fn allocator_ref_resolves_to_binding_stage() { logical_l4(windowed_scan()), ); let bind = PhysicalExpr::Committed(PostAsapPlan::LetBinding { - name: BindingName::new("shared"), + name: String::from("shared"), expr: Rc::new(PostAsapPlan::Summary(inner_agg)), child: Rc::new(PostAsapPlan::Ref { - name: BindingName::new("shared"), + name: String::from("shared"), }), }); let dag = StageAllocator diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 2ea40ab2..bd6fc106 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -9,10 +9,12 @@ use std::rc::Rc; use asap_aware_mapping::cost_model::Cost; use asap_aware_mapping::{ - plan_summary_maintenance_lifecycles, AccuracyEvidenceProvider, CostRate, DefaultAccuracyModel, - EqualSplitAllocator, Horizon, PropagationStats, SummaryMaintenanceCapabilities, - SummaryMaintenanceLifecycleCapabilities, SummaryMaintenanceLifecycleCostInputs, WorkloadDemand, + plan_summary_maintenance_lifecycles, AccuracyEvidenceProvider, CostRate, Horizon, + PropagationStats, SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCapabilities, + SummaryMaintenanceLifecycleCostInputs, WorkloadDemand, }; +#[cfg(test)] +use asap_aware_mapping::{DefaultAccuracyModel, EqualSplitAllocator}; use planner_types::post_asap::{ CompositionOperator, EvaluationSchedule, ExecutableDagCompilation, OutputRepresentation, PostAsapNodeId, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, @@ -34,7 +36,7 @@ use crate::query_plan::{ canonical_promql, FallbackPolicy, InstantExecution, MaterializationBinding, PhysicalGrouping, QueryPlan, QueryPlanEntry, }; -use crate::types_v2::AccuracyTarget; +use crate::types::AccuracyTarget; use planner_types::pre_asap::Source; pub const PLANNER_REVISION: &str = env!("ASAPPLANNER_REVISION"); @@ -272,6 +274,7 @@ pub use asap_types::producer_plan::{ TransmissionPlan, TransmissionPlanError, TransmissionRule, }; +#[cfg(test)] /// Build the fixed physical knob from the controller's canonical /// epsilon-floor allocator. Degenerate budgets/rates disable sampling. pub fn sampling_policy_from_accuracy_budget( @@ -297,6 +300,7 @@ pub fn sampling_policy_from_accuracy_budget( } } +#[cfg(test)] /// Allocate the deterministic staleness share with the same linear-peel /// composition used by `epsilon_alloc`. `None` means the selected sketch /// already consumes the budget or communication has no allocated weight. @@ -2200,6 +2204,7 @@ fn requires_exact_erp_fallback( }) } +#[cfg(test)] /// Planner-adapter selection step used before physical compilation. Keeping /// this separate makes the ownership boundary explicit: callers supply the /// selected post-ASAP DAG to [`PhysicalCompiler::compile`]. diff --git a/control_plane/src/physical/deployment_cost/delta.rs b/control_plane/src/physical/deployment_cost/delta.rs index 12f19f0f..0a1df2d1 100644 --- a/control_plane/src/physical/deployment_cost/delta.rs +++ b/control_plane/src/physical/deployment_cost/delta.rs @@ -483,7 +483,7 @@ mod tests { time_window: Duration::from_secs(300), repeat_every: Some(Duration::from_secs(10)), accuracy_sla: 0.01, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, diff --git a/control_plane/src/physical/deployment_cost/mod.rs b/control_plane/src/physical/deployment_cost/mod.rs index 564c430f..557ff4cf 100644 --- a/control_plane/src/physical/deployment_cost/mod.rs +++ b/control_plane/src/physical/deployment_cost/mod.rs @@ -127,7 +127,7 @@ pub fn score_with( cpu_micros_per_sample: costs.cpu_micros_per_sample, memory_bytes: memory, estimated_error: err, - meets_sla: matches!(w.accuracy, crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) + meets_sla: matches!(w.accuracy, crate::types::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) && err <= sla, } } @@ -165,7 +165,7 @@ pub fn score(plan: &CollectionPlan, w: &QueryWorkload) -> PlanScore { cpu_micros_per_sample: costs.cpu_micros_per_sample, memory_bytes: memory, estimated_error: err, - meets_sla: matches!(w.accuracy, crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) + meets_sla: matches!(w.accuracy, crate::types::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) && err <= sla, } } @@ -237,8 +237,7 @@ impl DeploymentCostPlanner { /// Pass `None` to use conservative defaults (1 000 series, 100 Hz, /// 100 B/sample, Zipf distribution, no memory budget). pub fn plan(&self, w: &QueryWorkload, wc: Option<&WorkloadCharacteristics>) -> CollectionPlan { - if !matches!(w.accuracy, crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) - { + if !matches!(w.accuracy, crate::types::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) { return self.inner.plan(w); } let default_wc; @@ -356,7 +355,7 @@ mod tests { time_window: Duration::from_secs(300), repeat_every: None, accuracy_sla: 0.01, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, @@ -407,7 +406,7 @@ mod tests { fn ddsketch_fails_tight_sla() { let w = QueryWorkload { accuracy_sla: 0.001, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.001), + accuracy: crate::types::AccuracyTarget::Epsilon(0.001), ..workload(vec![AggType::Quantile]) }; // Force 1% params despite tighter SLA. @@ -453,7 +452,7 @@ mod tests { fn kll_error_formula() { let w = QueryWorkload { accuracy_sla: 0.02, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.02), + accuracy: crate::types::AccuracyTarget::Epsilon(0.02), ..workload(vec![AggType::Quantile]) }; let mut plan = dummy_plan(SketchType::KLL); @@ -475,7 +474,7 @@ mod tests { ] { let w = QueryWorkload { accuracy_sla: sla, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(sla), + accuracy: crate::types::AccuracyTarget::Epsilon(sla), ..workload(vec![agg]) }; let plan = pl.plan(&w, None); @@ -492,7 +491,7 @@ mod tests { fn cost_model_prefers_lower_bandwidth_for_cardinality() { let w = QueryWorkload { accuracy_sla: 0.02, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.02), + accuracy: crate::types::AccuracyTarget::Epsilon(0.02), ..workload(vec![AggType::Cardinality]) }; let plan = DeploymentCostPlanner::new().plan(&w, None); diff --git a/control_plane/src/physical/deployment_cost/pareto.rs b/control_plane/src/physical/deployment_cost/pareto.rs index 11dcf819..29c2a0a7 100644 --- a/control_plane/src/physical/deployment_cost/pareto.rs +++ b/control_plane/src/physical/deployment_cost/pareto.rs @@ -100,7 +100,7 @@ pub fn pareto_frontier( weights: ObjectiveWeights, online_store: Option<&online::OnlineMetricsStore>, ) -> Vec { - if !matches!(workload.accuracy, crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) + if !matches!(workload.accuracy, crate::types::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) { return Vec::new(); } @@ -250,7 +250,7 @@ mod tests { time_window: Duration::from_secs(300), repeat_every: None, accuracy_sla: 0.02, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.02), + accuracy: crate::types::AccuracyTarget::Epsilon(0.02), latency_sla: None, sketch_type_override: None, exact_required: false, @@ -371,7 +371,7 @@ mod tests { fn tight_sla_excludes_inaccurate_sketches() { let w = QueryWorkload { accuracy_sla: 0.001, // very tight — only DDSketch at 0.1% accuracy can meet this - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.001), + accuracy: crate::types::AccuracyTarget::Epsilon(0.001), ..quantile_workload() }; let f = pareto_frontier(&w, &default_wc(), ObjectiveWeights::default(), None); diff --git a/control_plane/src/physical/deployment_cost/sketch_capability.rs b/control_plane/src/physical/deployment_cost/sketch_capability.rs index 28decc91..d30897da 100644 --- a/control_plane/src/physical/deployment_cost/sketch_capability.rs +++ b/control_plane/src/physical/deployment_cost/sketch_capability.rs @@ -20,6 +20,7 @@ use std::collections::HashMap; +#[cfg(test)] use serde::{Deserialize, Serialize}; use planner_types::post_asap::SketchAlgorithm; @@ -29,7 +30,7 @@ use planner_types::post_asap::SketchAlgorithm; /// Used by the optimizer to compare candidates and by the physical /// planner to check whether a sketch fits within a stage's budget. /// Populated from compiled-in defaults via [`default_capability_table`] -/// or overridden at runtime via [`load_capability_overrides`]. +/// with YAML override loading available to unit tests. #[derive(Debug, Clone)] pub struct SketchCapability { /// Insertion throughput (samples/sec at 1 core). @@ -53,6 +54,7 @@ pub struct SketchCapability { // ── YAML override loader ───────────────────────────────────────────────────── /// YAML-serialisable capability profile (matches `sketch_capabilities.yml`). +#[cfg(test)] #[derive(Debug, Clone, Deserialize, Serialize)] struct SketchCapabilityYaml { insert_throughput: f64, @@ -65,6 +67,7 @@ struct SketchCapabilityYaml { supports_sliding_window: bool, } +#[cfg(test)] impl SketchCapabilityYaml { fn to_capability(&self) -> SketchCapability { SketchCapability { @@ -82,6 +85,7 @@ impl SketchCapabilityYaml { /// YAML file structure for all sketch capabilities. Mirrors /// `control_plane/sketch_capabilities.yml` 1:1. +#[cfg(test)] #[derive(Debug, Clone, Deserialize, Serialize)] struct SketchCapabilitiesFile { ddsketch: SketchCapabilityYaml, @@ -162,6 +166,7 @@ pub fn default_capability_table() -> HashMap map } +#[cfg(test)] /// Load overrides from `CONTROLLER_SKETCH_CAPABILITIES`. Missing or malformed /// files fall back to [`default_capability_table`]. pub fn load_capability_overrides(path: &str) -> HashMap { diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 9db3de48..5240a7ed 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -298,7 +298,7 @@ impl asap_aware_mapping::AccuracyModel for ErpAccuracyModel<'_> { fn satisfies( &self, guarantee: &planner_types::post_asap::ResultGuarantee, - target: &crate::types_v2::AccuracyTarget, + target: &crate::types::AccuracyTarget, ) -> bool { asap_aware_mapping::DefaultAccuracyModel.satisfies(guarantee, target) } @@ -1181,7 +1181,7 @@ mod tests { )); assert!(!model.satisfies( &guarantee, - &crate::types_v2::AccuracyTarget::EpsilonDelta { + &crate::types::AccuracyTarget::EpsilonDelta { epsilon: 0.2, delta: 0.01 } diff --git a/control_plane/src/physical/plan_cache.rs b/control_plane/src/physical/plan_cache.rs index 58bbc345..a032c878 100644 --- a/control_plane/src/physical/plan_cache.rs +++ b/control_plane/src/physical/plan_cache.rs @@ -65,6 +65,7 @@ impl CachedDeploymentPlanner { self.cache.write().unwrap().remove(metric); } + #[cfg(test)] /// Return the metric names that have an established baseline. pub fn baseline_metrics(&self) -> Vec { self.cache.read().unwrap().keys().cloned().collect() @@ -89,7 +90,7 @@ mod tests { time_window: Duration::from_secs(300), repeat_every: None, accuracy_sla: 0.01, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, diff --git a/control_plane/src/physical/post_asap/cost_model.rs b/control_plane/src/physical/post_asap/cost_model.rs index 2fab2436..3a01b4f1 100644 --- a/control_plane/src/physical/post_asap/cost_model.rs +++ b/control_plane/src/physical/post_asap/cost_model.rs @@ -27,7 +27,7 @@ use planner_types::pre_asap::expr_ir::ColumnRef; use crate::physical::deployment_cost::wire::WireCostTable; use crate::physical::erp::{ErpParameterDecision, ErpPlanningInput}; use crate::planner_selection::FREQUENCY_EXT_KIND; -use crate::types_v2::AccuracyTarget; +use crate::types::AccuracyTarget; use planner_types::pre_asap::AggIntent; use serde::{Deserialize, Serialize}; @@ -303,17 +303,6 @@ impl ControlPlaneCostModel { self } - pub fn with_window_framework_costs( - mut self, - costs: Vec<(SummaryWindowFramework, Cost)>, - ) -> Self { - self.window_framework_costs = costs - .into_iter() - .map(|(framework, cost)| (None, framework, cost)) - .collect(); - self - } - pub fn with_summary_maintenance( mut self, lifecycle_costs: SummaryMaintenanceLifecycleCostInputs, @@ -835,99 +824,6 @@ impl CostModel for ForcedFamilyCostModel { } } -/// A `CostModel` that forces both the family AND the exact parameters -/// for whichever intent it's asked to rank/size, falling back to an -/// inner accuracy-driven [`ControlPlaneCostModel`] when nothing was -/// observed for the candidates on offer. -/// -/// This is the seam `data_plane`'s live-serving re-binding path -/// (`post_asap_planner.rs`) needs: planning already decided a family + params -/// for a metric (that decision is what's actually registered in the -/// `SketchStore`), so serving-time re-parsing the same query must -/// reproduce EXACTLY that plan, not size a fresh one from a guessed -/// accuracy target (`ForcedFamilyCostModel` above forces the family but -/// still re-derives params from `eps`/`delta` — the wrong tool here, -/// since re-deriving is exactly what caused the mismatch this type -/// exists to avoid; see `control_plane/docs/design-target-architecture.md`'s -/// "planning vs serving" split). `observed` is `None` whenever this -/// query's metric has no registered sid at all — `rank_candidates`/ -/// `size_params` then fall back to the accuracy-driven default, which -/// won't match anything registered either way, so the outcome -/// (`find_candidates` finds nothing) is unchanged. -/// -/// **Legacy metadata fallback:** -/// `post_asap_planner.rs` prefers reading planning's decision directly off an -/// installed SummaryCatalog materializations (no reconstruction needed -/// there — `Materialization.kind`/`.params` already ARE the pair -/// `observed` needs). This type's caller -/// (`observed_family_for_metric`, the `SketchStore`-metadata -/// reconstruction) is the fallback for deployments without a catalog snapshot -/// installed yet, or for metrics a partial/stale plan doesn't cover. -pub struct ObservedFamilyCostModel { - inner: ControlPlaneCostModel, - observed: Option<(SketchAlgorithm, SketchParams)>, -} - -impl ObservedFamilyCostModel { - pub fn new( - workload_accuracy: AccuracyTarget, - observed: Option<(SketchAlgorithm, SketchParams)>, - ) -> Self { - Self { - inner: ControlPlaneCostModel::new(workload_accuracy), - observed, - } - } -} - -impl CostModel for ObservedFamilyCostModel { - fn rank_candidates( - &self, - intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - match &self.observed { - Some((kind, _)) if candidates.contains(kind) => { - let mut ranked = self.inner.rank_candidates(intent, candidates); - let pos = ranked - .iter() - .position(|candidate| candidate == kind) - .expect("observed candidate was present before ranking"); - let observed = ranked.remove(pos); - ranked.insert(0, observed); - ranked - } - _ => self.inner.rank_candidates(intent, candidates), - } - } - - fn size_params( - &self, - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, - ) -> SketchParams { - match &self.observed { - Some((okind, oparams)) if *okind == kind => oparams.clone(), - _ => self.inner.size_params(kind, intent, eps, delta), - } - } - - fn realize_extension(&self, ext_kind: &str, payload: &serde_json::Value) -> Implementation { - self.inner.realize_extension(ext_kind, payload) - } - - fn readout_extension( - &self, - ext_kind: &str, - payload: &serde_json::Value, - col: &ColumnRef, - ) -> SketchQuery { - self.inner.readout_extension(ext_kind, payload, col) - } -} - /// Map an ε rank-error budget to a KLL stream-size `k`. Verbatim port of /// `bind_kll_quantile.rs::kll_k_for_eps` — power-of-two rungs (200, 400, /// 800, 2048, 8192) so the in-tree `algebra::directory` continues to diff --git a/control_plane/src/physical/post_asap/deployment_expr.rs b/control_plane/src/physical/post_asap/deployment_expr.rs index c55174e2..90893964 100644 --- a/control_plane/src/physical/post_asap/deployment_expr.rs +++ b/control_plane/src/physical/post_asap/deployment_expr.rs @@ -12,8 +12,6 @@ use std::time::Duration; use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryNode}; -use crate::types_v2::BindingName; - /// Backend-local wrapper around the Planner-owned post-ASAP tree. /// /// It adds only named sharing required during physical placement and emission. @@ -31,7 +29,7 @@ pub enum PostAsapPlan { /// shared by two `SummaryEstimate` parents reading different quantiles. LetBinding { /// Binding name; must be unique within the surrounding scope. - name: BindingName, + name: String, /// Bound sub-expression. expr: Rc, /// In-scope sub-tree — references the binding via `Ref`. @@ -42,7 +40,7 @@ pub enum PostAsapPlan { /// follows the surrounding `LetBinding` chain). Ref { /// Bound name. - name: BindingName, + name: String, }, } @@ -168,7 +166,7 @@ mod tests { measures: vec![planner_types::pre_asap::AggIntent::Quantile { col: None, q: 0.99, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), }], output_names: Vec::new(), having: None, diff --git a/control_plane/src/physical/post_asap/lower.rs b/control_plane/src/physical/post_asap/lower.rs index 632641d4..b9c69f72 100644 --- a/control_plane/src/physical/post_asap/lower.rs +++ b/control_plane/src/physical/post_asap/lower.rs @@ -38,7 +38,7 @@ use thiserror::Error; use crate::physical::post_asap::cost_model::ControlPlaneCostModel; use crate::physical::post_asap::deployment_expr::{PhysicalExpr, PostAsapPlan}; -use crate::types_v2::AccuracyTarget; +use crate::types::AccuracyTarget; use planner_types::pre_asap::{AggIntent, QueryExpr}; /// Errors surfaced by the `bind_query_expr` lowering. @@ -61,17 +61,7 @@ pub fn bind_query_expr( bind_query_expr_with_cost_model(expr, &cost_model) } -/// Like [`bind_query_expr`], but with an explicitly supplied [`CostModel`] -/// instead of the default planning-time [`ControlPlaneCostModel`]. -/// -/// This is the seam serving-time re-binding needs: `data_plane`'s -/// live-serving path (`post_asap_planner.rs`) must NOT re-derive family/params -/// choice independently of what was actually planned — it looks up what's -/// really registered in the `SketchStore` and hands in a cost model that -/// echoes that back, so the resulting `SummaryNode` matches reality by -/// construction rather than by a coincidental accuracy-target match. See -/// `control_plane/docs/design-target-architecture.md`'s "planning vs -/// serving" split. +/// Bind a query using the explicitly supplied planning cost model. pub fn bind_query_expr_with_cost_model( expr: &QueryExpr, cost_model: &dyn CostModel, diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index f23eafbd..15b19d88 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -14,7 +14,7 @@ use planner_types::pre_asap::expr_ir::ColumnRef; use crate::physical::post_asap::cost_model::ForcedFamilyCostModel; use crate::physical::post_asap::deployment_expr::{PhysicalExpr, PostAsapPlan}; use crate::physical::post_asap::lower::bind_query_expr; -use crate::types_v2::AccuracyTarget; +use crate::types::AccuracyTarget; use planner_types::pre_asap::{AggIntent, QueryExpr, Reduction, Schema, Source}; use planner_types::pre_asap::{Column, DataType}; diff --git a/control_plane/src/physical/runtime_capability.rs b/control_plane/src/physical/runtime_capability.rs index 92b68e30..ecbb1153 100644 --- a/control_plane/src/physical/runtime_capability.rs +++ b/control_plane/src/physical/runtime_capability.rs @@ -20,9 +20,10 @@ #![allow(dead_code)] use crate::physical::post_asap::matcher::sketch_family_satisfied; -use crate::types_v2::AccuracyTarget; +use crate::types::AccuracyTarget; use asap_types::AggregationType; pub use planner_types::post_asap::SketchAlgorithm; +#[cfg(test)] use planner_types::pre_asap::AggIntent; // ── Query-side capability tag ──────────────────────────────────────────────── @@ -365,6 +366,7 @@ fn multi_pop_satisfies_single(required: AggregationType, available: AggregationT // ── AggIntent → Capability bridge ──────────────────────────────────────────── +#[cfg(test)] /// Map a semantic [`AggIntent`] to the ASAP-tier [`Capability`] that can /// answer it. Returns `None` for intents that have no ASAP-tier sketch /// (Sum / Min / Max / Avg / Rate / Increase / every archive-only intent diff --git a/control_plane/src/physical/sketch_catalog.rs b/control_plane/src/physical/sketch_catalog.rs index 82ddd867..9be3c43f 100644 --- a/control_plane/src/physical/sketch_catalog.rs +++ b/control_plane/src/physical/sketch_catalog.rs @@ -89,6 +89,7 @@ pub fn sketch_type_for_op(op: &AggIntent) -> SketchType { } } +#[cfg(test)] /// Resolve the concrete [`SketchType`] for a `PerPartitionWrap` shape — /// delegates to the wrapped inner intent. Kept as a separate function so /// that the PerPartition structural collapse (Step γ) is a focused @@ -136,13 +137,7 @@ pub fn sketch_params_for_op(op: &AggIntent) -> SketchParams { } } -/// Derive [`SketchParams`] for a `PerPartitionWrap` — delegates to the -/// inner intent. Kept separate for the same Step γ reason as -/// [`sketch_type_for_per_partition`]. -pub fn sketch_params_for_per_partition(wrap: &PerPartitionWrap) -> SketchParams { - sketch_params_for_op(&wrap.inner) -} - +#[cfg(test)] /// Combined (type, params) lookup — convenience for callers that need both. pub fn sketch_type_and_params(op: &AggIntent) -> (SketchType, SketchParams) { (sketch_type_for_op(op), sketch_params_for_op(op)) @@ -177,6 +172,7 @@ pub fn estimated_sketch_memory_bytes(op: &AggIntent) -> u64 { } } +#[cfg(test)] /// Memory estimate for a `PerPartitionWrap` — scales the inner-intent /// estimate by `2 ** keys.len()` (capped at 2**10), matching the legacy /// `PerPartition` formula. @@ -278,7 +274,7 @@ mod tests { #[test] fn op_quantile_yields_ddsketch_type_and_params() { - use crate::types_v2::AccuracyTarget; + use crate::types::AccuracyTarget; let op = AggIntent::Quantile { col: None, q: 0.5, @@ -312,7 +308,7 @@ mod tests { #[test] fn memory_quantile() { - use crate::types_v2::AccuracyTarget; + use crate::types::AccuracyTarget; let op = AggIntent::Quantile { col: None, q: 0.5, diff --git a/control_plane/src/physical/stage_split.rs b/control_plane/src/physical/stage_split.rs index 3a182821..95b26d6e 100644 --- a/control_plane/src/physical/stage_split.rs +++ b/control_plane/src/physical/stage_split.rs @@ -101,7 +101,7 @@ mod l5_walk_propagation_tests { time_window: window, repeat_every: None, accuracy_sla: 0.01, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index d7a8ea3b..73566018 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -255,7 +255,7 @@ pub fn manifest( { let parsed = crate::query_parser::parse_query_expr_canonical( query, - crate::types_v2::AccuracyTarget::Exact, + crate::types::AccuracyTarget::Exact, ) .map_err(|error| invalid(error.to_string()))?; for metric in exact_source_metrics(&parsed)? { @@ -868,7 +868,7 @@ mod tests { q.query = planner_types::workload::Query("max_over_time(a[1m]) + max_over_time(b[1m])".into()); q.requirements.accuracy = planner_types::workload::AccuracyRequirement::Explicit( - crate::types_v2::AccuracyTarget::Exact, + crate::types::AccuracyTarget::Exact, ); let (request, environment) = snapshot.planning_request().unwrap(); let candidates = with_exact_alternative(request).unwrap(); @@ -962,7 +962,7 @@ mod tests { let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query("sum(rate(a{job=\"x\"}[1m])) / sum(rate(a{job!=\"x\"}[5m]))".into()); entry.requirements.accuracy = - AccuracyRequirement::Explicit(crate::types_v2::AccuracyTarget::Exact); + AccuracyRequirement::Explicit(crate::types::AccuracyTarget::Exact); let (request, environment) = snapshot.planning_request().unwrap(); let candidates = with_exact_alternative(request).unwrap(); assert!(candidates.len() >= 2); diff --git a/control_plane/src/physical/workload_planner.rs b/control_plane/src/physical/workload_planner.rs index d0b1ec14..a2531861 100644 --- a/control_plane/src/physical/workload_planner.rs +++ b/control_plane/src/physical/workload_planner.rs @@ -348,19 +348,12 @@ impl DeploymentPlanCompiler { } } - pub fn with_defaults(defaults: SketchDefaults) -> Self { - Self { - valid_for: DEFAULT_VALID_FOR, - sketch_defaults: defaults, - } - } - pub fn plan(&self, w: &QueryWorkload) -> CollectionPlan { // This legacy scalar cost path cannot certify a failure probability. // Exact/zero-error and EpsilonDelta use raw; the typed binder independently // checks the full requirement against Planner's family guarantees. if w.exact_required - || !matches!(w.accuracy, crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) + || !matches!(w.accuracy, crate::types::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) { return self.raw_passthrough_plan(w); } @@ -489,7 +482,7 @@ mod tests { time_window: Duration::from_secs(300), repeat_every: None, accuracy_sla: 0.01, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, @@ -606,7 +599,7 @@ mod tests { #[test] fn ddsketch_accuracy_params() { let mut w = workload(vec![AggType::Quantile]); - w.accuracy = crate::types_v2::AccuracyTarget::Epsilon(0.005); + w.accuracy = crate::types::AccuracyTarget::Epsilon(0.005); let plan = DeploymentPlanCompiler::new().plan(&w); match &plan.agent_config.sketch_params { SketchParams::DDSketch { @@ -619,7 +612,7 @@ mod tests { #[test] fn hll_precision_coarse_sla() { let mut w = workload(vec![AggType::Cardinality]); - w.accuracy = crate::types_v2::AccuracyTarget::Epsilon(0.03); + w.accuracy = crate::types::AccuracyTarget::Epsilon(0.03); let plan = DeploymentPlanCompiler::new().plan(&w); match &plan.agent_config.sketch_params { SketchParams::HLL { precision } => { @@ -705,7 +698,7 @@ mod tests { time_window: Duration::from_secs(300), repeat_every: None, accuracy_sla: 0.01, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, diff --git a/control_plane/src/pipeline.rs b/control_plane/src/pipeline.rs index 7b34b5ce..cc4a6d5b 100644 --- a/control_plane/src/pipeline.rs +++ b/control_plane/src/pipeline.rs @@ -4,8 +4,8 @@ use std::collections::{HashMap, HashSet}; use std::time::Duration; use crate::query_parser; +use crate::types::{AccuracyTarget, DataShape, QueryId, QueryLanguage, QueryShape}; use crate::types::{AggType, QueryWorkload, SketchType, WorkloadCharacteristics}; -use crate::types_v2::{AccuracyTarget, DataShape, QueryId, QueryLanguage, QueryShape}; // ── Public API ──────────────────────────────────────────────────────────────── @@ -124,7 +124,7 @@ impl Analyzer { pub fn analyze(&self, spec: QuerySpec) -> anyhow::Result { let accuracy = - crate::types_v2::resolve_accuracy_target(spec.accuracy.as_ref(), spec.accuracy_sla) + crate::types::resolve_accuracy_target(spec.accuracy.as_ref(), spec.accuracy_sla) .map_err(|error| anyhow!(error))?; // ── design.md L1: shape × data cross-product check ───────────────── @@ -822,7 +822,7 @@ mod tests { "data": "batch" }"#; let spec: QuerySpec = serde_json::from_str(json).unwrap(); - assert_eq!(spec.id.as_ref().unwrap().as_str(), "q-001"); + assert_eq!(spec.id.as_ref().unwrap().0.as_str(), "q-001"); assert_eq!(spec.language, Some(QueryLanguage::PromQl)); assert_eq!(spec.accuracy, Some(AccuracyTarget::Epsilon(0.02))); assert_eq!(spec.dollars, Some(0.001)); diff --git a/control_plane/src/planner_selection.rs b/control_plane/src/planner_selection.rs index e6029e25..0b2fc92c 100644 --- a/control_plane/src/planner_selection.rs +++ b/control_plane/src/planner_selection.rs @@ -6,7 +6,7 @@ use std::rc::Rc; -use crate::types_v2::AccuracyTarget; +use crate::types::AccuracyTarget; use asap_aware_mapping::{ AccuracyBudgetAllocator, AccuracyEvidenceProvider, AccuracyModel, CostModel, Replacement, ReplacementStrategy, SketchAlgorithmStrategy, TargetSubDAG, @@ -256,6 +256,7 @@ pub fn select_summary_default(expr: &QueryExpr) -> Result, Selec select_summary(expr, &asap_aware_mapping::DefaultCostModel) } +#[cfg(test)] /// Search a same-requirement workload cohort through Planner's canonical CSE /// and replacement inventory. Physical implementation compatibility is checked /// later, before publication; this function never assigns runtime identities. @@ -272,6 +273,7 @@ pub fn select_workload( ) } +#[cfg(test)] /// The entire cohort uses the same scoped accuracy certificate; callers must /// not spread one query's evidence to unrelated workload roots. pub fn select_workload_with_evidence( @@ -289,6 +291,7 @@ pub fn select_workload_with_evidence( ) } +#[cfg(test)] /// Keep replacement legality and workload-root validation on the same model. pub fn select_workload_with_accuracy_model( roots: Vec<(usize, Rc)>, @@ -476,21 +479,6 @@ pub fn select_summary_with_evidence( } } -/// Select a legal summary when Planner offers one, otherwise preserve the -/// subtree explicitly. This mirrors the removed single-tree binder's -/// conservative behavior and is appropriate for serving fallbacks; physical -/// compilation should use [`select_summary`] so an unimplementable target is -/// reported rather than silently committed. -pub fn select_summary_or_keep( - expr: &QueryExpr, - cost_model: &dyn CostModel, -) -> Result, SelectionError> { - match select_summary(expr, cost_model) { - Err(SelectionError::NoLegalCandidate) => keep_pre_asap(expr), - result => result, - } -} - #[cfg(test)] mod workload_tests { use super::*; diff --git a/control_plane/src/query_parser/mod.rs b/control_plane/src/query_parser/mod.rs index c7b9b2f6..d7837d0b 100644 --- a/control_plane/src/query_parser/mod.rs +++ b/control_plane/src/query_parser/mod.rs @@ -9,8 +9,8 @@ use std::collections::HashMap; use std::time::Duration; +use crate::types::AccuracyTarget; use crate::types::AggType; -use crate::types_v2::AccuracyTarget; use planner_types::pre_asap::AggIntent; use planner_types::pre_asap::{ColumnId, CompareOpKind, ScalarValue}; use planner_types::pre_asap::{Predicate, QueryExpr, Source}; @@ -498,7 +498,7 @@ mod doc_verify_all { // `asap_frontend_promql::lower_promql` produces — the only algebra IR // the parse path now emits. use super::parse_query_expr_canonical; - use crate::types_v2::AccuracyTarget; + use crate::types::AccuracyTarget; use planner_types::pre_asap::QueryExpr; use planner_types::pre_asap::{AggIntent, Reduction}; diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index bcce290d..20da91fb 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -13,31 +13,6 @@ use std::collections::BTreeMap; use std::collections::BTreeSet; use std::rc::Rc; -pub fn compile_bound( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - bind: F, -) -> Result -where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, -{ - compile_bound_mapped( - query_id, - canonical_query, - root, - instant, - fallback, - bind, - |_, _| {}, - ) -} - pub fn compile_bound_mapped( query_id: String, canonical_query: String, @@ -76,33 +51,6 @@ where }) } -pub fn compile_bound_relational( - query_id: String, - canonical_query: String, - root: &Rc, - fixed_evaluation: FixedEvaluationRange, - instant: InstantExecution, - fallback: FallbackPolicy, - bind: F, -) -> Result -where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, -{ - compile_bound_relational_mapped( - query_id, - canonical_query, - root, - fixed_evaluation, - instant, - fallback, - bind, - |_, _| {}, - ) -} - /// Preserve Planner-to-runtime node identities for installed SQL DAGs. pub fn compile_bound_relational_mapped( query_id: String, @@ -143,33 +91,6 @@ where }) } -/// Compile selected summary nodes and verified native residuals into one DAG. -/// This is a distinct physical alternative; native execution remains available. -pub fn compile_bound_composable( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - bind: F, -) -> Result -where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, -{ - compile_bound_composable_mapped( - query_id, - canonical_query, - root, - instant, - fallback, - bind, - |_, _| {}, - ) -} - /// Compile a composable query while exposing the stable mapping from /// Planner semantic nodes to installed query nodes. The control-plane /// physical compiler uses this to persist backend placement without diff --git a/control_plane/src/query_plan/clickhouse_exact.rs b/control_plane/src/query_plan/clickhouse_exact.rs index fa227841..8551ddfe 100644 --- a/control_plane/src/query_plan/clickhouse_exact.rs +++ b/control_plane/src/query_plan/clickhouse_exact.rs @@ -419,7 +419,7 @@ mod original_tests { let PhysicalExpr::Committed(PostAsapPlan::Summary(root)) = planned.physical else { panic!("missing selected SQL DAG") }; - let entry = crate::query_plan::compile_bound_relational( + let entry = crate::query_plan::compile_bound_relational_mapped( "test".into(), planned.canonical_sql, &root, @@ -435,6 +435,7 @@ mod original_tests { }, FallbackPolicy::ExactBackend, |_, _| Err(QueryPlanError::Invalid("unexpected summary binding".into())), + |_, _| {}, ) .unwrap(); assert!( diff --git a/control_plane/src/query_plan/logical.rs b/control_plane/src/query_plan/logical.rs index e802ea4b..2f7d25cc 100644 --- a/control_plane/src/query_plan/logical.rs +++ b/control_plane/src/query_plan/logical.rs @@ -632,7 +632,7 @@ mod hybrid_tests { .unwrap(); let selected = crate::planner_selection::select_summary_default(&canonical).unwrap(); let entry = - crate::query_plan::compile_bound_composable( + crate::query_plan::compile_bound_composable_mapped( "hybrid".into(), query.into(), &selected, @@ -658,6 +658,7 @@ mod hybrid_tests { readout_lookback_ms: Some(300_000), }) }, + |_, _| {}, ) .unwrap(); assert_eq!(entry.materialization_bindings().len(), 2); diff --git a/control_plane/src/query_planning.rs b/control_plane/src/query_planning.rs index 87bc1704..e9951201 100644 --- a/control_plane/src/query_planning.rs +++ b/control_plane/src/query_planning.rs @@ -50,6 +50,7 @@ pub struct QuerySetPlan { } impl QuerySetPlan { + #[cfg(test)] /// All sketch families this plan allocates, across every metric /// (de-duplicated, order-stable). Convenience for capacity sizing. pub fn all_sketches(&self) -> Vec { diff --git a/control_plane/src/replan.rs b/control_plane/src/replan.rs index cfc184e5..f2f313ee 100644 --- a/control_plane/src/replan.rs +++ b/control_plane/src/replan.rs @@ -184,6 +184,7 @@ impl Replanner { } } + #[cfg(test)] /// Remove every `(metric, role)` mapping for a disconnected agent. pub async fn unregister_agent(&self, agent_id: &str) { self.agent_to_metrics.write().await.remove(agent_id); @@ -894,7 +895,7 @@ mod tests { time_window: Duration::from_secs(300), repeat_every: None, accuracy_sla: 0.01, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, diff --git a/control_plane/src/store/workload.rs b/control_plane/src/store/workload.rs index 14f71f83..81fc0d15 100644 --- a/control_plane/src/store/workload.rs +++ b/control_plane/src/store/workload.rs @@ -115,7 +115,7 @@ mod tests { time_window: Duration::from_secs(300), repeat_every: None, accuracy_sla: 0.01, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, diff --git a/control_plane/src/types.rs b/control_plane/src/types.rs index 079f61f1..38b92b2a 100644 --- a/control_plane/src/types.rs +++ b/control_plane/src/types.rs @@ -260,7 +260,7 @@ pub struct QueryWorkload { pub time_window: Duration, pub repeat_every: Option, /// Authoritative query requirement; never reconstructed from the legacy view. - pub accuracy: crate::types_v2::AccuracyTarget, + pub accuracy: crate::types::AccuracyTarget, /// Deprecated confidence-style view retained for compatibility reporting only. pub accuracy_sla: f64, pub latency_sla: Option, @@ -280,9 +280,9 @@ impl QueryWorkload { /// Scalar sizing input for legacy cost formulas, not a confidence guarantee. pub fn error_bound(&self) -> f64 { match self.accuracy { - crate::types_v2::AccuracyTarget::Exact => 0.0, - crate::types_v2::AccuracyTarget::Epsilon(epsilon) - | crate::types_v2::AccuracyTarget::EpsilonDelta { epsilon, .. } => epsilon, + crate::types::AccuracyTarget::Exact => 0.0, + crate::types::AccuracyTarget::Epsilon(epsilon) + | crate::types::AccuracyTarget::EpsilonDelta { epsilon, .. } => epsilon, } } } @@ -622,16 +622,7 @@ pub struct StageResourceBudgets { pub precompute_memory_bytes: Option, } -impl StageResourceBudgets { - /// Derive budgets from [`WorkloadCharacteristics`]: propagates the agent - /// memory cap; other stages default to unbounded. - pub fn from_workload_chars(wc: &WorkloadCharacteristics) -> Self { - Self { - agent_memory_bytes: wc.memory_budget_bytes, - ..Default::default() - } - } -} +impl StageResourceBudgets {} // ── Collection plan ─────────────────────────────────────────────────────────── @@ -645,3 +636,240 @@ pub struct CollectionPlan { /// Bandwidth and overhead estimates for all three transmission strategies. pub transmission_cost_summary: TransmissionCostSummary, } + +pub use asap_types::QueryLanguage; + +// ── AccuracyTarget ──────────────────────────────────────────────────────────── + +/// Per-target accuracy SLA, shared with ASAPPlanner's `AggIntent` fields. +/// Uses Planner's externally tagged serde representation, e.g. +/// `{"Epsilon": 0.02}`; `EpsilonDelta` names its bound `epsilon`. +pub use planner_types::types::AccuracyTarget; + +/// Translate the legacy `accuracy_sla: f64` field -- a fractional +/// "1.0 = exact, 0.0 = anything goes" SLA -- into the typed form. +/// `accuracy_sla == 1.0` round-trips to `Exact`; everything else becomes +/// `Epsilon(1.0 - accuracy_sla)` (the implied error bound). Free function, +/// not `impl AccuracyTarget` -- Rust's orphan rules don't allow inherent +/// impls on a foreign type. +pub fn accuracy_target_from_legacy_accuracy_sla(accuracy_sla: f64) -> AccuracyTarget { + if accuracy_sla >= 1.0 { + AccuracyTarget::Exact + } else { + AccuracyTarget::Epsilon((1.0 - accuracy_sla).max(0.0)) + } +} + +/// Resolve the public compatibility input once, preserving typed delta/exact semantics. +pub fn resolve_accuracy_target( + typed: Option<&AccuracyTarget>, + legacy_confidence: f64, +) -> Result { + let target = if let Some(target) = typed { + target.clone() + } else { + if !legacy_confidence.is_finite() || !(0.0..=1.0).contains(&legacy_confidence) { + return Err("accuracy_sla must be finite and in [0,1]".into()); + } + accuracy_target_from_legacy_accuracy_sla(legacy_confidence) + }; + let valid = match target { + AccuracyTarget::Exact => true, + AccuracyTarget::Epsilon(epsilon) => epsilon.is_finite() && (0.0..=1.0).contains(&epsilon), + AccuracyTarget::EpsilonDelta { epsilon, delta } => { + epsilon.is_finite() + && (0.0..=1.0).contains(&epsilon) + && delta.is_finite() + && delta > 0.0 + && delta < 1.0 + } + }; + if valid { + Ok(target) + } else { + Err("invalid typed accuracy requirement".into()) + } +} + +// ── QueryShape ──────────────────────────────────────────────────────────────── + +/// How the query is *evaluated*: one-shot, continuous, or scheduled. +/// +/// Drives L4 binding (mergeable vs one-shot sketch family) and the L5 +/// wire format (`OneShot` emits config + result; `Streaming` and +/// `Periodic` emit a config that keeps running). Distinct from +/// [`DataShape`] — see `design.md` §6. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum QueryShape { + /// Evaluate once. Plan, execute, return result, discard state. + /// One-off PromQL via `POST /plan`. + #[default] + OneShot, + /// Continuous query — output stream that the executor keeps emitting + /// as new data arrives. No fixed cadence; the runtime emits whenever + /// the underlying state changes. Streaming dashboards, alerting + /// expressions evaluated by the agent rather than by a poller. + Streaming, + /// Re-evaluated at a fixed cadence — Prometheus recording rules, + /// scheduled dashboard panels, alerting evaluation cycles. The + /// planner amortises sketch / aggregate build cost across + /// evaluations within the cadence and reuses state between + /// adjacent windows. + Periodic { + /// Re-evaluation cadence. + every: Duration, + }, +} + +// ── DataShape ───────────────────────────────────────────────────────────────── + +/// Shape of the *data* feeding the query. Workload-level summary; per-leaf +/// detail rides on `Source::data_shape` per `design.md` §6 L3. +/// +/// Drives L4 binding choices: an `AppendOnlyStream` unlocks incremental, +/// mergeable sketches and retraction-free aggregation; `Batch` lets the +/// planner pick a non-mergeable estimator (e.g. exact percentile over a +/// sort) that wouldn't survive a distributed streaming setting; `Mutable` +/// requires retraction-aware operators (out of scope today — the planner +/// refuses sketch binding and falls back to re-scan). +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +#[serde(rename_all = "snake_case")] +pub enum DataShape { + /// Bounded relation, fully materialised at plan time. Parquet / CSV + /// files, in-process columnar tables. + Batch, + /// Append-only stream — events arrive over time, never updated or + /// deleted. Metrics, logs, event streams. The common case for the + /// asap-collector / asap-query deployments and so the control plane + /// default when the field is omitted. + #[default] + AppendOnlyStream, + /// Mutable relation — inserts + updates + deletes. Operational + /// databases, CRUD-style tables. Sketch binding is currently + /// refused for this shape (no retraction-aware sketches in the + /// catalog yet). + Mutable, + /// Join across sources of differing shape. The planner consults + /// `Source::data_shape` per leaf during L4; this variant exists so + /// callers don't have to flatten a workload-level summary. + Mixed, +} + +// ── QueryId ─────────────────────────────────────────────────────────────────── + +/// Stable identifier preserved across `replan` cycles so the runtime can +/// correlate plan outputs with the originating spec, and L4 reuse rules +/// can name shared producers across consumers in the same workload. +/// +/// Idiomatically a string here — the control plane already round-trips +/// metric names + agent IDs as strings (see `monitor::Endpoint`, +/// `opamp::AgentRole`), so adding a `Uuid` dependency for one field +/// with no DB-side semantics would be churn for no benefit. The HTTP +/// API is JSON; callers can supply any string they want, including a +/// stringified UUID. When omitted, `Analyzer::analyze` derives a +/// deterministic id from the parsed metric name + accuracy target. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[serde(transparent)] +pub struct QueryId(pub String); + +impl std::fmt::Display for QueryId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn query_language_serde_roundtrip() { + let variant = QueryLanguage::PromQl; + let json = serde_json::to_string(&variant).unwrap(); + let back: QueryLanguage = serde_json::from_str(&json).unwrap(); + assert_eq!(variant, back, "round-trip failed for {variant:?}"); + } + + #[test] + fn accuracy_target_serde_roundtrip() { + let cases = [ + AccuracyTarget::Exact, + AccuracyTarget::Epsilon(0.05), + AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }, + ]; + for variant in cases { + let json = serde_json::to_string(&variant).unwrap(); + let back: AccuracyTarget = serde_json::from_str(&json).unwrap(); + assert_eq!(variant, back, "round-trip failed for {variant:?}"); + } + } + + #[test] + fn accuracy_target_from_legacy() { + // 1.0 means exact in the legacy schema. + assert_eq!( + accuracy_target_from_legacy_accuracy_sla(1.0), + AccuracyTarget::Exact + ); + // 0.99 SLA → ε = 0.01. + match accuracy_target_from_legacy_accuracy_sla(0.99) { + AccuracyTarget::Epsilon(eps) => { + assert!((eps - 0.01).abs() < 1e-9, "got eps={eps}"); + } + other => panic!("expected Epsilon, got {other:?}"), + } + // Out-of-range guard — analyzer rejects these upstream, but the + // helper itself must not panic on a 0.0 SLA. + match accuracy_target_from_legacy_accuracy_sla(0.0) { + AccuracyTarget::Epsilon(eps) => assert!((eps - 1.0).abs() < 1e-9), + other => panic!("expected Epsilon, got {other:?}"), + } + } + + #[test] + fn query_shape_serde_roundtrip_and_default() { + assert_eq!(QueryShape::default(), QueryShape::OneShot); + let cases = [ + QueryShape::OneShot, + QueryShape::Streaming, + QueryShape::Periodic { + every: Duration::from_secs(60), + }, + ]; + for variant in cases { + let json = serde_json::to_string(&variant).unwrap(); + let back: QueryShape = serde_json::from_str(&json).unwrap(); + assert_eq!(variant, back, "round-trip failed for {variant:?}"); + } + } + + #[test] + fn data_shape_serde_roundtrip_and_default() { + assert_eq!(DataShape::default(), DataShape::AppendOnlyStream); + for variant in [ + DataShape::Batch, + DataShape::AppendOnlyStream, + DataShape::Mutable, + DataShape::Mixed, + ] { + let json = serde_json::to_string(&variant).unwrap(); + let back: DataShape = serde_json::from_str(&json).unwrap(); + assert_eq!(variant, back, "round-trip failed for {variant:?}"); + } + } + + #[test] + fn query_id_transparent_string_serde() { + let id = QueryId("metric-x@0.99".into()); + let json = serde_json::to_string(&id).unwrap(); + // `#[serde(transparent)]` should serialize as a bare string. + assert_eq!(json, "\"metric-x@0.99\""); + let back: QueryId = serde_json::from_str(&json).unwrap(); + assert_eq!(id, back); + assert_eq!(id.0, "metric-x@0.99"); + } +} diff --git a/control_plane/src/types_v2.rs b/control_plane/src/types_v2.rs deleted file mode 100644 index 24547544..00000000 --- a/control_plane/src/types_v2.rs +++ /dev/null @@ -1,303 +0,0 @@ -//! Typed schema fragments converging toward `control_plane/docs/design.md`. -//! -//! The design defines a typed `QuerySpec` with `id`, `language`, -//! `accuracy: AccuracyTarget`, `shape: QueryShape`, `data: DataShape`, etc. -//! The current `analyzer::QuerySpec` is a JSON-friendly impl that pre-dates -//! that design. -//! -//! Strategy: **additive extension**. The new types live here and are folded -//! into `analyzer::QuerySpec` as `#[serde(default)] Option<…>` fields so -//! existing JSON callers (the `/plan` HTTP endpoint, planner pre-population -//! from `workloads.yaml`, every test that constructs a `QuerySpec`) keep -//! working byte-for-byte. Code that wants to *consume* the new fields can -//! pattern-match on them; code that doesn't care can ignore them. -//! -//! Accuracy is shared with ASAPPlanner and resolved once at the compatibility -//! boundary. The remaining shape/deployment fields retain their existing -//! partial support; they are not silently promoted to executable capabilities. - -// Several types in this module (`BindingName`, the `new` / `as_str` helpers on `QueryId` -// and `BindingName`) are intentionally part of the public surface but -// have no in-tree consumers yet — they're targets for the downstream -// PR that wires the planner to consume the typed schema. Suppress the -// dead_code warnings until then so this module doesn't visually -// regress the lint baseline. -#![allow(dead_code)] - -use std::time::Duration; - -use serde::{Deserialize, Serialize}; - -pub use asap_types::QueryLanguage; - -// ── AccuracyTarget ──────────────────────────────────────────────────────────── - -/// Per-target accuracy SLA, shared with ASAPPlanner's `AggIntent` fields. -/// Uses Planner's externally tagged serde representation, e.g. -/// `{"Epsilon": 0.02}`; `EpsilonDelta` names its bound `epsilon`. -pub use planner_types::types::AccuracyTarget; - -/// Translate the legacy `accuracy_sla: f64` field -- a fractional -/// "1.0 = exact, 0.0 = anything goes" SLA -- into the typed form. -/// `accuracy_sla == 1.0` round-trips to `Exact`; everything else becomes -/// `Epsilon(1.0 - accuracy_sla)` (the implied error bound). Free function, -/// not `impl AccuracyTarget` -- Rust's orphan rules don't allow inherent -/// impls on a foreign type. -pub fn accuracy_target_from_legacy_accuracy_sla(accuracy_sla: f64) -> AccuracyTarget { - if accuracy_sla >= 1.0 { - AccuracyTarget::Exact - } else { - AccuracyTarget::Epsilon((1.0 - accuracy_sla).max(0.0)) - } -} - -/// Resolve the public compatibility input once, preserving typed delta/exact semantics. -pub fn resolve_accuracy_target( - typed: Option<&AccuracyTarget>, - legacy_confidence: f64, -) -> Result { - let target = if let Some(target) = typed { - target.clone() - } else { - if !legacy_confidence.is_finite() || !(0.0..=1.0).contains(&legacy_confidence) { - return Err("accuracy_sla must be finite and in [0,1]".into()); - } - accuracy_target_from_legacy_accuracy_sla(legacy_confidence) - }; - let valid = match target { - AccuracyTarget::Exact => true, - AccuracyTarget::Epsilon(epsilon) => epsilon.is_finite() && (0.0..=1.0).contains(&epsilon), - AccuracyTarget::EpsilonDelta { epsilon, delta } => { - epsilon.is_finite() - && (0.0..=1.0).contains(&epsilon) - && delta.is_finite() - && delta > 0.0 - && delta < 1.0 - } - }; - if valid { - Ok(target) - } else { - Err("invalid typed accuracy requirement".into()) - } -} - -// ── QueryShape ──────────────────────────────────────────────────────────────── - -/// How the query is *evaluated*: one-shot, continuous, or scheduled. -/// -/// Drives L4 binding (mergeable vs one-shot sketch family) and the L5 -/// wire format (`OneShot` emits config + result; `Streaming` and -/// `Periodic` emit a config that keeps running). Distinct from -/// [`DataShape`] — see `design.md` §6. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum QueryShape { - /// Evaluate once. Plan, execute, return result, discard state. - /// One-off PromQL via `POST /plan`. - #[default] - OneShot, - /// Continuous query — output stream that the executor keeps emitting - /// as new data arrives. No fixed cadence; the runtime emits whenever - /// the underlying state changes. Streaming dashboards, alerting - /// expressions evaluated by the agent rather than by a poller. - Streaming, - /// Re-evaluated at a fixed cadence — Prometheus recording rules, - /// scheduled dashboard panels, alerting evaluation cycles. The - /// planner amortises sketch / aggregate build cost across - /// evaluations within the cadence and reuses state between - /// adjacent windows. - Periodic { - /// Re-evaluation cadence. - every: Duration, - }, -} - -// ── DataShape ───────────────────────────────────────────────────────────────── - -/// Shape of the *data* feeding the query. Workload-level summary; per-leaf -/// detail rides on `Source::data_shape` per `design.md` §6 L3. -/// -/// Drives L4 binding choices: an `AppendOnlyStream` unlocks incremental, -/// mergeable sketches and retraction-free aggregation; `Batch` lets the -/// planner pick a non-mergeable estimator (e.g. exact percentile over a -/// sort) that wouldn't survive a distributed streaming setting; `Mutable` -/// requires retraction-aware operators (out of scope today — the planner -/// refuses sketch binding and falls back to re-scan). -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(rename_all = "snake_case")] -pub enum DataShape { - /// Bounded relation, fully materialised at plan time. Parquet / CSV - /// files, in-process columnar tables. - Batch, - /// Append-only stream — events arrive over time, never updated or - /// deleted. Metrics, logs, event streams. The common case for the - /// asap-collector / asap-query deployments and so the control plane - /// default when the field is omitted. - #[default] - AppendOnlyStream, - /// Mutable relation — inserts + updates + deletes. Operational - /// databases, CRUD-style tables. Sketch binding is currently - /// refused for this shape (no retraction-aware sketches in the - /// catalog yet). - Mutable, - /// Join across sources of differing shape. The planner consults - /// `Source::data_shape` per leaf during L4; this variant exists so - /// callers don't have to flatten a workload-level summary. - Mixed, -} - -// ── QueryId / BindingName ───────────────────────────────────────────────────── - -/// Stable identifier preserved across `replan` cycles so the runtime can -/// correlate plan outputs with the originating spec, and L4 reuse rules -/// can name shared producers across consumers in the same workload. -/// -/// Idiomatically a string here — the control plane already round-trips -/// metric names + agent IDs as strings (see `monitor::Endpoint`, -/// `opamp::AgentRole`), so adding a `Uuid` dependency for one field -/// with no DB-side semantics would be churn for no benefit. The HTTP -/// API is JSON; callers can supply any string they want, including a -/// stringified UUID. When omitted, `Analyzer::analyze` derives a -/// deterministic id from the parsed metric name + accuracy target. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] -#[serde(transparent)] -pub struct QueryId(pub String); - -impl QueryId { - pub fn new(id: impl Into) -> Self { - QueryId(id.into()) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl std::fmt::Display for QueryId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.0) - } -} - -/// Name of a hoisted shared sub-expression in a [`WorkloadPlan`]. Each -/// binding is referenced by ≥2 roots via `QueryExpr::Ref` (`design.md` -/// §6 batched-queries example). -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] -#[serde(transparent)] -pub struct BindingName(pub String); - -impl BindingName { - pub fn new(name: impl Into) -> Self { - BindingName(name.into()) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl std::fmt::Display for BindingName { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.0) - } -} - -// ── WorkloadPlan ────────────────────────────────────────────────────────────── - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn query_language_serde_roundtrip() { - let variant = QueryLanguage::PromQl; - let json = serde_json::to_string(&variant).unwrap(); - let back: QueryLanguage = serde_json::from_str(&json).unwrap(); - assert_eq!(variant, back, "round-trip failed for {variant:?}"); - } - - #[test] - fn accuracy_target_serde_roundtrip() { - let cases = [ - AccuracyTarget::Exact, - AccuracyTarget::Epsilon(0.05), - AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.001, - }, - ]; - for variant in cases { - let json = serde_json::to_string(&variant).unwrap(); - let back: AccuracyTarget = serde_json::from_str(&json).unwrap(); - assert_eq!(variant, back, "round-trip failed for {variant:?}"); - } - } - - #[test] - fn accuracy_target_from_legacy() { - // 1.0 means exact in the legacy schema. - assert_eq!( - accuracy_target_from_legacy_accuracy_sla(1.0), - AccuracyTarget::Exact - ); - // 0.99 SLA → ε = 0.01. - match accuracy_target_from_legacy_accuracy_sla(0.99) { - AccuracyTarget::Epsilon(eps) => { - assert!((eps - 0.01).abs() < 1e-9, "got eps={eps}"); - } - other => panic!("expected Epsilon, got {other:?}"), - } - // Out-of-range guard — analyzer rejects these upstream, but the - // helper itself must not panic on a 0.0 SLA. - match accuracy_target_from_legacy_accuracy_sla(0.0) { - AccuracyTarget::Epsilon(eps) => assert!((eps - 1.0).abs() < 1e-9), - other => panic!("expected Epsilon, got {other:?}"), - } - } - - #[test] - fn query_shape_serde_roundtrip_and_default() { - assert_eq!(QueryShape::default(), QueryShape::OneShot); - let cases = [ - QueryShape::OneShot, - QueryShape::Streaming, - QueryShape::Periodic { - every: Duration::from_secs(60), - }, - ]; - for variant in cases { - let json = serde_json::to_string(&variant).unwrap(); - let back: QueryShape = serde_json::from_str(&json).unwrap(); - assert_eq!(variant, back, "round-trip failed for {variant:?}"); - } - } - - #[test] - fn data_shape_serde_roundtrip_and_default() { - assert_eq!(DataShape::default(), DataShape::AppendOnlyStream); - for variant in [ - DataShape::Batch, - DataShape::AppendOnlyStream, - DataShape::Mutable, - DataShape::Mixed, - ] { - let json = serde_json::to_string(&variant).unwrap(); - let back: DataShape = serde_json::from_str(&json).unwrap(); - assert_eq!(variant, back, "round-trip failed for {variant:?}"); - } - } - - #[test] - fn query_id_transparent_string_serde() { - let id = QueryId::new("metric-x@0.99"); - let json = serde_json::to_string(&id).unwrap(); - // `#[serde(transparent)]` should serialize as a bare string. - assert_eq!(json, "\"metric-x@0.99\""); - let back: QueryId = serde_json::from_str(&json).unwrap(); - assert_eq!(id, back); - assert_eq!(id.as_str(), "metric-x@0.99"); - } -} diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs index 91c9269d..d851d32b 100644 --- a/control_plane/src/workload.rs +++ b/control_plane/src/workload.rs @@ -147,7 +147,7 @@ pub fn derive_agg_role(entry: &WorkloadEntry) -> AggRole { let Some(qs) = entry.query_string.as_ref() else { return AggRole::Other; }; - let accuracy = crate::types_v2::accuracy_target_from_legacy_accuracy_sla(entry.accuracy_sla); + let accuracy = crate::types::accuracy_target_from_legacy_accuracy_sla(entry.accuracy_sla); let Ok(expr) = crate::query_parser::parse_query_expr_canonical(qs, accuracy) else { return AggRole::Other; }; @@ -400,8 +400,8 @@ pub fn query_spec_for_entry(entry: &WorkloadEntry) -> crate::pipeline::QuerySpec accuracy: None, dollars: None, deployment_model: None, - shape: crate::types_v2::QueryShape::default(), - data: crate::types_v2::DataShape::default(), + shape: crate::types::QueryShape::default(), + data: crate::types::DataShape::default(), } } @@ -593,6 +593,7 @@ impl WorkloadRegistry { &self.entries } + #[cfg(test)] /// Returns workload entries assigned to a given role. pub fn for_role(&self, role: &str) -> Vec<&WorkloadEntry> { self.entries diff --git a/control_plane/tests/o11y_exact_fallback.rs b/control_plane/tests/o11y_exact_fallback.rs index e43508a0..3122ea08 100644 --- a/control_plane/tests/o11y_exact_fallback.rs +++ b/control_plane/tests/o11y_exact_fallback.rs @@ -1,7 +1,7 @@ use control_plane::{ physical::post_asap::{bind_query_expr, PhysicalExpr, PostAsapPlan}, query_parser::parse_query_expr_canonical, - types_v2::AccuracyTarget, + types::AccuracyTarget, }; use planner_types::post_asap::SummaryExpr; @@ -26,14 +26,14 @@ fn o11y_non_summary_roots_preserve_exact_query_semantics() { }; assert_eq!(original.as_ref(), &expr, "query semantics changed: {query}"); assert_eq!(node.schema.fields.len(), expr.output_schema().unwrap().columns.len()); - let executable = control_plane::query_plan::compile_bound( + let executable = control_plane::query_plan::compile_bound_mapped( "fixture".into(), query.into(), &node, control_plane::query_plan::InstantExecution { lookback_ms: 300_000, full_history: false, cumulative_readout: false, }, control_plane::query_plan::FallbackPolicy::Reject, |_, _| panic!("exact fallback must not request summary materializations"), - ).unwrap(); + |_, _| {}).unwrap(); assert!(matches!(executable.nodes[&executable.root], control_plane::query_plan::QueryPlanNode::ExactFallback { .. })); assert!(executable.materialization_bindings().is_empty()); diff --git a/control_plane/tests/offline_evidence.rs b/control_plane/tests/offline_evidence.rs index 0c4a936b..381bb10a 100644 --- a/control_plane/tests/offline_evidence.rs +++ b/control_plane/tests/offline_evidence.rs @@ -9,7 +9,7 @@ use control_plane::{ PostAsapPlan, }, query_parser::parse_query_expr_canonical, - types_v2::AccuracyTarget, + types::AccuracyTarget, }; use planner_types::{ post_asap::{SketchAlgorithm, SketchParams, SummaryExpr, SummaryFamilyType, SummaryNode}, @@ -362,7 +362,7 @@ fn binary_summary_has_explicit_warm_tier_fallback() { schema: child.schema.clone(), guarantee: None, }); - let plan = control_plane::query_plan::compile_bound( + let plan = control_plane::query_plan::compile_bound_mapped( "test".into(), "left / right".into(), &root, @@ -373,6 +373,7 @@ fn binary_summary_has_explicit_warm_tier_fallback() { }, FallbackPolicy::Reject, |_, _| panic!("unsupported binary plan must not bind a materialization"), + |_, _| {}, ) .unwrap(); assert!( diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index 52f27b24..c6ba3c04 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -196,14 +196,7 @@ pub struct AggregationIdInfo { pub aggregation_type_for_value: AggregationType, } -impl AggregationIdInfo { - pub fn policy_fp_for_key(&self) -> PolicyFingerprint { - PolicyFingerprint(self.aggregation_id_for_key) - } - pub fn policy_fp_for_value(&self) -> PolicyFingerprint { - PolicyFingerprint(self.aggregation_id_for_value) - } -} +impl AggregationIdInfo {} /// Compatibility name for legacy streaming-config and precompute call sites. /// New PhysicalPlan code should use [`PrecomputeMaterialization`]. @@ -354,11 +347,6 @@ impl PrecomputeMaterialization { self.policy_fingerprint().as_u64() } - pub fn with_original_yaml(mut self, yaml: String) -> Self { - self.original_yaml = yaml; - self - } - pub fn deserialize_from_json( data: &Value, ) -> Result> { diff --git a/crates/asap_types/src/aggregation_type.rs b/crates/asap_types/src/aggregation_type.rs index 2e03ef5c..a19734e7 100644 --- a/crates/asap_types/src/aggregation_type.rs +++ b/crates/asap_types/src/aggregation_type.rs @@ -73,29 +73,6 @@ impl AggregationType { | AggregationType::HydraKLL ) } - - /// Returns `true` if this type needs a paired key aggregation (SetAggregator / DeltaSetAggregator). - pub fn is_multi_population_value_type(self) -> bool { - matches!( - self, - AggregationType::MultipleSum - | AggregationType::MultipleMinMax - | AggregationType::MultipleIncrease - | AggregationType::CountMinSketch - | AggregationType::CountMinSketchWithHeap - | AggregationType::CountSketch - | AggregationType::CountSketchWithHeap - ) - } - - /// Returns `true` if this is a key-tracking aggregation type. - /// Retained as a stable predicate for downstream callers; the - /// historical `SetAggregator` / `DeltaSetAggregator` set-tracking - /// family has been retired (no `AggregationType` is key-tracking - /// today). - pub fn is_key_agg_type(self) -> bool { - false - } } impl fmt::Display for AggregationType { diff --git a/crates/asap_types/src/grouping_projection.rs b/crates/asap_types/src/grouping_projection.rs index 8bb955cb..49cd1cdd 100644 --- a/crates/asap_types/src/grouping_projection.rs +++ b/crates/asap_types/src/grouping_projection.rs @@ -372,6 +372,7 @@ pub fn encode_label_population_key( )) } +#[cfg(test)] /// Reject legacy, unknown-version, duplicate-name and noncanonical encodings. /// In particular, decoding must never silently collapse duplicate populations. pub fn decode_label_population_key( diff --git a/crates/asap_types/src/policy_registry.rs b/crates/asap_types/src/policy_registry.rs index 945556d2..af8ca349 100644 --- a/crates/asap_types/src/policy_registry.rs +++ b/crates/asap_types/src/policy_registry.rs @@ -55,6 +55,7 @@ impl PolicyRegistry { Self { policies } } + #[cfg(test)] /// Construct + report the count of duplicate fingerprints (entries /// where the source contained two configs producing the same /// fingerprint and the later one displaced the earlier). Zero in diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 8b0c48e8..7b4bcef3 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -299,46 +299,6 @@ pub struct InstantExecution { } impl QueryPlanEntry { - /// Replace an explicit planner fallback cut with a typed external-exact leaf. - /// The control plane chooses the cut; serving only executes the published DAG. - pub fn bind_external_exact_leaf( - &mut self, - node_id: QueryNodeId, - request: ExternalExactRequest, - ) -> Result<(), QueryPlanError> { - if request.language != self.language { - return Err(QueryPlanError::Invalid( - "external exact language differs from its query plan".into(), - )); - } - if !request.input_contracts.is_empty() { - return Err(QueryPlanError::Invalid( - "leaf binding cannot declare DAG input contracts".into(), - )); - } - match self.nodes.get(&node_id) { - Some(QueryPlanNode::ExactFallback { .. }) => {} - Some(_) => { - return Err(QueryPlanError::Invalid( - "external exact binding must replace a planner fallback cut".into(), - )) - } - None => { - return Err(QueryPlanError::Invalid( - "external exact cut node is absent".into(), - )) - } - } - self.nodes.insert( - node_id, - QueryPlanNode::ExternalExact { - request, - inputs: Vec::new(), - }, - ); - Ok(()) - } - /// Materializations this executable DAG reads, in stable node order. /// Serving uses this set for readiness accounting; it never performs a /// catalog candidate search to reconstruct dependencies. diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs index ce2c7e48..4d1ac062 100644 --- a/crates/asap_types/src/summary_catalog.rs +++ b/crates/asap_types/src/summary_catalog.rs @@ -62,6 +62,7 @@ pub enum SummaryCatalogError { } impl CatalogGeneration { + #[cfg(test)] /// Validate an untrusted wire reference against the installed immutable /// snapshot and its enclosing plan generation. pub fn validate_snapshot( diff --git a/data_plane/src/drivers/control_plane_client/config_fetcher.rs b/data_plane/src/drivers/control_plane_client/config_fetcher.rs deleted file mode 100644 index 0f86b4d1..00000000 --- a/data_plane/src/drivers/control_plane_client/config_fetcher.rs +++ /dev/null @@ -1,154 +0,0 @@ -//! Control-plane client. -//! -//! Fetches query configuration from the control plane's API -//! instead of reading a static `inference_config.yaml` file. -//! -//! This replaces the pattern-matching approach: the control plane already -//! knows which queries are precomputed and which sketches to use. - -use serde::Deserialize; -use tracing::{info, warn}; - -/// Configuration for the control-plane client. -#[derive(Debug, Clone)] -pub struct ControlPlaneClientConfig { - /// Base URL of the control plane (e.g., "http://control-plane:8080"). - pub base_url: String, - /// How often to poll for config updates (seconds). 0 = poll only on query miss. - pub poll_interval_secs: u64, -} - -impl Default for ControlPlaneClientConfig { - fn default() -> Self { - Self { - base_url: "http://localhost:8080".to_string(), - poll_interval_secs: 0, - } - } -} - -/// Slim plan status returned by `GET /api/v1/plan/:metric`. -/// -/// The GET endpoint only returns the metric name, sketch type, and -/// validity. Callers that need the richer plan (mode, aggregate_by, -/// staged_plan, etc.) must use [`ControlPlaneClient::create_plan`] -/// which invokes `POST /api/v1/plan`. -#[derive(Debug, Clone, Deserialize)] -pub struct ControlPlanePlanStatus { - pub metric: String, - pub sketch_type: String, - #[serde(default)] - pub valid_until: Option, -} - -/// Rich plan returned by `POST /api/v1/plan`. -#[derive(Debug, Clone, Deserialize)] -pub struct ControlPlanePlan { - pub metric: String, - pub sketch_type: String, - #[serde(default)] - pub mode: String, - #[serde(default)] - pub aggregate_by: Vec, - #[serde(default)] - pub delta_decision: serde_json::Value, - #[serde(default)] - pub staged_plan: Option, -} - -/// Client for the control plane's API. -pub struct ControlPlaneClient { - config: ControlPlaneClientConfig, - http: reqwest::Client, -} - -impl ControlPlaneClient { - pub fn new(config: ControlPlaneClientConfig) -> Self { - Self { - config, - http: reqwest::Client::new(), - } - } - - /// Look up the current plan status for a specific metric. - /// - /// Invokes `GET /api/v1/plan/:metric`. Returns `None` if the control - /// plane has no plan for this metric (404) or is unreachable. The - /// response is a slim `{ metric, sketch_type, valid_until }` shape — - /// for the full plan, use [`Self::create_plan`]. - pub async fn get_plan(&self, metric: &str) -> Option { - let url = format!("{}/api/v1/plan/{}", self.config.base_url, metric); - - match self.http.get(&url).send().await { - Ok(resp) => { - if resp.status().is_success() { - match resp.json::().await { - Ok(plan) => { - info!(metric = %metric, sketch_type = %plan.sketch_type, "Fetched plan status from control plane"); - Some(plan) - } - Err(e) => { - warn!(metric = %metric, error = %e, "Failed to parse control-plane plan status"); - None - } - } - } else { - None // 404 = no plan exists - } - } - Err(e) => { - warn!(url = %url, error = %e, "Control plane unreachable"); - None - } - } - } - - /// Create or refresh a plan by posting a `QuerySpec` to the control plane. - /// - /// Invokes `POST /api/v1/plan`. The `query_spec` is passed through as the - /// JSON body; refer to the control plane's `QuerySpec` type for the exact - /// schema. Returns the rich plan (mode, aggregate_by, staged plan, delta - /// decision, etc.) on success. - pub async fn create_plan(&self, query_spec: serde_json::Value) -> Option { - let url = format!("{}/api/v1/plan", self.config.base_url); - - match self.http.post(&url).json(&query_spec).send().await { - Ok(resp) if resp.status().is_success() => match resp.json::().await { - Ok(plan) => { - info!(metric = %plan.metric, sketch_type = %plan.sketch_type, "Created plan via control plane"); - Some(plan) - } - Err(e) => { - warn!(error = %e, "Failed to parse control-plane plan response"); - None - } - }, - Ok(resp) => { - warn!(status = %resp.status(), "Control plane rejected plan request"); - None - } - Err(e) => { - warn!(url = %url, error = %e, "Control plane unreachable"); - None - } - } - } - - /// Check if the control plane is healthy by listing connected agents. - pub async fn health_check(&self) -> bool { - let url = format!("{}/api/v1/agents", self.config.base_url); - match self.http.get(&url).send().await { - Ok(resp) => resp.status().is_success(), - Err(_) => false, - } - } - - /// Fetch the generated collector config YAML for a metric. - pub async fn get_config_yaml(&self, metric: &str) -> Option { - let url = format!("{}/api/v1/config/{}", self.config.base_url, metric); - match self.http.get(&url).send().await { - Ok(resp) if resp.status().is_success() => resp.text().await.ok(), - _ => None, - } - } -} diff --git a/data_plane/src/drivers/control_plane_client/mod.rs b/data_plane/src/drivers/control_plane_client/mod.rs index 6d37e7b8..b5dfbff5 100644 --- a/data_plane/src/drivers/control_plane_client/mod.rs +++ b/data_plane/src/drivers/control_plane_client/mod.rs @@ -3,7 +3,6 @@ //! Two cooperating submodules — bidirectional plumbing between the //! data plane and the control plane (`control_plane` crate): //! -//! * [`config_fetcher`] — outbound `GET /api/v1/plan/:metric` polling. //! Fetches plan config from the control plane. Optional; today's //! binary path does not consume it. //! * [`miss_notifier`] — outbound capability-miss notifications. The @@ -12,7 +11,6 @@ //! generate a new sketch plan and push it back via the streaming //! config endpoint. -pub mod config_fetcher; pub mod miss_notifier; pub use miss_notifier::{spawn_capability_miss_notify, ControlPlaneClient, HttpControlPlaneClient}; diff --git a/data_plane/src/drivers/ingest/series_resolver.rs b/data_plane/src/drivers/ingest/series_resolver.rs index d037fad6..938815ae 100644 --- a/data_plane/src/drivers/ingest/series_resolver.rs +++ b/data_plane/src/drivers/ingest/series_resolver.rs @@ -248,14 +248,6 @@ impl SeriesIdResolver { self.cache.get(&key).map(|v| *v) } - /// Reverse lookup: given a sid, is it known? Used at receive time - /// when an Export carries a sid != 0 with empty attributes — backend - /// must verify it knows the sid; otherwise stamp `unknown_series_ids` - /// in the response. - pub fn is_known(&self, sid: u64) -> bool { - self.cache.iter().any(|kv| *kv.value() == sid) - } - /// Number of registered identities. Used for telemetry / debugging. pub fn len(&self) -> usize { self.cache.len() diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index a5df79c2..bec47188 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -275,6 +275,7 @@ impl HttpServer { self } + #[cfg(test)] /// Plug an additional [`QueryEngine`] into the capability router. /// Used by the binary to register `GorillaQueryEngine` (cold /// archive) alongside the `ASAPQueryEngine` registered by `new`. diff --git a/data_plane/src/drivers/query/servers/metrics.rs b/data_plane/src/drivers/query/servers/metrics.rs index b480ef0d..46cbeeec 100644 --- a/data_plane/src/drivers/query/servers/metrics.rs +++ b/data_plane/src/drivers/query/servers/metrics.rs @@ -79,21 +79,3 @@ pub fn record_query_outcome(query_type: &str, status: &str) { .with_label_values(&[query_type, status]) .inc(); } - -pub fn start_ingest_timer(protocol: &str) -> HistogramTimer { - INGEST_BATCH_DURATION_SECONDS - .with_label_values(&[protocol]) - .start_timer() -} - -pub fn record_ingest_samples(protocol: &str, count: u64) { - INGEST_SAMPLES_TOTAL - .with_label_values(&[protocol]) - .inc_by(count as f64); -} - -pub fn record_ingest_decode_error(protocol: &str) { - INGEST_DECODE_ERRORS_TOTAL - .with_label_values(&[protocol]) - .inc(); -} diff --git a/data_plane/src/monitor/coordinator.rs b/data_plane/src/monitor/coordinator.rs index 8142dcaa..5b323e86 100644 --- a/data_plane/src/monitor/coordinator.rs +++ b/data_plane/src/monitor/coordinator.rs @@ -116,6 +116,7 @@ impl Monitor { pub fn round(&self) -> u64 { self.round } + #[cfg(test)] pub fn edge_count(&self) -> usize { self.edges.len() } diff --git a/data_plane/src/monitor/server.rs b/data_plane/src/monitor/server.rs index f24b7962..b5b8d9fe 100644 --- a/data_plane/src/monitor/server.rs +++ b/data_plane/src/monitor/server.rs @@ -54,6 +54,7 @@ impl MonitorCoordinator { }) } + #[cfg(test)] /// Number of configured monitors (test/observability). pub fn monitor_count(&self) -> usize { self.cfgs.read().unwrap().len() diff --git a/data_plane/src/precompute_engine/accumulator_factory.rs b/data_plane/src/precompute_engine/accumulator_factory.rs index c033ada4..22fdea46 100644 --- a/data_plane/src/precompute_engine/accumulator_factory.rs +++ b/data_plane/src/precompute_engine/accumulator_factory.rs @@ -899,6 +899,7 @@ impl AccumulatorUpdater for HydraKllAccumulatorUpdater { // Config helpers // --------------------------------------------------------------------------- +#[cfg(test)] /// Return `true` if `config` produces a keyed (MultipleSubpopulation) updater, /// without allocating an updater object. /// diff --git a/data_plane/src/precompute_engine/coordination_checkpoint.rs b/data_plane/src/precompute_engine/coordination_checkpoint.rs index 1381ba79..249fad53 100644 --- a/data_plane/src/precompute_engine/coordination_checkpoint.rs +++ b/data_plane/src/precompute_engine/coordination_checkpoint.rs @@ -228,6 +228,7 @@ impl SummaryCoordinationCheckpointStore { }) } + #[cfg(test)] pub fn publish_if_absent(&self, key: AtomicPublicationKey) -> io::Result { key.validate()?; self.mutate(|document| { diff --git a/data_plane/src/precompute_engine/group_key.rs b/data_plane/src/precompute_engine/group_key.rs index dd3c5684..44212278 100644 --- a/data_plane/src/precompute_engine/group_key.rs +++ b/data_plane/src/precompute_engine/group_key.rs @@ -45,6 +45,7 @@ impl GroupKey { } } + #[cfg(test)] pub fn canonical_bytes(&self) -> &[u8] { &self.canonical } diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index ab1f35d4..0f5149d2 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -842,6 +842,7 @@ fn evaluate_frozen_maintenance_sink( Ok((state, digest)) } +#[cfg(test)] /// Execute an installed, single-population maintenance subDAG from frozen /// base panes and publish its complete output through the durable part path. /// This initial entry point accepts non-overlapping output windows; sliding diff --git a/data_plane/src/precompute_engine/metrics.rs b/data_plane/src/precompute_engine/metrics.rs index 3e703359..4bf1e844 100644 --- a/data_plane/src/precompute_engine/metrics.rs +++ b/data_plane/src/precompute_engine/metrics.rs @@ -33,6 +33,7 @@ pub struct ThroughputTotals { } impl ThroughputTotals { + #[cfg(test)] pub fn since(self, previous: Self) -> Self { Self { accepted_samples: self @@ -48,6 +49,7 @@ impl ThroughputTotals { } } +#[cfg(test)] pub fn throughput_totals() -> ThroughputTotals { ThroughputTotals { accepted_samples: ACCEPTED_SAMPLES_TOTAL.get(), diff --git a/data_plane/src/precompute_engine/multisource_coordinator.rs b/data_plane/src/precompute_engine/multisource_coordinator.rs index d64cf643..0b305fa8 100644 --- a/data_plane/src/precompute_engine/multisource_coordinator.rs +++ b/data_plane/src/precompute_engine/multisource_coordinator.rs @@ -87,6 +87,7 @@ impl MultiSourceCoordinator { }) } + #[cfg(test)] /// Bind the complete installed producer roster before accepting any input /// or barrier. This validates scope; transport authentication and durable /// source payload publication remain caller obligations. @@ -280,6 +281,7 @@ impl MultiSourceCoordinator { self.checkpoint_store.advance_watermark(barrier) } + #[cfg(test)] pub fn ready_batches(&self) -> io::Result> { let _transition = self .transition @@ -332,30 +334,6 @@ impl MultiSourceCoordinator { Ok(ready) } - pub fn publication_key( - &self, - batch: &ReadyInputBatch, - sink_node_id: impl Into, - instance_id: SummaryInstanceId, - output_definition: PolicyFingerprint, - output_lineage: Vec, - state_reference: SummaryStateReference, - ) -> AtomicPublicationKey { - AtomicPublicationKey { - catalog_generation: self.spec.catalog_generation.clone(), - dag_id: self.spec.dag_id.clone(), - sink_node_id: sink_node_id.into(), - instance_id, - coordinates: SummaryInstanceCoordinates { - summary_definition_id: output_definition.into(), - time_range: batch.time_range, - group_values: batch.group_values.clone(), - }, - output_lineage, - state_reference, - } - } - fn validate_input(&self, input: &StagedSummaryInput) -> io::Result<()> { if input.catalog_generation != self.spec.catalog_generation || input.dag_id != self.spec.dag_id diff --git a/data_plane/src/precompute_engine/operators/edge_runtime_adapter.rs b/data_plane/src/precompute_engine/operators/edge_runtime_adapter.rs index d55b4b82..fba53304 100644 --- a/data_plane/src/precompute_engine/operators/edge_runtime_adapter.rs +++ b/data_plane/src/precompute_engine/operators/edge_runtime_adapter.rs @@ -267,6 +267,7 @@ pub fn encode_ddsketch_envelope(sk: &asap_sketchlib::DdSketch) -> Vec { buf } +#[cfg(test)] /// Merge two `DdSketch` instances by routing through asap-precompute-rs's /// runtime `Sketch::merge`. The result is byte-identical to /// `asap_sketchlib::DdSketch::merge_refs(&[a, b])` because diff --git a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs b/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs index d0bad06e..c1d59aa1 100644 --- a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs @@ -23,10 +23,6 @@ impl MultipleIncreaseAccumulator { } } - pub fn new_with_increases(increases: HashMap) -> Self { - Self { increases } - } - pub fn update(&mut self, key: KeyByLabelValues, accumulator: IncreaseAccumulator) { self.increases.insert(key, accumulator); } diff --git a/data_plane/src/precompute_engine/operators/multiple_min_max_accumulator.rs b/data_plane/src/precompute_engine/operators/multiple_min_max_accumulator.rs index 46c9277d..e65766c4 100644 --- a/data_plane/src/precompute_engine/operators/multiple_min_max_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/multiple_min_max_accumulator.rs @@ -36,14 +36,6 @@ impl MultipleMinMaxAccumulator { Self::new("max".to_string()) } - pub fn new_with_values(values: HashMap, sub_type: String) -> Self { - if sub_type != "min" && sub_type != "max" { - panic!("sub_type must be 'min' or 'max'"); - } - - Self { values, sub_type } - } - pub fn update(&mut self, key: KeyByLabelValues, value: f64) { match self.sub_type.as_str() { "min" => { @@ -62,6 +54,7 @@ impl MultipleMinMaxAccumulator { } } + #[cfg(test)] pub fn add_value(&mut self, key: KeyByLabelValues, value: f64) { self.values.insert(key, value); } diff --git a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs b/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs index 5b7e1011..85a9982d 100644 --- a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs @@ -22,10 +22,6 @@ impl MultipleSumAccumulator { } } - pub fn new_with_sums(sums: HashMap) -> Self { - Self { sums } - } - pub fn update(&mut self, key: KeyByLabelValues, value: f64) { *self.sums.entry(key).or_insert(0.0) += value; } diff --git a/data_plane/src/precompute_engine/operators/sketch_envelope_accumulator.rs b/data_plane/src/precompute_engine/operators/sketch_envelope_accumulator.rs index 1264122f..08956e0a 100644 --- a/data_plane/src/precompute_engine/operators/sketch_envelope_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/sketch_envelope_accumulator.rs @@ -56,11 +56,6 @@ impl SketchEnvelopeAccumulator { sketch_type, }) } - - /// Return the decoded envelope (re-decodes from bytes each time). - pub fn decode_envelope(&self) -> Result { - SketchEnvelope::decode(self.payload.as_slice()) - } } // --------------------------------------------------------------------------- diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index a26876eb..e1187041 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -15,13 +15,6 @@ use tracing::{debug_span, warn}; /// /metrics scrape never silently loses policy-miss drops. static GLOBAL_DROPPED_POLICY_MISS: AtomicU64 = AtomicU64::new(0); -/// Read the process-global output-sink policy-miss drop count. Exposed -/// so the /metrics surface can fold it in for sinks not yet wired to an -/// `IngestObservability`. -pub fn global_dropped_policy_miss() -> u64 { - GLOBAL_DROPPED_POLICY_MISS.load(Ordering::Relaxed) -} - /// Trait for emitting completed window outputs. pub trait OutputSink: Send + Sync { fn emit_batch( @@ -91,6 +84,7 @@ impl SketchStoreSink { } } + #[cfg(test)] /// CQ-6 — attach a shared `IngestObservability` so this sink's /// policy-miss drops increment the same counter the ingest path /// reports. Builder-style (returns `self`) so the `new()` signature diff --git a/data_plane/src/precompute_engine/series_buffer.rs b/data_plane/src/precompute_engine/series_buffer.rs index a6631423..2d8d8e48 100644 --- a/data_plane/src/precompute_engine/series_buffer.rs +++ b/data_plane/src/precompute_engine/series_buffer.rs @@ -42,6 +42,7 @@ impl SeriesBuffer { self.watermark_ms } + #[cfg(test)] /// Read all samples in `[start_ms, end_ms)` — inclusive start, exclusive end. /// Returns them in timestamp order. pub fn read_range(&self, start_ms: i64, end_ms: i64) -> Vec<(i64, f64)> { @@ -51,6 +52,7 @@ impl SeriesBuffer { .collect() } + #[cfg(test)] /// Drain (remove and return) all samples with `timestamp_ms < up_to_ms`. pub fn drain_up_to(&mut self, up_to_ms: i64) -> Vec<(i64, f64)> { let mut drained = Vec::new(); diff --git a/data_plane/src/precompute_engine/series_router.rs b/data_plane/src/precompute_engine/series_router.rs index de878604..751f47e5 100644 --- a/data_plane/src/precompute_engine/series_router.rs +++ b/data_plane/src/precompute_engine/series_router.rs @@ -241,18 +241,6 @@ impl SeriesRouter { Ok(()) } - /// Atomically reserve queue capacity for an entire request and then - /// publish it. If any target worker is full or closed, every reservation - /// is dropped and no message is enqueued. Remote Write uses this to turn - /// bounded-queue pressure into a retryable HTTP response without leaving - /// an untracked partial request behind. - pub fn try_route_group_batch_atomic( - &self, - messages: Vec, - ) -> Result<(), TryRouteError> { - self.try_route_group_batch_with_admission(messages, || Ok(None)) - } - pub fn try_route_group_batch_with_admission( &self, messages: Vec, @@ -306,17 +294,6 @@ impl SeriesRouter { Ok(()) } - /// Broadcast shutdown to all workers. - pub async fn broadcast_shutdown(&self) -> Result<(), Box> { - for (i, sender) in self.senders.iter().enumerate() { - sender - .send(WorkerMessage::Shutdown) - .await - .map_err(|e| format!("Failed to send shutdown to worker {}: {}", i, e))?; - } - Ok(()) - } - /// Caller must stop new input before invoking this finite-source barrier. pub async fn drain(&self) -> Result<(), String> { let mut replies = Vec::new(); diff --git a/data_plane/src/precompute_engine/window_manager.rs b/data_plane/src/precompute_engine/window_manager.rs index ed0b84b9..f198103d 100644 --- a/data_plane/src/precompute_engine/window_manager.rs +++ b/data_plane/src/precompute_engine/window_manager.rs @@ -178,6 +178,7 @@ impl WindowManager { origin + (timestamp_ms - origin).div_euclid(self.pane_interval_ms) * self.pane_interval_ms } + #[cfg(test)] /// All pane starts composing a window, in ascending order. /// A window `[ws, ws + window_size)` is composed of /// `window_size / slide_interval` consecutive panes. diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index b7a819ab..bdead2c9 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -1335,6 +1335,7 @@ fn precomputed_output_for_group( output } +#[cfg(test)] /// Extract the metric name from a series key like `"metric_name{key1=\"val1\"}"`. pub fn extract_metric_name(series_key: &str) -> &str { match series_key.find('{') { @@ -1343,6 +1344,7 @@ pub fn extract_metric_name(series_key: &str) -> &str { } } +#[cfg(test)] /// Extract grouping label values from a series key string based on the /// aggregation config's `grouping_labels`. /// diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs index 26183c1f..d4ecfa38 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs @@ -200,27 +200,6 @@ fn complete_pane_coverage( pane_ms > 0 && first_end.saturating_sub(pane_ms) <= requested.0 && last_end >= requested.1 } -/// Executes only the published physical DAG; serving performs no parsing, -/// summary selection, or materialization candidate search. -pub fn execute_sql_dag( - index: &SketchStore, - entry: &QueryPlanEntry, - sds: &SummaryCatalog, - t0_ms: u64, - t1_ms: u64, - is_cumulative: bool, -) -> ClickHouseDagOutcome { - execute_sql_dag_with_external( - index, - entry, - sds, - &PreparedExternalLeaves::new(), - t0_ms, - t1_ms, - is_cumulative, - ) -} - pub fn execute_sql_dag_with_external( index: &SketchStore, entry: &QueryPlanEntry, diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index e3ad2944..5a4f8d96 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -490,6 +490,7 @@ impl ASAPQueryEngine { .filter(|plan| plan.plan_id() != 0) } + #[cfg(test)] /// Phase-5 hybrid-stitch builder — attach an archive engine the /// `QueryEngine` trait adapter will dispatch to when the ASAP-tier /// reducer reports a coverage narrower than the requested range. @@ -528,34 +529,6 @@ impl ASAPQueryEngine { self } - /// Resolve the timeline of agg-signatures for a metric over a - /// query range. Reads exclusively from the sid catalog via - /// [`crate::storage_engines::sketch_db::query::timeline::timeline_for_metric`] - /// — schema retirement routed this away from the now-deleted - /// per-metric schema registry. - /// - /// When no `SketchStore` is wired (test contexts that never - /// installed one via [`Self::with_sketch_index`]) returns an - /// empty vector; downstream dispatch then bails to the default - /// single-agg path, identical to the pre-retirement behaviour - /// where an empty schema registry produced no segments. - pub fn timeline_for_query( - &self, - metric: &str, - t1_ms: u64, - t2_ms: u64, - ) -> Vec { - let Some(idx) = self.sketch_index.as_ref() else { - return Vec::new(); - }; - crate::storage_engines::sketch_db::query::timeline::timeline_for_metric( - idx.as_ref(), - metric, - t1_ms, - t2_ms, - ) - } - /// Build a minimal `QueryRequirements` from a bare PromQL string — /// used by the no-sketch-index miss branch in modern `execute()`, /// where we don't have a parsed candidate (analysis was skipped) @@ -695,19 +668,10 @@ impl ASAPQueryEngine { step_ms, ) } - Err(reason) => Err(crate::query_engines::asap_query_engine::post_asap_planner::LoweringSkip::QueryNotPlanned(reason.to_string())), + Err(reason) => Err(crate::query_engines::asap_query_engine::post_asap_readout::LoweringSkip::QueryNotPlanned(reason.to_string())), }, - #[cfg(test)] - None => crate::query_engines::asap_query_engine::live_serve::serve_from_summary_executor( - idx, - query, - start_ms, - end_ms, - false, - control_plane::types_v2::AccuracyTarget::Epsilon(0.01), - ), - #[cfg(not(test))] - None => Err(crate::query_engines::asap_query_engine::post_asap_planner::LoweringSkip::QueryNotPlanned( + + None => Err(crate::query_engines::asap_query_engine::post_asap_readout::LoweringSkip::QueryNotPlanned( "no active physical QueryPlan".into(), )), }; @@ -721,7 +685,7 @@ impl ASAPQueryEngine { requirement.max_window_ms, ); let Some(active) = self.active_physical_plan.as_ref() else { - return Err(crate::query_engines::asap_query_engine::post_asap_planner::LoweringSkip::MaterializationNotReady( + return Err(crate::query_engines::asap_query_engine::post_asap_readout::LoweringSkip::MaterializationNotReady( "physical readiness registry is unavailable".into(), )); }; @@ -732,7 +696,7 @@ impl ASAPQueryEngine { &requirement.materializations, result.coverage, ); - return Err(crate::query_engines::asap_query_engine::post_asap_planner::LoweringSkip::MaterializationNotReady( + return Err(crate::query_engines::asap_query_engine::post_asap_readout::LoweringSkip::MaterializationNotReady( format!("coverage {:?} does not completely and freshly cover [{start_ms}, {end_ms}]", result.coverage), )); } @@ -748,7 +712,7 @@ impl ASAPQueryEngine { &requirement.materializations, coverage, ) { - return Err(crate::query_engines::asap_query_engine::post_asap_planner::LoweringSkip::MaterializationNotReady( + return Err(crate::query_engines::asap_query_engine::post_asap_readout::LoweringSkip::MaterializationNotReady( "physical generation changed while checking readiness".into(), )); } @@ -1049,14 +1013,10 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu idx, query_entry, now_ms, ) }, - Err(reason) => Err(crate::query_engines::asap_query_engine::post_asap_planner::LoweringSkip::QueryNotPlanned(reason.to_string())), + Err(reason) => Err(crate::query_engines::asap_query_engine::post_asap_readout::LoweringSkip::QueryNotPlanned(reason.to_string())), }, - #[cfg(test)] - None => crate::query_engines::asap_query_engine::live_serve::serve_instant_from_summary_executor( - idx, query, now_ms, - ), - #[cfg(not(test))] - None => Err(crate::query_engines::asap_query_engine::post_asap_planner::LoweringSkip::QueryNotPlanned( + + None => Err(crate::query_engines::asap_query_engine::post_asap_readout::LoweringSkip::QueryNotPlanned( "no active physical QueryPlan".into(), )), }; @@ -1070,7 +1030,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu requirement.max_window_ms, ); let Some(active) = self.active_physical_plan.as_ref() else { - return Err(crate::query_engines::asap_query_engine::post_asap_planner::LoweringSkip::MaterializationNotReady( + return Err(crate::query_engines::asap_query_engine::post_asap_readout::LoweringSkip::MaterializationNotReady( "physical readiness registry is unavailable".into(), )); }; @@ -1081,7 +1041,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu &requirement.materializations, result.coverage, ); - return Err(crate::query_engines::asap_query_engine::post_asap_planner::LoweringSkip::MaterializationNotReady( + return Err(crate::query_engines::asap_query_engine::post_asap_readout::LoweringSkip::MaterializationNotReady( format!("coverage {:?} does not completely and freshly cover [{t0_ms}, {now_ms}]", result.coverage), )); } @@ -1097,7 +1057,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu &requirement.materializations, coverage, ) { - return Err(crate::query_engines::asap_query_engine::post_asap_planner::LoweringSkip::MaterializationNotReady( + return Err(crate::query_engines::asap_query_engine::post_asap_readout::LoweringSkip::MaterializationNotReady( "physical generation changed while checking readiness".into(), )); } @@ -1619,18 +1579,7 @@ mod aux_pushdown_tests { // HLL-Count contract; closes the wire-side ingest gap diagnosis. // =========================================================================== // =========================================================================== -// Capability matching — Rate over CountMinSketch (PR #111 honest-gap -// closure). With the new `Statistic::Rate` arm in -// `compatible_agg_types`, `rate([])` against a CMS-only -// agg config now matches. -// =========================================================================== -/// `QueryEngine::execute` ASAP-tier classification tests. -/// Pre-Phase-5 the trait adapter unconditionally delegated to -/// `handle_query`. After Phase 5 wire-in, when a `SketchStore` is -/// attached, the adapter classifies first and surfaces -/// `EngineError::CapabilityMiss(SketchStore, ...)` on Ghost / Unknown -/// / no-instance outcomes so the EngineRouter can fall -/// through to the archive engine. +/// Installed plans bind runtime reads and enforce complete coverage. #[cfg(test)] mod asap_tier_classify_tests { use super::*; @@ -1643,180 +1592,6 @@ mod asap_tier_classify_tests { use crate::storage_engines::types::{CleanupPolicy, HotReloadStreamingConfig}; use std::collections::{BTreeMap, BTreeSet}; - fn build_engine_with_index(idx: Arc) -> ASAPQueryEngine { - let streaming_config = Arc::new(crate::storage_engines::types::StreamingConfig::default()); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_config); - ASAPQueryEngine::new(15000).with_sketch_index(idx) - } - - fn dd_meta(sid: u64, metric: &str, group_by: &[&str]) -> SketchInstanceMetadata { - let cfg = SketchConfig::DDSketch { - relative_accuracy: 0.01, - }; - SketchInstanceMetadata { - sid, - metric_name: metric.to_string(), - group_by_keys: group_by - .iter() - .map(|s| s.to_string()) - .collect::>(), - capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), - agg_kind: crate::storage_engines::sketch_db::index::AggKind::Sketch { - algorithm: SketchAlgorithm::DDSketch, - config: cfg.clone(), - spatial_filter_canonical: String::new(), - }, - accuracy: Some(AccuracyBound::from_config(&cfg)), - first_seen_unix_ms: 0, - retired_at_ms: None, - expires_at_ms: None, - policy_fp: asap_types::PolicyFingerprint::UNSET, - } - } - - #[tokio::test] - async fn execute_returns_capability_miss_when_no_instance_matches() { - // No instance for `unknown_metric` is registered → adapter must - // capability-miss rather than burn a `handle_query` round-trip. - let idx = Arc::new(SketchStore::new()); - let engine = build_engine_with_index(idx); - let err = engine - .execute("unknown_metric{zone=\"z0\"}") - .await - .expect_err("ASAP-tier with no matching instance must yield CapabilityMiss"); - match err { - EngineError::CapabilityMiss { engine_id, .. } => { - assert_eq!( - engine_id, - crate::storage_engines::types::StorageBackend::SketchStore.data_source_id() - ); - } - other => panic!("expected CapabilityMiss, got {other:?}"), - } - } - - #[tokio::test] - async fn execute_returns_capability_miss_when_classify_is_ghost() { - // Register instance metadata but never call append_sample. With - // `dd_meta`'s `PolicyFingerprint::UNSET`, the engine's policy-fp - // lookup misses entirely — there's no policy in the (empty) - // registry to bind the sid to. Prior to the legacy-fallback - // removal, the engine would walk `instances_matching` and find - // the registered sid, classify it as Ghost (no sample state), - // and produce a "ghost/unknown" detail. After removal, the - // ghost lookup short-circuits at the policy-resolution step. - // The CapabilityMiss outcome is preserved; we just don't - // pin the detail string. - let idx = Arc::new(SketchStore::new()); - idx.register(dd_meta(1, "http_latency_ms", &["zone"])); - let engine = build_engine_with_index(idx); - let err = engine - .execute("quantile_over_time(0.99, http_latency_ms{zone=\"z0\"}[5m])") - .await - .expect_err("ghost sid registration must yield CapabilityMiss"); - match err { - EngineError::CapabilityMiss { engine_id, .. } => { - assert_eq!( - engine_id, - crate::storage_engines::types::StorageBackend::SketchStore.data_source_id() - ); - } - other => panic!("expected CapabilityMiss, got {other:?}"), - } - } - - #[tokio::test] - async fn execute_bare_selector_falls_over_to_archive() { - // A bare selector has no aggregate root and fails over to archive before - // policy lookup, regardless of the registered DDSketch policy. - let idx = Arc::new(SketchStore::new()); - idx.register(dd_meta(2, "http_latency_ms", &["zone"])); - idx.append_sample( - 2, - BTreeMap::from([("zone".to_string(), "z0".to_string())]), - (1_000, 1_010), - SketchSampleState { - bytes: vec![0], - encoding: crate::storage_engines::sketch_db::index::SketchEncoding::ProtoFull, - }, - ); - - let engine = build_engine_with_index(idx); - let result = engine.execute("http_latency_ms{zone=\"z0\"}").await; - match result { - Err(EngineError::CapabilityMiss { detail, .. }) => { - assert!( - detail.contains("ExactAgg(Sum)") - || detail.contains("no policy") - || detail.contains("NotRealized"), - "expected a capability-miss fall-over to archive: {detail}" - ); - } - other => panic!("expected CapabilityMiss fall-over to archive, got {other:?}"), - } - } - - /// Schema-retirement #5 regression: a sketch sid registered with - /// a wider-than-requested `group_by_keys` and `policy_fp=UNSET` - /// must still be findable by the query path. Mirrors the MVP - /// end-to-end failure (issue #271 / tracking #272): the agent - /// emits DDSketch DPs carrying every wire attribute, so the sid - /// catalog ends up with `group_by_keys=[zone,rack,node,pod,...]` - /// and `derive_sketch_policy_fp` returns `UNSET` because no - /// streaming-config policy has that exact key set. The query - /// asks for `grouping=[zone]` — a subset. With the policy-fp-only - /// lookup the query returned `CapabilityMiss → archive`; with the - /// `instances_matching` fallback restored it resolves to the sid - /// (and bottoms out at the reducer's sample-state check rather - /// than at sid resolution). - #[tokio::test] - async fn full_attr_sketch_sid_findable_via_subset_grouping() { - let idx = Arc::new(SketchStore::new()); - // Register with the SUPERSET of attrs the agent would emit: - // zone, rack, node, pod — none of which the streaming-config - // would list directly in `grouping_labels=[zone]`. - idx.register(dd_meta( - 42, - "http_latency_ms", - &["node", "pod", "rack", "zone"], - )); - idx.append_sample( - 42, - BTreeMap::from([ - ("zone".to_string(), "z0".to_string()), - ("rack".to_string(), "r0".to_string()), - ("node".to_string(), "n0".to_string()), - ("pod".to_string(), "p0".to_string()), - ]), - (1_000, 1_010), - SketchSampleState { - bytes: vec![0], - encoding: crate::storage_engines::sketch_db::index::SketchEncoding::ProtoFull, - }, - ); - - let engine = build_engine_with_index(idx); - // The query asks for grouping=[zone] (subset of registered - // group_by_keys). Pre-fix this returned CapabilityMiss because - // `series_ids_for_policy(UNSET)` is empty; post-fix the fallback - // finds sid 42 via `instances_matching` and the request - // proceeds to the reducer. - let result = engine - .execute("quantile_over_time(0.99, http_latency_ms{zone=\"z0\"}[5m])") - .await; - // The reducer can't produce a real quantile from the canned - // payload (just `vec![0]`), but it MUST reach the reducer — - // the sid-resolution-step CapabilityMiss with "no policy for - // metric" detail is the regression we're guarding against. - if let Err(EngineError::CapabilityMiss { detail, .. }) = &result { - assert!( - !detail.contains("has no policy for metric"), - "regression: sid was lost at policy-resolution step \ - instead of being found via instances_matching: {detail}" - ); - } - } - /// `sum by (zone) (http_requests_total)` end-to-end via the /// `execute(&str)` adapter. Mirrors the MVP acceptance test's Axis-C /// failure: ExactAgg(Sum) sids existed for `http_requests_total` @@ -1839,10 +1614,7 @@ mod asap_tier_classify_tests { let zones = ["z0", "z1", "z2", "z3"]; // Anchor windows so the engine's instant-query default // lookback (5 min) reaches them. - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); + let now_ms = 600_000_u64; let window_start = now_ms.saturating_sub(60_000); let window_end = now_ms.saturating_sub(30_000); @@ -1875,9 +1647,29 @@ mod asap_tier_classify_tests { ); } - let engine = build_engine_with_index(idx); + use crate::query_engines::asap_query_engine::test_plan; + use asap_types::query_plan::*; + let config = test_plan::materialization( + "http_requests_total", + "Sum", + serde_json::json!({}), + &["zone"], + 30000, + ); + let query = "sum by (zone) (http_requests_total)"; + let entry = test_plan::entry( + query, + &config, + PhysicalGrouping::Reduce(vec!["zone".into()]), + 30000, + QueryPlanNode::ExactReadout { + input: QueryNodeId(0), + readout: ExactReadout::Sum, + }, + ); + let engine = test_plan::engine(idx, config, vec![9000, 9001, 9002, 9003], entry); let result = engine - .execute("sum by (zone) (http_requests_total)") + .execute_at(query, window_end) .await .expect("sum by (zone) must dispatch to ExactAgg reducer, not capability-miss"); @@ -2008,16 +1800,79 @@ mod asap_tier_classify_tests { /// so the executor returns the HLL distinct-count directly. A single /// FULL HLL frame (~500 users) is used so /// the instant projection reads the real estimate. + /// Stored sketches cannot authorize a query without an installed QueryPlan. + #[tokio::test] + async fn unplanned_queries_fail_closed_even_with_matching_sketches() { + let idx = Arc::new(SketchStore::new()); + idx.register(hll_meta(7500, "unique_users_per_min")); + idx.append_sample( + 7500, + BTreeMap::new(), + (1_000, 2_000), + SketchSampleState { + bytes: encode_hll_with_cardinality(14, 500), + encoding: crate::storage_engines::sketch_db::index::SketchEncoding::ProtoFull, + }, + ); + let engine = ASAPQueryEngine::new(15000).with_sketch_index(idx); + let query = "count(unique_users_per_min)"; + let instant = engine.execute_at(query, 2_000).await; + let range = engine + .execute_range_promql_modern(query, 1_000, 2_000, 1_000) + .await; + assert!( + instant.is_err(), + "instant execution must require an installed plan" + ); + assert!( + range.is_err(), + "range execution must require an installed plan" + ); + } + + /// An installed plan authorizes only its exact query identity and still + /// requires readable materializations before serving. + #[tokio::test] + async fn installed_plan_rejects_unknown_queries_and_missing_samples() { + use crate::query_engines::asap_query_engine::test_plan; + use asap_types::query_plan::*; + let idx = Arc::new(SketchStore::new()); + idx.register(hll_meta(7500, "unique_users_per_min")); + let config = test_plan::materialization( + "unique_users_per_min", + "HLL", + serde_json::json!({"precision":14}), + &[], + 1000, + ); + let query = "count(unique_users_per_min)"; + let entry = test_plan::entry( + query, + &config, + PhysicalGrouping::Reduce(vec![]), + 1000, + QueryPlanNode::SummaryEstimate { + input: QueryNodeId(0), + query: QueryReadout::Cardinality, + }, + ); + let engine = test_plan::engine(idx, config, vec![7500], entry); + for query in [query, "count(other_metric)"] { + assert!(engine.execute_at(query, 2000).await.is_err()); + assert!(engine + .execute_range_promql_modern(query, 2000, 2000, 1000) + .await + .is_err()); + } + } + #[tokio::test] async fn execute_count_hll_returns_cardinality_not_rowcount() { let idx = Arc::new(SketchStore::new()); let sid = 7500u64; idx.register(hll_meta(sid, "unique_users_per_min")); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); + let now_ms = 600_000_u64; idx.append_sample( sid, BTreeMap::new(), @@ -2028,8 +1883,28 @@ mod asap_tier_classify_tests { }, ); - let engine = build_engine_with_index(idx); - let result = engine.execute("count(unique_users_per_min)").await.expect( + use crate::query_engines::asap_query_engine::test_plan; + use asap_types::query_plan::*; + let config = test_plan::materialization( + "unique_users_per_min", + "HLL", + serde_json::json!({"precision":14}), + &[], + 1000, + ); + let query = "count(unique_users_per_min)"; + let entry = test_plan::entry( + query, + &config, + PhysicalGrouping::Reduce(vec![]), + 1000, + QueryPlanNode::SummaryEstimate { + input: QueryNodeId(0), + query: QueryReadout::Cardinality, + }, + ); + let engine = test_plan::engine(idx, config, vec![7500], entry); + let result = engine.execute_at(query, now_ms - 2000).await.expect( "count(hll_metric) must dispatch to the Cardinality family \ via the candidate capability (empty trace function) and \ return the HLL distinct-count, NOT capability-miss", @@ -2094,10 +1969,7 @@ mod asap_tier_classify_tests { #[tokio::test] async fn execute_count_hll_global_merges_registers_across_series() { let idx = Arc::new(SketchStore::new()); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); + let now_ms = 600_000_u64; let w_start = now_ms.saturating_sub(3_000); let w_end = now_ms.saturating_sub(2_000); @@ -2127,9 +1999,29 @@ mod asap_tier_classify_tests { ); } - let engine = build_engine_with_index(idx); + use crate::query_engines::asap_query_engine::test_plan; + use asap_types::query_plan::*; + let config = test_plan::materialization( + "unique_users_global", + "HLL", + serde_json::json!({"precision":14}), + &[], + 1000, + ); + let query = "count(unique_users_global)"; + let entry = test_plan::entry( + query, + &config, + PhysicalGrouping::Reduce(vec![]), + 1000, + QueryPlanNode::SummaryEstimate { + input: QueryNodeId(0), + query: QueryReadout::Cardinality, + }, + ); + let engine = test_plan::engine(idx, config, vec![8200, 8201], entry); let result = engine - .execute("count(unique_users_global)") + .execute_at(query, w_end) .await .expect("global count(hll_metric) must answer, not capability-miss"); @@ -2182,13 +2074,10 @@ mod asap_tier_classify_tests { let sid = 7100u64; idx.register(kll_meta(sid, "http_requests_total_latency_ms")); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - // A 10s window ending 5s ago — comfortably inside the 30s range. - let window_start = now_ms.saturating_sub(15_000); - let window_end = now_ms.saturating_sub(5_000); + let now_ms = 600_000_u64; + // One complete pane covers the requested 30 second lookback. + let window_start = now_ms.saturating_sub(30_000); + let window_end = now_ms; let items: Vec = (1..=50).map(|i| i as f64).collect(); let bytes = encode_kll_items_proto(269, &items); @@ -2226,16 +2115,34 @@ mod asap_tier_classify_tests { async fn engine_quantile_result( idx: Arc, - _now_ms: u64, + now_ms: u64, ) -> crate::query_engines::query_result::QueryResult { - let engine = build_engine_with_index(idx); - engine - .execute("quantile_over_time(0.99, http_requests_total_latency_ms[30s])") - .await - .expect( - "quantile_over_time over a Hit KLL sid must NOT capability-miss \ + use crate::query_engines::asap_query_engine::test_plan; + use asap_types::query_plan::*; + let config = test_plan::materialization( + "http_requests_total_latency_ms", + "DatasketchesKLL", + serde_json::json!({"k":269}), + &[], + 30_000, + ); + let query = "quantile_over_time(0.99, http_requests_total_latency_ms[30s])"; + let entry = test_plan::entry( + query, + &config, + PhysicalGrouping::PerEntity, + 30_000, + QueryPlanNode::SummaryEstimate { + input: QueryNodeId(0), + query: QueryReadout::Quantile { q: 0.99 }, + }, + ); + let sids = idx.snapshot_instances().iter().map(|m| m.sid).collect(); + let engine = test_plan::engine(idx, config, sids, entry); + engine.execute_at(query, now_ms).await.expect( + "quantile_over_time over a Hit KLL sid must NOT capability-miss \ (if it does, the bug is upstream of the reducer)", - ) + ) } fn result_nonempty(r: &crate::query_engines::query_result::QueryResult) -> bool { @@ -2266,10 +2173,7 @@ mod asap_tier_classify_tests { let sid = 7400u64; idx.register(kll_meta(sid, "http_requests_total_latency_ms")); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); + let now_ms = 600_000_u64; let items: Vec = (1..=50).map(|i| i as f64).collect(); // Full at now-60s..now-55s — OUTSIDE the 30s window. @@ -2282,11 +2186,11 @@ mod asap_tier_classify_tests { encoding: crate::storage_engines::sketch_db::index::SketchEncoding::ProtoFull, }, ); - // Delta at now-15s..now-5s — INSIDE the window. + // A delta pane covers the complete requested lookback. idx.append_sample( sid, BTreeMap::new(), - (now_ms.saturating_sub(15_000), now_ms.saturating_sub(5_000)), + (now_ms.saturating_sub(30_000), now_ms), SketchSampleState { bytes: encode_kll_items_proto(269, &items), encoding: crate::storage_engines::sketch_db::index::SketchEncoding::ProtoDelta, @@ -2320,15 +2224,12 @@ mod asap_tier_classify_tests { let sid = 7300u64; idx.register(kll_meta(sid, "http_requests_total_latency_ms")); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); + let now_ms = 600_000_u64; let items: Vec = (1..=50).map(|i| i as f64).collect(); idx.append_sample( sid, BTreeMap::new(), - (now_ms.saturating_sub(15_000), now_ms.saturating_sub(5_000)), + (now_ms.saturating_sub(30_000), now_ms), SketchSampleState { bytes: encode_kll_items_proto(269, &items), encoding: crate::storage_engines::sketch_db::index::SketchEncoding::ProtoDelta, @@ -2343,148 +2244,6 @@ mod asap_tier_classify_tests { ); } - /// `rate(http_requests_total[5m])` end-to-end via `execute(&str)`. - /// The Planner input marks the inner aggregate as `AggIntent::Rate`; - /// bare `rate(...)` is one of the - /// shapes `SummaryExecutor` self-excludes before ever binding - /// (`LoweringSkip::RateShape` -- it has no rate-division logic), and - /// there's no legacy reducer left to fall through to. This test used - /// to pin the ExactAgg-rate reducer dispatch; now it pins the - /// accepted replacement outcome: capability-miss, failing over to - /// archive. - #[tokio::test] - async fn execute_rate_capability_misses_no_legacy_fallback() { - use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; - use crate::storage_engines::sketch_db::data::AggregationType; - - let idx = Arc::new(SketchStore::new()); - // Two zones, each its own sid, two windows each. The windows - // span `[now-150s, now-30s]` = 120s of ACTUAL coverage inside - // the requested 300s `[5m]` lookback. Post-#301 the rate divisor - // is the actual coverage span (`min(300, 120) = 120`), NOT the - // nominal 300 — so z0 = (600+600)/120 = 10.0; z1 = - // (900+900)/120 = 15.0. (Both windows stay strictly inside - // `[engine_now-300_000, engine_now]` so the window-contained - // range query captures them regardless of the small skew between - // the test's captured `now_ms` and the engine's query-time now.) - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - let w1_start = now_ms.saturating_sub(150_000); - let w1_end = now_ms.saturating_sub(90_000); - let w2_start = w1_end; - let w2_end = now_ms.saturating_sub(30_000); - - for (i, (zone, per_window)) in [("z0", 600.0_f64), ("z1", 900.0)].iter().enumerate() { - let sid = 11_000 + i as u64; - idx.register(SketchInstanceMetadata { - sid, - metric_name: "http_requests_total".to_string(), - group_by_keys: ["zone".to_string()].into_iter().collect(), - capability: Some(Capability::ExactAgg(AggregationType::Sum)), - agg_kind: crate::storage_engines::sketch_db::index::AggKind::ExactAgg { - agg_type: AggregationType::Sum, - parameters_canonical: String::new(), - spatial_filter_canonical: String::new(), - }, - accuracy: None, - first_seen_unix_ms: 0, - retired_at_ms: None, - expires_at_ms: None, - policy_fp: asap_types::PolicyFingerprint::UNSET, - }); - for (ws, we) in [(w1_start, w1_end), (w2_start, w2_end)] { - let mut lm = BTreeMap::new(); - lm.insert("zone".to_string(), zone.to_string()); - idx.append_precompute( - sid, - lm, - (ws, we), - Box::new(SumAccumulator::with_sum(*per_window)), - ); - } - } - - let engine = build_engine_with_index(idx); - let result = engine.execute("rate(http_requests_total[5m])").await; - assert!( - matches!(result, Err(EngineError::CapabilityMiss { .. })), - "rate() must capability-miss with no legacy reducer fallback, got {result:?}" - ); - } - - /// Regression (issue #301, decision (a)): `sum_over_time(counter[r])` - /// shares `Capability::ExactAgg(Sum)` with `rate`/`increase`/`sum`, - /// but its PromQL semantic (Σ of CUMULATIVE sample values in `[r]`, - /// a quadratic) CANNOT be reconstructed from the per-window deltas - /// asap stores. Rather than fabricate a wrong number, the engine - /// recognizes exact Sum over a Planner `TimeRange` and returns a - /// capability-miss so the query routes to the archive tier. Before - /// #301 this returned the delta-sum (1500 here) — a wrong answer the - /// caller couldn't distinguish from a correct one. - #[tokio::test] - async fn execute_sum_over_time_over_counter_capability_misses_to_archive() { - use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; - use crate::storage_engines::sketch_db::data::AggregationType; - - let idx = Arc::new(SketchStore::new()); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - let w1_start = now_ms.saturating_sub(120_000); - let w1_end = now_ms.saturating_sub(60_000); - let w2_start = w1_end; - let w2_end = now_ms.saturating_sub(1_000); - - for (i, (zone, per_window)) in [("z0", 600.0_f64), ("z1", 900.0)].iter().enumerate() { - let sid = 13_000 + i as u64; - idx.register(SketchInstanceMetadata { - sid, - metric_name: "http_requests_total".to_string(), - group_by_keys: ["zone".to_string()].into_iter().collect(), - capability: Some(Capability::ExactAgg(AggregationType::Sum)), - agg_kind: crate::storage_engines::sketch_db::index::AggKind::ExactAgg { - agg_type: AggregationType::Sum, - parameters_canonical: String::new(), - spatial_filter_canonical: String::new(), - }, - accuracy: None, - first_seen_unix_ms: 0, - retired_at_ms: None, - expires_at_ms: None, - policy_fp: asap_types::PolicyFingerprint::UNSET, - }); - for (ws, we) in [(w1_start, w1_end), (w2_start, w2_end)] { - let mut lm = BTreeMap::new(); - lm.insert("zone".to_string(), zone.to_string()); - idx.append_precompute( - sid, - lm, - (ws, we), - Box::new(SumAccumulator::with_sum(*per_window)), - ); - } - } - - let engine = build_engine_with_index(idx); - let err = engine - .execute("sum_over_time(http_requests_total[5m])") - .await - .expect_err( - "sum_over_time over a counter sid MUST capability-miss → archive \ - (issue #301 decision (a)); it must NOT fabricate a delta-sum", - ); - assert!( - matches!( - err, - crate::query_engines::EngineError::CapabilityMiss { .. } - ), - "expected CapabilityMiss for sum_over_time over counter, got {err:?}" - ); - } - /// Issue #301 Layer 3: instant `sum(counter)` must return the /// cumulative-since-storage value (Σ of ALL windows' deltas), NOT /// the most-recent window's delta. Two windows of 600/900 per zone @@ -2499,14 +2258,11 @@ mod asap_tier_classify_tests { use crate::storage_engines::sketch_db::data::AggregationType; let idx = Arc::new(SketchStore::new()); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); + let now_ms = 600_000_u64; let w1_start = now_ms.saturating_sub(120_000); let w1_end = now_ms.saturating_sub(60_000); let w2_start = w1_end; - let w2_end = now_ms.saturating_sub(1_000); + let w2_end = now_ms; for (i, (zone, per_window)) in [("z0", 600.0_f64), ("z1", 900.0)].iter().enumerate() { let sid = 14_000 + i as u64; @@ -2538,10 +2294,30 @@ mod asap_tier_classify_tests { } } - let engine = build_engine_with_index(idx); - let result = engine - .execute("sum by (zone) (http_requests_total)") - .await + use crate::query_engines::asap_query_engine::test_plan; + use asap_types::query_plan::*; + let config = test_plan::materialization( + "http_requests_total", + "Sum", + serde_json::json!({}), + &["zone"], + 60000, + ); + let query = "sum by (zone) (http_requests_total)"; + let entry = test_plan::entry( + query, + &config, + PhysicalGrouping::Reduce(vec!["zone".into()]), + 120000, + QueryPlanNode::ExactReadout { + input: QueryNodeId(0), + readout: ExactReadout::Sum, + }, + ); + let engine = test_plan::engine(idx, config, vec![14000, 14001], entry); + let result = engine + .execute_at(query, now_ms) + .await .expect("instant sum by zone must succeed"); let vector = match result { QueryResult::Vector(v) => v, @@ -2570,203 +2346,6 @@ mod asap_tier_classify_tests { ); } - /// `sketch_reducer.rs` retirement, strict-matching decision: Sum and - /// Increase are the same physical accumulator, but planning's - /// decision for THIS metric (what got registered) is `Sum`, not - /// `Increase` -- and serving must reproduce exactly what was - /// planned, not treat the two labels as interchangeable. A sid - /// registered as `ExactAgg(Sum)` therefore correctly capability-misses - /// an `increase(...)` query and fails over to archive, rather than - /// silently answering under a label planning never chose. - #[tokio::test] - async fn execute_increase_over_sum_registered_sid_capability_misses() { - use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; - use crate::storage_engines::sketch_db::data::AggregationType; - - let idx = Arc::new(SketchStore::new()); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - let w1_start = now_ms.saturating_sub(120_000); - let w1_end = now_ms.saturating_sub(60_000); - let w2_start = w1_end; - let w2_end = now_ms.saturating_sub(1_000); - - for (i, (zone, per_window)) in [("z0", 600.0_f64), ("z1", 900.0)].iter().enumerate() { - let sid = 15_000 + i as u64; - idx.register(SketchInstanceMetadata { - sid, - metric_name: "http_requests_total".to_string(), - group_by_keys: ["zone".to_string()].into_iter().collect(), - capability: Some(Capability::ExactAgg(AggregationType::Sum)), - agg_kind: crate::storage_engines::sketch_db::index::AggKind::ExactAgg { - agg_type: AggregationType::Sum, - parameters_canonical: String::new(), - spatial_filter_canonical: String::new(), - }, - accuracy: None, - first_seen_unix_ms: 0, - retired_at_ms: None, - expires_at_ms: None, - policy_fp: asap_types::PolicyFingerprint::UNSET, - }); - for (ws, we) in [(w1_start, w1_end), (w2_start, w2_end)] { - let mut lm = BTreeMap::new(); - lm.insert("zone".to_string(), zone.to_string()); - idx.append_precompute( - sid, - lm, - (ws, we), - Box::new(SumAccumulator::with_sum(*per_window)), - ); - } - } - - let engine = build_engine_with_index(idx); - let result = engine.execute("increase(http_requests_total[5m])").await; - assert!( - matches!(result, Err(EngineError::CapabilityMiss { .. })), - "increase() over a Sum-registered sid must capability-miss \ - (strict planning/serving match), got {result:?}" - ); - } - - /// An inner Rate intent excludes this composed query from summary execution; - /// the engine must report a capability miss. - #[tokio::test] - async fn execute_sum_by_zone_rate_capability_misses_no_legacy_fallback() { - use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; - use crate::storage_engines::sketch_db::data::AggregationType; - - let idx = Arc::new(SketchStore::new()); - // Four zones. Two windows each spanning `[now-150s, now-30s]` = - // 120s of actual coverage inside the 300s `[5m]` lookback. - // Post-#301 the rate divisor is the actual coverage span - // (`min(300, 120) = 120`), not the nominal 300. - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - let w1_start = now_ms.saturating_sub(150_000); - let w1_end = now_ms.saturating_sub(90_000); - let w2_start = w1_end; - let w2_end = now_ms.saturating_sub(30_000); - - let zones = ["z0", "z1", "z2", "z3"]; - for (i, zone) in zones.iter().enumerate() { - let sid = 12_000 + i as u64; - idx.register(SketchInstanceMetadata { - sid, - metric_name: "http_requests_total".to_string(), - group_by_keys: ["zone".to_string()].into_iter().collect(), - capability: Some(Capability::ExactAgg(AggregationType::Sum)), - agg_kind: crate::storage_engines::sketch_db::index::AggKind::ExactAgg { - agg_type: AggregationType::Sum, - parameters_canonical: String::new(), - spatial_filter_canonical: String::new(), - }, - accuracy: None, - first_seen_unix_ms: 0, - retired_at_ms: None, - expires_at_ms: None, - policy_fp: asap_types::PolicyFingerprint::UNSET, - }); - // per_window: 300, 600, 900, 1200 → per-zone totals 600, - // 1200, 1800, 2400 → rates over the 120s coverage are - // 5, 10, 15, 20. - let per_window = ((i + 1) * 300) as f64; - for (ws, we) in [(w1_start, w1_end), (w2_start, w2_end)] { - let mut lm = BTreeMap::new(); - lm.insert("zone".to_string(), zone.to_string()); - idx.append_precompute( - sid, - lm, - (ws, we), - Box::new(SumAccumulator::with_sum(per_window)), - ); - } - } - - let engine = build_engine_with_index(idx); - let result = engine - .execute("sum by (zone) (rate(http_requests_total[5m]))") - .await; - assert!( - matches!(result, Err(EngineError::CapabilityMiss { .. })), - "composed sum-by-rate must capability-miss with no legacy reducer \ - fallback, got {result:?}" - ); - } - - /// TopK over grouped rate fails over to archive for both K ≥ n and K < n. - /// The inner Rate intent excludes the entire query from summary execution. - #[tokio::test] - async fn execute_topk_over_sum_by_zone_rate_capability_misses_no_legacy_fallback() { - use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; - use crate::storage_engines::sketch_db::data::AggregationType; - - let idx = Arc::new(SketchStore::new()); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - let w_start = now_ms.saturating_sub(150_000); - let w_end = now_ms.saturating_sub(30_000); - - let zones = ["z0", "z1", "z2", "z3"]; - for (i, zone) in zones.iter().enumerate() { - let sid = 13_000 + i as u64; - idx.register(SketchInstanceMetadata { - sid, - metric_name: "http_requests_total".to_string(), - group_by_keys: ["zone".to_string()].into_iter().collect(), - capability: Some(Capability::ExactAgg(AggregationType::Sum)), - agg_kind: crate::storage_engines::sketch_db::index::AggKind::ExactAgg { - agg_type: AggregationType::Sum, - parameters_canonical: String::new(), - spatial_filter_canonical: String::new(), - }, - accuracy: None, - first_seen_unix_ms: 0, - retired_at_ms: None, - expires_at_ms: None, - policy_fp: asap_types::PolicyFingerprint::UNSET, - }); - let per_window = ((i + 1) * 300) as f64; - let mut lm = BTreeMap::new(); - lm.insert("zone".to_string(), zone.to_string()); - idx.append_precompute( - sid, - lm, - (w_start, w_end), - Box::new(SumAccumulator::with_sum(per_window)), - ); - } - - let engine = build_engine_with_index(idx); - - // K ≥ n (5 ≥ 4 zones). - let result = engine - .execute("topk(5, sum by (zone) (rate(http_requests_total[5m])))") - .await; - assert!( - matches!(result, Err(EngineError::CapabilityMiss { .. })), - "topk(5, ...) over sum-rate must capability-miss with no legacy \ - fallback, got {result:?}" - ); - - // K < n (2 < 4 zones) -- same shape, different K, same outcome. - let result = engine - .execute("topk(2, sum by (zone) (rate(http_requests_total[5m])))") - .await; - assert!( - matches!(result, Err(EngineError::CapabilityMiss { .. })), - "topk(2, ...) over sum-rate must capability-miss with no legacy \ - fallback, got {result:?}" - ); - } - // ── P1-1 / P2-6 — rate over FrequencyEstimate (CMS) + keyed safe-miss ── /// A FrequencyEstimate (CountMin) sid with `total` inserts in its @@ -2842,94 +2421,41 @@ mod asap_tier_classify_tests { .encode_to_vec() } - fn now_ms_for_test() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) - } - - #[tokio::test] - async fn rate_over_cms_frequency_capability_misses_no_legacy_fallback() { - // Rate is excluded from summary execution even with a registered frequency - // sid, so this request must capability-miss. - let now = now_ms_for_test(); - let idx = Arc::new(SketchStore::new()); - register_cms_freq_sid(&idx, 7000, "cms_metric", &[], "", 600, now); - - let engine = build_engine_with_index(idx); - let result = engine.execute("rate(cms_metric[5m])").await; - assert!( - matches!(result, Err(EngineError::CapabilityMiss { .. })), - "rate() over a registered CMS frequency sid must still capability-miss \ - with no legacy fallback, got {result:?}" - ); - } - - #[tokio::test] - async fn rate_over_cms_frequency_misses_when_no_freq_sid() { - // No CMS sid registered → the fallback finds nothing and the - // query fails over to archive (CapabilityMiss), unchanged. - let idx = Arc::new(SketchStore::new()); - let engine = build_engine_with_index(idx); - let err = engine - .execute("rate(cms_metric[5m])") - .await - .expect_err("no freq sid → capability-miss to archive"); - assert!(matches!(err, EngineError::CapabilityMiss { .. })); - } - - #[tokio::test] - async fn keyed_cms_frequency_reads_point_estimate_instead_of_total() { - use crate::query_engines::query_result::QueryResult; - // A per-item selector is bound to Planner's typed PointCount readout. - // This synthetic matrix stores 600 only in cell zero; item X hashes - // elsewhere, so its estimate is 0. Returning 600 would prove the old - // misleading whole-bucket-total behavior had regressed. - let now = now_ms_for_test(); - let idx = Arc::new(SketchStore::new()); - register_cms_freq_sid(&idx, 7100, "cms_metric", &[], "", 600, now); - - let engine = build_engine_with_index(idx); - let result = engine - .execute("count_over_time(cms_metric{item=\"X\"}[5m])") - .await - .expect("keyed CMS frequency should execute through PointCount"); - let QueryResult::Vector(vector) = result else { - panic!("expected instant vector") - }; - assert_eq!(vector.values.len(), 1); - assert_eq!(vector.values[0].value, 0.0); - } - - #[tokio::test] - async fn ordinary_label_filter_is_not_bound_as_frequency_item_key() { - let now = now_ms_for_test(); - let idx = Arc::new(SketchStore::new()); - register_cms_freq_sid(&idx, 7150, "cms_metric", &[], "", 600, now); - - let engine = build_engine_with_index(idx); - let result = engine - .execute("count_over_time(cms_metric{region=\"west\"}[5m])") - .await; - assert!( - matches!(result, Err(EngineError::CapabilityMiss { .. })), - "a spatial label must not be reinterpreted as a sketch item key: {result:?}" - ); - } - #[tokio::test] async fn bare_cms_frequency_still_answers_after_p2_6() { use crate::query_engines::query_result::QueryResult; // Regression guard: the working `count_over_time(cms_metric[5m])` // demo (NO item key, empty spatial filter) must still be answered // by the warm tier after the P2-6 safe-miss was added. - let now = now_ms_for_test(); + let now = 600_000_u64; let idx = Arc::new(SketchStore::new()); register_cms_freq_sid(&idx, 7200, "cms_metric", &[], "", 600, now); - let engine = build_engine_with_index(idx); - let result = engine.execute("count_over_time(cms_metric[5m])").await; + use crate::query_engines::asap_query_engine::test_plan; + use asap_types::query_plan::*; + let config = test_plan::materialization( + "cms_metric", + "CountMinSketch", + serde_json::json!({"w":512,"d":5}), + &[], + 30000, + ); + let query = "count_over_time(cms_metric[30s])"; + let entry = test_plan::entry( + query, + &config, + PhysicalGrouping::PerEntity, + 30000, + QueryPlanNode::SummaryEstimate { + input: QueryNodeId(0), + query: QueryReadout::PointCount { + key: planner_types::pre_asap::ColumnRef::SampleValue, + value: None, + }, + }, + ); + let engine = test_plan::engine(idx, config, vec![7200], entry); + let result = engine.execute_at(query, now - 30000).await; assert!( matches!( result, @@ -2941,14 +2467,7 @@ mod asap_tier_classify_tests { } // =========================================================================== -// Engine-level integration test for issue #296 — `max by (zone) -// (quantile_over_time(0.99, m[5m]))` over per-zone ExactAgg(Sum) sids -// must reach the reducer (not capability-miss). The asap engine's -// `evaluate` path returns DDSketch-decoded quantile values per zone; -// the outer-agg fold then collapses each zone's single row into a -// single value (identity). Pre-fix the query produced a CapabilityMiss -// because no analyzer-side composition existed. -// =========================================================================== +/// A registered series without samples must not hide a readable bound series. #[cfg(test)] mod outer_agg_integration_tests { use super::*; @@ -2964,12 +2483,6 @@ mod outer_agg_integration_tests { use asap_sketchlib::MessagePackCodec; use std::collections::{BTreeMap, BTreeSet}; - fn build_engine_with_index(idx: Arc) -> ASAPQueryEngine { - let streaming_config = Arc::new(crate::storage_engines::types::StreamingConfig::default()); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_config); - ASAPQueryEngine::new(15000).with_sketch_index(idx) - } - fn dd_sketch_with_values(values: &[f64]) -> Vec { // The msgpack encoding round-trips through // `DdSketch::deserialize_msgpack` on the engine side — simpler @@ -3006,55 +2519,6 @@ mod outer_agg_integration_tests { } } - /// An outer Max requires its own MinMax state in this summary plan. - /// An inner quantile sid alone cannot satisfy the whole tree; return a - /// capability miss so archive execution can answer the composed query. - #[tokio::test] - async fn execute_max_by_zone_over_quantile_over_time_capability_misses_pending_asapcontroller_171( - ) { - let idx = Arc::new(SketchStore::new()); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - let w_start = now_ms.saturating_sub(60_000); - let w_end = now_ms.saturating_sub(30_000); - - for (i, (zone, vals)) in [ - ( - "z0", - vec![1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0], - ), - ("z1", vec![100.0_f64, 200.0, 300.0, 400.0, 500.0]), - ] - .iter() - .enumerate() - { - let sid = 21_000 + i as u64; - idx.register(dd_meta_for(sid, "http_latency_ms", &["zone"])); - let bytes = dd_sketch_with_values(vals); - idx.append_sample( - sid, - BTreeMap::from([("zone".to_string(), zone.to_string())]), - (w_start, w_end), - SketchSampleState { - bytes, - encoding: SketchEncoding::MsgpackFull, - }, - ); - } - - let engine = build_engine_with_index(idx); - let result = engine - .execute("max by (zone) (quantile_over_time(0.99, http_latency_ms[5m]))") - .await; - assert!( - matches!(result, Err(EngineError::CapabilityMiss { .. })), - "max by (zone) over quantile_over_time must capability-miss \ - pending ASAPController#171, got {result:?}" - ); - } - /// Regression: a candidate's sid set legitimately contains a MIX of /// `Ghost` (retired-then-evicted or merged-away, no data) and `Hit` /// (Active, carrying live sketch state) sids under the same metric. @@ -3068,10 +2532,7 @@ mod outer_agg_integration_tests { #[tokio::test] async fn ghost_sid_does_not_mask_active_hit_sid_for_quantile() { let idx = Arc::new(SketchStore::new()); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); + let now_ms = 600_000_u64; let w_start = now_ms.saturating_sub(60_000); let w_end = now_ms.saturating_sub(30_000); @@ -3091,14 +2552,31 @@ mod outer_agg_integration_tests { }, ); - let engine = build_engine_with_index(idx); - let result = engine - .execute("quantile_over_time(0.99, http_latency_ms[5m])") - .await - .expect( - "a dataless Ghost sid must not abort the query when an \ + use crate::query_engines::asap_query_engine::test_plan; + use asap_types::query_plan::*; + let config = test_plan::materialization( + "http_latency_ms", + "DDSketch", + serde_json::json!({"alpha":0.01}), + &["zone"], + 30000, + ); + let query = "quantile_over_time(0.99, http_latency_ms[30s])"; + let entry = test_plan::entry( + query, + &config, + PhysicalGrouping::PerEntity, + 30000, + QueryPlanNode::SummaryEstimate { + input: QueryNodeId(0), + query: QueryReadout::Quantile { q: 0.99 }, + }, + ); + let engine = test_plan::engine(idx, config, vec![1, 2], entry); + let result = engine.execute_at(query, w_end).await.expect( + "a dataless Ghost sid must not abort the query when an \ Active Hit sid under the same metric can answer it", - ); + ); let vector = match result { QueryResult::Vector(v) => v, other => panic!("expected Vector, got {other:?}"), @@ -3114,63 +2592,6 @@ mod outer_agg_integration_tests { vector.values[0].value ); } - - /// `avg by (zone) (quantile_over_time(0.99, m[5m]))` — same - /// `sketch_reducer.rs`-retirement gap as - /// `execute_max_by_zone_over_quantile_over_time_capability_misses_pending_asapcontroller_171` - /// above, except `AggIntent::Avg` maps to `Implementation::PassThrough` - /// rather than an accumulator commitment - /// (`asap_aware_mapping::boundary::implementation_for_with`), so - /// `implement_tree_in_with`'s conservative fallback wraps the WHOLE - /// tree — including the otherwise-realizable inner quantile — as one - /// opaque `Logical` blob. Same accepted-gap outcome either way: - /// capability-miss, pending - /// https://github.com/ProjectASAP/ASAPController/issues/171. - #[tokio::test] - async fn execute_avg_by_zone_over_quantile_over_time_capability_misses_pending_asapcontroller_171( - ) { - let idx = Arc::new(SketchStore::new()); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - let w_start = now_ms.saturating_sub(60_000); - let w_end = now_ms.saturating_sub(30_000); - - for (i, (zone, vals)) in [ - ( - "z0", - vec![1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0], - ), - ("z1", vec![50.0_f64, 100.0, 150.0, 200.0, 250.0]), - ] - .iter() - .enumerate() - { - let sid = 22_000 + i as u64; - idx.register(dd_meta_for(sid, "http_latency_ms", &["zone"])); - let bytes = dd_sketch_with_values(vals); - idx.append_sample( - sid, - BTreeMap::from([("zone".to_string(), zone.to_string())]), - (w_start, w_end), - SketchSampleState { - bytes, - encoding: SketchEncoding::MsgpackFull, - }, - ); - } - - let engine = build_engine_with_index(idx); - let result = engine - .execute("avg by (zone) (quantile_over_time(0.99, http_latency_ms[5m]))") - .await; - assert!( - matches!(result, Err(EngineError::CapabilityMiss { .. })), - "avg by (zone) over quantile_over_time must capability-miss \ - pending ASAPController#171, got {result:?}" - ); - } } // =========================================================================== @@ -3278,15 +2699,7 @@ mod hybrid_stitch_tests { } // --------------------------------------------------------------------------- -// FIX 3 — RANGE-query warm+archive hybrid stitch. -// -// The instant path already stitches; the range path historically returned -// warm-only, so a `[start, end]` request whose warm sketches only cover a -// suffix lost the prefix. These tests drive `execute_range_promql_modern` -// with an archive engine wired and warm coverage narrower than the request, -// and assert the stitched matrix covers the FULL range (prefix from archive, -// suffix from warm). -// --------------------------------------------------------------------------- +/// Planned range execution requires complete warm coverage at every step. #[cfg(test)] mod range_stitch_tests { use super::*; @@ -3338,7 +2751,7 @@ mod range_stitch_tests { sketch_envelope, CountMinState, CounterType, SketchEnvelope, }; use prost::Message; - let (rows, cols) = (2u32, 4u32); + let (rows, cols) = (5u32, 512u32); let mut counts_int = vec![0i64; (rows * cols) as usize]; counts_int[0] = total; let state = CountMinState { @@ -3378,165 +2791,59 @@ mod range_stitch_tests { } } - /// Warm DDSketch covers only the SUFFIX of the requested range - /// (two windows near `end`); archive returns a full-range matrix - /// including the prefix. The stitched matrix must span the FULL request: - /// prefix timestamps come from archive, suffix from warm (warm wins on - /// any overlap). + /// Incomplete installed materializations must fail closed for archive routing, + /// regardless of whether this engine also has an archive client attached. #[tokio::test] - async fn range_stitches_archive_prefix_with_warm_suffix() { - let idx = Arc::new(SketchStore::new()); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - - // Requested range: [now-600s, now]. - let start_ms = now_ms.saturating_sub(600_000); - let end_ms = now_ms; - - // Warm windows only in the suffix: [now-200s], [now-100s]. - // `count_over_time` over a CountMin sid emits one PER-WINDOW total, - // so warm contributes a value at BOTH window-ends. - let warm_w1_end = now_ms.saturating_sub(200_000); - let warm_w2_end = now_ms.saturating_sub(100_000); - let warm_w1_total = 100.0_f64; - let sid = 9100u64; - idx.register(cms_meta(sid, "req_count")); - for (w_end, total) in [(warm_w1_end, 100i64), (warm_w2_end, 200i64)] { + async fn planned_range_rejects_partial_warm_coverage() { + use crate::query_engines::asap_query_engine::test_plan; + use asap_types::query_plan::*; + for with_archive in [false, true] { + let idx = Arc::new(SketchStore::new()); + idx.register(cms_meta(9100, "req_count")); idx.append_sample( - sid, + 9100, BTreeMap::new(), - (w_end.saturating_sub(30_000), w_end), + (30_000, 60_000), SketchSampleState { - bytes: cms_bytes(total), + bytes: cms_bytes(100), encoding: SketchEncoding::ProtoFull, }, ); + let config = test_plan::materialization( + "req_count", + "CountMinSketch", + serde_json::json!({"w":512,"d":5}), + &[], + 30_000, + ); + let query = "count_over_time(req_count[30s])"; + let entry = test_plan::entry( + query, + &config, + PhysicalGrouping::PerEntity, + 30_000, + QueryPlanNode::SummaryEstimate { + input: QueryNodeId(0), + query: QueryReadout::PointCount { + key: planner_types::pre_asap::ColumnRef::SampleValue, + value: None, + }, + }, + ); + let mut engine = test_plan::engine(idx, config, vec![9100], entry); + if with_archive { + engine = engine.with_archive_engine(Arc::new(FakeArchive { + matrix: QueryResult::matrix(vec![]), + })); + } + let result = engine + .execute_range_promql_modern(query, 30_000, 60_000, 30_000) + .await; + assert!( + matches!(result, Err(EngineError::CapabilityMiss { .. })), + "missing first pane must route the entire request to archive: {result:?}" + ); } - - // Archive provides the WHOLE range, including the prefix the warm - // tier can't cover. Use the bare empty-label series the DD reducer - // emits (so labels line up for the stitch merge). - let labels = KeyByLabelValues::new_with_labels(Vec::new()); - let mut arch_el = RangeVectorElement::new(labels); - // Prefix samples (before warm coverage) + a suffix sample warm will win. - let prefix_ts = now_ms.saturating_sub(500_000) as i64; - let mid_ts = now_ms.saturating_sub(300_000) as i64; - arch_el.samples.push(Sample::new(prefix_ts as u64, 999.0)); - arch_el.samples.push(Sample::new(mid_ts as u64, 998.0)); - arch_el.samples.push(Sample::new(warm_w1_end, 1.0)); // overlap: warm should win - let archive = Arc::new(FakeArchive { - matrix: QueryResult::matrix(vec![arch_el]), - }); - - let streaming_config = Arc::new(crate::storage_engines::types::StreamingConfig::default()); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_config); - let engine = ASAPQueryEngine::new(15000) - .with_sketch_index(idx) - .with_archive_engine(archive); - - let result = engine - .execute_range_promql_modern("count_over_time(req_count[5m])", start_ms, end_ms, 15_000) - .await - .expect("range query must answer (stitched), not error"); - - let m = match result { - QueryResult::Matrix(m) => m, - other => panic!("expected Matrix, got {other:?}"), - }; - assert_eq!(m.values.len(), 1, "one merged series: {m:?}"); - let samples = &m.values[0].samples; - let ts: std::collections::BTreeSet = - samples.iter().map(|s| s.timestamp as i64).collect(); - - // The PREFIX timestamps (only the archive has them) must be present — - // this is the whole point of the fix (warm-only would have dropped - // them). - assert!( - ts.contains(&prefix_ts), - "archive prefix sample (t={prefix_ts}) must survive the stitch: {ts:?}" - ); - assert!( - ts.contains(&mid_ts), - "archive mid sample (t={mid_ts}) must survive the stitch: {ts:?}" - ); - // The SUFFIX warm windows must be present too. - assert!( - ts.contains(&(warm_w1_end as i64)) && ts.contains(&(warm_w2_end as i64)), - "warm suffix windows must be present: {ts:?}" - ); - - // Warm wins on the overlapping timestamp: at warm_w1_end the value - // must be the warm per-window total (100), NOT the archive sentinel 1.0. - let overlap = samples - .iter() - .find(|s| s.timestamp == warm_w1_end) - .expect("overlap sample present"); - assert!( - (overlap.value - warm_w1_total).abs() < 1e-6, - "warm must win on overlap (expected warm total {warm_w1_total}, got {})", - overlap.value - ); - } - - /// Control: when warm coverage already spans the request exactly - /// (`cov_lo == start_ms && cov_hi == end_ms`), no stitch is needed and the - /// warm-only matrix is returned unchanged — the archive is NOT consulted - /// even though it's wired. Per-window coverage is window-end-point-based, - /// Control: with NO archive engine wired, the range path returns the - /// warm-only matrix (no stitch, no error) even when warm coverage is - /// narrower than the request — the stitch is gated on a configured - /// archive. This pins that the fix doesn't disturb the archive-less - /// deployment (the warm tier answers what it can). - #[tokio::test] - async fn range_warm_only_when_no_archive_engine() { - let idx = Arc::new(SketchStore::new()); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - let start_ms = now_ms.saturating_sub(600_000); - let end_ms = now_ms; - // Warm covers only one suffix window — narrower than the request. - let w_end = now_ms.saturating_sub(100_000); - let sid = 9200u64; - idx.register(cms_meta(sid, "req_count")); - idx.append_sample( - sid, - BTreeMap::new(), - (w_end.saturating_sub(30_000), w_end), - SketchSampleState { - bytes: cms_bytes(42), - encoding: SketchEncoding::ProtoFull, - }, - ); - - // No `.with_archive_engine(...)` — stitch must NOT fire. - let streaming_config = Arc::new(crate::storage_engines::types::StreamingConfig::default()); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_config); - let engine = ASAPQueryEngine::new(15000).with_sketch_index(idx); - - let result = engine - .execute_range_promql_modern("count_over_time(req_count[5m])", start_ms, end_ms, 15_000) - .await - .expect("range query must answer warm-only"); - let m = match result { - QueryResult::Matrix(m) => m, - other => panic!("expected Matrix, got {other:?}"), - }; - // Warm-only: exactly the single warm window-end sample, no archive - // prefix injected. - let ts: Vec = m - .values - .iter() - .flat_map(|el| el.samples.iter().map(|s| s.timestamp)) - .collect(); - assert_eq!( - ts, - vec![w_end], - "warm-only result must carry just the warm window sample: {ts:?}" - ); } #[tokio::test] diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 95a36b7d..e5bc38ef 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -1,63 +1,15 @@ -//! Execute sketch-tier queries through `SummaryExecutor`. Returning `None` -//! means this path cannot answer and the caller must fail over to archive. +//! Execute installed sketch-tier plans; errors allow routing to the exact backend. use std::collections::BTreeMap; -use control_plane::types_v2::AccuracyTarget; - -use crate::query_engines::asap_query_engine::post_asap_planner::LoweringSkip; +use crate::query_engines::asap_query_engine::post_asap_readout::LoweringSkip; use crate::query_engines::asap_query_engine::post_asap_readout::{ - execute_post_asap_instant, execute_post_asap_readout, execute_query_plan_instant, - execute_query_plan_readout, fold_coverage, + execute_query_plan_instant, fold_coverage, }; use crate::storage_engines::sketch_db::index::SketchStore; use crate::storage_engines::sketch_db::query::ASAPTierResult; -/// Fallback accuracy target used only when `post_asap_planner.rs`'s -/// observed-family lookup finds nothing registered for the query's -/// metric (in which case no family/params choice here can matter — the -/// query can't be served either way). `data_plane` doesn't carry a -/// per-workload `AccuracyTarget` today (see the design doc's "Rollout" -/// section). -const DEFAULT_LIVE_EPSILON: f64 = 0.01; - -fn live_accuracy() -> AccuracyTarget { - live_accuracy_from_values( - std::env::var("ASAP_SUMMARY_EXECUTOR_EPSILON") - .ok() - .as_deref(), - std::env::var("ASAP_SUMMARY_EXECUTOR_DELTA").ok().as_deref(), - ) -} - -fn live_accuracy_from_values(epsilon: Option<&str>, delta: Option<&str>) -> AccuracyTarget { - let epsilon = epsilon - .and_then(|value| value.trim().parse::().ok()) - .filter(|value| value.is_finite() && *value > 0.0 && *value < 1.0) - .unwrap_or(DEFAULT_LIVE_EPSILON); - match delta - .and_then(|value| value.trim().parse::().ok()) - .filter(|value| value.is_finite() && *value > 0.0 && *value < 1.0) - { - Some(delta) => AccuracyTarget::EpsilonDelta { epsilon, delta }, - None => AccuracyTarget::Epsilon(epsilon), - } -} - -/// Whether the serving cutover is enabled for this process. The env var -/// stays as a kill switch (`ASAP_SUMMARY_EXECUTOR_LIVE=0`/`false`/`off`) — -/// with it set, every query fails over to archive rather than being -/// served from the sketch tier at all (there's no legacy reducer left to -/// fall through to). -/// -/// Default flipped to **on** (control_plane/docs/design-target-architecture.md -/// §4/Part A): both unit tests and a real HTTP-level e2e test -/// (`live_serve_hll_global_count_merges_across_sids`, -/// `live_serve_actually_answers_ddsketch_quantile`) already prove -/// correctness for the shapes `try_serve_from_summary_executor` covers, -/// and the grouping-ambiguity problem that gated this default off -/// (empty-`by` ambiguity) is resolved via the real `Reduction::{Reduce, -/// PerEntity}` IR signal, not a heuristic. +/// Disable warm execution with ASAP_SUMMARY_EXECUTOR_LIVE=0, false, or off. pub fn summary_executor_live_enabled() -> bool { let value = std::env::var("ASAP_SUMMARY_EXECUTOR_LIVE").ok(); summary_executor_live_value(value.as_deref()) @@ -69,86 +21,6 @@ fn summary_executor_live_value(value: Option<&str>) -> bool { }) } -/// Try to serve `query` through `SummaryExecutor`. Return `None` if disabled, -/// excluded by shape, missing a matching sid, or unable to lower or execute. -/// The caller then falls back to archive. -pub fn try_serve_from_summary_executor( - index: &SketchStore, - query: &str, - t0_ms: u64, - t1_ms: u64, - is_cumulative: bool, -) -> Option { - if !summary_executor_live_enabled() { - return None; - } - - serve_from_summary_executor(index, query, t0_ms, t1_ms, is_cumulative, live_accuracy()) - .map_err(|skip| { - tracing::debug!( - query, - ?skip, - "live: query not servable from SummaryExecutor, falling back" - ); - }) - .ok() -} - -pub fn serve_from_summary_executor( - index: &SketchStore, - query: &str, - t0_ms: u64, - t1_ms: u64, - is_cumulative: bool, - accuracy: AccuracyTarget, -) -> Result { - if !summary_executor_live_enabled() { - return Err(LoweringSkip::Disabled); - } - let outcome = execute_post_asap_readout(index, query, t0_ms, t1_ms, is_cumulative, accuracy)?; - - tracing::debug!(query, "live: served from SummaryExecutor"); - Ok(ASAPTierResult { - series: outcome.series, - coverage: outcome.coverage, - }) -} - -pub fn serve_instant_from_summary_executor( - index: &SketchStore, - query: &str, - now_ms: u64, -) -> Result<(ASAPTierResult, u64), LoweringSkip> { - if !summary_executor_live_enabled() { - return Err(LoweringSkip::Disabled); - } - let (outcome, t0_ms) = execute_post_asap_instant(index, query, now_ms, live_accuracy())?; - Ok(( - ASAPTierResult { - series: outcome.series, - coverage: outcome.coverage, - }, - t0_ms, - )) -} - -pub fn serve_from_query_plan( - index: &SketchStore, - entry: &asap_types::query_plan::QueryPlanEntry, - t0_ms: u64, - t1_ms: u64, - is_cumulative: bool, -) -> Result { - if !summary_executor_live_enabled() { - return Err(LoweringSkip::Disabled); - } - let outcome = execute_query_plan_readout(index, entry, t0_ms, t1_ms, is_cumulative)?; - Ok(ASAPTierResult { - series: outcome.series, - coverage: outcome.coverage, - }) -} - /// Evaluate a planned range query at the exact Prometheus timestamps /// `start, start+step, ... <= end`. Each point executes the same immutable /// QueryPlan DAG with its own lookback interval; native summary window-close @@ -258,86 +130,8 @@ mod tests { use super::*; use std::collections::BTreeMap; - use crate::storage_engines::sketch_db::data::{AggKind, SketchConfig}; - use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchAlgorithm, SketchInstanceMetadata, SketchSampleState, - }; - - fn ddsketch_fixture() -> SketchStore { - let idx = SketchStore::new(); - let cfg = SketchConfig::DDSketch { - relative_accuracy: 0.01, - }; - idx.register(SketchInstanceMetadata { - sid: 1, - metric_name: "latency_ms".to_string(), - group_by_keys: std::collections::BTreeSet::new(), - capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), - agg_kind: AggKind::Sketch { - algorithm: SketchAlgorithm::DDSketch, - config: cfg.clone(), - spatial_filter_canonical: String::new(), - }, - accuracy: Some(AccuracyBound::from_config(&cfg)), - first_seen_unix_ms: 0, - retired_at_ms: None, - expires_at_ms: None, - policy_fp: asap_types::PolicyFingerprint::UNSET, - }); - use asap_sketchlib::{DdSketch, MessagePackCodec}; - let mut sk = DdSketch::new(0.01); - for i in 1..=100 { - sk.update(i as f64); - } - idx.append_sample( - 1, - BTreeMap::new(), - (1_000, 2_000), - SketchSampleState { - bytes: sk.to_msgpack().expect("encode DDSketch"), - encoding: crate::storage_engines::sketch_db::index::SketchEncoding::MsgpackFull, - }, - ); - idx - } - - fn register_hll(idx: &SketchStore, sid: u64, service: &str, items: &[&str]) { - let cfg = SketchConfig::Hll { precision: 14 }; - let mut group_by_keys = std::collections::BTreeSet::new(); - group_by_keys.insert("service".to_string()); - idx.register(SketchInstanceMetadata { - sid, - metric_name: "unique_users".to_string(), - group_by_keys, - capability: Some(Capability::CardinalityApprox), - agg_kind: AggKind::Sketch { - algorithm: SketchAlgorithm::Hll, - config: cfg.clone(), - spatial_filter_canonical: String::new(), - }, - accuracy: Some(AccuracyBound::from_config(&cfg)), - first_seen_unix_ms: 0, - retired_at_ms: None, - expires_at_ms: None, - policy_fp: asap_types::PolicyFingerprint::UNSET, - }); - use asap_sketchlib::{HllSketch, HllVariant, MessagePackCodec}; - let mut sk = HllSketch::new(HllVariant::Regular, 14); - for item in items { - sk.update(item.as_bytes()); - } - let mut labels = BTreeMap::new(); - labels.insert("service".to_string(), service.to_string()); - idx.append_sample( - sid, - labels, - (1_000, 2_000), - SketchSampleState { - bytes: sk.to_msgpack().expect("encode HLL"), - encoding: crate::storage_engines::sketch_db::index::SketchEncoding::MsgpackFull, - }, - ); - } + use crate::storage_engines::sketch_db::data::AggKind; + use crate::storage_engines::sketch_db::index::{Capability, SketchInstanceMetadata}; #[test] fn flag_off_spellings_disable_live_serve() { @@ -348,49 +142,7 @@ mod tests { #[test] fn unset_flag_defaults_to_serving() { - // Default flipped to on (design-target-architecture.md §4/Part A) - // -- an unset env var must serve, not fall back to the legacy - // path, for a shape this executor already proves safe. assert!(summary_executor_live_value(None)); - let idx = ddsketch_fixture(); - let result = try_serve_from_summary_executor( - &idx, - "quantile_over_time(0.99, latency_ms[1m])", - 1_000, - 2_000, - true, - ); - assert!(result.is_some(), "unset flag must default to serving"); - } - - #[test] - fn live_accuracy_accepts_explicit_valid_epsilon_delta() { - assert_eq!( - live_accuracy_from_values(Some("0.02"), Some("0.04")), - AccuracyTarget::EpsilonDelta { - epsilon: 0.02, - delta: 0.04, - } - ); - assert_eq!( - live_accuracy_from_values(Some("invalid"), Some("1.0")), - AccuracyTarget::Epsilon(DEFAULT_LIVE_EPSILON) - ); - } - - #[test] - fn flag_on_safe_shape_serves() { - let idx = ddsketch_fixture(); - let result = try_serve_from_summary_executor( - &idx, - "quantile_over_time(0.99, latency_ms[1m])", - 1_000, - 2_000, - true, - ); - let result = result.expect("unambiguous single-series quantile must serve"); - assert_eq!(result.series.len(), 1); - assert!(!result.is_empty()); } #[test] @@ -465,42 +217,4 @@ mod tests { vec![(1_000, 1.0), (2_000, 2.0), (3_000, 3.0)] ); } - - #[test] - fn flag_on_global_merge_shape_is_served_merged_not_declined() { - // Previously `flag_on_ambiguous_shape_falls_back`, asserting - // `result.is_none()`: the grouping-ambiguity gate declined this - // shape because an empty `by` couldn't be told apart from "reduce - // everything" (ASAPController#163). With `Reduction` (#165) the - // executor resolves it -- `count(...)` lowers to `Reduce([])`, both - // sids share one group key, and the new path serves the correctly - // merged answer instead of falling back. - let idx = SketchStore::new(); - register_hll(&idx, 1, "svc-a", &["a", "b", "c"]); - register_hll(&idx, 2, "svc-b", &["d", "e", "f"]); - let result = - try_serve_from_summary_executor(&idx, "count(unique_users)", 1_000, 2_000, true); - let result = result - .expect("global-merge shape is no longer ambiguous -- it must be served, not declined"); - assert_eq!( - result.series.len(), - 1, - "a by-less count() must merge both sids into ONE series, got {:?}", - result.series - ); - // Disjoint item sets {a,b,c} + {d,e,f} -> merged cardinality ~6. - let card = result.series[0].1[0].1; - assert!( - (4.0..=8.0).contains(&card), - "merged cardinality {card} should be ~6 (both sids), not ~3 (one sid)" - ); - } - - #[test] - fn flag_on_unservable_query_falls_back() { - let idx = SketchStore::new(); - let result = - try_serve_from_summary_executor(&idx, "rate(http_requests_total[5m])", 0, 1000, true); - assert!(result.is_none()); - } } diff --git a/data_plane/src/query_engines/asap_query_engine/mod.rs b/data_plane/src/query_engines/asap_query_engine/mod.rs index 0bef6e4b..3fc00fb4 100644 --- a/data_plane/src/query_engines/asap_query_engine/mod.rs +++ b/data_plane/src/query_engines/asap_query_engine/mod.rs @@ -7,7 +7,6 @@ mod exact_subqueries; pub mod live_serve; pub mod logical_dag; pub mod physical_dag; -pub mod post_asap_planner; pub mod post_asap_readout; pub mod summary_exec; pub mod summary_executor; @@ -18,3 +17,6 @@ pub use crate::storage_engines::sketch_db::query as asap_tier; pub mod tests; pub use engine::ASAPQueryEngine; + +#[cfg(test)] +mod test_plan; diff --git a/data_plane/src/query_engines/asap_query_engine/physical_dag.rs b/data_plane/src/query_engines/asap_query_engine/physical_dag.rs index ea6d9f63..178965e5 100644 --- a/data_plane/src/query_engines/asap_query_engine/physical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/physical_dag.rs @@ -94,6 +94,7 @@ pub fn execute_from( }) } +#[cfg(test)] pub async fn execute_async( entry: &QueryPlanEntry, runtime: &R, diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs deleted file mode 100644 index d6695ed4..00000000 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs +++ /dev/null @@ -1,709 +0,0 @@ -//! PromQL string → `planner_types::post_asap::SummaryNode` bridge, shared by the actual -//! serving cutover (`live_serve.rs`, via `post_asap_readout.rs`). -//! -//! `control_plane` runs in-process with `data_plane` in this deployment -//! (see `data_plane/Cargo.toml`'s "Phase 9" comment), so this is a -//! same-binary library call, not a new planning implementation living -//! here. -//! -//! ## Serving time must not re-plan -//! -//! Parsing and canonicalization are safe to re-run at serving time: they are -//! pure, deterministic transformations of the query text. Selecting the -//! post-ASAP implementation (which summary family and parameters to use) is -//! a genuine planning decision, and planning already made it once, for -//! real, when this metric's workload was planned — that decision is what -//! `data_plane`'s ingest path actually registered in the `SketchStore` -//! (`AggKind::Sketch { algorithm: kind, config, .. }`). Serving time must reproduce -//! THAT decision, not independently re-derive a fresh one from a -//! hardcoded accuracy target: doing so picks whatever family/params an -//! accuracy-driven cost model prefers in the abstract (e.g. DDSketch -//! over Kll for quantiles, unconditionally), with no guarantee it matches -//! what's actually registered — and `SummaryExecutor::find_candidates` -//! requires an exact `(SketchAlgorithm, SketchParams)` match, by design (see -//! `summary_executor.rs::summary_params_match`'s doc: this deployment -//! chose strict equality over silently serving an answer under a looser -//! guarantee than what was planned). -//! -//! So before binding, [`plan_promql_to_post_asap`] looks up what's actually -//! registered for the query's target metric and constructs an -//! [`ObservedFamilyCostModel`] that echoes that back — the resulting -//! `SummaryNode` matches reality by construction, not by a coincidental -//! accuracy-target match. When nothing is registered for the metric (or -//! this deployment's family/param mapping doesn't recognize the -//! registered shape), `observed` is `None` and binding falls back to the -//! same accuracy-driven `ControlPlaneCostModel` behavior as before — it -//! won't find a match either way, so the outcome (`find_candidates` finds -//! nothing) is unchanged, just for a more honest reason. - -use std::rc::Rc; - -use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryExpr, SummaryNode}; - -use control_plane::physical::post_asap::cost_model::ObservedFamilyCostModel; -use control_plane::physical::post_asap::{ - bind_query_expr_with_cost_model, BindingError, PhysicalExpr, PostAsapPlan, -}; -use control_plane::types_v2::AccuracyTarget; - -use crate::query_engines::asap_query_engine::summary_executor::find_metric_in_query_expr; -use crate::storage_engines::sketch_db::data::{AggKind, SketchConfig}; -use crate::storage_engines::sketch_db::index::SketchStore; - -/// Why a query couldn't be answered through the `SummaryNode`/`SummaryExecutor` -/// path — covers both `plan_promql_to_post_asap`'s own failure to produce a -/// tree, AND (via `post_asap_readout.rs`'s `execute_post_asap_readout`) a failure of -/// `crate::query_engines::asap_query_engine::summary_exec::execute()` on a tree that DID lower successfully. -/// None of these are errors in the alarming sense — every variant is an -/// expected, frequent outcome for *some* fraction of live traffic; the -/// caller's only obligation is "fall back to the legacy path," never -/// "log this as a problem." -#[derive(Debug)] -pub enum LoweringSkip { - /// Operational kill switch disabled warm DAG execution. - Disabled, - /// No exact identity exists in the active, control-plane-compiled - /// QueryPlan. This is a catalog miss, not an invitation to re-plan. - QueryNotPlanned(String), - /// The installed QueryPlan entry could not be reconstructed or failed - /// its internal contract. The request must fail closed. - InvalidQueryPlan(String), - /// The QueryPlan binding exists, but its warm materializations do not yet - /// cover the requested evaluation interval with complete, fresh windows. - MaterializationNotReady(String), - /// `parse_query_expr_canonical` failed — same failure mode the legacy - /// parsing path tolerates and reports as an archive fallback reason. - ParseFailed(String), - /// The query contains `rate(...)`/`irate(...)` (`OuterFn::Rate`, per - /// `control_plane::asap_tier_analysis`'s own candidate analysis). - /// `lower.rs`'s `bind_recursive` rewrites `AggIntent::Rate` → - /// `Increase` before binding, so this WOULD otherwise bind - /// successfully to a valid `SummaryAgg{Increase}` tree — but - /// `summary_executor.rs` has no rate-division logic (dividing by a - /// coverage-clamped range), so comparing against it would produce a - /// spurious mismatch, not a real one. Must be excluded before ever - /// calling into `control_plane`'s binder, not just deprioritized. - RateShape, - /// An exact Sum materialization stores counter deltas and cannot answer - /// PromQL's sum-over-cumulative-samples semantics. - CounterSumOverTime, - /// The current CMS/CountSketch serving adapter cannot safely resolve a - /// string-keyed point lookup from the maintained state. - KeyedFrequency, - /// `bind_query_expr` itself failed (a genuine `BindingError`, e.g. - /// post-ASAP implementation/schema-derivation failure). - Implement(String), - /// `bind_query_expr` returned a `PhysicalExpr` variant other than - /// a committed post-ASAP summary. Per `bind_query_expr`'s own doc - /// this shouldn't happen in practice (it never picks a Phase ε.1 - /// placement), but the match is kept exhaustive and defensive rather - /// than assuming. - UnsupportedPhysicalShape, - /// The root node is `SummaryExpr::Logical(_)` — the query didn't - /// realize to any sketch/exact-agg binding at all (e.g. - /// `topk(K, sum by(...)(rate(m[r])))`: `implement_tree_in_with` only - /// recurses through `Aggregate` nodes, so hitting the outer - /// `Sort`/`Limit` wraps the WHOLE tree as one opaque `Logical` blob - /// even though the inner aggregate would bind fine on its own — see - /// this crate's design doc). Not an error: this is exactly the - /// existing `SummaryExecutorError::Logical`/"no candidate bound" - /// outcome, just detected one step earlier so the caller can skip - /// without even constructing a `QueryExecutionContext`. - NotRealized, - /// ASAPPlanner produced a maintained-summary plan, but no installed - /// QueryPlan binding has a ready materialization with the required - /// source, family and parameters. - NoWarmRoute(String), - /// The tree lowered successfully, but `crate::query_engines::asap_query_engine::summary_exec::execute()` - /// itself returned `Err` (`NoCandidates`, `MergeKindParamsMismatch`, - /// a decode/merge failure surfaced from `summary_executor.rs`, ...). - /// Always safe to just fall back — this means "can't answer this way - /// right now" (e.g. the sid catalog doesn't have an exact - /// `(SketchAlgorithm, SketchParams)` match), never "answered wrong." - ExecuteFailed(String), -} - -// `PostAsapPlan` and `PhysicalExpr` are backend compatibility/placement wrappers. -// The semantic tree returned by ASAPPlanner is `post_asap::SummaryNode`; this -// module does not claim or recreate an ASAPPlanner "L4" IR. - -/// Map a registered sid's `(SketchAlgorithm, SketchConfig)` — the -/// durable record of what planning actually decided for this metric — to -/// the `(SketchAlgorithm, SketchParams)` pair `ObservedFamilyCostModel` -/// needs to reproduce that decision exactly. `None` for shapes this -/// deployment doesn't map (e.g. an unsupported algorithm, which is an -/// analysis-time wildcard that's never actually registered on a sid). -/// -/// Heap-bearing kinds (`CmsWithHeap`/`CountSketchWithHeap`) reuse their -/// heap-less base's `SketchConfig` shape for identity (no `heap_size` -/// field exists on `SketchConfig` at all — mirrors `to_delta_kind`'s same -/// note), so `heap_size` here is a placeholder; `summary_params_match` -/// only compares `width`/`depth` for these kinds, so it doesn't affect -/// matching. -fn observed_summary_params( - kind: SketchAlgorithm, - config: &SketchConfig, -) -> Option<(SketchAlgorithm, SketchParams)> { - const PLACEHOLDER_HEAP_SIZE: u32 = 100; - match (kind, config) { - (SketchAlgorithm::DDSketch, SketchConfig::DDSketch { relative_accuracy }) => Some(( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { - alpha: *relative_accuracy, - }, - )), - (SketchAlgorithm::Kll, SketchConfig::Kll { k }) => { - Some((SketchAlgorithm::Kll, SketchParams::Kll { k: *k })) - } - (SketchAlgorithm::Hll, SketchConfig::Hll { precision }) => Some(( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: *precision as u8, - }, - )), - (SketchAlgorithm::Cms, SketchConfig::CountMin { rows, cols }) => Some(( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: *cols as u32, - depth: *rows as u32, - }, - )), - (SketchAlgorithm::CmsWithHeap, SketchConfig::CountMin { rows, cols }) => Some(( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: *cols as u32, - depth: *rows as u32, - heap_size: PLACEHOLDER_HEAP_SIZE, - }, - )), - (SketchAlgorithm::CountSketch, SketchConfig::CountSketch { rows, cols }) => Some(( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { - width: *cols as u32, - depth: *rows as u32, - }, - )), - (SketchAlgorithm::CountSketchWithHeap, SketchConfig::CountSketch { rows, cols }) => Some(( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: *cols as u32, - depth: *rows as u32, - heap_size: PLACEHOLDER_HEAP_SIZE, - }, - )), - _ => None, - } -} - -/// Look up what family/params is ACTUALLY registered for `metric` in -/// `index` — the durable record of planning's real decision (see this -/// module's docs). Checks every sid registered for the metric (no -/// group-by filter — an empty `required_keys` set matches any -/// registration, since we only need to know the FAMILY here, not resolve -/// a specific series) and returns the first sketch-typed one found. -/// `None` when nothing is registered (or only `ExactAgg` sids are — -/// those never consult `CostModel` at all, so there's nothing to -/// observe for them). -fn observed_family_for_metric( - index: &SketchStore, - metric: &str, -) -> Option<(SketchAlgorithm, SketchParams)> { - for sid in index.instances_matching(metric, &Default::default()) { - let found = index.with_instance(sid, |m| match &m.agg_kind { - AggKind::Sketch { - algorithm: kind, - config, - .. - } => observed_summary_params(kind.clone(), config), - AggKind::ExactAgg { .. } => None, - }); - if let Some(Some(observed)) = found { - return Some(observed); - } - } - None -} - -fn query_expr_contains_time_range(qe: &planner_types::pre_asap::QueryExpr) -> bool { - use planner_types::pre_asap::QueryExpr; - match qe { - QueryExpr::TimeRange { .. } => true, - QueryExpr::Filter { child, .. } - | QueryExpr::Project { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::TimeShift { child, .. } - | QueryExpr::SQLWindowFunc { child, .. } => query_expr_contains_time_range(child), - QueryExpr::Concat { children, .. } => children.iter().any(query_expr_contains_time_range), - QueryExpr::Join { left, right, .. } - | QueryExpr::SetOp { left, right, .. } - | QueryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } => query_expr_contains_time_range(left) || query_expr_contains_time_range(right), - _ => false, - } -} - -fn query_expr_contains_rate(qe: &planner_types::pre_asap::QueryExpr) -> bool { - use planner_types::pre_asap::{AggIntent, QueryExpr}; - match qe { - QueryExpr::Aggregate { - measures, child, .. - } => { - measures - .iter() - .any(|m| matches!(m, AggIntent::Rate | AggIntent::IRate)) - || query_expr_contains_rate(child) - } - QueryExpr::Filter { child, .. } - | QueryExpr::Project { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::TimeRange { child, .. } - | QueryExpr::TimeShift { child, .. } - | QueryExpr::SQLWindowFunc { child, .. } => query_expr_contains_rate(child), - QueryExpr::Concat { children, .. } => children.iter().any(query_expr_contains_rate), - QueryExpr::Join { left, right, .. } - | QueryExpr::SetOp { left, right, .. } - | QueryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } => query_expr_contains_rate(left) || query_expr_contains_rate(right), - _ => false, - } -} - -fn query_expr_has_filter(qe: &planner_types::pre_asap::QueryExpr) -> bool { - use planner_types::pre_asap::QueryExpr; - match qe { - QueryExpr::Scan { predicates, .. } => !predicates.is_empty(), - QueryExpr::Filter { .. } => true, - QueryExpr::Project { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::TimeRange { child, .. } - | QueryExpr::TimeShift { child, .. } - | QueryExpr::SQLWindowFunc { child, .. } => query_expr_has_filter(child), - QueryExpr::Concat { children, .. } => children.iter().any(query_expr_has_filter), - QueryExpr::Join { left, right, .. } - | QueryExpr::SetOp { left, right, .. } - | QueryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } => query_expr_has_filter(left) || query_expr_has_filter(right), - _ => false, - } -} - -fn query_expr_max_time_range_ms(qe: &planner_types::pre_asap::QueryExpr) -> Option { - use planner_types::pre_asap::QueryExpr; - let child_max = - |child: &planner_types::pre_asap::QueryExpr| query_expr_max_time_range_ms(child); - match qe { - QueryExpr::TimeRange { range, child } => { - Some((range.as_millis() as u64).max(child_max(child).unwrap_or_default())) - } - QueryExpr::PromqlSubquery { range, child, .. } => { - Some((range.as_millis() as u64).max(child_max(child).unwrap_or_default())) - } - QueryExpr::Filter { child, .. } - | QueryExpr::Project { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::TimeShift { child, .. } - | QueryExpr::SQLWindowFunc { child, .. } => child_max(child), - QueryExpr::Concat { children, .. } => children - .iter() - .filter_map(query_expr_max_time_range_ms) - .max(), - QueryExpr::Join { left, right, .. } - | QueryExpr::SetOp { left, right, .. } - | QueryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } => child_max(left).into_iter().chain(child_max(right)).max(), - _ => None, - } -} - -fn summary_contains_time_range(node: &SummaryNode) -> bool { - match &node.expr { - SummaryExpr::KeepPreAsap(qe) => query_expr_contains_time_range(qe), - SummaryExpr::SummaryAgg { child, .. } => summary_contains_time_range(child), - SummaryExpr::SummaryEstimate { summary_input, .. } => { - summary_contains_time_range(summary_input) - } - SummaryExpr::SummaryMerge { children } => { - children.iter().any(|n| summary_contains_time_range(n)) - } - _ => false, - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PostAsapExecutionHints { - pub lookback_ms: Option, - pub cumulative_readout: bool, - pub full_history: bool, -} - -/// Derive execution-time window semantics from the Planner DAG rather than -/// from a second PromQL capability analyzer. -pub fn execution_hints(node: &SummaryNode) -> PostAsapExecutionHints { - use planner_types::post_asap::{ExactKind, SketchQuery, SummaryFamilyType}; - - fn visit( - node: &SummaryNode, - lookback_ms: &mut Option, - has_cardinality: &mut bool, - has_plain_sum: &mut bool, - ) { - match &node.expr { - SummaryExpr::KeepPreAsap(qe) => { - if let Some(range) = query_expr_max_time_range_ms(qe) { - *lookback_ms = Some(lookback_ms.unwrap_or_default().max(range)); - } - } - SummaryExpr::SummaryAgg { child, family, .. } => { - if matches!(family, SummaryFamilyType::ExactAggregate(ExactKind::Sum, _)) { - *has_plain_sum = true; - } - visit(child, lookback_ms, has_cardinality, has_plain_sum); - } - SummaryExpr::SummaryEstimate { - summary_input, - query, - } => { - *has_cardinality |= matches!(query, SketchQuery::Cardinality); - visit(summary_input, lookback_ms, has_cardinality, has_plain_sum); - } - SummaryExpr::SummaryMerge { children } => { - for child in children { - visit(child, lookback_ms, has_cardinality, has_plain_sum); - } - } - _ => {} - } - } - - let mut lookback_ms = None; - let mut has_cardinality = false; - let mut has_plain_sum = false; - visit( - node, - &mut lookback_ms, - &mut has_cardinality, - &mut has_plain_sum, - ); - PostAsapExecutionHints { - lookback_ms, - cumulative_readout: lookback_ms.is_some() || has_cardinality, - full_history: lookback_ms.is_none() && has_plain_sum, - } -} - -/// Runtime support is derived from the Planner DAG itself. This replaces the -/// former PromQL candidate analyzer gate, so unsupported semantics cannot -/// diverge from the plan the executor will actually run. -fn ensure_warm_runtime_support( - node: &SummaryNode, - source_has_filter: bool, -) -> Result<(), LoweringSkip> { - use planner_types::post_asap::{ExactKind, SketchQuery, SummaryFamilyType}; - match &node.expr { - SummaryExpr::SummaryAgg { child, family, .. } => { - match family { - SummaryFamilyType::ExactAggregate(ExactKind::Rate | ExactKind::IRate, _) => { - return Err(LoweringSkip::RateShape) - } - SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) - if summary_contains_time_range(child) => - { - return Err(LoweringSkip::CounterSumOverTime) - } - _ => {} - } - ensure_warm_runtime_support(child, source_has_filter) - } - SummaryExpr::SummaryEstimate { - summary_input, - query: SketchQuery::PointCount { value: None, .. }, - } if source_has_filter => Err(LoweringSkip::KeyedFrequency), - SummaryExpr::SummaryEstimate { summary_input, .. } => { - ensure_warm_runtime_support(summary_input, source_has_filter) - } - SummaryExpr::SummaryMerge { children } => { - for child in children { - ensure_warm_runtime_support(child, source_has_filter)?; - } - Ok(()) - } - SummaryExpr::KeepPreAsap(_) => Ok(()), - _ => Err(LoweringSkip::NoWarmRoute( - "post-ASAP operator is not executable by the warm runtime".into(), - )), - } -} - -/// Bind the conventional PromQL `item="..."` equality matcher to a frequency -/// point readout. The Planner DAG owns the `SketchQuery`; this adapter only -/// supplies the literal value that the PromQL frontend currently leaves as -/// `None`. -fn bind_point_count_filter(node: &mut Rc, key: &str, value: &str) -> bool { - let node = Rc::make_mut(node); - match &mut node.expr { - SummaryExpr::SummaryEstimate { - query: - planner_types::post_asap::SketchQuery::PointCount { - key: point_key, - value: point_value, - }, - .. - } if point_value.is_none() => { - *point_key = planner_types::pre_asap::ColumnRef::Named(key.to_string()); - *point_value = Some(value.to_string()); - true - } - SummaryExpr::SummaryAgg { child, .. } => bind_point_count_filter(child, key, value), - SummaryExpr::SummaryEstimate { summary_input, .. } => { - bind_point_count_filter(summary_input, key, value) - } - SummaryExpr::SummaryMerge { children } => children - .iter_mut() - .any(|child| bind_point_count_filter(child, key, value)), - _ => false, - } -} - -/// Lower a raw PromQL query string to the `SummaryNode` tree -/// `crate::query_engines::asap_query_engine::summary_exec::execute`/`SummaryExecutor` needs — the actual -/// serving cutover (`live_serve.rs`). Returns `Err` for any shape serving -/// shouldn't attempt (parse failure, `rate()`, or anything that doesn't -/// realize to a concrete sketch/exact-agg binding) — see -/// `LoweringSkip`'s variants. -pub fn plan_promql_to_post_asap( - index: &SketchStore, - query: &str, - accuracy: AccuracyTarget, -) -> Result, LoweringSkip> { - let qe = control_plane::query_parser::parse_query_expr_canonical(query, accuracy.clone()) - .map_err(|e| LoweringSkip::ParseFailed(e.to_string()))?; - if query_expr_contains_rate(&qe) { - return Err(LoweringSkip::RateShape); - } - let source_has_filter = query_expr_has_filter(&qe); - - // This dynamic lowering path is retained for isolated executor tests. - // Production serving executes the installed QueryPlan and resolves its - // SummaryDefinitionId bindings through SummaryCatalog. - let metric = find_metric_in_query_expr(&qe); - let mut observed = Vec::new(); - if let Some(family) = metric - .as_deref() - .and_then(|metric| observed_family_for_metric(index, metric)) - { - observed.push(family); - } - // No observed family means the test-only planner may use its - // accuracy-driven default. - let candidates: Vec<_> = if observed.is_empty() { - vec![None] - } else { - observed.into_iter().map(Some).collect() - }; - let mut last_skip = LoweringSkip::NotRealized; - for observed in candidates { - let cost_model = ObservedFamilyCostModel::new(accuracy.clone(), observed); - let physical = match bind_query_expr_with_cost_model(&qe, &cost_model) { - Ok(physical) => physical, - Err(BindingError::Implement( - control_plane::planner_selection::SelectionError::NoLegalCandidate, - )) => { - last_skip = LoweringSkip::NotRealized; - continue; - } - Err(other) => { - last_skip = LoweringSkip::Implement(other.to_string()); - continue; - } - }; - - match physical { - PhysicalExpr::Committed(PostAsapPlan::Summary(mut node)) => { - if matches!(node.expr, SummaryExpr::KeepPreAsap(_)) { - last_skip = LoweringSkip::NotRealized; - } else { - if let Ok(parsed) = - control_plane::query_parser::parse_query(query, accuracy.clone()) - { - if parsed.label_filters.len() == 1 { - if let Some(value) = parsed.label_filters.get("item") { - bind_point_count_filter(&mut node, "item", value); - } - } - } - if let Err(skip) = ensure_warm_runtime_support(&node, source_has_filter) { - last_skip = skip; - continue; - } - return Ok(node); - } - } - _ => last_skip = LoweringSkip::UnsupportedPhysicalShape, - } - } - Err(last_skip) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn accuracy() -> AccuracyTarget { - AccuracyTarget::Epsilon(0.01) - } - - /// Empty index -- every test below exercises a shape that either - /// self-excludes before ever consulting the `SketchStore`, or (for - /// `frequency_intent_realizes_via_bind_query_expr`) relies on - /// `ObservedFamilyCostModel` falling back to the accuracy-driven - /// default when nothing is registered. - fn empty_index() -> SketchStore { - SketchStore::new() - } - - #[test] - fn rate_query_is_skipped_before_binding() { - let idx = empty_index(); - let result = plan_promql_to_post_asap(&idx, "rate(http_requests_total[5m])", accuracy()); - assert!( - matches!(result, Err(LoweringSkip::RateShape)), - "expected RateShape, got {result:?}" - ); - } - - #[test] - fn irate_query_is_skipped_before_binding() { - let idx = empty_index(); - let result = plan_promql_to_post_asap(&idx, "irate(http_requests_total[5m])", accuracy()); - assert!( - matches!(result, Err(LoweringSkip::RateShape)), - "expected RateShape, got {result:?}" - ); - } - - #[test] - fn unparseable_query_is_skipped() { - let idx = empty_index(); - let result = plan_promql_to_post_asap(&idx, "this is not promql (((", accuracy()); - assert!( - matches!(result, Err(LoweringSkip::ParseFailed(_))), - "expected ParseFailed, got {result:?}" - ); - } - - #[test] - fn bare_selector_is_not_realized() { - // L1 adoption (design-target-architecture.md Part B), accepted - // behavior change: `asap_frontend_promql::lower_promql` no longer - // wraps a bare selector in an implicit `Aggregate { Sum }` (see - // control_plane's - // `asap_tier_implement::bare_selector_has_no_aggregate_root_to_implement` - // and `asap_tier_analysis::bare_selector_is_no_longer_asap_tier_answerable` - // for the sibling fixes). With no `Aggregate` node anywhere in the - // tree, `implement_tree_in_with` has nothing to bind and the whole - // expression stays one opaque `Logical` blob, which this module - // surfaces as `NotRealized`. - let idx = empty_index(); - let result = plan_promql_to_post_asap(&idx, "http_requests_total", accuracy()); - assert!( - matches!(result, Err(LoweringSkip::NotRealized)), - "expected NotRealized, got {result:?}" - ); - } - - #[test] - fn frequency_intent_realizes_via_bind_query_expr() { - // The exact shape `asap_tier_implement.rs`'s own - // `implement_frequency_as_agg_test` pins as a KNOWN, documented gap - // for `implement_promql_for_asap_tier`/`DefaultCostModel` (asserts - // it stays `Logical` "for now"). `bind_query_expr`/ - // `ControlPlaneCostModel` is exactly the fix -- via - // `realize_extension`/`readout_extension` (ASAPController#150) -- - // so this must realize to a real binding here, confirming this - // module picked the seam that actually handles Frequency. No sid - // is registered for this metric, so `ObservedFamilyCostModel` - // falls back to the accuracy-driven default -- same outcome as - // before this module started consulting the `SketchStore`. - let idx = empty_index(); - let node = - plan_promql_to_post_asap(&idx, "count_over_time(http_requests_total[5m])", accuracy()) - .expect("Frequency intent must realize via bind_query_expr/ControlPlaneCostModel"); - assert!( - !matches!(node.expr, SummaryExpr::KeepPreAsap(_)), - "expected a real SummaryAgg/SummaryEstimate binding, got Logical (the gap \ - this module exists to avoid): {:?}", - node.expr - ); - } - - #[test] - fn execution_hints_come_from_post_asap_dag() { - let idx = empty_index(); - let ranged = - plan_promql_to_post_asap(&idx, "quantile_over_time(0.99, latency[5m])", accuracy()) - .expect("quantile plan"); - assert_eq!( - execution_hints(&ranged), - PostAsapExecutionHints { - lookback_ms: Some(300_000), - cumulative_readout: true, - full_history: false, - } - ); - - let plain_sum = - plan_promql_to_post_asap(&idx, "sum(requests)", accuracy()).expect("sum plan"); - assert_eq!( - execution_hints(&plain_sum), - PostAsapExecutionHints { - lookback_ms: None, - cumulative_readout: false, - full_history: true, - } - ); - } - - #[test] - fn topk_over_rate_is_not_realized() { - // The outer `Sort{Limit{Aggregate}}` shape: `implement_tree_in_with` - // only recurses through `Aggregate`, so the whole tree wraps as - // one opaque `Logical` blob -- self-excludes via `NotRealized`, - // no special-case detection needed for this shape specifically. - let idx = empty_index(); - let result = plan_promql_to_post_asap( - &idx, - "topk(5, sum by (host) (rate(http_requests_total[5m])))", - accuracy(), - ); - assert!( - matches!(result, Err(LoweringSkip::NotRealized) | Err(LoweringSkip::RateShape)), - "expected NotRealized or RateShape (both are valid skips for this shape), got {result:?}" - ); - } -} diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 8632aba0..862c86c8 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -1,19 +1,21 @@ -//! `SummaryNode` lowering + execution + conversion into `ASAPTierResult`'s -//! `(series, coverage)` shape — the core `live_serve.rs` (the serving -//! cutover) calls into. +//! Execute installed query DAGs and project their values and coverage. use std::collections::BTreeMap; use crate::utils::arithmetic::evaluate_float64_arithmetic as arithmetic; -use crate::query_engines::asap_query_engine::summary_exec::{execute, ExecOutcome}; use asap_types::query_plan::{QueryNodeId, QueryPlanNode}; -use control_plane::types_v2::AccuracyTarget; use crate::query_engines::asap_query_engine::physical_dag::{self, QueryNodeRuntime}; -use crate::query_engines::asap_query_engine::post_asap_planner::{ - execution_hints, plan_promql_to_post_asap, LoweringSkip, -}; +/// A planned warm query cannot be served; callers may route to an exact backend. +#[derive(Debug)] +pub enum LoweringSkip { + Disabled, + QueryNotPlanned(String), + InvalidQueryPlan(String), + MaterializationNotReady(String), + ExecuteFailed(String), +} use crate::query_engines::asap_query_engine::summary_executor::{ GroupState, QueryExecutionContext, SummaryExecutorError, SummaryValue, }; @@ -33,27 +35,7 @@ pub struct PostAsapReadoutOutcome { pub coverage: Option<(u64, u64)>, } -/// Ask ASAPPlanner for the post-ASAP representation of `query`, execute it -/// against `index` over `[t0_ms, t1_ms]`, and -/// convert the result into `PostAsapReadoutOutcome`. `Err` covers every reason -/// this couldn't produce a trustworthy answer — see `LoweringSkip`'s -/// variants; every one of them means "fall back to the legacy path," -/// never "the legacy path is wrong." -pub fn execute_post_asap_readout( - index: &SketchStore, - query: &str, - t0_ms: u64, - t1_ms: u64, - is_cumulative: bool, - accuracy: AccuracyTarget, -) -> Result { - let node = plan_promql_to_post_asap(index, query, accuracy.clone())?; - execute_planned_post_asap(index, &node, query, accuracy, t0_ms, t1_ms, is_cumulative) -} - -/// Execute an already-bound QueryPlan entry. This is the production serving -/// path: no PromQL lowering, planner cost model, observed-family lookup, or -/// Installed QueryPlan materialization resolution occurs before this legacy test helper. +/// Execute the materializations and readouts bound in an installed QueryPlan entry. pub fn execute_query_plan_readout( index: &SketchStore, entry: &asap_types::query_plan::QueryPlanEntry, @@ -604,80 +586,6 @@ fn execute_physical_query_payload( result } -/// Plan and execute an instant query without consulting the legacy candidate -/// analyzer. Lookback and cumulative-vs-per-window behavior come from the -/// post-ASAP DAG itself. -pub fn execute_post_asap_instant( - index: &SketchStore, - query: &str, - now_ms: u64, - accuracy: AccuracyTarget, -) -> Result<(PostAsapReadoutOutcome, u64), LoweringSkip> { - const DEFAULT_LOOKBACK_MS: u64 = 5 * 60 * 1000; - let node = plan_promql_to_post_asap(index, query, accuracy.clone())?; - let hints = execution_hints(&node); - let t0_ms = if hints.full_history { - 0 - } else { - now_ms.saturating_sub(hints.lookback_ms.unwrap_or(DEFAULT_LOOKBACK_MS)) - }; - let outcome = execute_planned_post_asap( - index, - &node, - query, - accuracy, - t0_ms, - now_ms, - hints.cumulative_readout, - )?; - Ok((outcome, t0_ms)) -} - -fn execute_planned_post_asap( - index: &SketchStore, - node: &planner_types::post_asap::SummaryNode, - query: &str, - accuracy: AccuracyTarget, - t0_ms: u64, - t1_ms: u64, - is_cumulative: bool, -) -> Result { - let _ = (query, accuracy); - let allowed_materializations = None; - let ctx = QueryExecutionContext { - index, - t0_ms, - t1_ms, - is_cumulative, - allowed_materializations, - }; - - match execute(node, &ctx) { - Ok(ExecOutcome::Value(values)) => { - let mut coverage: Option<(u64, u64)> = None; - let mut series = Vec::new(); - for (group_key, value) in &values { - fold_coverage(&mut coverage, value.coverage()); - series.extend(summary_value_to_series(group_key, value)); - } - Ok(PostAsapReadoutOutcome { series, coverage }) - } - Ok(ExecOutcome::State(groups)) => { - let mut coverage: Option<(u64, u64)> = None; - let mut series = Vec::new(); - for (group_key, state, _family) in &groups { - fold_coverage(&mut coverage, state.exact_coverage()); - let Some(value) = state.exact_value(&None) else { - continue; - }; - series.push((group_key.clone(), vec![(t1_ms as i64, value)])); - } - Ok(PostAsapReadoutOutcome { series, coverage }) - } - Err(e) => Err(LoweringSkip::ExecuteFailed(format!("{e:?}"))), - } -} - /// `SummaryValue::Points`/`TopK` -> `ASAPTierResult.series`'s row shape. /// `TopK`'s ranked-list-per-timestamp shape is pivoted into one row per /// item (each row = the group's label map plus an `item` label, one point @@ -722,6 +630,8 @@ pub(crate) fn fold_coverage(coverage: &mut Option<(u64, u64)>, next: Option<(u64 #[cfg(test)] mod tests { use super::*; + use crate::query_engines::asap_query_engine::test_plan; + use asap_types::query_plan::{ExactReadout, PhysicalGrouping, QueryReadout}; use planner_types::pre_asap::ArithmeticOpKind; fn exact_points(metric: &str, service: &str, value: f64) -> PhysicalQueryOutput { @@ -835,10 +745,6 @@ mod tests { AccuracyBound, Capability, SketchAlgorithm, SketchInstanceMetadata, SketchSampleState, }; - fn accuracy() -> AccuracyTarget { - AccuracyTarget::Epsilon(0.01) - } - fn register_hll(idx: &SketchStore, sid: u64, service: &str, items: &[&str]) { // precision 14 -- what `ControlPlaneCostModel` actually picks for // `AccuracyTarget::Epsilon(0.01)` (confirmed by inspecting the @@ -1013,48 +919,60 @@ mod tests { let idx = SketchStore::new(); register_hll(&idx, 1, "api", &["a", "b"]); register_hll(&idx, 2, "worker", &["b", "c"]); - let node = plan_promql_to_post_asap(&idx, "count(unique_users)", accuracy()) - .expect("compile-stage fixture"); - let canonical = asap_types::query_plan::canonical_promql("count(unique_users)").unwrap(); - let entry = control_plane::query_plan::compile_bound( - "q-cardinality".into(), - canonical, - &node, - asap_types::query_plan::InstantExecution { - lookback_ms: 60_000, - full_history: false, - cumulative_readout: true, - }, - asap_types::query_plan::FallbackPolicy::ExactBackend, - |_node, _family| { - Ok(asap_types::query_plan::MaterializationBinding { - item_labels: Vec::new(), - materialization: asap_types::PolicyFingerprint(123).into(), - output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, - window_ms: 60_000, - pane_origin_ms: Some(2_000), - readout_lookback_ms: Some(60_000), - }) + let config = crate::query_engines::asap_query_engine::test_plan::materialization( + "unique_users", + "HLL", + serde_json::json!({"precision": 14}), + &["service"], + 1000, + ); + let entry = crate::query_engines::asap_query_engine::test_plan::entry( + "count(unique_users)", + &config, + asap_types::query_plan::PhysicalGrouping::PerEntity, + 1000, + QueryPlanNode::SummaryEstimate { + input: QueryNodeId(0), + query: asap_types::query_plan::QueryReadout::Cardinality, }, - ) - .unwrap(); + ); + crate::query_engines::asap_query_engine::test_plan::install( + &idx, + &[(config, vec![1])], + vec![entry.clone()], + ); let result = execute_query_plan_readout(&idx, &entry, 1_000, 2_000, true) .expect("execute formal QueryPlan"); - assert!(!result.series.is_empty()); + assert_eq!(result.series.len(), 1, "unbound policy must not contribute"); + assert_eq!( + result.series[0].0.get("service").map(String::as_str), + Some("api") + ); + assert!((result.series[0].1[0].1 - 2.0).abs() < 0.1); } #[test] fn bare_range_function_keeps_one_series_per_entity() { let idx = ddsketch_fixture(); - let outcome = execute_post_asap_readout( - &idx, - "quantile_over_time(0.99, latency_ms[1m])", - 1_000, - 2_000, - true, - accuracy(), - ) - .expect("should execute"); + let config = test_plan::materialization( + "latency_ms", + "DDSketch", + serde_json::json!({"alpha": 0.01}), + &[], + 1000, + ); + let entry = test_plan::entry( + "quantile_over_time(0.99, latency_ms[1s])", + &config, + PhysicalGrouping::PerEntity, + 1000, + QueryPlanNode::SummaryEstimate { + input: QueryNodeId(0), + query: QueryReadout::Quantile { q: 0.99 }, + }, + ); + test_plan::install(&idx, &[(config, vec![1])], vec![entry.clone()]); + let outcome = execute_query_plan_readout(&idx, &entry, 1000, 2000, true).unwrap(); assert_eq!(outcome.series.len(), 1); } @@ -1065,9 +983,25 @@ mod tests { let idx = SketchStore::new(); register_hll(&idx, 1, "svc-a", &["a", "b", "c"]); register_hll(&idx, 2, "svc-b", &["d", "e", "f"]); - let outcome = - execute_post_asap_readout(&idx, "count(unique_users)", 1_000, 2_000, true, accuracy()) - .expect("should execute"); + let config = test_plan::materialization( + "unique_users", + "HLL", + serde_json::json!({"precision": 14}), + &["service"], + 1000, + ); + let entry = test_plan::entry( + "count(unique_users)", + &config, + PhysicalGrouping::Reduce(vec![]), + 1000, + QueryPlanNode::SummaryEstimate { + input: QueryNodeId(0), + query: QueryReadout::Cardinality, + }, + ); + test_plan::install(&idx, &[(config, vec![1, 2])], vec![entry.clone()]); + let outcome = execute_query_plan_readout(&idx, &entry, 1000, 2000, true).unwrap(); assert_eq!( outcome.series.len(), 1, @@ -1111,9 +1045,20 @@ mod tests { (1_000, 2_000), Box::new(crate::precompute_engine::operators::SumAccumulator::with_sum(42.0)), ); - let outcome = - execute_post_asap_readout(&idx, "sum(bytes_total)", 1_000, 2_000, true, accuracy()) - .expect("should execute"); + let config = + test_plan::materialization("bytes_total", "Sum", serde_json::json!({}), &[], 1000); + let entry = test_plan::entry( + "sum(bytes_total)", + &config, + PhysicalGrouping::Reduce(vec![]), + 1000, + QueryPlanNode::ExactReadout { + input: QueryNodeId(0), + readout: ExactReadout::Sum, + }, + ); + test_plan::install(&idx, &[(config, vec![1])], vec![entry.clone()]); + let outcome = execute_query_plan_readout(&idx, &entry, 1000, 2000, true).unwrap(); // Window-end-only coverage: a single window (1_000, 2_000) is // keyed by its end (2_000) alone, so both bounds equal 2_000 -- // same semantics as `SummaryValue::coverage()`, reconfirmed for diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index e03e702d..5b4b583a 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -1348,7 +1348,7 @@ fn find_metric(node: &SummaryNode) -> Option { /// Walk a canonical `QueryExpr` down to its first `Scan { /// source: Source::TimeSeries { metric }, .. }` to recover the target -/// metric name. Shared with `post_asap_planner.rs`'s observed-family lookup +/// metric name. /// (serving time must know which metric to check the `SketchStore` /// against BEFORE binding — see that module's docs). pub(crate) fn find_metric_in_query_expr(qe: &QueryExpr) -> Option { diff --git a/data_plane/src/query_engines/asap_query_engine/test_plan.rs b/data_plane/src/query_engines/asap_query_engine/test_plan.rs new file mode 100644 index 00000000..f25bbd4b --- /dev/null +++ b/data_plane/src/query_engines/asap_query_engine/test_plan.rs @@ -0,0 +1,150 @@ +//! Explicit installed plans for runtime tests. Fixtures choose the materialization +//! and readout; query text is only canonicalized, never planned here. + +use super::engine::ASAPQueryEngine; +use crate::drivers::query::servers::http::{ + build_active_physical_plan, PhysicalPlanInstallRequest, +}; +use crate::storage_engines::sketch_db::index::SketchStore; +use crate::storage_engines::types::{BackendStorageRouting, HotReloadActivePhysicalPlan}; +use asap_types::precompute_plan::{PlanEnvelope, PrecomputePlan, BACKEND_COMPAT}; +use asap_types::query_plan::*; +use asap_types::PrecomputeMaterialization; +use control_plane::physical::compiler::PLANNER_REVISION; +use std::{collections::BTreeMap, sync::Arc}; + +pub(super) fn materialization( + metric: &str, + aggregation: &str, + parameters: serde_json::Value, + labels: &[&str], + pane_ms: u64, +) -> PrecomputeMaterialization { + assert_eq!(pane_ms % 1000, 0); + serde_json::from_value(serde_json::json!({ + "aggregation_type": aggregation, "aggregation_sub_type": "", + "metric": metric, "window_size": pane_ms / 1000, "slide_interval": pane_ms / 1000, + "window_type": "tumbling", "num_aggregates_to_retain": 100, + "parameters": parameters, "pane_origin_ms": 0, + "partitioning": "per_entity", + "window_layout": {"kind": "pane", "pane_secs": pane_ms / 1000}, + "grouping_labels": {"labels": labels}, "aggregated_labels": {"labels": []}, + "rollup_labels": {"labels": []}, "spatial_filter": "", + "spatial_filter_normalized": "", "original_yaml": "" + })) + .unwrap() +} + +pub(super) fn entry( + query: &str, + config: &PrecomputeMaterialization, + grouping: PhysicalGrouping, + lookback_ms: u64, + readout: QueryPlanNode, +) -> QueryPlanEntry { + let canonical = canonical_promql(query).unwrap(); + QueryPlanEntry { + language: QueryLanguage::PromQl, + query_id: canonical.clone(), + canonical_query: canonical, + fixed_evaluation: None, + root: QueryNodeId(1), + nodes: BTreeMap::from([ + ( + QueryNodeId(0), + QueryPlanNode::ReadMaterialization { + binding: MaterializationBinding { + materialization: config.policy_fingerprint().into(), + output_grouping: grouping, + item_labels: config.aggregated_labels.labels.clone(), + window_ms: config.slide_interval * 1000, + pane_origin_ms: config.pane_origin_ms, + readout_lookback_ms: Some(lookback_ms), + }, + }, + ), + (QueryNodeId(1), readout), + ]), + instant: InstantExecution { + lookback_ms, + full_history: false, + cumulative_readout: true, + }, + fallback: FallbackPolicy::ExactBackend, + } +} + +pub(super) fn install( + index: &SketchStore, + configs: &[(PrecomputeMaterialization, Vec)], + entries: Vec, +) -> HotReloadActivePhysicalPlan { + let envelope = PlanEnvelope { + plan_id: 1, + plan_version: 1, + generated_at_unix_ms: 0, + activation_unix_ms: 0, + expiry_unix_ms: None, + backend_compat: BACKEND_COMPAT.into(), + planner_revision: PLANNER_REVISION.into(), + capability_snapshot_id: "runtime-fixture".into(), + }; + let materializations: Vec<_> = configs.iter().map(|(c, _)| c.clone()).collect(); + let catalog = + asap_types::summary_catalog::SummaryCatalog::from_materializations(1, 1, &materializations) + .unwrap(); + let mut precompute = + PrecomputePlan::build(envelope.clone(), materializations, &["fixture".into()]).unwrap(); + precompute.summary_catalog = Some(catalog.reference().unwrap()); + let mut transmission = control_plane::physical::compiler::compile_transmission_plan( + envelope, + &precompute, + &BTreeMap::new(), + ) + .unwrap(); + transmission.summary_catalog = Some(catalog.reference().unwrap()); + index + .install_summary_catalog(Arc::new(catalog.clone())) + .unwrap(); + for (config, sids) in configs { + for sid in sids { + let mut metadata = (*index.instance(*sid).expect("registered fixture sid")).clone(); + metadata.policy_fp = config.policy_fingerprint(); + index.register(metadata); + } + } + let active = build_active_physical_plan( + PhysicalPlanInstallRequest { + summary_catalog: catalog, + collector_plans: vec![], + precompute_plan: precompute, + transmission_plan: transmission, + query_plan: QueryPlan { + plan_id: 1, + plan_version: 1, + clickhouse_context: None, + entries: entries + .into_iter() + .map(|e| (e.canonical_query.clone(), e)) + .collect(), + }, + storage_routing: None, + adaptation_evidence: vec![], + }, + Arc::new(BackendStorageRouting::empty()), + ) + .unwrap(); + HotReloadActivePhysicalPlan::new(active) +} + +pub(super) fn engine( + index: Arc, + config: PrecomputeMaterialization, + sids: Vec, + query_entry: QueryPlanEntry, +) -> ASAPQueryEngine { + let active = install(&index, &[(config, sids)], vec![query_entry]); + ASAPQueryEngine::new(15_000) + .with_sketch_index(index) + .with_active_physical_plan(active) +} diff --git a/data_plane/src/query_engines/query_result.rs b/data_plane/src/query_engines/query_result.rs index 30b6743c..08eb75e0 100644 --- a/data_plane/src/query_engines/query_result.rs +++ b/data_plane/src/query_engines/query_result.rs @@ -48,6 +48,7 @@ impl QueryResult { }) } + #[cfg(test)] /// Construct an instant vector with a non-empty warnings list. /// Used by the schema-timeline dispatcher when the query spans /// a reconfigure boundary and one or more segments could not @@ -124,16 +125,6 @@ impl QueryResult { QueryResult::Matrix(m) => m.window_used, } } - - /// Attach the actual window range that produced this answer. - /// Chainable, mirroring `with_accuracy`. - pub fn with_window_used(mut self, window: (u64, u64)) -> Self { - match &mut self { - QueryResult::Vector(iv) => iv.window_used = Some(window), - QueryResult::Matrix(m) => m.window_used = Some(window), - } - self - } } /// Instant vector - a set of time series containing a single sample for each time series, all sharing the same timestamp diff --git a/data_plane/src/query_engines/routing/backend_storage_routing.rs b/data_plane/src/query_engines/routing/backend_storage_routing.rs index 04498d14..3ee0c2d9 100644 --- a/data_plane/src/query_engines/routing/backend_storage_routing.rs +++ b/data_plane/src/query_engines/routing/backend_storage_routing.rs @@ -347,6 +347,7 @@ impl BackendStorageRouting { } } + #[cfg(test)] /// Return a copy of this routing table re-scoped to `tenant`. /// Used by tests / per-tenant hot-reload to retag a table built /// from a tenant-agnostic JSON / YAML payload. diff --git a/data_plane/src/query_engines/thanos_query_engine/forward.rs b/data_plane/src/query_engines/thanos_query_engine/forward.rs index ad67d49e..67abdbf2 100644 --- a/data_plane/src/query_engines/thanos_query_engine/forward.rs +++ b/data_plane/src/query_engines/thanos_query_engine/forward.rs @@ -197,6 +197,7 @@ impl ThanosQueryEngine { ] } + #[cfg(test)] /// The infos a forwarded-but-failed answer carries. Includes /// the quirk line so the upcoming Step-2.4 e2e demo can pin /// fail-loud behaviour. @@ -654,9 +655,8 @@ pub fn engine_from_env() -> Result, ThanosQueryError> // Test helpers // --------------------------------------------------------------------------- -#[doc(hidden)] -#[cfg(any(test, feature = "extra_debugging"))] -pub mod test_support { +#[cfg(test)] +pub(crate) mod test_support { //! Test-only helpers for spinning up an in-process mock //! `thanos-query` sidecar. Used by the unit + integration //! tests below and by the `routing/query_engine_routing.rs` tests diff --git a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs index a61f0099..2a9d28bf 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs @@ -794,6 +794,7 @@ impl BackfillRegistry { Coverage::Missing } + #[cfg(test)] /// Remove any terminal job older than `older_than_ms`. Used by /// the eventual retention sweep; returns the number of jobs /// evicted. Non-terminal jobs are never evicted. diff --git a/data_plane/src/storage_engines/sketch_db/backfill/prometheus_reader.rs b/data_plane/src/storage_engines/sketch_db/backfill/prometheus_reader.rs index ea071d5a..035af397 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/prometheus_reader.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/prometheus_reader.rs @@ -80,24 +80,6 @@ impl PrometheusReader { } } - /// Override the evaluation step. Prometheus returns one - /// `(ts, value)` point every `step` within the query window, - /// so setting `step` below the scrape interval wastes - /// bandwidth and setting it above drops samples. - pub fn with_step(mut self, step: Duration) -> Self { - self.step = step; - self - } - - /// Override the HTTP client timeout. Defaults to none. - pub fn with_timeout(mut self, timeout: Duration) -> Self { - self.http = reqwest::Client::builder() - .timeout(timeout) - .build() - .unwrap_or_else(|_| reqwest::Client::new()); - self - } - fn url(&self) -> String { format!("{}/api/v1/query_range", self.base_url.trim_end_matches('/')) } diff --git a/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs b/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs index c49f27a3..7f9208a1 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs @@ -150,6 +150,7 @@ impl MockRawSampleReader { Self { samples } } + #[cfg(test)] /// Total number of samples the reader was seeded with — /// independent of any range / filter. Exposed so tests can /// assert reader setup without round-tripping through diff --git a/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs b/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs index f4bffd51..4cb979fe 100644 --- a/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs +++ b/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs @@ -694,28 +694,6 @@ impl

SealedEpoch

{ } } - /// Count of distinct time windows in this sealed epoch — O(N) - /// scan (entries are sorted, so consecutive dupes are adjacent). - pub fn distinct_window_count(&self) -> usize { - let mut count = 0usize; - let mut last: Option = None; - for (w, _, _) in &self.entries { - if last != Some(*w) { - count += 1; - last = Some(*w); - } - } - count - } - - /// Sorted-deduplicated windows. Used by the legacy SketchStore to - /// surface the windows that were dropped on epoch eviction. - pub fn unique_windows(&self) -> Vec { - let mut windows: Vec = self.entries.iter().map(|(w, _, _)| *w).collect(); - windows.dedup(); - windows - } - /// Remove all entries whose window is in `windows`. O(N) scan; /// preserves sortedness since `retain` keeps relative order. pub fn remove_windows(&mut self, windows: &[TimestampRange]) { diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 84e8a207..0ee018b7 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -277,14 +277,6 @@ impl SketchInstanceMetadata { AggKind::ExactAgg { .. } => None, } } - - /// Sketch-config accessor mirroring [`Self::sketch_algorithm`]. - pub fn sketch_config(&self) -> Option<&SketchConfig> { - match &self.agg_kind { - AggKind::Sketch { config, .. } => Some(config), - AggKind::ExactAgg { .. } => None, - } - } } /// Per-sid storage value — wraps `SidStoreData` in an `RwLock` so the @@ -644,7 +636,7 @@ pub struct SketchStore { /// Reverse index: `policy_fp → {sids}`. Lets the query path resolve /// "which sids belong to this policy?" in O(1) without walking /// `instances`. Maintained by [`Self::register`] / - /// [`Self::remove_instance`] / [`Self::remove_instances_for_agg_config`]. + /// [`Self::remove_instance`]. /// Entries with `PolicyFingerprint::UNSET` are NOT recorded (the /// sentinel means "no source config"); legacy callers that mint /// sids without a fingerprint reach those sids through @@ -1094,14 +1086,6 @@ impl SketchStore { } } - /// The per-item attribute name recorded for `sid`, if any. `Some` - /// means the sketch hashes that label's VALUE (so `estimate(value)` - /// is meaningful); `None` means per-attribute-set keying (only the - /// bucket total is meaningful — keyed selectors must safe-miss). - pub fn item_label_for(&self, sid: u64) -> Option { - self.item_labels.read().unwrap().get(&sid).cloned() - } - /// Resolve a policy fingerprint to the set of sids it has minted. /// Returns an empty vector when no sid is bound to the fingerprint /// (e.g. fresh policy with no ingest activity yet) or when the @@ -1142,6 +1126,7 @@ impl SketchStore { .is_some_and(|generation| binding.catalog_generation.as_deref() == Some(generation)) } + #[cfg(test)] /// Live policy count — number of distinct fingerprints with at /// least one sid. Useful for telemetry / `/runtime` introspection /// (mirrors the legacy "active aggregation count" metric). @@ -1172,6 +1157,7 @@ impl SketchStore { )) } + #[cfg(test)] pub fn descriptor_counts(&self) -> (usize, usize) { ( self.descriptors.summary_count(), @@ -2409,6 +2395,7 @@ impl SketchStore { self.series.len() } + #[cfg(test)] /// Total count of in-memory SEALED epochs across all sids — i.e. /// epochs sealed (pending flush) but not yet evicted to disk. `0` /// once the flusher has drained everything. Used by tests and @@ -3160,51 +3147,6 @@ impl SketchStore { }; accepted.then_some(sid) } - - /// Phase 5 M2.3.6d — eviction-side helper. Removes every sid in the - /// index whose metadata was registered against `agg_cfg`, i.e. - /// shares the same metric, agg_type, parameters canonicalization, - /// and grouping-keys set the `SketchStoreSink` used at write time. - /// Returns how many sids were removed. Used by - /// `SchemaEvictionService` to drop a retired schema's residual sid - /// state. - pub fn remove_instances_for_agg_config( - &self, - agg_cfg: &asap_types::aggregation_config::AggregationConfig, - ) -> usize { - let target_metric = agg_cfg.metric.as_str(); - let target_agg_type = agg_cfg.aggregation_type; - let target_params = canonical_parameters(&agg_cfg.parameters); - let target_group_keys: BTreeSet = agg_cfg.grouping_labels.iter().cloned().collect(); - - // Collect the matching sids under a short read lock; then call - // `remove_instance` per sid (which takes its own write lock). - let to_remove: Vec = { - let g = self.instances.read().unwrap(); - g.iter() - .filter(|(_, m)| { - if m.metric_name != target_metric { - return false; - } - if m.group_by_keys != target_group_keys { - return false; - } - matches!( - &m.agg_kind, - AggKind::ExactAgg { agg_type, parameters_canonical, .. } - if *agg_type == target_agg_type - && parameters_canonical == &target_params - ) - }) - .map(|(sid, _)| *sid) - .collect() - }; - let count = to_remove.len(); - for sid in to_remove { - self.remove_instance(sid); - } - count - } } /// Persistence harness for `SketchStore` — Phase 5 M2.3.6c. diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index 14c9ea2d..5a59e5ed 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -61,14 +61,6 @@ impl SchemaEvictionService { } } - /// Override the retirement-retention duration. Tests use this to - /// drive lifecycle transitions deterministically without waiting - /// out the 24-hour default. - pub fn with_retirement_retention(mut self, retention: Duration) -> Self { - self.retirement_retention = retention; - self - } - /// Spawn as a tokio task. Returns a handle whose `shutdown` /// oneshot stops the loop cleanly on ctrl-c. pub fn spawn(self) -> SchemaEvictionHandle { diff --git a/data_plane/src/storage_engines/sketch_db/persistence/cache.rs b/data_plane/src/storage_engines/sketch_db/persistence/cache.rs index 5d1e4723..0c235a45 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/cache.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/cache.rs @@ -75,8 +75,3 @@ impl PartCache { self.inner.as_ref().map(|c| c.entry_count()).unwrap_or(0) } } - -/// Convenience helper: given a disk_path, return the parts root. -pub fn parts_root_of(disk_path: &Path) -> std::path::PathBuf { - disk_path.join("parts") -} diff --git a/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs b/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs index ee66370c..6018bdeb 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs @@ -150,71 +150,6 @@ impl FlusherHandle { } } - /// Wake the flusher early (e.g., on `hard_cap_bytes` pressure). - pub fn wake(&self) { - self.inner.pressure_cv.notify_all(); - } - - /// Block the calling thread until `mem_counter.load() < cap` or - /// `max_wait` elapses, whichever comes first. Wakes the flusher - /// once on entry and then sleeps on the pressure condvar until - /// the flusher's per-tick `notify_all` fires — typical wait is - /// one flush tick. - /// - /// Returns `true` if the wait finished with memory under the cap, - /// `false` if the timeout expired first (so the caller can log - /// and proceed rather than hanging forever on a stuck flusher). - /// - /// Safe to call from the insert hot path — the only synchronization - /// held during the wait is the flusher's dedicated pressure mutex, - /// which the flusher thread itself never takes (it only calls - /// `notify_all` on the condvar, which requires no lock). - pub fn wait_for_memory_under( - &self, - mem_counter: &std::sync::atomic::AtomicUsize, - cap: usize, - max_wait: Duration, - ) -> bool { - if mem_counter.load(Ordering::Relaxed) < cap { - return true; - } - // Slow path: the cap was crossed and we are about to block. - // Bump a counter so tests can assert that back-pressure fired - // without relying on wall-clock thresholds (the actual wait is - // ≤1 flush tick, which is sub-millisecond on fast CI runners - // and makes wall-clock-based signals flaky). - self.inner - .back_pressure_wait_count - .fetch_add(1, Ordering::Relaxed); - // Kick the flusher once so it tries to drain right now instead - // of waiting for its own interval tick. - self.inner.pressure_cv.notify_all(); - - let deadline = Instant::now() + max_wait; - let mut guard = match self.inner.pressure_mutex.lock() { - Ok(g) => g, - Err(_) => return false, // poisoned — give up rather than deadlock - }; - loop { - if mem_counter.load(Ordering::Relaxed) < cap { - return true; - } - if self.inner.shutdown.load(Ordering::Acquire) { - return false; - } - let now = Instant::now(); - if now >= deadline { - return false; - } - let remaining = deadline - now; - let (g, _) = match self.inner.pressure_cv.wait_timeout(guard, remaining) { - Ok(v) => v, - Err(_) => return false, - }; - guard = g; - } - } - /// Access to the manifest for the query read-through path. pub fn manifest(&self) -> Arc { Arc::clone(&self.inner.manifest) diff --git a/data_plane/src/storage_engines/sketch_db/persistence/manifest.rs b/data_plane/src/storage_engines/sketch_db/persistence/manifest.rs index e126c98a..c6bde084 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/manifest.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/manifest.rs @@ -256,6 +256,7 @@ impl Manifest { Ok(()) } + #[cfg(test)] /// Rewrite the snapshot from the current in-memory live set and /// truncate the log. Atomic: the new snapshot goes to a tmp file /// first, then rename, then the log is truncated. diff --git a/data_plane/src/storage_engines/sketch_db/query/delta_apply.rs b/data_plane/src/storage_engines/sketch_db/query/delta_apply.rs index 593aeb63..26341f94 100644 --- a/data_plane/src/storage_engines/sketch_db/query/delta_apply.rs +++ b/data_plane/src/storage_engines/sketch_db/query/delta_apply.rs @@ -514,18 +514,6 @@ impl SummaryState { } } - /// Borrow the inner HLL sketch when this rolling state is HLL-backed. - /// Used by the GLOBAL cardinality rollup (`count(hll_metric)` with no - /// `by`), which must MERGE the per-series HLL registers (register-wise - /// max) across all matched series and estimate ONCE — summing per-series - /// distinct estimates would double-count items present in multiple series. - pub fn as_hll(&self) -> Option<&HllSketch> { - match self { - SummaryState::Hll(sk) => Some(sk), - _ => None, - } - } - /// Merge `other` into `self` in place — both must be the same sketch /// family. Used to combine several sids' reconstructed states /// (`cumulative_summary_state`/`per_window_summary_states`) into one @@ -619,6 +607,7 @@ pub fn cumulative_summary_state( Ok(rolling) } +#[cfg(test)] /// Walk a sorted-by-window-end slice of samples in time order and /// produce ONE per-window scalar `(window_end_ms, scalar)`. /// @@ -737,79 +726,6 @@ pub fn per_window_summary_states( Ok((out, skipped)) } -/// Cumulative-mode rollup: fold every window in `[t0, t1]` into a -/// single rolling state and emit one scalar at the latest -/// `window_end_ms` seen (or the largest if all were Deltas that got -/// skipped). Used by `quantile_over_time` / `count_distinct_over_time`. -/// -/// Returns `Ok((window_end, scalar), skipped_leading_deltas)`. Returns -/// `Ok(None, _)` if every sample was a leading delta (no Full ever -/// landed in the range). -pub fn cumulative_evaluate( - samples: &[(i64, &SketchSampleState)], - kind: DeltaSketchKind, - eval: E, -) -> Result<(Option<(i64, f64)>, usize), String> -where - E: Fn(&SummaryState) -> f64, -{ - let mut rolling: Option = None; - let mut latest_end = i64::MIN; - let mut skipped = 0usize; - - for (window_end, state) in samples { - if *window_end > latest_end { - latest_end = *window_end; - } - match state.encoding { - SketchEncoding::ProtoFull | SketchEncoding::MsgpackFull => { - let new_state = decode_full(&kind, &state.bytes, state.encoding)?; - // Merge new_state into any existing rolling state — a - // mid-range Full effectively "restarts" the window in - // the cumulative roll-up if the agent flushed a new - // snapshot. Merging keeps the answer monotonic in - // sample inclusion. - rolling = Some(match (rolling.take(), new_state) { - (None, n) => n, - (Some(SummaryState::Dd(mut a)), SummaryState::Dd(b)) => { - a.merge(&b).map_err(|e| format!("cum merge DD: {e}"))?; - SummaryState::Dd(a) - } - (Some(SummaryState::Hll(mut a)), SummaryState::Hll(b)) => { - a.merge(&b).map_err(|e| format!("cum merge HLL: {e}"))?; - SummaryState::Hll(a) - } - (Some(SummaryState::Kll(mut a)), SummaryState::Kll(b)) => { - a.merge(&b).map_err(|e| format!("cum merge KLL: {e}"))?; - SummaryState::Kll(a) - } - (Some(_), _) => { - return Err("cumulative merge across sketch family mismatch".to_string()) - } - }); - } - SketchEncoding::ProtoDelta | SketchEncoding::MsgpackDelta => { - // PWR: the range may have NO carry-in Full (it starts - // mid-stream), so the first frame is a delta-from-empty. - // Bootstrap an empty base and apply onto it. Under PWR - // each window's frames are increments-since-its-own-base; - // folding them all (Full-fragment merge for DD/KLL, - // register-max for HLL) yields the union over the range, - // which is the cumulative (`*_over_time`) answer. - if rolling.is_none() { - rolling = Some(kind.bootstrap_empty()); - } - match rolling.as_mut() { - Some(rs) => rs.apply_delta_bytes(&state.bytes, state.encoding)?, - None => skipped += 1, - } - } - } - } - let out = rolling.map(|rs| (latest_end, eval(&rs))); - Ok((out, skipped)) -} - // --------------------------------------------------------------------------- // Proto-envelope decoders — P2-4: ONE decoder per family. // diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index 88786343..1bcc2833 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -334,7 +334,7 @@ impl PhysicalPlanLifecycle { } /// Roll back a failed publication without touching active readers or state. - pub fn discard_staged( + pub(crate) fn discard_staged( &self, plan_id: u64, plan_version: u64, @@ -433,7 +433,7 @@ impl PhysicalPlanLifecycle { /// Mark a superseded generation retired after all readers of the old /// immutable snapshot have drained. - pub fn retire_drained(&self, plan_id: u64, plan_version: u64) { + pub(crate) fn retire_drained(&self, plan_id: u64, plan_version: u64) { if let Some(status) = self .state .lock() @@ -481,7 +481,7 @@ impl HotReloadActivePhysicalPlan { old } - pub fn materialization_statuses(&self) -> Vec { + pub(crate) fn materialization_statuses(&self) -> Vec { self.readiness .lock() .expect("materialization readiness lock poisoned") diff --git a/data_plane/src/storage_engines/types/key_by_label_values.rs b/data_plane/src/storage_engines/types/key_by_label_values.rs index 9b282bf8..34bc8489 100644 --- a/data_plane/src/storage_engines/types/key_by_label_values.rs +++ b/data_plane/src/storage_engines/types/key_by_label_values.rs @@ -49,6 +49,7 @@ impl KeyByLabelValues { self.labels.join(";") } + #[cfg(test)] /// Decode a semicolon-joined string back into a KeyByLabelValues. pub fn from_semicolon_str(s: &str) -> Self { Self { diff --git a/data_plane/src/storage_engines/types/precomputed_output.rs b/data_plane/src/storage_engines/types/precomputed_output.rs index 130f761b..4d268dc4 100644 --- a/data_plane/src/storage_engines/types/precomputed_output.rs +++ b/data_plane/src/storage_engines/types/precomputed_output.rs @@ -126,18 +126,6 @@ impl PrecomputedOutput { policy_fp, } } - - pub fn get_freshness_debug_string(&self) -> String { - let current_time = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - let freshness = current_time.saturating_sub(self.end_timestamp); - format!( - "end_timestamp: {}, current_time: {}, freshness: {}", - self.end_timestamp, current_time, freshness - ) - } } // ─── Origin tag tests ───────────────────────────────────────────── diff --git a/data_plane/tests/all_sketches_process_oracle_e2e.rs b/data_plane/tests/all_sketches_process_oracle_e2e.rs index b78a42ca..9c8e442f 100644 --- a/data_plane/tests/all_sketches_process_oracle_e2e.rs +++ b/data_plane/tests/all_sketches_process_oracle_e2e.rs @@ -38,8 +38,7 @@ const CMS_ROWS: usize = 5; const CMS_COLS: usize = 2048; // ASAPPlanner's CountSketch guarantee is L2-based: epsilon=sqrt(3/width), // with an odd Hoeffding-median depth. The current runtime's packed sign hash -// limits this width to five rows, which satisfies the explicit delta=0.8 -// contract used only by the CountSketch child process below. +// limits this width to five rows. The fixture installs these parameters explicitly. const COUNT_SKETCH_ROWS: usize = 5; const COUNT_SKETCH_COLS: usize = 32_768; @@ -108,7 +107,7 @@ fn envelope(metric: &str, data: Data) -> ExportMetricsServiceRequest { } } -async fn start_backend(config_yaml: &str, live_delta: Option<&str>) -> Backend { +async fn start_backend(config_yaml: &str) -> Backend { let query_port = unused_port(); let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); @@ -165,9 +164,6 @@ async fn start_backend(config_yaml: &str, live_delta: Option<&str>) -> Backend { .arg("100") .stdout(Stdio::null()) .stderr(Stdio::null()); - if let Some(delta) = live_delta { - command.env("ASAP_SUMMARY_EXECUTOR_DELTA", delta); - } let child = command.spawn().expect("start production data-plane binary"); let mut child = ChildGuard(child); let client = reqwest::Client::new(); @@ -449,11 +445,7 @@ fn assert_frequency_point_oracle(response: &Value, raw: &[&str], item: &str) { #[tokio::test] async fn production_kll_matches_raw_quantile_oracle() { let metric = "oracle_kll_latency"; - let backend = start_backend( - &config(metric, "DatasketchesKLL", &format!(" k: {K}")), - None, - ) - .await; + let backend = start_backend(&config(metric, "DatasketchesKLL", &format!(" k: {K}"))).await; let raw: Vec = (1..=101).map(f64::from).collect(); let timestamp = now_ns().saturating_sub(2_000_000_000); post(&backend, kll_export(metric, timestamp, &raw)).await; @@ -475,10 +467,11 @@ async fn production_kll_matches_raw_quantile_oracle() { #[tokio::test] async fn production_hll_matches_raw_distinct_oracle() { let metric = "oracle_hll_users"; - let backend = start_backend( - &config(metric, "HLL", &format!(" precision: {HLL_PRECISION}")), - None, - ) + let backend = start_backend(&config( + metric, + "HLL", + &format!(" precision: {HLL_PRECISION}"), + )) .await; let owned: Vec = (0..2_000).map(|i| format!("user-{i}")).collect(); let mut raw: Vec<&str> = owned.iter().map(String::as_str).collect(); @@ -512,7 +505,7 @@ fn frequency_fixture() -> Vec<&'static str> { async fn production_cms_matches_raw_frequency_oracle() { let metric = "oracle_cms_frequency"; let params = format!(" w: {CMS_COLS}\n d: {CMS_ROWS}"); - let backend = start_backend(&config(metric, "CountMinSketch", ¶ms), None).await; + let backend = start_backend(&config(metric, "CountMinSketch", ¶ms)).await; let raw = frequency_fixture(); let timestamp = now_ns().saturating_sub(2_000_000_000); post(&backend, cms_export(metric, timestamp, &raw)).await; @@ -535,7 +528,7 @@ async fn production_count_sketch_matches_raw_frequency_oracle() { // current packed-hash runtime can execute at this width. Keep the default // production SLA untouched and declare the weaker contract explicitly // for this isolated algorithm-oracle process. - let backend = start_backend(&config(metric, "CountSketch", ¶ms), Some("0.8")).await; + let backend = start_backend(&config(metric, "CountSketch", ¶ms)).await; let raw = frequency_fixture(); let timestamp = now_ns().saturating_sub(2_000_000_000); post(&backend, count_sketch_export(metric, timestamp, &raw)).await; diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index caf5a21f..3bcbeba3 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -167,7 +167,7 @@ fn build_workload_with_override( aggregations, time_window, repeat_every: None, - accuracy: control_plane::types_v2::AccuracyTarget::Epsilon(accuracy_sla), + accuracy: control_plane::types::AccuracyTarget::Epsilon(accuracy_sla), accuracy_sla, latency_sla: None, sketch_type_override, @@ -2346,20 +2346,7 @@ async fn live_serve_actually_answers_ddsketch_quantile() { // merges them itself. The gate is gone; this SHOULD exercise the new // path serving the shape directly, not a fallback. // -// Correction (sketch_reducer.rs retirement): that claim above wasn't -// actually true until now. This shape was ALSO hitting a real -// family/params mismatch on `SummaryExecutor` (serving time picked -// precision from a hardcoded default accuracy, not what this workload -// was actually planned/registered with) -- the legacy reducer's -// `evaluate_cardinality_global` fallback silently masked that miss, so -// the test passed via the fallback, not the new path. With the reducer -// gone, the params mismatch is fixed (see `ObservedFamilyCostModel`), -// but that unmasked a SECOND, independent bug: `effective_is_cumulative` -// classifies a bare `count(...)` as non-cumulative, so `readout` -// evaluates per-window instead of merging the whole range -- this test's -// later "watermark" sample (a distinct, more recent window) then wins -// over the real data instead of being merged with it. Tracked as -// https://github.com/ProjectASAP/ASAPQuery-backend/issues/431. +// The installed cardinality readout merges all bound series and windows. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn live_serve_hll_global_count_merges_across_sids() { let _live = LiveServeEnvGuard::enable();