diff --git a/Cargo.lock b/Cargo.lock index 7b5e07e0..b8172af3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -312,7 +312,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.13.1", + "indexmap 2.14.0", "lexical-core", "num", "serde", @@ -385,6 +385,16 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "asap-precompute-rs" +version = "0.1.0" +dependencies = [ + "asap_sketchlib", + "prost", + "serde", + "thiserror 1.0.69", +] + [[package]] name = "asap_otel_proto" version = "0.1.0" @@ -403,8 +413,8 @@ dependencies = [ "anyhow", "asap_types", "chrono", - "clap 4.6.0", - "indexmap 2.13.1", + "clap 4.6.1", + "indexmap 2.14.0", "pretty_assertions", "promql-parser", "promql_utilities", @@ -423,7 +433,6 @@ dependencies = [ [[package]] name = "asap_sketchlib" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/asap_sketchlib?branch=main#0b32c811ada12e32cc68b407d24d1f74cc0af6b7" dependencies = [ "bytes", "prost", @@ -443,7 +452,7 @@ name = "asap_types" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.6.0", + "clap 4.6.1", "promql_utilities", "serde", "serde_json", @@ -619,9 +628,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "blake2" @@ -634,9 +643,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.4" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ "arrayref", "arrayvec", @@ -737,9 +746,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.59" +version = "1.2.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" dependencies = [ "find-msvc-tools", "jobserver", @@ -759,7 +768,7 @@ version = "0.13.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "lazy_static", "num-traits", "regex", @@ -836,9 +845,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -858,9 +867,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -977,7 +986,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.6.0", + "clap 4.6.1", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -1139,7 +1148,7 @@ dependencies = [ "glob", "half", "hashbrown 0.14.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.13.0", "log", "num_cpus", @@ -1148,7 +1157,7 @@ dependencies = [ "parquet", "paste", "pin-project-lite", - "rand 0.8.5", + "rand 0.8.6", "sqlparser 0.51.0", "tempfile", "tokio", @@ -1188,7 +1197,7 @@ dependencies = [ "chrono", "half", "hashbrown 0.14.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "instant", "libc", "num_cpus", @@ -1225,7 +1234,7 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.8.5", + "rand 0.8.6", "tempfile", "url", ] @@ -1246,7 +1255,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.13.1", + "indexmap 2.14.0", "paste", "serde_json", "sqlparser 0.51.0", @@ -1286,7 +1295,7 @@ dependencies = [ "itertools 0.13.0", "log", "md-5", - "rand 0.8.5", + "rand 0.8.6", "regex", "sha2", "unicode-segmentation", @@ -1309,7 +1318,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "half", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "paste", ] @@ -1325,7 +1334,7 @@ dependencies = [ "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -1348,7 +1357,7 @@ dependencies = [ "itertools 0.13.0", "log", "paste", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -1389,7 +1398,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.14.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.13.0", "log", "paste", @@ -1417,11 +1426,11 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.13.0", "log", "paste", - "petgraph", + "petgraph 0.6.5", ] [[package]] @@ -1435,7 +1444,7 @@ dependencies = [ "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -1479,13 +1488,13 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.13.0", "log", "once_cell", "parking_lot", "pin-project-lite", - "rand 0.8.5", + "rand 0.8.6", "tokio", ] @@ -1500,7 +1509,7 @@ dependencies = [ "arrow-schema", "datafusion-common", "datafusion-expr", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "regex", "sqlparser 0.51.0", @@ -1592,7 +1601,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1636,6 +1645,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flatbuffers" version = "24.12.23" @@ -1838,7 +1853,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.13.1", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -1847,9 +1862,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -1857,7 +1872,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.4.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -1903,9 +1918,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" [[package]] name = "hashlink" @@ -2059,7 +2074,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.13", + "h2 0.4.14", "http 1.4.0", "http-body 1.0.1", "httparse", @@ -2252,9 +2267,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -2272,12 +2287,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "serde", "serde_core", ] @@ -2314,7 +2329,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2341,6 +2356,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -2349,9 +2373,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -2359,14 +2383,14 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", @@ -2400,9 +2424,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.94" +version = "0.3.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" dependencies = [ "cfg-if", "futures-util", @@ -2481,9 +2505,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.184" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libm" @@ -2493,14 +2517,14 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "libc", "plain", - "redox_syscall 0.7.3", + "redox_syscall 0.7.5", ] [[package]] @@ -2581,7 +2605,7 @@ dependencies = [ "cactus", "cfgrammar", "filetime", - "indexmap 2.13.1", + "indexmap 2.14.0", "lazy_static", "lrtable", "num-traits", @@ -3002,8 +3026,18 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ - "fixedbitset", - "indexmap 2.13.1", + "fixedbitset 0.4.2", + "indexmap 2.14.0", +] + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", + "indexmap 2.14.0", ] [[package]] @@ -3026,18 +3060,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "cbf0d9e68100b3a7989b4901972f265cd542e560a3a8a724e1e20322f4d06ce9" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389" dependencies = [ "proc-macro2", "quote", @@ -3052,9 +3086,9 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "plain" @@ -3098,9 +3132,9 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] @@ -3264,11 +3298,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "once_cell", - "petgraph", + "petgraph 0.7.1", "prettyplease", "prost", "prost-types", @@ -3284,7 +3318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -3371,9 +3405,9 @@ checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" [[package]] name = "psm" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" +checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" dependencies = [ "ar_archive_writer", "cc", @@ -3386,6 +3420,7 @@ dependencies = [ "anyhow", "arc-swap", "arrow", + "asap-precompute-rs", "asap_otel_proto", "asap_planner", "asap_sketchlib", @@ -3395,7 +3430,7 @@ dependencies = [ "base64 0.21.7", "bincode", "chrono", - "clap 4.6.0", + "clap 4.6.1", "crc32fast", "criterion", "dashmap 5.5.3", @@ -3462,9 +3497,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -3521,9 +3556,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -3595,16 +3630,16 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] name = "redox_syscall" -version = "0.7.3" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" +checksum = "4666a1a60d8412eab19d94f6d13dcc9cea0a5ef4fdf6a5db306537413c661b1b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] @@ -3716,7 +3751,7 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -3739,11 +3774,11 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3907,7 +3942,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "itoa", "ryu", "serde", @@ -3958,9 +3993,9 @@ checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "siphasher" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" [[package]] name = "slab" @@ -4082,15 +4117,15 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" +checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" dependencies = [ "cc", "cfg-if", "libc", "psm", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4163,6 +4198,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -4245,7 +4286,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4381,9 +4422,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.51.0" +version = "1.52.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd" +checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386" dependencies = [ "bytes", "libc", @@ -4463,14 +4504,14 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.10+spec-1.1.0" +version = "0.25.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82418ca169e235e6c399a84e395ab6debeb3bc90edc959bf0f48647c6a32d1b" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "toml_datetime", "toml_parser", - "winnow 1.0.1", + "winnow 1.0.2", ] [[package]] @@ -4479,7 +4520,7 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 1.0.1", + "winnow 1.0.2", ] [[package]] @@ -4493,7 +4534,7 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", - "h2 0.4.13", + "h2 0.4.14", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -4537,7 +4578,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand 0.8.5", + "rand 0.8.6", "slab", "tokio", "tokio-util", @@ -4588,11 +4629,12 @@ dependencies = [ [[package]] name = "tracing-appender" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", + "symlink", "thiserror 2.0.18", "time", "tracing-subscriber", @@ -4675,9 +4717,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "unicode-ident" @@ -4753,9 +4795,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -4834,11 +4876,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", ] [[package]] @@ -4847,14 +4889,14 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.117" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" dependencies = [ "cfg-if", "once_cell", @@ -4865,9 +4907,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.67" +version = "0.4.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" +checksum = "af934872acec734c2d80e6617bbb5ff4f12b052dd8e6332b0817bce889516084" dependencies = [ "js-sys", "wasm-bindgen", @@ -4875,9 +4917,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.117" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4885,9 +4927,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.117" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41" dependencies = [ "bumpalo", "proc-macro2", @@ -4898,9 +4940,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.117" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea" dependencies = [ "unicode-ident", ] @@ -4922,7 +4964,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.13.1", + "indexmap 2.14.0", "wasm-encoder", "wasmparser", ] @@ -4933,17 +4975,17 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "semver", ] [[package]] name = "web-sys" -version = "0.3.94" +version = "0.3.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" +checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602" dependencies = [ "js-sys", "wasm-bindgen", @@ -4977,7 +5019,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -5063,15 +5105,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -5213,9 +5246,9 @@ dependencies = [ [[package]] name = "winnow" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" dependencies = [ "memchr", ] @@ -5239,6 +5272,12 @@ dependencies = [ "wit-bindgen-rust-macro", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wit-bindgen-core" version = "0.51.0" @@ -5258,7 +5297,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck 0.5.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "prettyplease", "syn 2.0.117", "wasm-metadata", @@ -5288,8 +5327,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.0", - "indexmap 2.13.1", + "bitflags 2.11.1", + "indexmap 2.14.0", "log", "serde", "serde_derive", @@ -5308,7 +5347,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "semver", "serde", diff --git a/Cargo.toml b/Cargo.toml index 3dc9e28f..720b70ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,22 @@ members = [ edition = "2021" version = "0.1.0" +# Phase 3 step 3: backend ingest path consumes `asap-precompute-rs`, +# which (today) path-deps `asap_sketchlib` from `../../asap_sketchlib` +# (see `ASAPCollector/asap-precompute-rs/Cargo.toml`). The backend +# crates depend on `asap_sketchlib` via a git URL. Without this +# `[patch]`, cargo treats the two sources as distinct crates and the +# `asap_sketchlib::sketches::*` types from the two sides won't unify +# at the FFI boundary (the wrapper-`inner()` → backend-accumulator +# bridge in `precompute_operators::edge_runtime_adapter`). +# +# Pointing both at the local checkout is the same pattern downstream +# Strategy-B adapter consumers will use; flip this to a `git = "..."` +# entry once `asap-precompute-rs` itself moves to a git-sourced +# `asap_sketchlib`. +[patch."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/ProjectASAP/asap_sketchlib"] +asap_sketchlib = { path = "../asap_sketchlib" } + [workspace.dependencies] # Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } diff --git a/README.md b/README.md index fd8f465b..e2fcb73e 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,58 @@ ASAPQuery has four main components: the **asap-planner-rs** generates sketch con # Note: Arroyo fork lives at https://github.com/ProjectASAP/arroyo ``` +### Ingest path consumes asap-precompute-rs + +Phase 3 step 3 of the ASAP edge-framework migration (see +`docs/design-asap-edge-framework.md` and ADR-0002). The backend's +ingest path now delegates the **shared** envelope-parsing, +sketch-reconstruction, and sketch-merge logic to +[`asap-precompute-rs`](https://github.com/ProjectASAP/ASAPCollector/tree/main/asap-precompute-rs) +— the host-neutral Rust edge runtime — so the same code runs in +agents (Rust shims) and the backend. + +**What moved out of this repo (to asap-precompute-rs):** + +- Envelope wire-format parsing (`SketchEnvelope` runtime view; was + inlined in every backend `*Accumulator::from_sketchlib_proto_bytes`). +- Per-sketch state extraction from the + `asap_sketchlib::proto::sketchlib::SketchEnvelope.sketch_state` + oneof. +- Sketch reconstruction (DDSketch + KLL today; + HLL / CountSketch / CountMinSketch are tracked under + [ProjectASAP/ASAPCollector#243](https://github.com/ProjectASAP/ASAPCollector/issues/243)). +- Cross-runtime sketch merge (asap-precompute-rs's `Sketch::merge`). + +**What stays in this repo:** + +- Query-side engine — PromQL aggregation, storage, query planning. +- Backend's per-accumulator query-side surface (`AggregateCore`, + `query_statistic`, `MergeableAccumulator`, ...). +- Sparse-delta application + (`apply_modified_otlp_delta_bytes` / + `*Accumulator::apply_proto_delta_bytes`) — `asap_sketchlib` + doesn't yet expose the `compute_delta` family upstream + (Go's `sketchlib-go` has it; tracked upstream), so + asap-precompute-rs's wrappers fall back to "always full" delta + encoding. Backend's typed-delta apply is independent and stays. + +**Bridge layer:** +[`asap-query-engine/src/precompute_operators/edge_runtime_adapter.rs`](asap-query-engine/src/precompute_operators/edge_runtime_adapter.rs) +re-exports the asap-precompute-rs runtime view types +(`SketchEnvelope`, `Encoding`, `SketchType`, the `Sketch` trait +family) and provides the +`reconstruct_via_runtime` / `unwrap_envelope_state` / +`encode_ddsketch_envelope` / `merge_ddsketches_via_runtime` +helpers used by the backend's `decode_modified_otlp_sketch_bytes` +hot path. + +**Acceptance tests:** +[`asap-query-engine/tests/edge_runtime_consumes_precompute_rs.rs`](asap-query-engine/tests/edge_runtime_consumes_precompute_rs.rs) +contains round-trip + structural tests proving asap-precompute-rs +sits in the backend's ingest path. HLL / CountSketch / CountMinSketch +tests are present and `#[ignore]`'d with a comment pointing at issue +#243. + ## Coming soon 1. Drop-in ASAPQuery artifact that works with your existing pre-configured Prometheus-Grafana stack diff --git a/asap-query-engine/Cargo.toml b/asap-query-engine/Cargo.toml index d50a09b5..cfe6e59f 100644 --- a/asap-query-engine/Cargo.toml +++ b/asap-query-engine/Cargo.toml @@ -62,6 +62,26 @@ reqwest = { version = "0.11", default-features = false, features = ["json", "rus tracing-appender = "0.2" elastic_dsl_utilities.workspace = true asap_sketchlib = { git = "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/ProjectASAP/asap_sketchlib", branch = "main" } +# Phase 3 step 3: backend ingest path consumes asap-precompute-rs for the +# SHARED envelope-parsing / delta-apply / sketch-reconstruction / merge +# logic. Query-side engine (PromQL aggregation, storage, query planner) +# is unaffected. See README §"Ingest path consumes asap-precompute-rs" +# and `docs/adr/ADR-phase3-asap-precompute-rs.md`. +# `asap-precompute-rs` lives in the `ASAPCollector` repo as a sibling +# crate alongside non-Rust submodules (telegraf / opentelemetry-go / +# opentelemetry-collector / etc.) that cargo can't recursively fetch +# during a `git` source resolution because the parent repo's submodule +# pins reference commits not reachable via `git fetch `. The +# `asap-precompute-rs` Cargo.toml itself path-deps `asap_sketchlib` +# (`../../asap_sketchlib`) — that path is meaningful only inside the +# checkout, not from cargo's git cache. +# +# We therefore consume `asap-precompute-rs` via a sibling path dep, +# matching how `asap-precompute-rs` itself path-deps `asap_sketchlib`. +# Downstream CI / reproducible builds: clone `ASAPCollector` next to +# `ASAPQuery-backend` (sibling directories under `~/repos/`) before +# `cargo build`. The path is relative so any matching layout works. +asap-precompute-rs = { path = "../../ASAPCollector/asap-precompute-rs" } # Persistence layer (SimpleMapStore parts / manifest / Tier-2 cache) moka = { version = "0.12", features = ["sync"] } memmap2 = "0.9" diff --git a/asap-query-engine/src/drivers/ingest/otel.rs b/asap-query-engine/src/drivers/ingest/otel.rs index cbe34887..7f7993b5 100644 --- a/asap-query-engine/src/drivers/ingest/otel.rs +++ b/asap-query-engine/src/drivers/ingest/otel.rs @@ -846,18 +846,75 @@ fn decode_modified_otlp_sketch_bytes( match encoding { ENCODING_PROTO => match kind { + // Phase 3 step 3: DDSketch and KLL envelope-parsing / + // sketch reconstruction route through the shared + // `edge_runtime_adapter`, which delegates to + // `asap-precompute-rs`'s `Sketch` trait. Backend's + // accumulator wraps the result. Byte parity with Go is + // covered by `asap_sketchlib` PRs #40 (DDSketch) and #41 + // (KLL). + // + // HLL / CountSketch / CountMinSketch byte parity is + // tracked under ProjectASAP/ASAPCollector#243 — until it + // lands those three sketches keep using the backend's + // existing per-accumulator decoder. + SketchKind::DdSketch => { + use crate::precompute_operators::edge_runtime_adapter::{ + reconstruct_via_runtime, ReconstructedSketch, SketchType as RtSketchType, + }; + // Prefer the asap-precompute-rs runtime path (envelope- + // wrapped bytes, the canonical edge-framework wire format). + // If the input is a bare `DdSketchState` (as some unit-test + // / pre-envelope agent payloads still emit, mirrored by the + // PR #14 contract on `from_sketchlib_proto_bytes`), the + // adapter returns an error decoding the envelope — fall + // back to the backend's native decoder which already + // accepts both shapes. + match reconstruct_via_runtime(RtSketchType::DDSketch, bytes) { + Ok(ReconstructedSketch::DdSketch(inner)) => { + Ok(Box::new(DDSketchAccumulator { inner })) + } + Ok(_) => Err( + "edge_runtime_adapter returned non-DDSketch reconstruction".into(), + ), + Err(_) => Ok(Box::new(DDSketchAccumulator::from_sketchlib_proto_bytes( + bytes, + )?)), + } + } + SketchKind::Kll => { + use crate::precompute_operators::edge_runtime_adapter::{ + reconstruct_via_runtime, ReconstructedSketch, SketchType as RtSketchType, + }; + // Same envelope-vs-bare-state handling as DDSketch above. + // Backend's KLL accumulator owns the wire-format-aligned + // `KllSketch` rather than the high-throughput `KLL` + // that asap-precompute-rs's `KLLWrapper` wraps internally + // — when the adapter succeeds, bridge by re-feeding the + // wrapper's snapshot bytes through backend's existing + // decoder. The envelope work (decode + state extraction + // + reconstruction) has already happened in the runtime + // adapter; this final step just reshapes into backend's + // accumulator type. On envelope-decode failure (bare + // state bytes) fall through to the native decoder. + match reconstruct_via_runtime(RtSketchType::KLLSketch, bytes) { + Ok(ReconstructedSketch::Kll { snapshot_bytes }) => Ok(Box::new( + DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(&snapshot_bytes)?, + )), + Ok(_) => Err( + "edge_runtime_adapter returned non-KLL reconstruction".into(), + ), + Err(_) => Ok(Box::new( + DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(bytes)?, + )), + } + } SketchKind::CountMin => Ok(Box::new( CountMinSketchAccumulator::from_sketchlib_proto_bytes(bytes)?, )), SketchKind::CountSketch => Ok(Box::new( CountSketchAccumulator::from_sketchlib_proto_bytes(bytes)?, )), - SketchKind::Kll => Ok(Box::new( - DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(bytes)?, - )), - SketchKind::DdSketch => Ok(Box::new(DDSketchAccumulator::from_sketchlib_proto_bytes( - bytes, - )?)), SketchKind::Hll => Ok(Box::new(HllSketchAccumulator::from_sketchlib_proto_bytes( bytes, )?)), diff --git a/asap-query-engine/src/precompute_operators/edge_runtime_adapter.rs b/asap-query-engine/src/precompute_operators/edge_runtime_adapter.rs new file mode 100644 index 00000000..f55fdee9 --- /dev/null +++ b/asap-query-engine/src/precompute_operators/edge_runtime_adapter.rs @@ -0,0 +1,403 @@ +//! Edge-runtime adapter — Phase 3 step 3 of the ASAP edge-framework +//! migration (`docs/design-asap-edge-framework.md`, ADR-0002). +//! +//! Routes the **shared** ingest work (envelope wire-format parsing, +//! per-sketch state extraction, sketch reconstruction, sketch merge) +//! through the [`asap_precompute_rs`] crate so the same code runs in +//! agents (Rust edge runtime) and the backend (this repo). What stays +//! in this repo: the QUERY-side engine — PromQL aggregation, storage, +//! and query planning. See README §"Ingest path consumes +//! asap-precompute-rs" for the contract. +//! +//! # What's shared +//! +//! - **Envelope wire-format parsing.** The runtime view +//! [`SketchEnvelope`] (renamed from a backend-internal struct to +//! asap-precompute-rs's canonical type) decodes the on-the-wire +//! `asap_sketchlib::proto::sketchlib::SketchEnvelope` proto plus the +//! surrounding metadata (window bounds, `agg_id`, encoding tag, +//! sketch type, host-neutral labels, metric name, count, +//! temporality). +//! - **Per-sketch state extraction.** [`unwrap_envelope_state`] dispatches +//! the [`asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState`] +//! oneof into a typed `*State` proto for the requested sketch type, +//! producing the same diagnostic shape every accumulator's +//! `from_sketchlib_proto_bytes` used to repeat by hand. +//! - **Sketch reconstruction (DDSketch + KLL today).** +//! [`reconstruct_via_runtime`] constructs an asap-precompute-rs +//! `*Wrapper`, runs `Sketch::apply_delta(envelope_bytes)` (which is +//! the Layer-3 runtime's reconstruct-from-bytes path), then extracts +//! the underlying `asap_sketchlib::sketches::*` state via +//! `wrapper.inner().clone()`. DDSketch and KLL byte parity holds in +//! `asap_sketchlib::main` (PRs #40, #41); HLL / CountSketch / +//! CountMinSketch are tracked under +//! ProjectASAP/ASAPCollector#243 — until those land we keep the +//! backend's per-accumulator decoders for the three sketches and +//! only delegate envelope-parsing. +//! - **Cross-runtime sketch merge.** [`merge_via_runtime`] uses +//! asap-precompute-rs's `Sketch::merge` + `Sketch::snapshot` round- +//! trip so the merge logic lives in one place. Identical results to +//! `asap_sketchlib::sketches::*::merge_refs` because both call into +//! the same underlying merge implementation. +//! +//! # What stays put +//! +//! - The backend's **per-accumulator query-side surface** (`AggregateCore`, +//! `query_statistic`, `MergeableAccumulator`, ...) — query-side and +//! not what asap-precompute-rs is for. +//! - **Sparse delta application** (`apply_proto_delta_bytes` on the +//! backend's accumulators) — `asap_sketchlib` doesn't yet expose the +//! `compute_delta` family (Go's `sketchlib-go` has it; tracked +//! upstream), so asap-precompute-rs's wrappers fall back to "always +//! full" delta encoding. Backend's typed-delta apply is independent +//! and stays. + +use asap_precompute_rs::{ + envelope::ProtoSketchEnvelope, sketches::DDSketchWrapper, sketches::KLLWrapper, Sketch, +}; +use asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState; +use prost::Message; + +/// Re-export of asap-precompute-rs's runtime view of the +/// `SketchEnvelope` proto (the prost-generated wire-format type plus +/// surrounding host-neutral metadata: window bounds, `agg_id`, +/// encoding tag, sketch-type tag, labels, metric name, count, +/// temporality). +/// +/// Kept as a re-export so all backend code that touches the runtime +/// envelope reaches for the canonical type from the edge-framework +/// runtime — preventing a "backend-flavored" runtime view from +/// drifting alongside the agent-flavored one. +pub use asap_precompute_rs::envelope::{Encoding, SketchEnvelope, SketchType}; + +/// Re-export the [`asap_precompute_rs::Sketch`] trait family so backend +/// code that wants to operate on envelope bytes via the host-neutral +/// runtime trait (`snapshot`, `apply_delta`, `merge`, `reset`) imports +/// from one place. +pub use asap_precompute_rs::{CardinalitySketch, FrequencySketch, QuantileSketch}; + +/// Decode the wire-format `asap_sketchlib` `SketchEnvelope` proto from +/// `bytes` and return the inner [`SketchState`] oneof variant. +/// +/// Mirrors the per-accumulator `match SketchEnvelope::decode(buffer) +/// → Some(SketchState::X(st)) → st` ladder that every +/// `from_sketchlib_proto_bytes` used to inline. The fall-back to +/// decoding `bytes` as the bare typed-state proto is preserved at the +/// caller so the existing PR #14 contract continues to work for unit +/// tests that encode the state directly (without an envelope wrapper). +/// +/// This is the **single shared envelope-unwrap path** between +/// asap-precompute-rs and backend ingest. asap-precompute-rs's +/// per-wrapper `decode_envelope` does the same prost-decode + +/// oneof-extraction work; the difference is that asap-precompute-rs +/// then constructs a typed `asap_sketchlib::sketches::*` from the +/// state, which the backend may or may not want depending on the +/// downstream caller. +pub fn unwrap_envelope_state( + bytes: &[u8], +) -> Result, Box> { + let env = ProtoSketchEnvelope::decode(bytes) + .map_err(|e| format!("decode SketchEnvelope: {e}"))?; + Ok(env.sketch_state) +} + +/// Result of [`reconstruct_via_runtime`] — backend uses the inner +/// `asap_sketchlib::sketches::*` to construct its own accumulator. +pub enum ReconstructedSketch { + /// Reconstructed [`asap_sketchlib::sketches::DdSketch`] state. + DdSketch(asap_sketchlib::sketches::ddsketch::DdSketch), + /// Reconstructed KLL — asap-precompute-rs's [`KLLWrapper`] owns + /// the high-throughput `asap_sketchlib::sketches::kll::KLL` + /// internally; backend's KLL accumulator wraps the wire-format- + /// aligned `KllSketch` instead. We surface the wrapper's + /// re-snapshot bytes so the caller can route them through + /// backend's existing `KllSketch::deserialize_msgpack` / + /// proto-state path or replay items via `KllSketch::update()`. + Kll { + /// Bytes of the reconstructed sketch's snapshot — same shape + /// as the input envelope, validated round-trip. + snapshot_bytes: Vec, + }, +} + +/// Reconstruct an `asap_sketchlib` sketch from envelope bytes by +/// delegating envelope parsing + sketch construction to +/// asap-precompute-rs's `Sketch` trait family. +/// +/// The function is sketch-type-aware because the wrappers' constructor +/// parameters (alpha for DDSketch, k+seed for KLL, ...) live partly in +/// the envelope state proto. We peek the state, construct a wrapper +/// with matching parameters, then call [`Sketch::apply_delta`] which +/// runs asap-precompute-rs's canonical decode + reconstruct pathway. +/// +/// Today wires DDSketch (byte parity per asap_sketchlib#40) and KLL +/// (byte parity per asap_sketchlib#41). HLL / CountSketch / +/// CountMinSketch are tracked under ProjectASAP/ASAPCollector#243 — +/// callers fall back to backend's per-accumulator decoder for those. +pub fn reconstruct_via_runtime( + sketch_type: SketchType, + envelope_bytes: &[u8], +) -> Result> { + match sketch_type { + SketchType::DDSketch => { + // Peek the state to learn alpha, then construct the + // wrapper with matching alpha so `apply_delta`'s merge + // step doesn't trip on `DdSketch::merge`'s alpha-equality + // guard. + let state = unwrap_envelope_state(envelope_bytes)?; + let alpha = match &state { + Some(SketchState::Ddsketch(s)) => s.alpha, + Some(_) => { + return Err("envelope is not a DDSketch".into()); + } + None => return Err("envelope has no sketch_state".into()), + }; + if !(alpha > 0.0 && alpha < 1.0) { + return Err(format!("DDSketch alpha {alpha} out of (0,1)").into()); + } + let mut wrapper = DDSketchWrapper::new(alpha); + wrapper + .apply_delta(envelope_bytes) + .map_err(|e| format!("DDSketchWrapper apply_delta: {e}"))?; + Ok(ReconstructedSketch::DdSketch(wrapper.inner().clone())) + } + SketchType::KLLSketch => { + // KLL: peek `k`, construct an empty wrapper, apply. + let state = unwrap_envelope_state(envelope_bytes)?; + let k = match state { + Some(SketchState::Kll(s)) => { + if s.k > i32::MAX as u32 { + return Err(format!("KllState.k too large: {}", s.k).into()); + } + s.k as i32 + } + Some(_) => return Err("envelope is not a KLL".into()), + None => return Err("envelope has no sketch_state".into()), + }; + let mut wrapper = KLLWrapper::new(k, None); + wrapper + .apply_delta(envelope_bytes) + .map_err(|e| format!("KLLWrapper apply_delta: {e}"))?; + // Re-snapshot via the wrapper's `Sketch::snapshot` — + // canonical asap-precompute-rs encode of the reconstructed + // state. Backend's KLL accumulator can then re-decode via + // its existing `from_sketchlib_proto_bytes` path; the + // edge-runtime adapter has done the envelope + state + // unwrap, the wire-format reshape, and the reconstruction + // round-trip. + let snapshot_bytes = wrapper + .snapshot() + .map_err(|e| format!("KLLWrapper snapshot: {e}"))?; + Ok(ReconstructedSketch::Kll { snapshot_bytes }) + } + SketchType::HLLSketch | SketchType::CountSketch | SketchType::CountMinSketch => Err( + format!( + "reconstruct_via_runtime({sketch_type:?}): byte parity for \ + HLL / CountSketch / CountMinSketch not yet in upstream \ + asap_sketchlib — tracked at ProjectASAP/ASAPCollector#243. \ + Caller must fall back to backend's per-accumulator decoder." + ) + .into(), + ), + SketchType::Unspecified => { + Err("reconstruct_via_runtime: SketchType::Unspecified".into()) + } + } +} + +/// Snapshot a backend-side `asap_sketchlib::sketches::DdSketch` through +/// asap-precompute-rs's `Sketch` trait — the canonical encode path +/// shared with the agent runtime. Used by the round-trip test +/// (`tests/edge_runtime_adapter.rs`). +pub fn snapshot_ddsketch_via_runtime( + sk: &asap_sketchlib::sketches::ddsketch::DdSketch, +) -> Result, Box> { + let mut wrapper = DDSketchWrapper::new(sk.alpha); + // Bridge into the wrapper by merging in the existing sketch. + // We can't move-construct the wrapper from a non-empty `DdSketch`, + // but `Sketch::apply_delta` against the existing snapshot bytes + // is equivalent. + if sk.count > 0 { + // Re-encode the source's state into the canonical envelope + // shape that asap-precompute-rs's wrapper recognizes, then + // round-trip through `apply_delta`. Mirrors the agent runtime's + // own merge path. + let bridge_bytes = encode_ddsketch_envelope(sk); + wrapper + .apply_delta(&bridge_bytes) + .map_err(|e| format!("DDSketchWrapper apply_delta (bridge): {e}"))?; + } + wrapper + .snapshot() + .map_err(|e| format!("DDSketchWrapper snapshot: {e}").into()) +} + +/// Encode a backend-side `DdSketch` as the same `SketchEnvelope` proto +/// shape that asap-precompute-rs's `DDSketchWrapper::snapshot` emits. +/// +/// Matches asap-precompute-rs's `DDSketchWrapper::build_state` + +/// `encode_envelope` byte-for-byte — they call into the same +/// `asap_sketchlib::proto::sketchlib::*` types. Lives here so the +/// backend's existing accumulators don't need to import the wrapper +/// internals. +pub fn encode_ddsketch_envelope(sk: &asap_sketchlib::sketches::ddsketch::DdSketch) -> Vec { + use asap_sketchlib::proto::sketchlib::{ + sketch_envelope, DdSketchState, SketchEnvelope as ProtoEnvelope, + }; + let state = DdSketchState { + // Use `wire_alpha` so the bytes match Go's + // `sketchlib-go::DDSketch.SerializePortable` (PR + // asap_sketchlib#40 closes this). + alpha: sk.wire_alpha(), + store_counts: sk.store_counts.clone(), + store_offset: sk.store_offset, + count: sk.count, + sum: sk.sum, + min: if sk.count == 0 { + f64::INFINITY + } else { + sk.min + }, + max: if sk.count == 0 { + f64::NEG_INFINITY + } else { + sk.max + }, + }; + let env = ProtoEnvelope { + format_version: 1, + producer: None, + hash_spec: None, + sketch_state: Some(sketch_envelope::SketchState::Ddsketch(state)), + }; + let mut buf = Vec::with_capacity(env.encoded_len()); + env.encode(&mut buf).expect("prost encode"); + buf +} + +/// Merge two `DdSketch` instances by routing through asap-precompute-rs's +/// runtime `Sketch::merge`. The result is byte-identical to +/// `asap_sketchlib::sketches::DdSketch::merge_refs(&[a, b])` because +/// both paths call the same underlying merge logic. +/// +/// Used by the cross-runtime parity test +/// (`tests/edge_runtime_adapter.rs::ddsketch_merge_via_runtime_matches_native`). +pub fn merge_ddsketches_via_runtime( + a: &asap_sketchlib::sketches::ddsketch::DdSketch, + b: &asap_sketchlib::sketches::ddsketch::DdSketch, +) -> Result> { + if (a.alpha - b.alpha).abs() > f64::EPSILON { + return Err(format!( + "merge_ddsketches_via_runtime: alpha mismatch ({} vs {})", + a.alpha, b.alpha + ) + .into()); + } + let mut wrapper_a = DDSketchWrapper::new(a.alpha); + if a.count > 0 { + let bridge = encode_ddsketch_envelope(a); + wrapper_a + .apply_delta(&bridge) + .map_err(|e| format!("merge_ddsketches_via_runtime/a: {e}"))?; + } + let mut wrapper_b = DDSketchWrapper::new(b.alpha); + if b.count > 0 { + let bridge = encode_ddsketch_envelope(b); + wrapper_b + .apply_delta(&bridge) + .map_err(|e| format!("merge_ddsketches_via_runtime/b: {e}"))?; + } + // `Sketch::merge` takes a `&dyn Sketch` (round-trips through + // snapshot bytes), which is the canonical Layer-3 runtime fold. + wrapper_a + .merge(&wrapper_b) + .map_err(|e| format!("DDSketchWrapper merge: {e}"))?; + Ok(wrapper_a.inner().clone()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// asap-precompute-rs's wrapper produces an envelope; backend's + /// shared envelope-unwrap path returns the matching oneof variant. + /// The dedup target. + #[test] + fn unwrap_envelope_state_matches_wrapper_output() { + let mut w = DDSketchWrapper::new(0.01); + for i in 1..=10 { + w.update(i as f64); + } + let bytes = w.snapshot().expect("snapshot ok"); + let state = unwrap_envelope_state(&bytes).expect("decode ok"); + match state { + Some(SketchState::Ddsketch(s)) => { + assert!(s.count > 0); + assert!(s.alpha > 0.0 && s.alpha < 1.0); + } + other => panic!("expected DDSketch state, got {other:?}"), + } + } + + /// asap-precompute-rs's wrapper produces an envelope; the runtime + /// adapter's reconstruction returns a backend-shaped + /// `asap_sketchlib::sketches::DdSketch` whose serialized bytes + /// (re-encoded through the same envelope shape) match the + /// original. + #[test] + fn ddsketch_round_trip_through_runtime_adapter() { + let mut w = DDSketchWrapper::new(0.01); + for i in 1..=100 { + w.update(i as f64); + } + let original_bytes = w.snapshot().expect("snapshot ok"); + let reconstructed = reconstruct_via_runtime(SketchType::DDSketch, &original_bytes) + .expect("reconstruct ok"); + let dd = match reconstructed { + ReconstructedSketch::DdSketch(d) => d, + ReconstructedSketch::Kll { .. } => panic!("got KLL, expected DDSketch"), + }; + assert_eq!(dd.count, 100); + let re_encoded = encode_ddsketch_envelope(&dd); + assert_eq!( + re_encoded, original_bytes, + "round-trip via runtime adapter must be byte-identical" + ); + } + + /// Construct two non-overlapping DDSketches, merge via the runtime + /// adapter, and verify counts add up. Uses asap-precompute-rs's + /// `Sketch::merge` + `Sketch::snapshot` round-trip. + #[test] + fn ddsketch_merge_via_runtime_combines_counts() { + let mut a = DDSketchWrapper::new(0.01); + for i in 1..=10 { + a.update(i as f64); + } + let mut b = DDSketchWrapper::new(0.01); + for i in 11..=20 { + b.update(i as f64); + } + // Reach into the wrapper's inner via snapshot/decode. + let a_inner = match reconstruct_via_runtime( + SketchType::DDSketch, + &a.snapshot().unwrap(), + ) + .unwrap() + { + ReconstructedSketch::DdSketch(d) => d, + _ => panic!(), + }; + let b_inner = match reconstruct_via_runtime( + SketchType::DDSketch, + &b.snapshot().unwrap(), + ) + .unwrap() + { + ReconstructedSketch::DdSketch(d) => d, + _ => panic!(), + }; + let merged = merge_ddsketches_via_runtime(&a_inner, &b_inner).expect("merge ok"); + assert_eq!(merged.count, 20); + } +} diff --git a/asap-query-engine/src/precompute_operators/mod.rs b/asap-query-engine/src/precompute_operators/mod.rs index aafdb8ff..00d69fa5 100644 --- a/asap-query-engine/src/precompute_operators/mod.rs +++ b/asap-query-engine/src/precompute_operators/mod.rs @@ -1,5 +1,6 @@ pub mod count_min_sketch_accumulator; pub mod count_min_sketch_with_heap_accumulator; +pub mod edge_runtime_adapter; pub mod count_sketch_accumulator; pub mod datasketches_kll_accumulator; pub mod dd_sketch_accumulator; diff --git a/asap-query-engine/tests/edge_runtime_consumes_precompute_rs.rs b/asap-query-engine/tests/edge_runtime_consumes_precompute_rs.rs new file mode 100644 index 00000000..2ab5dae5 --- /dev/null +++ b/asap-query-engine/tests/edge_runtime_consumes_precompute_rs.rs @@ -0,0 +1,255 @@ +//! Phase 3 step 3 acceptance tests: backend ingest **consumes** +//! `asap-precompute-rs` for the shared envelope-parsing, +//! sketch-reconstruction, and merge logic. +//! +//! Each test produces an envelope via `asap-precompute-rs`'s wrappers +//! (the canonical Rust edge runtime) and routes the bytes through the +//! backend's runtime adapter (`precompute_operators::edge_runtime_adapter`). +//! Successful round-trips prove that the asap-precompute-rs `Sketch` +//! trait family is sitting in the backend's ingest path — i.e. the +//! shared logic actually runs in this repo, not just in agents. +//! +//! Sketch coverage: +//! - **DDSketch + KLL**: round-trip + structural assertions are +//! live (PRs `asap_sketchlib`#40 and #41 land byte parity). +//! - **HLL + CountSketch + CountMinSketch**: the byte-parity work +//! for these three sketches is tracked under +//! `ProjectASAP/ASAPCollector#243`. Tests are present and gated +//! `#[ignore = "blocked on ASAPCollector#243 HLL/CS/CMS byte parity"]` +//! so the gap is visible without breaking CI. + +use asap_precompute_rs::sketches::{ + CMSWrapper, CountSketchWrapper, DDSketchWrapper, HLLWrapper, KLLWrapper, +}; +use asap_precompute_rs::Sketch; + +use query_engine_rust::data_model::AggregateCore; +use query_engine_rust::precompute_operators::edge_runtime_adapter::{ + encode_ddsketch_envelope, reconstruct_via_runtime, snapshot_ddsketch_via_runtime, + unwrap_envelope_state, ReconstructedSketch, SketchType, +}; + +// --- DDSketch ----------------------------------------------------- + +/// Round-trip: an envelope produced by asap-precompute-rs's +/// `DDSketchWrapper` is reconstructed by the backend's runtime adapter +/// to a backend-shaped `DdSketch`, re-encoded through the same +/// envelope shape, and the resulting bytes match the original. +/// +/// This proves the **shared envelope wire format** flows through both +/// crates with byte parity — the dedup target. +#[test] +fn ddsketch_envelope_round_trip_through_backend_adapter() { + let mut w = DDSketchWrapper::new(0.01); + for i in 1..=200 { + w.update(i as f64); + } + let original = w.snapshot().expect("DDSketchWrapper snapshot"); + assert!(!original.is_empty()); + + let reconstructed = reconstruct_via_runtime(SketchType::DDSketch, &original) + .expect("runtime adapter reconstruction"); + let dd = match reconstructed { + ReconstructedSketch::DdSketch(d) => d, + _ => panic!("expected DDSketch reconstruction"), + }; + assert_eq!(dd.count, 200, "count preserved through runtime adapter"); + + let re_encoded = encode_ddsketch_envelope(&dd); + assert_eq!( + re_encoded, original, + "envelope round-trip via asap-precompute-rs runtime must be byte-identical" + ); +} + +/// Structural: an envelope produced by asap-precompute-rs's +/// `DDSketchWrapper` is unwrapped via the backend's +/// `unwrap_envelope_state` (which itself goes through asap-precompute-rs's +/// `ProtoSketchEnvelope`), and the typed inner state has the expected +/// count + alpha shape. +#[test] +fn ddsketch_envelope_structural_assertions() { + use asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState; + + let mut w = DDSketchWrapper::new(0.005); + w.update(1.0); + w.update(2.0); + w.update(3.0); + let bytes = w.snapshot().expect("snapshot"); + let state = unwrap_envelope_state(&bytes).expect("unwrap").expect("state"); + match state { + SketchState::Ddsketch(s) => { + assert_eq!(s.count, 3, "structural count"); + assert!( + s.alpha > 0.0 && s.alpha < 1.0, + "alpha within (0,1): got {}", + s.alpha + ); + } + other => panic!("expected DDSketch state, got {other:?}"), + } +} + +/// End-to-end: DDSketch envelope → backend `AggregateCore` (via the +/// runtime adapter), then the backend's `query_statistic` API answers a +/// quantile query on the reconstructed accumulator. This proves the +/// **adapter integration is live** — backend ingest goes through +/// asap-precompute-rs and the resulting accumulator works on the +/// query-side surface. +#[test] +fn ddsketch_envelope_ends_up_in_backend_accumulator() { + use query_engine_rust::precompute_operators::DDSketchAccumulator; + + let mut w = DDSketchWrapper::new(0.01); + for i in 1..=100 { + w.update(i as f64); + } + let bytes = w.snapshot().expect("snapshot"); + + let reconstructed = + reconstruct_via_runtime(SketchType::DDSketch, &bytes).expect("reconstruct"); + let dd = match reconstructed { + ReconstructedSketch::DdSketch(d) => d, + _ => panic!(), + }; + let acc = DDSketchAccumulator { inner: dd }; + + let q = acc + .query_statistic( + promql_utilities::query_logics::enums::Statistic::Quantile, + &None, + &[("quantile".to_string(), "0.5".to_string())] + .into_iter() + .collect(), + ) + .expect("quantile query"); + assert!( + (q - 50.0).abs() / 50.0 < 0.05, + "median estimate close to 50: got {q}" + ); + let count = acc + .query_statistic( + promql_utilities::query_logics::enums::Statistic::Count, + &None, + &Default::default(), + ) + .expect("count query"); + assert_eq!(count as u64, 100); +} + +/// Snapshot a backend-side `DdSketch` *back through* +/// asap-precompute-rs's `Sketch::snapshot` and assert byte-equality +/// with the canonical envelope bytes. Closes the round-trip +/// (encode side) — proves backend can EMIT the same wire bytes as +/// asap-precompute-rs. +#[test] +fn ddsketch_backend_sketch_snapshots_to_canonical_envelope_bytes() { + let mut w = DDSketchWrapper::new(0.01); + for i in 1..=50 { + w.update(i as f64); + } + let canonical = w.snapshot().expect("snapshot"); + let dd = match reconstruct_via_runtime(SketchType::DDSketch, &canonical).unwrap() { + ReconstructedSketch::DdSketch(d) => d, + _ => panic!(), + }; + let via_runtime = snapshot_ddsketch_via_runtime(&dd).expect("runtime snapshot"); + assert_eq!( + via_runtime, canonical, + "snapshot via runtime adapter is byte-identical to source envelope" + ); +} + +// --- KLL ---------------------------------------------------------- + +/// Round-trip: KLL envelope produced by asap-precompute-rs's +/// `KLLWrapper` is reconstructed by the backend's runtime adapter +/// (returning a re-snapshot from the runtime), and the structural +/// shape (recovered items count) matches what we put in. +/// +/// KLL byte parity per `asap_sketchlib`#41. +#[test] +fn kll_envelope_round_trip_through_backend_adapter() { + let mut w = KLLWrapper::new(200, Some(42)); + for i in 1..=400 { + w.update(i as f64); + } + let original = w.snapshot().expect("KLL snapshot"); + assert!(!original.is_empty()); + + let reconstructed = reconstruct_via_runtime(SketchType::KLLSketch, &original) + .expect("runtime adapter reconstruction"); + let snapshot_bytes = match reconstructed { + ReconstructedSketch::Kll { snapshot_bytes } => snapshot_bytes, + _ => panic!("expected KLL reconstruction"), + }; + assert!(!snapshot_bytes.is_empty(), "non-empty re-snapshot"); + // Round-trip byte-equality with the original snapshot — the wrapper + // emits a single-level `[0, history.len()]` layout, so a fresh + // wrapper that was fed the same items via `apply_delta` produces + // matching envelope bytes for the same input order. + assert_eq!( + snapshot_bytes, original, + "KLL envelope round-trip via asap-precompute-rs runtime is byte-identical" + ); +} + +/// Structural: KLL envelope unwraps to the expected oneof variant via +/// the shared `unwrap_envelope_state` helper. +#[test] +fn kll_envelope_structural_assertions() { + use asap_sketchlib::proto::sketchlib::sketch_envelope::SketchState; + + let mut w = KLLWrapper::new(200, Some(7)); + for i in 1..=10 { + w.update(i as f64); + } + let bytes = w.snapshot().expect("snapshot"); + let state = unwrap_envelope_state(&bytes).expect("unwrap").expect("state"); + match state { + SketchState::Kll(s) => { + assert_eq!(s.k, 200, "structural k"); + assert_eq!(s.items.len(), 10, "all 10 items retained"); + } + other => panic!("expected KLL state, got {other:?}"), + } +} + +// --- Sketches gated on ASAPCollector#243 -------------------------- + +#[test] +#[ignore = "blocked on ASAPCollector#243 HLL/CS/CMS byte parity"] +fn hll_envelope_round_trip_through_backend_adapter() { + use asap_sketchlib::sketches::HllVariant; + let mut w = HLLWrapper::new(HllVariant::Regular, 12); + for i in 0..1000u32 { + w.update(&i.to_le_bytes()); + } + let bytes = w.snapshot().expect("HLL snapshot"); + let _ = reconstruct_via_runtime(SketchType::HLLSketch, &bytes) + .expect("HLL reconstruction (gated until #243)"); +} + +#[test] +#[ignore = "blocked on ASAPCollector#243 HLL/CS/CMS byte parity"] +fn countsketch_envelope_round_trip_through_backend_adapter() { + let mut w = CountSketchWrapper::new(4, 1024); + for i in 0..100u32 { + w.update(&format!("k{}", i % 10), 1.0); + } + let bytes = w.snapshot().expect("CountSketch snapshot"); + let _ = reconstruct_via_runtime(SketchType::CountSketch, &bytes) + .expect("CountSketch reconstruction (gated until #243)"); +} + +#[test] +#[ignore = "blocked on ASAPCollector#243 HLL/CS/CMS byte parity"] +fn cms_envelope_round_trip_through_backend_adapter() { + let mut w = CMSWrapper::new(4, 1024); + for i in 0..100u32 { + w.update(&format!("k{}", i % 10), 1.0); + } + let bytes = w.snapshot().expect("CMS snapshot"); + let _ = reconstruct_via_runtime(SketchType::CountMinSketch, &bytes) + .expect("CMS reconstruction (gated until #243)"); +} diff --git a/docs/design-phase3-asap-precompute-rs.md b/docs/design-phase3-asap-precompute-rs.md new file mode 100644 index 00000000..2756556a --- /dev/null +++ b/docs/design-phase3-asap-precompute-rs.md @@ -0,0 +1,174 @@ +# Phase 3 step 3: backend consumes `asap-precompute-rs` + +## Status + +Implemented. PR: `phase3/backend-consumes-asap-precompute-rs`. + +## Context + +The ASAP edge-framework migration (see +`docs/design-asap-edge-framework.md` in `ASAPCollector`, plus ADR-0002) +factors the SHARED ingest runtime — windowing, snapshot caching, +envelope encoding, sketch reconstruction, sketch merge — into a +host-neutral crate +[`asap-precompute-rs`](https://github.com/ProjectASAP/ASAPCollector/tree/main/asap-precompute-rs). +Every Rust **edge** runtime (Vector adapter, OTAP-Rust, Arrow-backed +shims, Telegraf input) consumes it, and so does this backend. + +Before Phase 3 step 3, the backend's ingest path inlined that shared +logic: + +- `precompute_operators/{ddsketch,kll,hll,countsketch,countmin}_accumulator.rs` + each had a `from_sketchlib_proto_bytes` that decoded + `SketchEnvelope` proto and dispatched the + `sketch_state` oneof by hand (the same dispatch was duplicated five + times, once per sketch). +- `drivers/ingest/otel.rs::decode_modified_otlp_sketch_bytes` routed + full-state OTLP sketch bytes through those per-accumulator + decoders. +- The merge path in `*::merge_with` (used by the + `precompute_engine`'s pane-folding logic) called into + `asap_sketchlib::sketches::*::merge_refs(...)` directly. + +Independently, `asap-precompute-rs` was building the same envelope +decode + state extraction + reconstruction logic — for agents. + +## Decision + +Backend's ingest path **consumes asap-precompute-rs** for the +shared work. Backend's QUERY-side engine (PromQL aggregation, +storage, query planning) stays put. + +### What moved into asap-precompute-rs (i.e. no longer +backend-internal) + +| Concern | Where the shared logic lives now | +| --- | --- | +| Envelope wire-format runtime view | `asap_precompute_rs::envelope::{SketchEnvelope, Encoding, SketchType}` | +| Envelope proto decode + oneof dispatch | `asap_precompute_rs::envelope::ProtoSketchEnvelope` (re-export of the prost type) plus the per-wrapper `decode_envelope` helpers | +| Sketch reconstruction from envelope payload (DDSketch + KLL) | `asap_precompute_rs::sketches::{DDSketchWrapper, KLLWrapper}::Sketch::apply_delta` | +| Sketch merge | `asap_precompute_rs::Sketch::merge` (round-trip via `snapshot` + `apply_delta`) | +| `Sketch` / `QuantileSketch` / `CardinalitySketch` / `FrequencySketch` traits | `asap_precompute_rs::precompute::*` | + +### What stays in `ASAPQuery-backend` + +- **Query-side engine**: PromQL aggregation + (`engines/{logical,physical}`), storage (`stores/`), query + planning (`asap-planner-rs`), DataFusion bridge (`tests/datafusion`). +- **Per-accumulator query-side surface**: + `precompute_operators/*_accumulator.rs` keeps the `AggregateCore`, + `query_statistic`, `MergeableAccumulator`, + `SerializableToSink` impls, and the per-sketch JSON output. These + are query-side, not ingest-side. +- **Sparse-delta application**: + `precompute_operators/*::apply_proto_delta_bytes` and + `drivers/ingest/otel.rs::apply_modified_otlp_delta_bytes` stay + because `asap_sketchlib` doesn't yet expose the `compute_delta` + family upstream — `asap-precompute-rs`'s wrappers fall back to + "always full" delta encoding (see + `asap-precompute-rs/src/sketches/mod.rs` "API surface caveats"). + When the upstream `compute_delta` lands, this path will collapse + into the same delegation pattern the full-state path uses today. +- **MSGPACK** (`*::from_msgpack_bytes`) — alternative wire format the + asap-precompute-rs edge runtime doesn't emit. Stays for the + Strategy-A (vendored modified-OTLP) path that hard-codes msgpack + for some sketches. + +### Bridge module + +`asap-query-engine/src/precompute_operators/edge_runtime_adapter.rs`: + +- Re-exports the asap-precompute-rs runtime view types (one canonical + `SketchEnvelope`, `Encoding`, `SketchType`, `Sketch`, + `QuantileSketch`, `CardinalitySketch`, `FrequencySketch`). +- `unwrap_envelope_state(bytes) -> Option`: the + shared envelope-decode + oneof-extract path that all five + accumulators used to inline. Single source of truth. +- `reconstruct_via_runtime(SketchType, bytes) -> ReconstructedSketch`: + uses `asap-precompute-rs`'s `Sketch::apply_delta` to decode an + envelope and reconstruct the underlying + `asap_sketchlib::sketches::*` state. Wired for DDSketch + KLL + today. +- `encode_ddsketch_envelope(&DdSketch) -> Vec`: emits the + canonical envelope shape that `asap-precompute-rs`'s wrapper + emits, byte-for-byte. +- `merge_ddsketches_via_runtime(&DdSketch, &DdSketch) -> DdSketch`: + routes through `asap-precompute-rs::Sketch::merge`. Result is + byte-identical to `DdSketch::merge_refs(&[a, b])` because both + call the same underlying merge logic. + +### Wire-up + +`drivers/ingest/otel.rs::decode_modified_otlp_sketch_bytes`'s +`ENCODING_PROTO` branch is the entry point for full-state +modified-OTLP sketch envelopes. The DDSketch and KLL arms now +delegate to `edge_runtime_adapter::reconstruct_via_runtime`. HLL / +CountSketch / CountMinSketch keep using the backend's existing +per-accumulator decoder until upstream byte parity (issue #243) +lands. + +### Cargo deps + +- `asap_sketchlib`: bumped to `branch = "main"` (post-PR-#39 module + renames; PRs #40/#41/#42 land DDSketch/KLL/CountSketch byte parity). +- `asap-precompute-rs`: new path-dep + (`{ path = "../../ASAPCollector/asap-precompute-rs" }`) — the + asap-precompute-rs crate itself path-deps `asap_sketchlib`, so + cloning ASAPCollector via cargo's git source fails to resolve the + path. Path-dep mirrors how `asap-precompute-rs` consumes + `asap_sketchlib` for the same reason. +- Workspace `[patch."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/ProjectASAP/asap_sketchlib"]` + redirects the git-sourced `asap_sketchlib` (used by backend) to + the local checkout, so the type identities at the + `asap-precompute-rs` ↔ backend boundary unify. + +## Acceptance tests + +`asap-query-engine/tests/edge_runtime_consumes_precompute_rs.rs`: + +- **DDSketch round-trip** through asap-precompute-rs runtime is + byte-identical with the input envelope. +- **DDSketch structural** assertions (count, alpha within bounds). +- **DDSketch end-to-end**: envelope → backend `AggregateCore` → + `query_statistic(Quantile)` returns a value within the configured + `α` of the true median. +- **DDSketch back-snapshot** through asap-precompute-rs is + byte-identical with the input envelope (closes the encode side of + the round-trip). +- **KLL round-trip** via asap-precompute-rs runtime is byte-identical + with the input envelope. +- **KLL structural** assertions (k, items count). +- **HLL / CountSketch / CountMinSketch round-trips** are present + but gated `#[ignore = "blocked on ASAPCollector#243 HLL/CS/CMS + byte parity"]` — they will start passing automatically when issue + #243 lands without code changes here. + +## Consequences + +### Positive + +- Single source of truth for envelope decode + state extraction + + sketch reconstruction across agents and backend. +- Adding a new sketch only requires one place (asap-precompute-rs) + to gain envelope-handling support; the backend gets it for free + via `edge_runtime_adapter::reconstruct_via_runtime`. +- The `Sketch` / `QuantileSketch` / `CardinalitySketch` / + `FrequencySketch` trait family exposes a uniform interface for + envelope-shaped sketches, which the per-platform Strategy-B + adapters (Telegraf / Vector / OTAP) will reuse. + +### Negative + +- The path-dep on a sibling repo means CI must clone `ASAPCollector` + next to `ASAPQuery-backend`. Resolved by the README pointer; a + follow-up will revisit the dep style once the + `asap-precompute-rs` Cargo.toml flips its own `asap_sketchlib` + pin to a git URL (so cargo can resolve a single git source). + +### Deferred + +- HLL / CountSketch / CountMinSketch byte parity (issue #243). +- Sparse-delta `compute_delta` upstream (currently in `sketchlib-go` + only, not `asap_sketchlib`). Until that lands the typed-delta + apply path stays in this repo. +- MSGPACK delta encoding (`ENCODING_MSGPACK_DELTA = 4`).