From 4eac552ad09ade659dfffddbf715feec5a1e870e Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 24 May 2026 10:26:46 -0600 Subject: [PATCH] chore: delete the superseded GORILLA1 format (Rust store + legacy archive leg) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The custom GORILLA1 container format read side is fully superseded by Path A2: agents emit XOR-chunk fragments -> backend gorilla-merger -> TSDB blocks -> S3 -> thanos-query, answered by ThanosQueryEngine (env ASAP_THANOS_QUERY_URL). Path A2 is verified end-to-end, so the "Phase δ deletes this leg after Path A2 is verified" cleanup is now done. Removed: * data_plane/src/storage_engines/gorilla_object_store/ in full -- GorillaS3Store + decode_block (the GORILLA1 block decoder via the asap_gorilla crate), the in-process GorillaQueryEngine + ExactExecutor (archive_query.rs), postings sidecar, and the GORILLA1-only S3 cost wrapper (s3_cost.rs) + its /metrics append and /internal/s3_cost.csv endpoint (the counters were only ever populated by GorillaS3Store). * The legacy archive leg in main.rs: the Ok(None) fallback that built GorillaS3Store/GorillaQueryEngine is gone; when ASAP_THANOS_QUERY_URL is unset the binary now registers the NoDataArchiveEngine stub. * The now-unused asap-gorilla path-dep plus the rust-s3 / lru crates from data_plane/Cargo.toml (and their transitive entries in Cargo.lock). Kept intact: * StorageBackend::GorillaObjectStore (asap_types) -- the archive-tier routing slot the live ThanosQueryEngine still registers under (data_source_id = thanos_query); only the answering engine changed. * The sketch / thanos query paths and the gorilla-merger story. Build: cargo build -p data_plane --all-targets green; routing (77), thanos (15) and http-server (53) lib unit tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 402 +----- data_plane/Cargo.toml | 25 +- .../src/drivers/query/fallback/metrics.rs | 4 +- data_plane/src/drivers/query/servers/http.rs | 22 - data_plane/src/main.rs | 68 +- data_plane/src/query_engines/mod.rs | 7 +- data_plane/src/query_engines/routing/mod.rs | 2 +- .../thanos_query_engine/forward.rs | 32 +- .../query_engines/thanos_query_engine/mod.rs | 7 +- .../gorilla_object_store/archive_query.rs | 921 ------------ .../gorilla_object_store/mod.rs | 354 ----- .../gorilla_object_store/postings.rs | 164 --- .../gorilla_object_store/s3_cost.rs | 259 ---- .../gorilla_object_store/store.rs | 1249 ----------------- .../gorilla_object_store/tests.rs | 862 ------------ data_plane/src/storage_engines/mod.rs | 13 +- 16 files changed, 60 insertions(+), 4331 deletions(-) delete mode 100644 data_plane/src/storage_engines/gorilla_object_store/archive_query.rs delete mode 100644 data_plane/src/storage_engines/gorilla_object_store/mod.rs delete mode 100644 data_plane/src/storage_engines/gorilla_object_store/postings.rs delete mode 100644 data_plane/src/storage_engines/gorilla_object_store/s3_cost.rs delete mode 100644 data_plane/src/storage_engines/gorilla_object_store/store.rs delete mode 100644 data_plane/src/storage_engines/gorilla_object_store/tests.rs diff --git a/Cargo.lock b/Cargo.lock index b3ada6729..70e8b5c23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,12 +31,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - [[package]] name = "android-tzdata" version = "0.1.1" @@ -346,16 +340,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "asap-gorilla" -version = "0.1.0" -dependencies = [ - "crc32fast", - "serde", - "serde_json", - "thiserror 1.0.69", -] - [[package]] name = "asap-precompute-rs" version = "0.1.0" @@ -456,22 +440,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "attohttpc" -version = "0.30.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" -dependencies = [ - "base64 0.22.1", - "http 1.4.0", - "log", - "rustls 0.23.40", - "serde", - "serde_json", - "url", - "webpki-roots 1.0.7", -] - [[package]] name = "atty" version = "0.2.14" @@ -489,54 +457,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "aws-creds" -version = "0.39.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3b85155d265df828f84e53886ed9e427aed979dd8a39f5b8b2162c77e142d7" -dependencies = [ - "attohttpc", - "home", - "log", - "quick-xml", - "rust-ini", - "serde", - "thiserror 2.0.18", - "time", - "url", -] - -[[package]] -name = "aws-lc-rs" -version = "1.16.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.40.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", -] - -[[package]] -name = "aws-region" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "838b36c8dc927b6db1b6c6b8f5d05865f2213550b9e83bf92fa99ed6525472c0" -dependencies = [ - "thiserror 2.0.18", -] - [[package]] name = "axum" version = "0.7.9" @@ -802,15 +722,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] - [[package]] name = "colorchoice" version = "1.0.5" @@ -1037,7 +948,6 @@ dependencies = [ "anyhow", "arc-swap", "arrow", - "asap-gorilla", "asap-precompute-rs", "asap_otel_proto", "asap_sketchlib", @@ -1057,7 +967,6 @@ dependencies = [ "futures", "hex", "lazy_static", - "lru", "memmap2", "moka", "prometheus", @@ -1067,7 +976,6 @@ dependencies = [ "regex", "reqwest 0.11.27", "rusqlite", - "rust-s3", "serde", "serde_json", "serde_yaml", @@ -1092,7 +1000,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ "powerfmt", - "serde_core", ] [[package]] @@ -1103,7 +1010,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "subtle", ] [[package]] @@ -1117,21 +1023,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "dlv-list" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" -dependencies = [ - "const-random", -] - -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "either" version = "1.15.0" @@ -1245,12 +1136,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.32" @@ -1469,8 +1354,6 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "allocator-api2", - "equivalent", "foldhash", ] @@ -1525,24 +1408,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "http" version = "0.2.12" @@ -1734,7 +1599,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core", ] [[package]] @@ -2137,15 +2002,6 @@ dependencies = [ "vob", ] -[[package]] -name = "lru" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" -dependencies = [ - "hashbrown 0.15.5", -] - [[package]] name = "lru-slab" version = "0.1.2" @@ -2167,23 +2023,6 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" -[[package]] -name = "maybe-async" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "md5" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" - [[package]] name = "memchr" version = "2.8.0" @@ -2249,15 +2088,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" -[[package]] -name = "ntapi" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" -dependencies = [ - "winapi", -] - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2356,25 +2186,6 @@ dependencies = [ "libc", ] -[[package]] -name = "objc2-core-foundation" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" -dependencies = [ - "bitflags 2.11.1", -] - -[[package]] -name = "objc2-io-kit" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" -dependencies = [ - "libc", - "objc2-core-foundation", -] - [[package]] name = "once_cell" version = "1.21.4" @@ -2393,16 +2204,6 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" -[[package]] -name = "ordered-multimap" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" -dependencies = [ - "dlv-list", - "hashbrown 0.14.5", -] - [[package]] name = "packedvec" version = "1.2.5" @@ -2433,7 +2234,7 @@ dependencies = [ "libc", "redox_syscall 0.5.18", "smallvec", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -2794,16 +2595,6 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" -[[package]] -name = "quick-xml" -version = "0.38.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quinn" version = "0.11.9" @@ -3056,7 +2847,6 @@ dependencies = [ "base64 0.22.1", "bytes", "futures-core", - "futures-util", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -3076,14 +2866,12 @@ dependencies = [ "sync_wrapper 1.0.2", "tokio", "tokio-rustls 0.26.4", - "tokio-util", "tower 0.5.3", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", "web-sys", "webpki-roots 1.0.7", ] @@ -3135,50 +2923,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "rust-ini" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" -dependencies = [ - "cfg-if", - "ordered-multimap", -] - -[[package]] -name = "rust-s3" -version = "0.37.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeedb13abdaa7e48d391de05b0569b37fa0a7a64a668dff6ffb2141ad0c2527e" -dependencies = [ - "async-trait", - "aws-creds", - "aws-region", - "base64 0.22.1", - "bytes", - "cfg-if", - "futures-util", - "hex", - "hmac", - "http 1.4.0", - "log", - "maybe-async", - "md5", - "percent-encoding", - "quick-xml", - "reqwest 0.12.28", - "serde", - "serde_derive", - "serde_json", - "sha2", - "sysinfo", - "thiserror 2.0.18", - "time", - "tokio", - "tokio-stream", - "url", -] - [[package]] name = "rustc-hash" version = "2.1.2" @@ -3238,8 +2982,6 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ - "aws-lc-rs", - "log", "once_cell", "ring", "rustls-pki-types", @@ -3283,7 +3025,6 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3431,17 +3172,6 @@ dependencies = [ "digest", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -3625,20 +3355,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "sysinfo" -version = "0.37.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f" -dependencies = [ - "libc", - "memchr", - "ntapi", - "objc2-core-foundation", - "objc2-io-kit", - "windows", -] - [[package]] name = "system-configuration" version = "0.5.1" @@ -4419,19 +4135,6 @@ dependencies = [ "wasmparser", ] -[[package]] -name = "wasm-streams" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "wasmparser" version = "0.244.0" @@ -4510,41 +4213,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.61.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" -dependencies = [ - "windows-collections", - "windows-core 0.61.2", - "windows-future", - "windows-link 0.1.3", - "windows-numerics", -] - -[[package]] -name = "windows-collections" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" -dependencies = [ - "windows-core 0.61.2", -] - -[[package]] -name = "windows-core" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.1.3", - "windows-result 0.3.4", - "windows-strings 0.4.2", -] - [[package]] name = "windows-core" version = "0.62.2" @@ -4553,20 +4221,9 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - -[[package]] -name = "windows-future" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" -dependencies = [ - "windows-core 0.61.2", - "windows-link 0.1.3", - "windows-threading", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] @@ -4591,53 +4248,19 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "windows-link" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-numerics" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" -dependencies = [ - "windows-core 0.61.2", - "windows-link 0.1.3", -] - -[[package]] -name = "windows-result" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" -dependencies = [ - "windows-link 0.1.3", -] - [[package]] name = "windows-result" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link 0.2.1", -] - -[[package]] -name = "windows-strings" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" -dependencies = [ - "windows-link 0.1.3", + "windows-link", ] [[package]] @@ -4646,7 +4269,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -4673,7 +4296,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -4707,15 +4330,6 @@ dependencies = [ "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-threading" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" -dependencies = [ - "windows-link 0.1.3", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index aa6bd69f8..56ca21f88 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -86,25 +86,12 @@ asap-precompute-rs = { path = "../../ASAPCollector/asap-precompute-rs" } moka = { version = "0.12", features = ["sync"] } memmap2 = "0.9" crc32fast = "1.4" -# Phase 3 — Gorilla-S3 cold engine. `asap-gorilla` (path-dep, mirrors -# the `asap-precompute-rs` sibling-checkout pattern) provides the -# `GORILLA1` block decoder + per-hour `index.json` catalog. `rust-s3` -# (`s3` package on crates.io) is a single-crate S3-compatible client -# with first-class MinIO support; we hold it behind -# `default-features = false` + `tokio-rustls-tls` so it shares the -# rustls backend already pulled in by `reqwest`. `lru` powers the -# Phase 3 `ChunkCache` keyed on chunk object key. -asap-gorilla = { path = "../../ASAPCollector/asap-gorilla-rust" } -# mvp/v5 NOTE FOR REVIEWERS: this PR depends on the matching -# `mvp/v5-postings-compactor` PR in ASAPCollector — specifically the -# `asap_gorilla::Postings` type and the `IndexEntry::byte_offset / -# byte_length / object_key` fields. CI machines that build from a -# `/home/.../ASAPCollector` checkout pinned to `origin/main` will -# fail compilation with `no Postings in the root` until the -# collector PR lands. Local dev: `git fetch && git checkout -# mvp/v5-postings-compactor` in the sibling ASAPCollector repo. -s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls"] } -lru = "0.12" +# NOTE: the `asap-gorilla` path-dep + the `s3` (rust-s3) / `lru` +# crates were dropped when the superseded GORILLA1 read side +# (`gorilla_object_store`: GorillaS3Store + decode_block + the +# in-process GorillaQueryEngine) was deleted. The archive tier is +# served by the ThanosQueryEngine (Path A2) over HTTP, which needs +# none of them. [dev-dependencies] tempfile = "3.20.0" diff --git a/data_plane/src/drivers/query/fallback/metrics.rs b/data_plane/src/drivers/query/fallback/metrics.rs index 16dc1a57e..3021020b9 100644 --- a/data_plane/src/drivers/query/fallback/metrics.rs +++ b/data_plane/src/drivers/query/fallback/metrics.rs @@ -8,8 +8,8 @@ //! //! * **Hot** = the `ASAPQueryEngine` handled the query from live //! sketch-backed state. -//! * **Cold** = the query was answered from the Gorilla archive -//! tier ([`crate::storage_engines::gorilla_object_store::GorillaQueryEngine`]). +//! * **Cold** = the query was answered from the archive tier +//! ([`crate::query_engines::thanos_query_engine::ThanosQueryEngine`], Path A2). //! //! The "shape" label is the parsed query's root op (`sum`, //! `count`, `avg`, `selector`, ...) — low-cardinality by design, diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 137e17a49..744eb1c72 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -423,10 +423,6 @@ impl HttpServer { .route(runtime_info_path, get(handle_runtime_info)) .route(runtime_info_path, post(handle_runtime_info)) .route("/metrics", get(handle_metrics)) - // mvp/v5: dump the S3 cost-tracking counters as CSV. - // The demo's `run_mvp_demo.sh` curls this for each - // baseline; missing counters render as zeros. - .route("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/internal/s3_cost.csv", get(handle_s3_cost_csv)) // Control plane integration endpoints .route("/api/v1/precompute", post(handle_precompute_job)) // Control plane's `PrecomputeClient` (control_plane/src/emit/precompute.rs) @@ -1621,11 +1617,6 @@ async fn handle_metrics() -> impl IntoResponse { let mut buffer = Vec::new(); prometheus::Encoder::encode(&encoder, &metric_families, &mut buffer) .unwrap_or_else(|e| tracing::error!("Failed to encode metrics: {}", e)); - // mvp/v5: append the S3 cost counters in Prometheus text - // exposition. Mirrors `/internal/s3_cost.csv` — the CSV is for - // the demo, this is for live dashboards. - let counters = crate::storage_engines::gorilla_object_store::global_s3_cost_counters(); - buffer.extend_from_slice(counters.render_prometheus().as_bytes()); ( [( axum::http::header::CONTENT_TYPE, @@ -1635,19 +1626,6 @@ async fn handle_metrics() -> impl IntoResponse { ) } -/// mvp/v5: CSV dump of the S3 cost counters. -/// -/// Renders ONE header row + ONE data row. Empty when no S3 -/// operations have been issued (the counters default to zero, so -/// the CSV is still well-formed). -async fn handle_s3_cost_csv() -> impl IntoResponse { - let counters = crate::storage_engines::gorilla_object_store::global_s3_cost_counters(); - ( - [(axum::http::header::CONTENT_TYPE, "text/csv; charset=utf-8")], - counters.render_csv(), - ) -} - // ============================================================ // Range Query Handlers // ============================================================ diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 12d0dfe62..41f064187 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -242,7 +242,7 @@ struct Args { /// (`{metric_name: storage_backend}` map). Loaded at startup and /// consulted by the HTTP query handler on every PromQL request to /// pick the right engine (`ASAPQueryEngine` for ASAP-tier sketches, - /// `GorillaQueryEngine` for the cold archive, etc.). Without + /// `ThanosQueryEngine` for the cold archive, etc.). Without /// this flag the handler falls back to the streaming-config /// single axis (always `SketchStore`) and the EngineRouter is /// effectively bypassed — the issue-46 v2 demo's criterion ⑤ @@ -608,23 +608,21 @@ async fn main() -> Result<()> { server = server.with_backend_storage_routing(Arc::new(bootstrap_routing)); // Phase-5/6 + Step-2.3: register the Thanos query engine on the - // capability router. Two operating modes, selected at startup: + // capability router. Path A2 is the only archive path now: when + // `ASAP_THANOS_QUERY_URL` is set, the backend forwards + // archive-tier PromQL queries to a `thanos-query` sidecar via the + // [`ThanosQueryEngine`], registered under the single public id + // `thanos_query`. // - // * **Path A2 mode** — when `ASAP_THANOS_QUERY_URL` is set, the - // backend forwards archive-tier PromQL queries to a - // `thanos-query` sidecar via the - // [`ThanosQueryEngine`], registered under the single public id - // `thanos_query`. The legacy in-process `GorillaQueryEngine` - // is skipped in this mode. - // * **Legacy mode** — when `ASAP_THANOS_QUERY_URL` is unset, the - // in-process `GorillaQueryEngine` answers archive queries - // from per-hour Gorilla chunks landed on S3 / MinIO via the - // `GorillaS3Store`. This is the dev path and is preserved - // verbatim until Phase δ deletes it after Path A2 is verified - // end-to-end. + // The superseded legacy in-process `GorillaQueryEngine` / + // `GorillaS3Store` leg (which read the custom GORILLA1 container + // format from per-hour chunks on S3 / MinIO) has been deleted now + // that Path A2 is verified end-to-end (agents emit XOR-chunk + // fragments → backend gorilla-merger → TSDB blocks → S3 → + // thanos-query). // - // When neither env-var family is configured the binary registers - // a `NoDataArchiveEngine` stub under `thanos_query` + // When `ASAP_THANOS_QUERY_URL` is not configured the binary + // registers a `NoDataArchiveEngine` stub under `thanos_query` // so cold queries succeed with an empty result instead of // surfacing as `503 NoEngineRegistered`. Operators that want the // original fail-loud behaviour can opt back in by setting @@ -635,43 +633,17 @@ async fn main() -> Result<()> { use data_plane::query_engines::routing::QueryEngine; info!( upstream = thanos.base_url(), - "Path A2: registering ThanosQueryEngine for the archive tier (data_source_id=thanos_query); legacy in-process GorillaQueryEngine skipped", + "Path A2: registering ThanosQueryEngine for the archive tier (data_source_id=thanos_query)", ); let thanos_arc: Arc = Arc::new(thanos); server = server.with_archive_query_engine(thanos_arc); archive_registered = true; } - Ok(None) => match data_plane::storage_engines::gorilla_object_store::GorillaS3Config::from_env() { - Ok(s3_cfg) => { - match data_plane::storage_engines::gorilla_object_store::GorillaS3Store::with_default_backend( - s3_cfg, - ) { - Ok(store) => { - use data_plane::storage_engines::{GorillaEngineConfig, GorillaQueryEngine}; - use data_plane::query_engines::routing::QueryEngine; - let gorilla = Arc::new(GorillaQueryEngine::with_gorilla_s3( - Arc::new(store), - GorillaEngineConfig::default(), - )); - info!( - "Registering legacy in-process GorillaQueryEngine on the archive slot (canonical data_source_id=thanos_query); set ASAP_THANOS_QUERY_URL to use the intended Thanos archive path", - ); - server = server.with_archive_query_engine(gorilla as Arc); - archive_registered = true; - } - Err(e) => { - warn!( - "ASAP_GORILLA_S3_* env vars present but GorillaS3Store failed to build ({e}); router will not have an archive engine", - ); - } - } - } - Err(_) => { - info!( - "ASAP_GORILLA_S3_* env vars not configured — router serves ASAP-tier metrics only (set ASAP_GORILLA_S3_BUCKET + ASAP_GORILLA_S3_REGION to enable archive routing, or set ASAP_THANOS_QUERY_URL to enable Path A2 thanos forwarding)", - ); - } - }, + Ok(None) => { + info!( + "ASAP_THANOS_QUERY_URL not configured — router serves ASAP-tier metrics only (set ASAP_THANOS_QUERY_URL to enable Path A2 thanos archive forwarding)", + ); + } Err(e) => { warn!( "ASAP_THANOS_QUERY_URL set but ThanosQueryEngine failed to build ({e}); router will not have an archive engine", diff --git a/data_plane/src/query_engines/mod.rs b/data_plane/src/query_engines/mod.rs index 5f17ceeaf..722fc96dd 100644 --- a/data_plane/src/query_engines/mod.rs +++ b/data_plane/src/query_engines/mod.rs @@ -2,10 +2,9 @@ //! //! The public query-engine surface is intentionally small: //! [`asap_query_engine`] answers from ASAP's sketch store, and -//! [`thanos_query_engine`] forwards exact/archive queries to `thanos-query`. -//! Gorilla object storage lives under [`crate::storage_engines::gorilla_object_store`] -//! because it is a storage implementation detail, not a public query-engine -//! family. +//! [`thanos_query_engine`] forwards exact/archive queries to `thanos-query` +//! (Path A2 — the only archive path now that the superseded in-process +//! Gorilla executor / GORILLA1 store has been deleted). //! //! ## Public surface //! diff --git a/data_plane/src/query_engines/routing/mod.rs b/data_plane/src/query_engines/routing/mod.rs index 540d69dcf..9b13de4ea 100644 --- a/data_plane/src/query_engines/routing/mod.rs +++ b/data_plane/src/query_engines/routing/mod.rs @@ -3,7 +3,7 @@ //! This module is the dispatch boundary between the HTTP query //! handler and the tier-co-located engines (warm sketch tier in //! [`crate::query_engines::asap_query_engine`], archive tier in -//! [`crate::storage_engines::gorilla_object_store`]). Two cooperating pieces: +//! [`crate::query_engines::thanos_query_engine`]). Two cooperating pieces: //! //! * [`backend_storage_routing`] — config loader + multi-target //! per-metric lookup (`metric → [(backend, query-shape filter), ...]`). diff --git a/data_plane/src/query_engines/thanos_query_engine/forward.rs b/data_plane/src/query_engines/thanos_query_engine/forward.rs index e3aa7c725..3abc47709 100644 --- a/data_plane/src/query_engines/thanos_query_engine/forward.rs +++ b/data_plane/src/query_engines/thanos_query_engine/forward.rs @@ -7,30 +7,21 @@ //! Step-2.3 (this file) wires the backend to forward archive-tier //! PromQL queries to that sidecar over HTTP. //! -//! Operating modes are selected by the -//! [`ASAP_THANOS_QUERY_URL_ENV`] env var, consulted at backend -//! startup: +//! The engine is selected by the [`ASAP_THANOS_QUERY_URL_ENV`] env +//! var, consulted at backend startup: //! //! * **Path A2 mode** (env set) — `ThanosQueryEngine` is //! registered in the [`crate::query_engines::routing::EngineRouter`]. Archive //! queries POST to `${ASAP_THANOS_QUERY_URL}/api/v1/query` and //! the answer is wrapped in ASAP's standard //! [`crate::query_engines::QueryResult`] shape. -//! * **Legacy mode** (env unset) — the in-process -//! [`crate::storage_engines::gorilla_object_store::GorillaQueryEngine`] -//! handles archive queries from the per-hour Gorilla chunks that -//! [`crate::storage_engines::gorilla_object_store::GorillaS3Store`] streams -//! from S3 / MinIO. -//! Phase δ deletes this leg after Path A2 is verified -//! end-to-end. //! -//! The two modes are mutually exclusive: when Path A2 is active, -//! both the legacy id (`thanos_query`) and the alias id -//! (`thanos_query`) point at the same `ThanosQueryEngine` -//! instance, so the per-metric `BackendStorageRouting` config can -//! target either name without surprise. See the binary's -//! `register_thanos_or_thanos_query` helper for the -//! registration site. +//! Path A2 is now the only archive path. The superseded legacy +//! in-process Gorilla executor (custom GORILLA1 container format, +//! read from per-hour chunks on S3 / MinIO) has been deleted after +//! Path A2 was verified end-to-end. When the env var is unset, the +//! binary registers a `NoDataArchiveEngine` stub on the archive +//! slot instead. use std::time::{Duration, Instant}; @@ -55,8 +46,7 @@ use crate::storage_engines::sketch_db::accuracy::{AccuracyEnvelope, AccuracyProf /// Env var consulted at backend startup. When set, the binary /// registers a [`ThanosQueryEngine`] pointing at the URL and the /// router dispatches archive-tier queries to it. When unset, the -/// legacy in-process [`crate::storage_engines::gorilla_object_store::GorillaQueryEngine`] -/// handles archive queries. +/// binary registers a `NoDataArchiveEngine` stub on the archive slot. pub const ASAP_THANOS_QUERY_URL_ENV: &str = "ASAP_THANOS_QUERY_URL"; /// Default upstream URL when `ASAP_THANOS_QUERY_URL` is set to the @@ -535,8 +525,8 @@ pub enum ThanosQueryError { /// Convenience combinator the binary uses at startup: try /// [`ThanosQueryEngine::from_env`]; if it returns `None`, the -/// caller falls through to the legacy in-process -/// [`crate::storage_engines::gorilla_object_store::GorillaQueryEngine`] path. +/// caller registers a `NoDataArchiveEngine` stub on the archive +/// slot (the legacy in-process Gorilla path has been deleted). /// /// Returning `Result, ...>` instead of unwrapping in /// `main.rs` keeps the construction failure (bad URL / bad TLS diff --git a/data_plane/src/query_engines/thanos_query_engine/mod.rs b/data_plane/src/query_engines/thanos_query_engine/mod.rs index 85df48a44..beb60b381 100644 --- a/data_plane/src/query_engines/thanos_query_engine/mod.rs +++ b/data_plane/src/query_engines/thanos_query_engine/mod.rs @@ -1,8 +1,9 @@ //! Thanos query-engine wrapper. //! -//! This module owns the public archive query engine, [`ThanosQueryEngine`]. -//! Gorilla object storage and the legacy in-process Gorilla executor live -//! under [`crate::storage_engines::gorilla_object_store`]. +//! This module owns the public archive query engine, [`ThanosQueryEngine`] +//! (Path A2). The archive tier reuses the `StorageBackend::GorillaObjectStore` +//! routing slot, but the answering engine is Thanos; the superseded +//! in-process Gorilla executor (custom GORILLA1 format) has been deleted. pub mod forward; diff --git a/data_plane/src/storage_engines/gorilla_object_store/archive_query.rs b/data_plane/src/storage_engines/gorilla_object_store/archive_query.rs deleted file mode 100644 index f8f27ad99..000000000 --- a/data_plane/src/storage_engines/gorilla_object_store/archive_query.rs +++ /dev/null @@ -1,921 +0,0 @@ -//! Gorilla archive query planner + per-statistic exact executor. -//! -//! Step-1 of the JSONL-deprecation refactor merged the previous -//! `query_planner.rs` (PromQL → [`QueryPlan`]) and `exact_executor.rs` -//! (per-statistic `[`ExactExecutor`]` dispatch) into a single -//! `archive_query.rs` so the archive data flow is readable top-to-bottom -//! in one file: parse PromQL → plan → execute. -//! -//! The two halves keep their existing structure inside the merged -//! file: -//! -//! * **Planner** (top) — translates a PromQL string into a -//! [`QueryPlan`] (`metric, time_range_ms, statistic, -//! label_matchers`). Supports `*_over_time`, `rate`, `increase`, -//! `quantile_over_time`, `topk`, and the v7 `last_over_time` -//! freshness-probe spelling. -//! * **Executor** (bottom) — takes a [`QueryPlan`] + an -//! `Arc` and returns an -//! [`super::ExecutionOutcome`]. Two strategies, picked by -//! [`QueryStatistic::is_streaming_additive`]: -//! -//! - **Streaming-additive** — `Sum / Count / Avg / Min / Max / -//! Rate / Increase / Last`. One chunk at a time, fold into a -//! tiny accumulator, drop the decoded chunk before fetching -//! the next one. Memory is O(1) per query. -//! - **Buffered** — `Quantile / TopK / Cardinality`. Materialise -//! every in-range sample, then sort or otherwise post-process. -//! Bounded by [`super::GorillaEngineConfig::max_buffered_samples`]; -//! over-budget queries fail fast with -//! [`super::EngineError::TooManySamples`] rather than OOM. - -use std::sync::Arc; -use std::time::SystemTime; - -use chrono::Utc; -use promql_parser::parser::{ - AggregateExpr, Call, Expr, FunctionArgs, MatrixSelector, NumberLiteral, ParenExpr, - VectorSelector, -}; -use tracing::debug; - -use super::store::{ChunkRef, RawSample, Store, StoreError}; -use super::{EngineError, ExecutionOutcome, GorillaEngineConfig}; - -// ===================================================================== -// Planner -// ===================================================================== - -/// Statistic to compute, alongside any extra parameters -/// (quantile φ, top-k k). -#[derive(Debug, Clone, PartialEq)] -pub enum QueryStatistic { - /// `sum_over_time(m[range])` - SumOverTime, - /// `count_over_time(m[range])` - CountOverTime, - /// `avg_over_time(m[range])` (= sum / count) - AvgOverTime, - /// `min_over_time(m[range])` - MinOverTime, - /// `max_over_time(m[range])` - MaxOverTime, - /// `rate(m[range])` — `(last - first) / range_seconds` - Rate, - /// `increase(m[range])` — `last - first` - Increase, - /// `quantile_over_time(φ, m[range])` - QuantileOverTime { phi: f64 }, - /// `topk(k, sum_over_time(m[range]))`-style aggregation. The - /// MVP Phase 4 implementation returns the sum of the top-`k` - /// sample values in the range — once Phase 5 adds spatial - /// grouping the executor will return a per-group vector. - TopK { k: usize }, - /// **v7**: `last_over_time(m[range])` — value of the - /// largest-timestamp sample in the range. Used by issue #46 - /// criterion ⑥ freshness probes; counter-shaped probes encode - /// `unix_ts_ms_of_emission` in their cumulative value, and - /// `last_over_time(...)` returns that value so the replay - /// client can compute per-path freshness deltas. - LastOverTime, -} - -impl QueryStatistic { - /// True iff the executor can answer this statistic via the - /// streaming-additive path; false → buffered path (everything - /// has to be in memory before producing the answer). - pub fn is_streaming_additive(&self) -> bool { - matches!( - self, - Self::SumOverTime - | Self::CountOverTime - | Self::AvgOverTime - | Self::MinOverTime - | Self::MaxOverTime - | Self::Rate - | Self::Increase - | Self::LastOverTime - ) - } -} - -/// One label-equality matcher extracted from the PromQL AST. mvp/v5 -/// uses these to drive the postings-aware chunk-pruning path. -/// -/// The MVP only supports exact equality (`label = "value"`). Regex -/// (`=~`) and inequality (`!=`, `!~`) matchers fall through to a -/// post-decode filter — the postings file holds *exact* values per -/// label, not patterns. The fall-through is correct (just slower) -/// and is signalled to callers via -/// [`QueryPlan::has_unsupported_matchers`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct LabelMatcher { - /// Label name, e.g. `"service"`. - pub name: String, - /// Label value, e.g. `"api"`. - pub value: String, -} - -/// Output of [`plan_query`]. -#[derive(Debug, Clone, PartialEq)] -pub struct QueryPlan { - pub metric: String, - /// Half-open `[start_ms, end_ms)` request window. Computed as - /// `(now_ms - range_ms, now_ms)` from the matrix selector's - /// `[range]` duration. - pub time_range_ms: (i64, i64), - pub statistic: QueryStatistic, - /// **mvp/v5**: exact-equality label matchers extracted from the - /// vector selector. Empty for `metric[range]` (no predicate). - /// Non-empty for `metric{label="value"}[range]`. Used by the - /// postings-aware chunk filter; `=~` / `!=` / `!~` matchers are - /// dropped from this list and signalled via - /// [`Self::has_unsupported_matchers`]. - pub label_matchers: Vec, - /// **mvp/v5**: `true` iff the original PromQL had at least one - /// matcher we couldn't translate into a postings lookup (regex, - /// inequality). The caller must still apply those matchers - /// post-decode; we surface the flag so `data_source_quirk` - /// annotations make it back to the client. - pub has_unsupported_matchers: bool, -} - -/// Parse `query` and produce a [`QueryPlan`]. `now` defaults to -/// the system clock; the [`plan_query_at`] variant lets tests pin -/// a deterministic timestamp. -pub fn plan_query(query: &str) -> Result { - let now_ms = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as i64) - .unwrap_or_else(|_| Utc::now().timestamp_millis()); - plan_query_at(query, now_ms) -} - -/// As [`plan_query`], with a caller-supplied `now_ms`. -pub fn plan_query_at(query: &str, now_ms: i64) -> Result { - let ast = promql_parser::parser::parse(query).map_err(|e| format!("parse: {e}"))?; - plan_from_ast(&ast, now_ms) -} - -fn plan_from_ast(ast: &Expr, now_ms: i64) -> Result { - match ast { - Expr::Paren(ParenExpr { expr }) => plan_from_ast(expr, now_ms), - Expr::Call(call) => plan_from_call(call, now_ms), - Expr::Aggregate(agg) => plan_from_aggregate(agg, now_ms), - other => Err(format!( - "unsupported top-level expression: {:?}; the Gorilla engine \ - expects a single function call (rate/increase/*_over_time) \ - or topk(k, ...) aggregation", - std::mem::discriminant(other) - )), - } -} - -fn plan_from_call(call: &Call, now_ms: i64) -> Result { - let name = call.func.name.to_lowercase(); - match name.as_str() { - "sum_over_time" | "count_over_time" | "avg_over_time" | "min_over_time" - | "max_over_time" | "last_over_time" | "rate" | "increase" => { - let ms = expect_single_matrix_arg(&call.args, &name)?; - let (metric, range_ms) = matrix_metric_and_range_ms(ms); - let stat = match name.as_str() { - "sum_over_time" => QueryStatistic::SumOverTime, - "count_over_time" => QueryStatistic::CountOverTime, - "avg_over_time" => QueryStatistic::AvgOverTime, - "min_over_time" => QueryStatistic::MinOverTime, - "max_over_time" => QueryStatistic::MaxOverTime, - "last_over_time" => QueryStatistic::LastOverTime, - "rate" => QueryStatistic::Rate, - "increase" => QueryStatistic::Increase, - _ => unreachable!(), - }; - let (label_matchers, has_unsupported) = extract_label_matchers(&ms.vs); - Ok(QueryPlan { - metric, - time_range_ms: (now_ms - range_ms, now_ms), - statistic: stat, - label_matchers, - has_unsupported_matchers: has_unsupported, - }) - } - "quantile_over_time" => { - // quantile_over_time(φ, m[range]) - if call.args.args.len() != 2 { - return Err(format!( - "quantile_over_time expects 2 args, got {}", - call.args.args.len() - )); - } - let phi = expect_number(&call.args.args[0], "quantile_over_time φ")?; - let ms = expect_matrix_selector(&call.args.args[1], "quantile_over_time")?; - let (metric, range_ms) = matrix_metric_and_range_ms(ms); - let (label_matchers, has_unsupported) = extract_label_matchers(&ms.vs); - Ok(QueryPlan { - metric, - time_range_ms: (now_ms - range_ms, now_ms), - statistic: QueryStatistic::QuantileOverTime { phi }, - label_matchers, - has_unsupported_matchers: has_unsupported, - }) - } - other => Err(format!( - "unsupported PromQL function: {other}; the Gorilla engine \ - supports rate/increase/*_over_time/quantile_over_time" - )), - } -} - -fn plan_from_aggregate(agg: &AggregateExpr, now_ms: i64) -> Result { - // PromQL grammar requires aggregation operators to take a - // vector — so the legal Phase-4 spellings are e.g. - // `topk(2, sum_over_time(m[10s]))`. We strip the outer - // aggregation, recurse into the inner call to recover the - // `(metric, range)` pair, then overlay the TopK statistic. - let op_str = format!("{}", agg.op); - if !op_str.eq_ignore_ascii_case("topk") { - return Err(format!( - "unsupported top-level aggregation: {op_str}; only `topk(k, ...)` \ - is supported in Phase 4" - )); - } - let k_expr = agg - .param - .as_deref() - .ok_or_else(|| "topk requires a numeric parameter (k)".to_string())?; - let k = expect_number(k_expr, "topk k")?; - if !k.is_finite() || k <= 0.0 { - return Err(format!("topk k must be positive, got {k}")); - } - // Recurse into the inner expression — it can be a matrix - // selector (handled by [`matrix_metric_and_range_ms`] - // directly) OR a vector-returning function call (the legal - // PromQL spelling). Either way we end up with a - // `(metric, range_ms)` pair we can overlay TopK on. - let inner_plan = match &*agg.expr { - Expr::MatrixSelector(ms) => { - let (metric, range_ms) = matrix_metric_and_range_ms(ms); - let (label_matchers, has_unsupported) = extract_label_matchers(&ms.vs); - QueryPlan { - metric, - time_range_ms: (now_ms - range_ms, now_ms), - statistic: QueryStatistic::SumOverTime, // overlay below - label_matchers, - has_unsupported_matchers: has_unsupported, - } - } - _ => plan_from_ast(&agg.expr, now_ms)?, - }; - Ok(QueryPlan { - metric: inner_plan.metric, - time_range_ms: inner_plan.time_range_ms, - statistic: QueryStatistic::TopK { k: k as usize }, - label_matchers: inner_plan.label_matchers, - has_unsupported_matchers: inner_plan.has_unsupported_matchers, - }) -} - -fn expect_single_matrix_arg<'a>( - args: &'a FunctionArgs, - fname: &str, -) -> Result<&'a MatrixSelector, String> { - if args.args.len() != 1 { - return Err(format!( - "{fname} expects 1 matrix-selector arg, got {}", - args.args.len() - )); - } - expect_matrix_selector(&args.args[0], fname) -} - -fn expect_matrix_selector<'a>(expr: &'a Expr, ctx: &str) -> Result<&'a MatrixSelector, String> { - match expr { - Expr::MatrixSelector(ms) => Ok(ms), - Expr::Paren(ParenExpr { expr }) => expect_matrix_selector(expr, ctx), - other => Err(format!( - "{ctx}: expected matrix selector `metric[range]`, got {:?}", - std::mem::discriminant(other) - )), - } -} - -fn expect_number(expr: &Expr, ctx: &str) -> Result { - match expr { - Expr::NumberLiteral(NumberLiteral { val }) => Ok(*val), - Expr::Paren(ParenExpr { expr }) => expect_number(expr, ctx), - other => Err(format!( - "{ctx}: expected numeric literal, got {:?}", - std::mem::discriminant(other) - )), - } -} - -fn matrix_metric_and_range_ms(ms: &MatrixSelector) -> (String, i64) { - let metric = vector_selector_metric(&ms.vs); - let range_ms = ms.range.as_millis() as i64; - (metric, range_ms) -} - -fn vector_selector_metric(vs: &VectorSelector) -> String { - if let Some(name) = &vs.name { - return name.clone(); - } - // Fallback: inspect matchers for an `__name__` exact match. - for m in vs.matchers.matchers.iter() { - if m.name == "__name__" { - return m.value.clone(); - } - } - String::new() -} - -/// **mvp/v5**: extract exact-equality label matchers from a vector -/// selector for postings-aware chunk pruning. -/// -/// Returns `(supported_matchers, has_unsupported_matchers)`. Supported -/// matchers are the `label = "value"` tuples the postings file can -/// answer directly. Anything else (regex, inequality, the implicit -/// `__name__` matcher) is excluded from `supported_matchers` and -/// flips the second return value to `true` — the executor still -/// applies them post-decode for correctness. -pub(crate) fn extract_label_matchers(vs: &VectorSelector) -> (Vec, bool) { - use promql_parser::label::MatchOp; - - let mut supported = Vec::new(); - let mut has_unsupported = false; - for m in vs.matchers.matchers.iter() { - // The implicit `__name__` matcher is the metric name itself - // — we already pulled that out of the selector elsewhere. - if m.name == "__name__" { - continue; - } - match &m.op { - MatchOp::Equal => { - supported.push(LabelMatcher { - name: m.name.clone(), - value: m.value.clone(), - }); - } - // Regex / inequality matchers are correctness-relevant - // but cannot be answered by an exact postings lookup. - // Surface the flag so the caller emits a quirk - // annotation; the actual filter is applied post-decode. - MatchOp::NotEqual | MatchOp::Re(_) | MatchOp::NotRe(_) => { - has_unsupported = true; - } - } - } - (supported, has_unsupported) -} - -// ===================================================================== -// Executor -// ===================================================================== - -/// Streaming-additive operation tag — what the per-sample fold -/// does. Pulled out so [`ExactExecutor::execute_streaming_additive`] -/// is a single function regardless of which stat is being computed. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum AdditiveOp { - Sum, - Count, - /// `(sum, count)` — the engine divides at the end. - Avg, - Min, - Max, - /// `last - first` over the time-ordered samples. - Increase, - /// `(last - first) / range_seconds`. - Rate, - /// **v7**: the value of the latest sample in the range. Used by - /// `last_over_time([])` — the freshness probe - /// queries from MVP v6 issue #46 criterion ⑥. The fold tracks - /// `(ts_ms, value)` pairs already; this op just returns `value` - /// of the largest-timestamp sample. - Last, -} - -/// Per-statistic executor. Holds an `Arc` so the -/// engine + executor share the same archive-tier handle without -/// re-implementing trait dispatch. -pub struct ExactExecutor { - store: Arc, - config: GorillaEngineConfig, -} - -impl ExactExecutor { - pub fn new(store: Arc, config: GorillaEngineConfig) -> Self { - Self { store, config } - } - - /// Top-level dispatch — picks streaming vs buffered based on - /// the plan's statistic. - pub async fn execute_plan(&self, plan: &QueryPlan) -> Result { - match &plan.statistic { - QueryStatistic::SumOverTime => { - self.execute_streaming_additive(plan, AdditiveOp::Sum).await - } - QueryStatistic::CountOverTime => { - self.execute_streaming_additive(plan, AdditiveOp::Count) - .await - } - QueryStatistic::AvgOverTime => { - self.execute_streaming_additive(plan, AdditiveOp::Avg).await - } - QueryStatistic::MinOverTime => { - self.execute_streaming_additive(plan, AdditiveOp::Min).await - } - QueryStatistic::MaxOverTime => { - self.execute_streaming_additive(plan, AdditiveOp::Max).await - } - QueryStatistic::Rate => { - self.execute_streaming_additive(plan, AdditiveOp::Rate) - .await - } - QueryStatistic::Increase => { - self.execute_streaming_additive(plan, AdditiveOp::Increase) - .await - } - QueryStatistic::LastOverTime => { - self.execute_streaming_additive(plan, AdditiveOp::Last) - .await - } - QueryStatistic::QuantileOverTime { phi } => self.execute_quantile(plan, *phi).await, - QueryStatistic::TopK { k } => self.execute_topk(plan, *k).await, - } - } - - /// Streaming additive path. Reads chunks one at a time, applies - /// the per-sample fold, drops the decoded chunk before fetching - /// the next one. Memory is O(1) per query, regardless of how - /// many samples the time range covers. - /// - /// **mvp/v5**: when the plan carries label matchers, the executor - /// first reads the postings sidecar to compute the matching - /// `series_ids`, then prunes the chunk list down to chunks - /// whose `label_hash` appears in that set. Falls back to the - /// scan-all path when postings are missing. - pub async fn execute_streaming_additive( - &self, - plan: &QueryPlan, - op: AdditiveOp, - ) -> Result { - let (start_ms, end_ms) = plan.time_range_ms; - let chunks = self - .store - .list_chunks(&plan.metric, start_ms, end_ms) - .await?; - let total_chunks = chunks.len(); - debug!( - metric = plan.metric.as_str(), - chunks = total_chunks, - op = ?op, - label_matchers = plan.label_matchers.len(), - "gorilla-engine: streaming additive over chunks" - ); - - let (filtered_chunks, postings_outcome) = self.apply_postings_filter(plan, &chunks).await; - - let mut acc = AdditiveAccumulator::new(op); - let mut samples_scanned: usize = 0; - let chunks_fetched = filtered_chunks.len(); - for chunk in filtered_chunks { - let samples = self.store.read_chunk(&chunk).await?; - for s in samples { - if s.ts_ms >= start_ms && s.ts_ms < end_ms { - if !self.sample_matches(plan, &s) { - continue; - } - acc.observe(&s); - samples_scanned += 1; - } - } - } - - let value = acc.finalize(plan, op); - Ok(ExecutionOutcome { - value, - samples_scanned, - chunks_fetched, - chunks_skipped_via_postings: total_chunks - chunks_fetched, - postings_filtered_series_count: postings_outcome.matched_series, - postings_missing: postings_outcome.postings_missing, - }) - } - - /// **mvp/v5**: apply the postings-aware filter to a chunk list. - /// Returns `(filtered_chunks, postings_outcome)` where the - /// outcome captures `(matched_series, postings_missing)` so the - /// caller can populate [`super::ExecutionOutcome`] without - /// re-querying. - async fn apply_postings_filter( - &self, - plan: &QueryPlan, - chunks: &[ChunkRef], - ) -> (Vec, PostingsOutcome) { - if plan.label_matchers.is_empty() { - // No predicate — the postings filter is a no-op. The - // postings file isn't consulted at all in this path. - return ( - chunks.to_vec(), - PostingsOutcome { - matched_series: 0, - postings_missing: false, - }, - ); - } - let matchers: Vec<(String, String)> = plan - .label_matchers - .iter() - .map(|LabelMatcher { name, value }| (name.clone(), value.clone())) - .collect(); - let (start_ms, end_ms) = plan.time_range_ms; - let hits = match self - .store - .list_postings_for(&plan.metric, start_ms, end_ms, &matchers) - .await - { - Ok(h) => h, - Err(StoreError::Unsupported(_)) => { - // Backend doesn't support postings at all. Surface - // as missing and fall through. - debug!("gorilla-engine: store does not support postings; falling back to scan-all"); - return ( - chunks.to_vec(), - PostingsOutcome { - matched_series: 0, - postings_missing: true, - }, - ); - } - Err(e) => { - // Transport/parse failure — log + fall through. We - // don't propagate the error because the scan-all - // path is still correct, just slower. - debug!(error = %e, "gorilla-engine: postings fetch failed; falling back to scan-all"); - return ( - chunks.to_vec(), - PostingsOutcome { - matched_series: 0, - postings_missing: true, - }, - ); - } - }; - - // If any bucket in range was missing postings we can't trust - // the filter to be complete; scan everything (correctness - // first, postings are a perf optimization). - if !hits.fully_covered() { - return ( - chunks.to_vec(), - PostingsOutcome { - matched_series: hits.series_ids.len(), - postings_missing: true, - }, - ); - } - - // Postings → series_ids → keep only chunks whose - // `label_hash` is in the set. Chunks with `label_hash == 0` - // are pre-mvp/v5 multi-series chunks that don't pin a - // single series — keep them (they may carry matching - // series; correctness > pruning). - let series_set: std::collections::BTreeSet = hits.series_ids.iter().copied().collect(); - let filtered: Vec = chunks - .iter() - .filter(|c| c.label_hash == 0 || series_set.contains(&c.label_hash)) - .cloned() - .collect(); - ( - filtered, - PostingsOutcome { - matched_series: hits.series_ids.len(), - postings_missing: false, - }, - ) - } - - /// Post-decode label-equality filter. Always-true when no - /// matchers are present (most common). Used as a safety net so - /// chunks with `label_hash = 0` (multi-series, can't be pruned - /// at the postings level) still respect the predicate. - fn sample_matches(&self, plan: &QueryPlan, s: &RawSample) -> bool { - if plan.label_matchers.is_empty() { - return true; - } - for LabelMatcher { name, value } in &plan.label_matchers { - match s.labels.get(name) { - Some(v) if v == value => {} - _ => return false, - } - } - true - } - - /// Buffered quantile path. Materialises every in-range sample - /// up to [`GorillaEngineConfig::max_buffered_samples`], sorts - /// the value column, and picks the φ-rank using a - /// nearest-rank rule (matches Prometheus's - /// `quantile_over_time` semantics for the linear-interp-free - /// midpoint case — the float index rounds to nearest). - pub async fn execute_quantile( - &self, - plan: &QueryPlan, - phi: f64, - ) -> Result { - let buffered = self.collect_buffered_samples(plan).await?; - if buffered.samples.is_empty() { - return Ok(ExecutionOutcome { - value: f64::NAN, - samples_scanned: 0, - chunks_fetched: buffered.chunks_fetched, - chunks_skipped_via_postings: buffered.chunks_skipped_via_postings, - postings_filtered_series_count: buffered.postings_filtered_series_count, - postings_missing: buffered.postings_missing, - }); - } - - let mut values: Vec = buffered.samples.iter().map(|s| s.value).collect(); - values.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - - let n = values.len() as f64; - let phi = phi.clamp(0.0, 1.0); - let raw_idx = ((n - 1.0) * phi).round() as i64; - let idx = raw_idx.clamp(0, values.len() as i64 - 1) as usize; - Ok(ExecutionOutcome { - value: values[idx], - samples_scanned: values.len(), - chunks_fetched: buffered.chunks_fetched, - chunks_skipped_via_postings: buffered.chunks_skipped_via_postings, - postings_filtered_series_count: buffered.postings_filtered_series_count, - postings_missing: buffered.postings_missing, - }) - } - - /// Buffered top-k path. Materialises every in-range sample, - /// sorts the value column descending, and returns the SUM of - /// the top-`k` values. The Phase-5 capability router will - /// extend this to per-group top-k once spatial grouping - /// lands; the MVP scalar return value matches the existing - /// `ExecutionOutcome` shape. - pub async fn execute_topk( - &self, - plan: &QueryPlan, - k: usize, - ) -> Result { - let buffered = self.collect_buffered_samples(plan).await?; - if buffered.samples.is_empty() { - return Ok(ExecutionOutcome { - value: f64::NAN, - samples_scanned: 0, - chunks_fetched: buffered.chunks_fetched, - chunks_skipped_via_postings: buffered.chunks_skipped_via_postings, - postings_filtered_series_count: buffered.postings_filtered_series_count, - postings_missing: buffered.postings_missing, - }); - } - - let mut values: Vec = buffered.samples.iter().map(|s| s.value).collect(); - values.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); - let take = k.min(values.len()); - let topk_sum: f64 = values.iter().take(take).sum(); - Ok(ExecutionOutcome { - value: topk_sum, - samples_scanned: values.len(), - chunks_fetched: buffered.chunks_fetched, - chunks_skipped_via_postings: buffered.chunks_skipped_via_postings, - postings_filtered_series_count: buffered.postings_filtered_series_count, - postings_missing: buffered.postings_missing, - }) - } - - /// Shared helper for the buffered paths: walk every chunk, - /// keep in-range samples, enforce the - /// [`GorillaEngineConfig::max_buffered_samples`] ceiling. - /// **mvp/v5**: also applies the postings-aware filter so the - /// quantile / topk paths share the same pruning as - /// streaming-additive. - async fn collect_buffered_samples( - &self, - plan: &QueryPlan, - ) -> Result { - let (start_ms, end_ms) = plan.time_range_ms; - let chunks = self - .store - .list_chunks(&plan.metric, start_ms, end_ms) - .await?; - let total_chunks = chunks.len(); - let (filtered_chunks, postings_outcome) = self.apply_postings_filter(plan, &chunks).await; - let chunks_fetched = filtered_chunks.len(); - let mut buffer: Vec = Vec::new(); - let limit = self.config.max_buffered_samples; - for chunk in filtered_chunks { - let samples = self.store.read_chunk(&chunk).await?; - for s in samples { - if s.ts_ms >= start_ms && s.ts_ms < end_ms { - if !self.sample_matches(plan, &s) { - continue; - } - if buffer.len() >= limit { - // Surface the over-budget sample count - // (limit + 1) so callers can pin - // `count > limit` in tests; we don't - // bother walking the rest of the chunks - // just to tighten the number. - return Err(EngineError::TooManySamples { - count: buffer.len() + 1, - limit, - }); - } - buffer.push(s); - } - } - } - Ok(BufferedScan { - samples: buffer, - chunks_fetched, - chunks_skipped_via_postings: total_chunks - chunks_fetched, - postings_filtered_series_count: postings_outcome.matched_series, - postings_missing: postings_outcome.postings_missing, - }) - } -} - -/// Output of [`ExactExecutor::collect_buffered_samples`]. -struct BufferedScan { - samples: Vec, - chunks_fetched: usize, - chunks_skipped_via_postings: usize, - postings_filtered_series_count: usize, - postings_missing: bool, -} - -/// Output of [`ExactExecutor::apply_postings_filter`]. -#[derive(Debug, Clone, Copy)] -struct PostingsOutcome { - matched_series: usize, - postings_missing: bool, -} - -/// Per-sample fold for the streaming-additive path. Fields are -/// kept in raw f64 (sum / min / max) + i64 (count) so the -/// finaliser can pick the right arithmetic per op. -/// -/// `op` is taken at `new` time rather than carried as a struct -/// field — the finaliser receives it as an argument, keeping the -/// struct itself op-agnostic and shrinking the per-fold footprint. -#[derive(Debug, Clone, Copy)] -struct AdditiveAccumulator { - sum: f64, - count: i64, - min: f64, - max: f64, - /// Earliest observed `(ts_ms, value)` — used by Rate / Increase. - first: Option<(i64, f64)>, - /// Latest observed `(ts_ms, value)` — used by Rate / Increase. - last: Option<(i64, f64)>, -} - -impl AdditiveAccumulator { - fn new(_op: AdditiveOp) -> Self { - Self { - sum: 0.0, - count: 0, - min: f64::INFINITY, - max: f64::NEG_INFINITY, - first: None, - last: None, - } - } - - fn observe(&mut self, s: &RawSample) { - self.sum += s.value; - self.count += 1; - if s.value < self.min { - self.min = s.value; - } - if s.value > self.max { - self.max = s.value; - } - match self.first { - None => self.first = Some((s.ts_ms, s.value)), - Some((ts, _)) if s.ts_ms < ts => self.first = Some((s.ts_ms, s.value)), - _ => {} - } - match self.last { - None => self.last = Some((s.ts_ms, s.value)), - Some((ts, _)) if s.ts_ms > ts => self.last = Some((s.ts_ms, s.value)), - _ => {} - } - } - - /// Convert the running accumulator into a final scalar. - /// Returns `NaN` for the empty-time-range case so downstream - /// formatting stays consistent. - fn finalize(&self, plan: &QueryPlan, op: AdditiveOp) -> f64 { - if self.count == 0 { - return match op { - AdditiveOp::Count => 0.0, - _ => f64::NAN, - }; - } - match op { - AdditiveOp::Sum => self.sum, - AdditiveOp::Count => self.count as f64, - AdditiveOp::Avg => self.sum / self.count as f64, - AdditiveOp::Min => self.min, - AdditiveOp::Max => self.max, - AdditiveOp::Increase => match (self.first, self.last) { - (Some((_, fv)), Some((_, lv))) => lv - fv, - _ => f64::NAN, - }, - AdditiveOp::Rate => match (self.first, self.last) { - (Some((_, fv)), Some((_, lv))) => { - let (start_ms, end_ms) = plan.time_range_ms; - let range_secs = ((end_ms - start_ms).max(1)) as f64 / 1000.0; - if range_secs <= 0.0 { - f64::NAN - } else { - (lv - fv) / range_secs - } - } - _ => f64::NAN, - }, - // v7 / issue #46 ⑥: return the value of the - // largest-timestamp sample. Counter-shaped freshness - // probes (http_freshness_probe_*) encode the unix_ts_ms - // of the most recent emission directly in the - // cumulative counter value, so `last_over_time(...)` - // returning that value lets the replay client subtract - // the polled timestamp and get a per-path freshness - // delta. - AdditiveOp::Last => match self.last { - Some((_, lv)) => lv, - None => f64::NAN, - }, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - const NOW: i64 = 1_715_000_000_000; - - #[test] - fn plans_sum_over_time() { - let plan = plan_query_at("sum_over_time(http_requests_total[5m])", NOW).unwrap(); - assert_eq!(plan.metric, "http_requests_total"); - assert_eq!(plan.statistic, QueryStatistic::SumOverTime); - assert_eq!(plan.time_range_ms, (NOW - 5 * 60_000, NOW)); - } - - #[test] - fn plans_quantile_over_time() { - let plan = plan_query_at("quantile_over_time(0.99, latency_ms[1m])", NOW).unwrap(); - assert_eq!(plan.metric, "latency_ms"); - assert!(matches!( - plan.statistic, - QueryStatistic::QuantileOverTime { phi } if (phi - 0.99).abs() < 1e-12 - )); - } - - #[test] - fn plans_topk() { - // Legal PromQL spelling: aggregation wraps a vector-returning - // function call. The Phase-4 planner peels off the outer - // `topk` and recovers the `(metric, range)` pair from the - // inner `sum_over_time(...)`. - let plan = plan_query_at("topk(3, sum_over_time(m[10s]))", NOW).unwrap(); - assert!(matches!(plan.statistic, QueryStatistic::TopK { k } if k == 3)); - assert_eq!(plan.metric, "m"); - assert_eq!(plan.time_range_ms, (NOW - 10_000, NOW)); - } - - #[test] - fn rejects_binary_expression() { - assert!(plan_query_at("foo + bar", NOW).is_err()); - } - - #[test] - fn streaming_classification() { - assert!(QueryStatistic::SumOverTime.is_streaming_additive()); - assert!(QueryStatistic::Rate.is_streaming_additive()); - assert!(QueryStatistic::LastOverTime.is_streaming_additive()); - assert!(!QueryStatistic::QuantileOverTime { phi: 0.5 }.is_streaming_additive()); - assert!(!QueryStatistic::TopK { k: 1 }.is_streaming_additive()); - } - - #[test] - fn plans_last_over_time_v7() { - // v7: `last_over_time(...)` translates to the streaming - // additive path, picking the value of the largest-timestamp - // sample in [now-range, now). Issue #46 ⑥ freshness probes - // ride this path. - let plan = plan_query_at("last_over_time(http_freshness_probe_warm[10s])", NOW).unwrap(); - assert_eq!(plan.metric, "http_freshness_probe_warm"); - assert_eq!(plan.statistic, QueryStatistic::LastOverTime); - assert_eq!(plan.time_range_ms, (NOW - 10_000, NOW)); - } -} diff --git a/data_plane/src/storage_engines/gorilla_object_store/mod.rs b/data_plane/src/storage_engines/gorilla_object_store/mod.rs deleted file mode 100644 index 61fa8f2ef..000000000 --- a/data_plane/src/storage_engines/gorilla_object_store/mod.rs +++ /dev/null @@ -1,354 +0,0 @@ -//! Gorilla object store and legacy in-process archive executor. -//! -//! This module owns the Gorilla chunk/object-store implementation. The -//! public archive query engine is [`crate::query_engines::thanos_query_engine`]; the -//! in-process [`GorillaQueryEngine`] remains here as a legacy fallback for -//! development deployments that still read per-hour Gorilla chunks directly. -//! -//! ## Module layout (post Step-1 refactor) -//! -//! * [`archive_query`] — query planner + per-statistic exact executor (the -//! merged form of the previous `query_planner.rs` + -//! `exact_executor.rs`). -//! * [`store`] — `GorillaS3Store` (the only `Store` impl after -//! the JSONL deletion) + `Store`/`ObjectStore` traits + -//! `RawSample` / `ChunkRef` types. -//! * [`postings`] — postings-sidecar cache + per-bucket -//! intersection helper. -//! * [`s3_cost`] — instrumented S3 client wrapper that ticks the -//! process-wide cost counters surfaced on -//! `/internal/s3_cost.csv` + `/metrics`. -//! -//! Result wrapping pins three things: -//! -//! 1. an [`crate::storage_engines::sketch_db::AccuracyEnvelope`] with -//! `kind = Exact`, ε = 0, δ = 0, -//! 2. a `data_source: thanos_query` info line, -//! 3. cheap diagnostics (`samples_scanned`, `chunks_fetched`). -//! -//! See `docs/design-gorilla-s3-cold-engine.md` §6. -//! -//! ## Two execution strategies -//! -//! Per-statistic dispatch in [`archive_query::ExactExecutor`]: -//! -//! * **Streaming-additive** — `Sum`, `Count`, `Min`, `Max`, `Rate`, -//! `Increase` (and `Avg` derived as Sum/Count). One chunk at a -//! time, fold into a small accumulator, drop the decoded samples -//! before fetching the next chunk. Memory cost is O(1) per group. -//! * **Buffered** — `Quantile`, `TopK`, `Cardinality`. Materialise -//! every in-range sample, then sort / count. Bounded by -//! [`GorillaEngineConfig::max_buffered_samples`]; over-budget -//! queries fail fast with [`EngineError::TooManySamples`]. - -pub mod archive_query; -pub mod postings; -pub mod s3_cost; -pub mod store; - -#[cfg(test)] -mod tests; - -use std::sync::Arc; -use std::time::Duration; - -use thiserror::Error; -use tokio::time::error::Elapsed; -use tracing::debug; - -use crate::storage_engines::types::KeyByLabelValues; -use crate::query_engines::query_result::{InstantVectorElement, QueryResult}; -use crate::storage_engines::sketch_db::accuracy::{AccuracyEnvelope, AccuracyProfile}; - -pub use archive_query::{ - plan_query, plan_query_at, AdditiveOp, ExactExecutor, LabelMatcher, QueryPlan, QueryStatistic, -}; -pub use postings::PostingsHits; -pub use s3_cost::{ - global_s3_cost_counters, S3CostCounters, S3CostSnapshot, S3CostTrackingObjectStore, -}; -pub use store::{ - ChunkRef, GorillaS3Config, GorillaS3ConfigError, GorillaS3Store, ObjectStore, RawSample, - S3ObjectStore, Store, StoreError, -}; -/// Marker line that every archive answer carries on its `infos` array. -/// The legacy in-process Gorilla executor is an implementation detail; -/// the public archive engine identity is Thanos. -pub const DATA_SOURCE_GORILLA_ARCHIVE: &str = "data_source: thanos_query"; - -/// Tunable runtime knobs for the Gorilla query engine. -/// -/// Call sites typically construct via `Default::default()`; tests -/// override `max_buffered_samples` to exercise the bounded-buffer -/// guard. -#[derive(Debug, Clone)] -pub struct GorillaEngineConfig { - /// Hard cap on the number of samples a buffered-aggregate - /// query (quantile / topk / cardinality) is allowed to - /// materialise in memory. Default `10_000_000` - /// (~160 MB at 16 B per `(ts, value)` pair). - pub max_buffered_samples: usize, - /// Wall-clock query timeout, in seconds. Default `30`. - pub query_timeout_secs: u64, -} - -impl Default for GorillaEngineConfig { - fn default() -> Self { - Self { - max_buffered_samples: 10_000_000, - query_timeout_secs: 30, - } - } -} - -/// Error surface returned by [`GorillaQueryEngine::execute`]. -#[derive(Debug, Error)] -pub enum EngineError { - /// PromQL string failed to parse, or used a construct outside - /// the archive query supported surface (see [`archive_query`]). - #[error("query planning failed: {0}")] - Plan(String), - /// Archive-store fetch / decode failed. - #[error("store error: {0}")] - Store(#[from] store::StoreError), - /// Buffered-aggregate budget exceeded — query asked for more - /// samples than [`GorillaEngineConfig::max_buffered_samples`] - /// will allow. The user should narrow the time range or - /// lower the cardinality. - #[error( - "buffered-aggregate budget exceeded: {count} samples > limit {limit}; \ - narrow the time range or lower the metric cardinality" - )] - TooManySamples { - /// Samples the engine attempted to materialise. - count: usize, - /// Configured ceiling. - limit: usize, - }, - /// Wall-clock timeout fired before the query finished. - #[error("query timed out after {0:?}")] - Timeout(Duration), -} - -impl From for EngineError { - fn from(_: Elapsed) -> Self { - Self::Timeout(Duration::from_secs(0)) - } -} - -/// Archive-tier exact engine. -/// -/// Holds an `Arc` rather than a concrete -/// `Arc` so tests can inject in-memory mocks -/// and so future archive backends (Prometheus-block format via -/// the planned Step-2 Thanos store-gateway, multi-region fan-out) -/// drop in without changing the engine surface. The production -/// constructor [`GorillaQueryEngine::with_gorilla_s3`] keeps the -/// design.md type signature working at the call site. -pub struct GorillaQueryEngine { - store: Arc, - config: GorillaEngineConfig, -} - -impl GorillaQueryEngine { - /// Build with an arbitrary `Store` implementation. Used by - /// tests + the capability router (which may swap the - /// concrete impl based on routing decisions). - pub fn new(store: Arc, config: GorillaEngineConfig) -> Self { - Self { store, config } - } - - /// Convenience constructor for the production - /// [`store::GorillaS3Store`] path. Mirrors the design.md type - /// signature. - pub fn with_gorilla_s3(store: Arc, config: GorillaEngineConfig) -> Self { - Self::new(store as Arc, config) - } - - /// Read-only access to the configured limits — useful for - /// diagnostics + the cost-aware router's cost estimator. - pub fn config(&self) -> &GorillaEngineConfig { - &self.config - } - - /// Test-only accessor for the underlying store. mvp/v5 - /// tests use this to construct a `ExactExecutor` that shares - /// the same mock without re-wrapping in a fresh `Arc`. - #[cfg(test)] - pub(super) fn store_for_tests(&self) -> Arc { - self.store.clone() - } - - /// Execute a parsed PromQL query against the archive tier. - /// - /// The query string is parsed via [`archive_query::plan_query`], - /// the resulting plan dispatches to either the streaming - /// additive or the buffered execution path, and the answer is - /// wrapped with the exact-accuracy envelope + the - /// `data_source: thanos_query` annotation. - pub async fn execute(&self, query: &str) -> Result { - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map(|d| d.as_millis() as i64) - .unwrap_or(0); - self.execute_at(query, now_ms).await - } - - /// Like [`Self::execute`], with a caller-supplied `now_ms` - /// pinning the right edge of the request window. Used by - /// tests + by callers that want to back-date a query against - /// historical chunks. - pub async fn execute_at(&self, query: &str, now_ms: i64) -> Result { - let timeout = Duration::from_secs(self.config.query_timeout_secs.max(1)); - tokio::time::timeout(timeout, self.execute_inner(query, now_ms)) - .await - .map_err(|_| EngineError::Timeout(timeout))? - } - - async fn execute_inner(&self, query: &str, now_ms: i64) -> Result { - let plan = archive_query::plan_query_at(query, now_ms).map_err(EngineError::Plan)?; - debug!( - metric = plan.metric.as_str(), - stat = ?plan.statistic, - start_ms = plan.time_range_ms.0, - end_ms = plan.time_range_ms.1, - "gorilla-engine: executing plan" - ); - - let executor = ExactExecutor::new(self.store.clone(), self.config.clone()); - let outcome = executor.execute_plan(&plan).await?; - - Ok(wrap_result(&plan, outcome)) - } -} - -/// Wrap a finished `(scalar value, sample / chunk counts)` into a -/// `QueryResult` with the exact-accuracy envelope + the -/// `data_source: thanos_query` info line. Pulled out so tests -/// can pin the wrapping shape independently of the executor. -pub fn wrap_result(plan: &QueryPlan, outcome: ExecutionOutcome) -> QueryResult { - // Result timestamp is the right edge of the requested range — - // mirrors `ASAPQueryEngine`'s convention for instant-vector queries - // against a closed time window. - let result_ts = plan.time_range_ms.1.max(0) as u64; - - let labels = KeyByLabelValues::new_with_labels(Vec::new()); - let element = InstantVectorElement::new(labels, outcome.value); - let envelope = AccuracyEnvelope::single(AccuracyProfile::exact()); - QueryResult::vector(vec![element], result_ts) - .with_accuracy(envelope) - // Window is the requested range, expressed in u64 ms. - .with_window_used(( - plan.time_range_ms.0.max(0) as u64, - plan.time_range_ms.1.max(0) as u64, - )) -} - -/// Output of an executed plan. Kept narrow on purpose — Phase 4's -/// MVP returns a single scalar per query. Higher-cardinality -/// (per-group) shapes will land in Phase 5+ once capability -/// routing decides which engine answers grouped queries. -#[derive(Debug, Clone, PartialEq)] -pub struct ExecutionOutcome { - /// Final scalar (e.g. `sum_over_time` total, `quantile_over_time` - /// φ-quantile). NaN when the time range carries no samples. - pub value: f64, - /// Number of raw samples that contributed to `value`. - pub samples_scanned: usize, - /// Number of chunks the executor fetched from the cold store. - pub chunks_fetched: usize, - /// **mvp/v5**: number of chunks the postings filter pruned — - /// the executor was able to skip these without a chunk-body - /// fetch. `0` when the postings-aware path didn't run (no label - /// matchers / postings missing). - pub chunks_skipped_via_postings: usize, - /// **mvp/v5**: number of series the postings file said matched - /// the label predicates. The executor uses this to decide - /// whether a chunk's `label_hash` is interesting before paying - /// for the chunk body. Surfaces in `infos` as - /// `postings_filtered_series_count`. - pub postings_filtered_series_count: usize, - /// **mvp/v5**: `true` when the engine hit a missing postings - /// sidecar in the request window and fell back to the scan-all - /// path. Drives the `data_source_quirk: postings_missing` - /// `infos` annotation. - pub postings_missing: bool, -} - -impl ExecutionOutcome { - /// "no data" sentinel — used when the time range is empty. - pub fn empty() -> Self { - Self { - value: f64::NAN, - samples_scanned: 0, - chunks_fetched: 0, - chunks_skipped_via_postings: 0, - postings_filtered_series_count: 0, - postings_missing: false, - } - } - - /// Build the `infos` array surfaced on the wire response. - /// Pulled out so tests can pin the exact strings. - pub fn info_lines(&self) -> Vec { - let mut out = vec![ - AccuracyProfile::exact().summary(), - DATA_SOURCE_GORILLA_ARCHIVE.to_string(), - format!("samples_scanned: {}", self.samples_scanned), - format!("chunks_fetched: {}", self.chunks_fetched), - ]; - // mvp/v5: surface postings-aware execution counters. - out.push(format!( - "chunks_skipped_via_postings: {}", - self.chunks_skipped_via_postings - )); - out.push(format!( - "postings_filtered_series_count: {}", - self.postings_filtered_series_count - )); - if self.postings_missing { - out.push("data_source_quirk: postings_missing".to_string()); - } - out - } -} - -// --------------------------------------------------------------------------- -// `QueryEngine` trait impl. -// -// Wraps `GorillaQueryEngine::execute` with the EngineError envelope the -// router speaks. Plan-time / parse-time failures fold into -// `EngineError::CapabilityMiss` (the engine cannot serve this query -// shape; router should fall through). Store / timeout / buffer-budget -// failures fold into `EngineError::Backend` (the engine could have served -// the query but its backend transiently failed; router should also fall -// through, typically to the ASAP-tier sketch path on `DoubleWrite`). -// --------------------------------------------------------------------------- - -#[async_trait::async_trait] -impl crate::query_engines::routing::query_engine_routing::QueryEngine for GorillaQueryEngine { - async fn execute(&self, query: &str) -> Result { - match GorillaQueryEngine::execute(self, query).await { - Ok(result) => Ok(result), - Err(EngineError::Plan(msg)) => Err(crate::query_engines::EngineError::capability_miss( - asap_types::StorageBackend::GorillaObjectStore.data_source_id(), - msg, - )), - Err(other) => Err(crate::query_engines::EngineError::backend( - asap_types::StorageBackend::GorillaObjectStore.data_source_id(), - other, - )), - } - } - - fn capabilities(&self) -> crate::query_engines::routing::query_engine_routing::EngineCapabilities { - crate::query_engines::routing::query_engine_routing::EngineCapabilities { - data_source_id: asap_types::StorageBackend::GorillaObjectStore.data_source_id(), - storage_backend: asap_types::StorageBackend::GorillaObjectStore, - // The buffered-aggregate budget gives a natural ceiling: each - // sample is ~16 B (i64 ts + f64 value), so the byte budget is - // ~16 × max_buffered_samples. - supports_streams_above_bytes: self.config.max_buffered_samples.saturating_mul(16), - } - } -} diff --git a/data_plane/src/storage_engines/gorilla_object_store/postings.rs b/data_plane/src/storage_engines/gorilla_object_store/postings.rs deleted file mode 100644 index e2eb86e11..000000000 --- a/data_plane/src/storage_engines/gorilla_object_store/postings.rs +++ /dev/null @@ -1,164 +0,0 @@ -//! Postings cache + sidecar fetch helper for the Gorilla archive -//! engine. -//! -//! The on-S3 postings sidecar is a JSON file emitted by the agent -//! `gorillas3processor` alongside each per-hour `index.json`: -//! `//YYYY/MM/DD/HH/postings-v1.json`. It maps -//! `(label_name, label_value) → [series_id, ...]` so the -//! [`super::archive_query::ExactExecutor`] can prune chunks by `label_hash` -//! without paying the chunk-body GET cost. -//! -//! Step-1 of the JSONL deprecation refactor pulled this code out -//! of `gorilla_s3.rs` so the cache + intersection logic has a -//! single home; the previous co-located version conflated three -//! responsibilities (S3 wiring, postings cache, intersection -//! algebra). With Step-2 (Prometheus-block format + Thanos -//! store-gateway) coming next, splitting now means the Thanos -//! impl can either reuse [`intersect_per_bucket_postings`] as-is -//! or replace it without touching the gorilla store. - -use std::collections::BTreeSet; -use std::num::NonZeroUsize; -use std::sync::Arc; - -use lru::LruCache; -use tokio::sync::Mutex; -use tracing::debug; - -use asap_gorilla::Postings; - -use super::store::{ObjectStore, StoreError}; - -/// Default LRU capacity for the postings cache. ~1 MiB per -/// postings file, so 64 entries ≈ 64 MiB worst-case. -pub const POSTINGS_CACHE_DEFAULT_CAPACITY: usize = 64; - -/// LRU cache for parsed postings sidecars. Keyed by the -/// `postings-v1.json` S3 key (one per `(metric, hour)`). -pub type PostingsCache = Mutex>>; - -/// Build a fresh empty postings cache with capacity `cap` (clamped -/// to ≥ 1). -pub fn new_postings_cache(cap: usize) -> PostingsCache { - let cap = NonZeroUsize::new(cap.max(1)).unwrap_or(NonZeroUsize::new(1).unwrap()); - Mutex::new(LruCache::new(cap)) -} - -/// Output of [`intersect_per_bucket_postings`]: the union of -/// series_ids matching every label predicate, plus per-bucket -/// coverage counters used by the engine to decide whether to emit -/// the `postings_missing` quirk on the response. -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct PostingsHits { - /// Series ids matching all label predicates, sorted ascending, - /// deduplicated. - pub series_ids: Vec, - /// Hour buckets in the request window. - pub buckets_in_range: usize, - /// Buckets that actually had a `postings-v1.json` sidecar. - pub buckets_with_postings: usize, -} - -impl PostingsHits { - /// `true` iff at least one hour bucket carried postings — - /// indicates the postings-aware filter ran on real data and the - /// caller should trust [`Self::series_ids`] as a complete answer. - pub fn fully_covered(&self) -> bool { - self.buckets_in_range > 0 && self.buckets_with_postings == self.buckets_in_range - } - - /// `true` iff postings were present for every bucket AND at - /// least one matched series. - pub fn nonempty_and_complete(&self) -> bool { - self.fully_covered() && !self.series_ids.is_empty() - } -} - -/// Walk every `postings-v1.json` key in `keys`, fetch + parse via -/// `object_store` (LRU-cached in `cache`), intersect the per-matcher -/// series-id lists within each bucket, and union the results -/// across buckets. -/// -/// Empty `matchers` ⇒ returns the union of every series_id across -/// every label in every bucket (the no-predicate short-circuit). -/// -/// Cross-bucket join is a UNION (a series might exist in one hour -/// but not the next); intra-bucket intersection across matchers is -/// an AND. -pub async fn intersect_per_bucket_postings( - object_store: &dyn ObjectStore, - cache: &PostingsCache, - keys: &[String], - matchers: &[(String, String)], -) -> Result { - let mut hits = PostingsHits { - series_ids: Vec::new(), - buckets_in_range: keys.len(), - buckets_with_postings: 0, - }; - let mut union_set: BTreeSet = BTreeSet::new(); - - for key in keys { - // LRU short-circuit. - let postings = { - let mut guard = cache.lock().await; - guard.get(key).cloned() - }; - let postings = match postings { - Some(p) => Some(p), - None => match object_store.get_object(key).await { - Ok(bytes) => match Postings::read(bytes.as_slice()) { - Ok(p) => { - let arc = Arc::new(p); - let mut guard = cache.lock().await; - guard.put(key.clone(), arc.clone()); - Some(arc) - } - Err(e) => { - // Treat a corrupt postings file as - // "missing" — the engine then falls - // through to the scan-all path with - // the postings_missing quirk. - debug!( - key = %key, - error = %e, - "gorilla-engine: postings parse failed; treating as missing" - ); - None - } - }, - Err(e) if object_store.object_missing(&e) => { - debug!(key = %key, "gorilla-engine: postings missing for hour bucket"); - None - } - Err(e) => return Err(e), - }, - }; - let Some(postings) = postings else { continue }; - hits.buckets_with_postings += 1; - - // Intersect across matchers within this bucket. - let bucket_set: BTreeSet = if matchers.is_empty() { - // Union of every series_id across every label. - let mut set = BTreeSet::new(); - for by_value in postings.by_label.values() { - for ids in by_value.values() { - set.extend(ids.iter().copied()); - } - } - set - } else { - let first = postings.lookup(&matchers[0].0, &matchers[0].1); - let mut acc: BTreeSet = first.iter().copied().collect(); - for (label_name, label_value) in &matchers[1..] { - let next = postings.lookup(label_name, label_value); - let next_set: BTreeSet = next.iter().copied().collect(); - acc = acc.intersection(&next_set).copied().collect(); - } - acc - }; - union_set.extend(bucket_set); - } - hits.series_ids = union_set.into_iter().collect(); - Ok(hits) -} diff --git a/data_plane/src/storage_engines/gorilla_object_store/s3_cost.rs b/data_plane/src/storage_engines/gorilla_object_store/s3_cost.rs deleted file mode 100644 index c56d66c3a..000000000 --- a/data_plane/src/storage_engines/gorilla_object_store/s3_cost.rs +++ /dev/null @@ -1,259 +0,0 @@ -//! mvp/v5 — instrumented S3 client wrapper. -//! -//! The compaction story for the MVP demo wants a measured (not -//! fabricated) S3 cost picture: per-baseline counts of PUT / GET / -//! HEAD / LIST / DELETE plus bytes-out per request, dumped to CSV -//! at end-of-run. -//! -//! This module provides a thin wrapper that delegates to the -//! existing `rust-s3` [`s3::Bucket`] but ticks a small counter set -//! before / after every operation. Live values are exposed via a -//! Prometheus gauge (`asap_backend_s3__total`) so dashboards -//! see them in real time, AND a CSV / JSON dump on demand. -//! -//! ## Boundary -//! -//! The wrapper sits at the lowest level — between the -//! [`GorillaS3Store`](super::store::GorillaS3Store)'s `ObjectStore` -//! impl and the actual `Bucket`. -//! Tests that don't need S3 (the in-memory mock path) never touch -//! it; production deployments wire `S3CostTrackingObjectStore` -//! around `S3ObjectStore`. - -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, OnceLock}; - -use async_trait::async_trait; - -/// Process-wide S3 cost counters. The HTTP server's -/// `/internal/s3_cost.csv` endpoint reads this; the -/// [`GorillaS3Store`](super::store::GorillaS3Store) constructor opts in via -/// [`S3CostTrackingObjectStore`]. Lazy-initialised on first access. -static GLOBAL_S3_COST: OnceLock> = OnceLock::new(); - -/// Access (and lazily create) the process-wide S3 cost counters. -pub fn global_s3_cost_counters() -> Arc { - GLOBAL_S3_COST - .get_or_init(|| Arc::new(S3CostCounters::new())) - .clone() -} - -use super::store::{ObjectStore, StoreError}; - -/// Per-operation counter set + cumulative bytes. -#[derive(Debug, Default)] -pub struct S3CostCounters { - /// PUT operations issued. - pub put_count: AtomicU64, - /// GET operations issued (full + range). - pub get_count: AtomicU64, - /// HEAD operations issued. - pub head_count: AtomicU64, - /// LIST operations issued. - pub list_count: AtomicU64, - /// DELETE operations issued. - pub delete_count: AtomicU64, - /// Bytes uploaded (PUT request bodies). - pub bytes_put: AtomicU64, - /// Bytes downloaded (GET response bodies). - pub bytes_got: AtomicU64, -} - -impl S3CostCounters { - /// Build a fresh zeroed counter set. - pub fn new() -> Self { - Self::default() - } - - /// Plain-old-data snapshot. - pub fn snapshot(&self) -> S3CostSnapshot { - S3CostSnapshot { - put_count: self.put_count.load(Ordering::Relaxed), - get_count: self.get_count.load(Ordering::Relaxed), - head_count: self.head_count.load(Ordering::Relaxed), - list_count: self.list_count.load(Ordering::Relaxed), - delete_count: self.delete_count.load(Ordering::Relaxed), - bytes_put: self.bytes_put.load(Ordering::Relaxed), - bytes_got: self.bytes_got.load(Ordering::Relaxed), - } - } - - /// Render Prometheus text-exposition lines for `/metrics`. - pub fn render_prometheus(&self) -> String { - let s = self.snapshot(); - format!( - concat!( - "# HELP asap_backend_s3_put_total S3 PUT count.\n", - "# TYPE asap_backend_s3_put_total counter\n", - "asap_backend_s3_put_total {}\n", - "asap_backend_s3_get_total {}\n", - "asap_backend_s3_head_total {}\n", - "asap_backend_s3_list_total {}\n", - "asap_backend_s3_delete_total {}\n", - "asap_backend_s3_bytes_put {}\n", - "asap_backend_s3_bytes_got {}\n", - ), - s.put_count, - s.get_count, - s.head_count, - s.list_count, - s.delete_count, - s.bytes_put, - s.bytes_got, - ) - } - - /// Render a CSV summary suitable for the demo's `s3_cost.csv`. - /// Single header + single data row. - pub fn render_csv(&self) -> String { - let s = self.snapshot(); - format!( - "put_count,get_count,head_count,list_count,delete_count,bytes_put,bytes_got\n\ - {},{},{},{},{},{},{}\n", - s.put_count, - s.get_count, - s.head_count, - s.list_count, - s.delete_count, - s.bytes_put, - s.bytes_got, - ) - } -} - -/// Plain-old-data snapshot returned by [`S3CostCounters::snapshot`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct S3CostSnapshot { - /// PUT count. - pub put_count: u64, - /// GET count. - pub get_count: u64, - /// HEAD count. - pub head_count: u64, - /// LIST count. - pub list_count: u64, - /// DELETE count. - pub delete_count: u64, - /// Bytes uploaded. - pub bytes_put: u64, - /// Bytes downloaded. - pub bytes_got: u64, -} - -/// `ObjectStore` wrapper that ticks the supplied counters. -/// -/// Note: only `get_object` is in the cold-store hot path today -/// (Phase 3 + Phase 4). PUT / LIST / DELETE / HEAD are recorded -/// even though current callers never go through them — having the -/// counter live makes follow-up MVP cost work additive. -pub struct S3CostTrackingObjectStore { - inner: Arc, - counters: Arc, -} - -impl S3CostTrackingObjectStore { - /// Wrap `inner` and a counter-set for the wrapper to update. - pub fn new(inner: Arc, counters: Arc) -> Self { - Self { inner, counters } - } - - /// Borrow the live counters — handy for HTTP exposition. - pub fn counters(&self) -> &Arc { - &self.counters - } -} - -#[async_trait] -impl ObjectStore for S3CostTrackingObjectStore { - async fn get_object(&self, key: &str) -> Result, StoreError> { - self.counters.get_count.fetch_add(1, Ordering::Relaxed); - let body = self.inner.get_object(key).await?; - self.counters - .bytes_got - .fetch_add(body.len() as u64, Ordering::Relaxed); - Ok(body) - } - - fn object_missing(&self, err: &StoreError) -> bool { - self.inner.object_missing(err) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::storage_engines::gorilla_object_store::store::ObjectStore as _; - use std::collections::HashMap; - use tokio::sync::Mutex; - - /// Minimal in-memory ObjectStore stand-in for the wrapper test — - /// we don't pull in `InMemoryObjectStore` because it's a - /// `#[cfg(test)]` type local to its own module. - #[derive(Default)] - struct StubStore { - inner: Mutex>>, - } - - #[async_trait] - impl ObjectStore for StubStore { - async fn get_object(&self, key: &str) -> Result, StoreError> { - let g = self.inner.lock().await; - match g.get(key) { - Some(b) => Ok(b.clone()), - None => Err(StoreError::Backend(format!("get {key}: not found"))), - } - } - } - - #[tokio::test] - async fn counters_increment_on_get() { - let inner = Arc::new(StubStore::default()); - inner - .inner - .lock() - .await - .insert("k1".to_string(), vec![0u8; 100]); - let counters = Arc::new(S3CostCounters::new()); - let wrapped = S3CostTrackingObjectStore::new(inner, counters.clone()); - - let _ = wrapped.get_object("k1").await.unwrap(); - let _ = wrapped.get_object("k1").await.unwrap(); - let snap = counters.snapshot(); - assert_eq!(snap.get_count, 2); - assert_eq!(snap.bytes_got, 200); - assert_eq!(snap.put_count, 0); - assert_eq!(snap.head_count, 0); - } - - #[tokio::test] - async fn missing_object_does_not_count_bytes() { - let inner = Arc::new(StubStore::default()); - let counters = Arc::new(S3CostCounters::new()); - let wrapped = S3CostTrackingObjectStore::new(inner, counters.clone()); - let _ = wrapped.get_object("missing").await; - let snap = counters.snapshot(); - assert_eq!(snap.get_count, 1); - assert_eq!(snap.bytes_got, 0); - } - - #[test] - fn render_csv_has_expected_columns() { - let c = S3CostCounters::new(); - c.put_count.store(3, Ordering::Relaxed); - c.get_count.store(7, Ordering::Relaxed); - c.bytes_got.store(1024, Ordering::Relaxed); - let csv = c.render_csv(); - assert!(csv.starts_with( - "put_count,get_count,head_count,list_count,delete_count,bytes_put,bytes_got\n" - )); - assert!(csv.contains("3,7,0,0,0,0,1024")); - } - - #[test] - fn render_prometheus_has_help_line() { - let c = S3CostCounters::new(); - let p = c.render_prometheus(); - assert!(p.contains("asap_backend_s3_put_total")); - assert!(p.contains("# TYPE asap_backend_s3_put_total counter")); - } -} diff --git a/data_plane/src/storage_engines/gorilla_object_store/store.rs b/data_plane/src/storage_engines/gorilla_object_store/store.rs deleted file mode 100644 index e658d2189..000000000 --- a/data_plane/src/storage_engines/gorilla_object_store/store.rs +++ /dev/null @@ -1,1249 +0,0 @@ -//! Gorilla-on-S3 archive store — the Phase-4 [`GorillaQueryEngine`](super::GorillaQueryEngine)'s -//! sole storage backend. -//! -//! Lists per-hour `index.json` catalogs out of an S3-compatible -//! bucket, prunes them by time range, then fetches + decodes the -//! selected `GORILLA1` chunks via the [`asap_gorilla`] crate -//! (`ASAPCollector` PR #281). -//! -//! Step-1 refactor (`refactor: tier-co-locate query-engines/{simple,gorilla}/`) -//! folded the previous `ColdStore` trait + `RawSample`/`ChunkRef` -//! types into this module. The legacy JSONL leg -//! (`LocalFsColdStore`, `parse_jsonl`, `ColdJsonlFallback`) was -//! deleted at the same commit; this is now the only `Store` impl -//! in the archive tier. -//! -//! # Object key layout -//! -//! `GorillaS3Store` is **agnostic** about the on-S3 chunk-key -//! shape. Two layouts are known to coexist (see PR #281): -//! -//! * design.md canonical: -//! `//YYYY/MM/DD/HH/part-NNNNNN.gor` -//! * Telegraf-side `gorilla_s3` output: -//! `/block---.gorilla` (random suffix) -//! -//! The per-hour `index.json` is the source of truth for what keys -//! exist; we treat [`asap_gorilla::IndexEntry::key`] as opaque and -//! do not try to parse it. The `prefix_template` config field -//! controls only where the **index** files live, not the chunks. -//! -//! # S3 client -//! -//! Backed by the `rust-s3` crate (`s3 = "0.37"`) — single-crate -//! dep, MinIO-friendly out of the box (no AWS-specific signing -//! quirks, supports custom endpoint URLs + path-style addressing). -//! Hidden behind the [`ObjectStore`] trait below so tests use an -//! in-memory mock and do not need a live MinIO. - -use std::collections::BTreeMap; -use std::num::NonZeroUsize; -use std::sync::Arc; - -#[cfg(test)] -use std::collections::HashMap; - -use async_trait::async_trait; -use chrono::{DateTime, Datelike, Timelike, Utc}; -use lru::LruCache; -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use tokio::sync::Mutex; -use tracing::debug; - -use asap_gorilla::{GorillaDecoder, IndexFile}; - -use super::postings::{intersect_per_bucket_postings, PostingsCache, PostingsHits}; -use super::s3_cost::{global_s3_cost_counters, S3CostTrackingObjectStore}; - -// ───────────────────────────────────────────────────────────────────── -// Public types — merged in from the deleted `cold_store/mod.rs` -// ───────────────────────────────────────────────────────────────────── - -/// A single raw observability sample as decoded out of a -/// `GORILLA1` chunk. `labels` is a `BTreeMap` so identical samples -/// hash deterministically (handy for golden tests + the postings -/// cross-check). -/// -/// Pre-Step-1 this lived in the JSONL `cold_store::format` module -/// and was the wire format the legacy `LocalFsColdStore` parsed. -/// JSONL is gone; the type stays as the in-memory shape every -/// gorilla-engine consumer (`exact_executor`, the postings filter, -/// the test mocks) speaks. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct RawSample { - pub ts_ms: i64, - pub labels: BTreeMap, - pub value: f64, -} - -/// Convenience alias: a label set as stored on a [`RawSample`]. -pub type LabelSet = BTreeMap; - -/// Error surface for archive-store operations. -#[derive(Debug, Error)] -pub enum StoreError { - #[error("I/O error: {0}")] - Io(#[from] std::io::Error), - #[error("malformed record: {0}")] - Malformed(String), - /// Backend-storage error (e.g. an S3 GET failed) that is not - /// itself a `std::io::Error`. - #[error("backend error: {0}")] - Backend(String), - /// A trait method that this `Store` impl does not support. - /// Reserved for forwards-compatible trait extensions. - #[error("unsupported store operation: {0}")] - Unsupported(&'static str), -} - -/// Descriptor for a single immutable chunk stored in the archive -/// tier. Returned by [`Store::list_chunks`]; carries enough -/// metadata for callers to prune by time / label without reading -/// the chunk body. -#[derive(Debug, Clone, PartialEq)] -pub struct ChunkRef { - /// Opaque object key (e.g. an S3 key). The Telegraf-side - /// `gorilla_s3` output uses - /// `/block---.gorilla`; the - /// design.md-style layout is `//YYYY/MM/DD/HH/ - /// part-NNNNNN.gor`. Either is fine — the index file is the - /// source of truth for what keys exist. - pub key: String, - /// Metric name the chunk was fetched against. Recovered from - /// the caller's `list_chunks` request rather than the on-wire - /// chunk metadata. - pub metric: String, - /// `(start_unix_ms, end_unix_ms)` covered by the chunk. - pub time_range_ms: (i64, i64), - /// 64-bit canonical-label-set hash — for prune-by-label-equality - /// without fetching the chunk. - pub label_hash: u64, - /// Number of samples in the chunk. - pub sample_count: u32, - /// On-wire size of the chunk object in bytes. - pub size_bytes: u32, -} - -/// Read-only view over the Gorilla archive tier. -/// -/// Trait-shaped (rather than collapsed onto `GorillaS3Store` -/// concretely) so tests can drop in an in-memory mock without -/// touching production S3 wiring. Step-2 of the JSONL deprecation -/// (Prometheus-block format + Thanos store-gateway) will plug a -/// second impl in under the same trait. -/// -/// Scans are `(metric, [start_ms, end_ms))` — inclusive start, -/// exclusive end — matching the half-open range convention used by -/// the rest of the engine. -#[async_trait] -pub trait Store: Send + Sync { - /// Return all samples for `metric` whose timestamp lies in - /// `[start_ms, end_ms)`. Ordering is not guaranteed. - async fn scan( - &self, - metric: &str, - start_ms: i64, - end_ms: i64, - ) -> Result, StoreError>; - - /// List chunk descriptors covering `[start_ms, end_ms)` without - /// decoding any bodies. - async fn list_chunks( - &self, - metric: &str, - start_ms: i64, - end_ms: i64, - ) -> Result, StoreError>; - - /// Decode a single chunk into an owned `Vec`. - async fn read_chunk(&self, chunk: &ChunkRef) -> Result, StoreError>; - - /// Load + intersect per-bucket postings under `(metric, - /// time_range)` for the supplied `(label_name, label_value)` - /// matchers. Default impl returns - /// [`StoreError::Unsupported`] so chunk-only stores keep - /// compiling without postings sidecars. - async fn list_postings_for( - &self, - _metric: &str, - _start_ms: i64, - _end_ms: i64, - _matchers: &[(String, String)], - ) -> Result { - Err(StoreError::Unsupported("list_postings_for")) - } -} - -// ───────────────────────────────────────────────────────────────────── -// Public config -// ───────────────────────────────────────────────────────────────────── - -/// Tunable configuration for [`GorillaS3Store`]. -/// -/// Use [`GorillaS3Config::from_env`] to pull values from environment -/// variables in deployment, or build manually for tests. -#[derive(Debug, Clone)] -pub struct GorillaS3Config { - /// `None` for AWS S3 (the SDK uses the standard regional - /// endpoint), `Some("http://minio:9000")` for MinIO / a custom - /// S3-compatible endpoint. - pub endpoint: Option, - /// Bucket name to list / read from. - pub bucket: String, - /// Tenant identifier prepended to every index-file prefix. - /// Empty string is allowed for single-tenant deployments. - pub tenant: String, - /// Prefix template for per-hour `index.json` files. Supports - /// the placeholders `{tenant}`, `{metric}`, `{year}`, `{month}`, - /// `{day}`, `{hour}` (zero-padded). Default: - /// `"{tenant}/{metric}/{year}/{month}/{day}/{hour}/"`. - pub prefix_template: String, - /// AWS-region the bucket lives in (e.g. `"us-east-1"`). For - /// MinIO any non-empty placeholder works. - pub region: String, - /// Optional static credential override. Both fields must be set - /// together; if either is `None` the underlying SDK falls back - /// to its environment / IMDS resolution. - pub access_key_id: Option, - /// See [`Self::access_key_id`]. - pub secret_access_key: Option, - /// LRU capacity (in number of decoded chunks). Default `256`. - pub cache_capacity: usize, - /// `false` switches the SDK to plain HTTP — required for local - /// MinIO / docker-compose smoke tests. Default `true`. - pub use_ssl: bool, -} - -impl Default for GorillaS3Config { - fn default() -> Self { - Self { - endpoint: None, - bucket: String::new(), - tenant: String::new(), - prefix_template: "{tenant}/{metric}/{year}/{month}/{day}/{hour}/".to_string(), - region: "us-east-1".to_string(), - access_key_id: None, - secret_access_key: None, - cache_capacity: 256, - use_ssl: true, - } - } -} - -/// Errors raised by [`GorillaS3Config::from_env`]. -#[derive(Debug, Error)] -pub enum GorillaS3ConfigError { - /// A required environment variable was missing. - #[error("missing required env var: {0}")] - MissingEnv(&'static str), - /// `ASAP_GORILLA_S3_CACHE_CAPACITY` could not be parsed as a - /// positive `usize`. - #[error("invalid env var {var}: {value} ({source})")] - InvalidEnv { - /// Variable name. - var: &'static str, - /// Raw value the user supplied. - value: String, - /// Underlying parse error. - source: std::num::ParseIntError, - }, -} - -impl GorillaS3Config { - /// Read a config from process environment variables. Required: - /// - /// * `ASAP_GORILLA_S3_BUCKET` - /// * `ASAP_GORILLA_S3_REGION` - /// - /// Optional (with defaults shown above): - /// - /// * `ASAP_GORILLA_S3_ENDPOINT` - /// * `ASAP_GORILLA_S3_TENANT` - /// * `ASAP_GORILLA_S3_PREFIX_TEMPLATE` - /// * `ASAP_GORILLA_S3_ACCESS_KEY_ID` / `..._SECRET_ACCESS_KEY` - /// * `ASAP_GORILLA_S3_CACHE_CAPACITY` - /// * `ASAP_GORILLA_S3_USE_SSL` (`"true"` / `"false"`, - /// case-insensitive) - pub fn from_env() -> Result { - let bucket = std::env::var("ASAP_GORILLA_S3_BUCKET") - .map_err(|_| GorillaS3ConfigError::MissingEnv("ASAP_GORILLA_S3_BUCKET"))?; - let region = std::env::var("ASAP_GORILLA_S3_REGION") - .map_err(|_| GorillaS3ConfigError::MissingEnv("ASAP_GORILLA_S3_REGION"))?; - let endpoint = std::env::var("ASAP_GORILLA_S3_ENDPOINT").ok(); - let tenant = std::env::var("ASAP_GORILLA_S3_TENANT").unwrap_or_default(); - let prefix_template = std::env::var("ASAP_GORILLA_S3_PREFIX_TEMPLATE") - .unwrap_or_else(|_| "{tenant}/{metric}/{year}/{month}/{day}/{hour}/".to_string()); - let access_key_id = std::env::var("ASAP_GORILLA_S3_ACCESS_KEY_ID").ok(); - let secret_access_key = std::env::var("ASAP_GORILLA_S3_SECRET_ACCESS_KEY").ok(); - let cache_capacity = match std::env::var("ASAP_GORILLA_S3_CACHE_CAPACITY") { - Ok(s) => s - .parse::() - .map_err(|e| GorillaS3ConfigError::InvalidEnv { - var: "ASAP_GORILLA_S3_CACHE_CAPACITY", - value: s, - source: e, - })?, - Err(_) => 256, - }; - let use_ssl = std::env::var("ASAP_GORILLA_S3_USE_SSL") - .map(|s| !matches!(s.trim().to_ascii_lowercase().as_str(), "false" | "0" | "no")) - .unwrap_or(true); - Ok(Self { - endpoint, - bucket, - tenant, - prefix_template, - region, - access_key_id, - secret_access_key, - cache_capacity, - use_ssl, - }) - } -} - -// ───────────────────────────────────────────────────────────────────── -// ObjectStore — internal trait so tests don't need real S3 -// ───────────────────────────────────────────────────────────────────── - -/// Minimal async object-fetch interface. -/// -/// Sized + `Send + Sync` so [`GorillaS3Store`] can hold one -/// behind an `Arc` regardless of how it's backed. -/// Production callers use [`S3ObjectStore`] (rust-s3); tests use the -/// in-memory mock at the bottom of this file. -#[async_trait] -pub trait ObjectStore: Send + Sync { - /// Fetch the full object body for `key`. - async fn get_object(&self, key: &str) -> Result, StoreError>; - - /// True iff `err` was raised because the requested key did not - /// exist (vs. a transport / permission failure). Used by the - /// list path to treat a missing `index.json` as "no chunks for - /// this hour" rather than a hard error. - fn object_missing(&self, err: &StoreError) -> bool { - matches!(err, StoreError::Backend(msg) if msg.contains("not found")) - } -} - -// ───────────────────────────────────────────────────────────────────── -// rust-s3 backed production ObjectStore -// ───────────────────────────────────────────────────────────────────── - -mod rust_s3_backend { - use super::*; - use s3::creds::Credentials; - use s3::region::Region as S3Region; - use s3::Bucket; - - /// `rust-s3`-backed [`ObjectStore`]. Default production choice. - pub struct S3ObjectStore { - bucket: Box, - } - - impl S3ObjectStore { - /// Build from a [`GorillaS3Config`]. Sets - /// `path_style = true` whenever a custom endpoint is - /// configured (MinIO mandates path-style addressing). - pub fn new(cfg: &GorillaS3Config) -> Result { - let region = match &cfg.endpoint { - Some(ep) => { - let endpoint = if ep.starts_with("http://") || ep.starts_with("https://") { - ep.clone() - } else if cfg.use_ssl { - format!("https://{}", ep) - } else { - format!("http://{}", ep) - }; - S3Region::Custom { - region: cfg.region.clone(), - endpoint, - } - } - None => cfg - .region - .parse::() - .map_err(|e| StoreError::Backend(format!("region parse: {e}")))?, - }; - let creds = match (&cfg.access_key_id, &cfg.secret_access_key) { - (Some(ak), Some(sk)) => Credentials::new(Some(ak), Some(sk), None, None, None) - .map_err(|e| StoreError::Backend(format!("credentials: {e}")))?, - _ => Credentials::default() - .map_err(|e| StoreError::Backend(format!("default credentials: {e}")))?, - }; - let bucket = Bucket::new(&cfg.bucket, region, creds) - .map_err(|e| StoreError::Backend(format!("bucket: {e}")))?; - // MinIO + most S3-compatibles require path-style addressing - // when a custom endpoint is in play. AWS S3 supports both, - // so leaving it on for the AWS path is safe but slightly - // less efficient — only flip when an endpoint is set. - let bucket = if cfg.endpoint.is_some() { - bucket.with_path_style() - } else { - bucket - }; - Ok(Self { bucket }) - } - } - - #[async_trait] - impl ObjectStore for S3ObjectStore { - async fn get_object(&self, key: &str) -> Result, StoreError> { - let resp = self - .bucket - .get_object(key) - .await - .map_err(|e| StoreError::Backend(format!("s3 get {key}: {e}")))?; - if resp.status_code() == 404 { - return Err(StoreError::Backend(format!("s3 get {key}: not found"))); - } - if !(200..300).contains(&resp.status_code()) { - return Err(StoreError::Backend(format!( - "s3 get {key}: status {}", - resp.status_code() - ))); - } - Ok(resp.to_vec()) - } - } -} - -pub use rust_s3_backend::S3ObjectStore; - -// ───────────────────────────────────────────────────────────────────── -// GorillaS3Store -// ───────────────────────────────────────────────────────────────────── - -/// LRU cache keyed by chunk object key. Stored values are -/// pre-decoded `RawSample` lists so repeated reads of the same -/// chunk skip the Gorilla decode pass entirely. -type ChunkCache = Mutex>>>; - -/// **mvp/v5**: LRU cache for parsed `index.json` files (per -/// `(metric, hour)`). Same capacity tier as the postings cache. -type IndexCache = Mutex>>; - -/// `Store` adapter that reads `GORILLA1`-format chunks out of an -/// S3-compatible bucket. See module docs for layout + S3 client -/// notes. -/// -/// Step-1 rename (`GorillaS3ColdStore` → `GorillaS3Store`) reflects -/// the JSONL deprecation: there is no longer a "warm/cold" split -/// inside the archive tier; this is *the* archive store. -pub struct GorillaS3Store { - object_store: Arc, - config: GorillaS3Config, - cache: ChunkCache, - /// **mvp/v5**: postings sidecar cache. - postings_cache: PostingsCache, - /// **mvp/v5**: index.json cache. Reserved for the upcoming - /// `Range:`-based partial-read path that fetches chunk bytes - /// out of compactor-merged objects — the cache is wired now to - /// match the backend's hot-path layout but the caller doesn't - /// yet route partial reads through it. The current `read_chunk` - /// already uses an LRU on samples, which is the dominant cost. - #[allow(dead_code)] - index_cache: IndexCache, -} - -impl GorillaS3Store { - /// Build with an explicit object-store backend. The production - /// constructor [`Self::with_default_backend`] wires up - /// `S3ObjectStore` from `cfg`; tests inject the in-memory mock. - pub fn new(object_store: Arc, config: GorillaS3Config) -> Self { - let cap = NonZeroUsize::new(config.cache_capacity.max(1)) - .unwrap_or(NonZeroUsize::new(1).unwrap()); - // mvp/v5: postings + index caches scale with the chunk - // cache (one entry per hour-bucket, mirrors typical query - // cardinality). - let pc_cap = NonZeroUsize::new(cap.get().max(64)).unwrap_or(NonZeroUsize::new(64).unwrap()); - Self { - object_store, - config, - cache: Mutex::new(LruCache::new(cap)), - postings_cache: Mutex::new(LruCache::new(pc_cap)), - index_cache: Mutex::new(LruCache::new(pc_cap)), - } - } - - /// Build from a [`GorillaS3Config`] using the default - /// `rust-s3`-backed [`ObjectStore`]. - /// - /// **mvp/v5**: the underlying `S3ObjectStore` is wrapped in an - /// [`S3CostTrackingObjectStore`] tied to the global counter - /// set, so the HTTP server's `/internal/s3_cost.csv` + - /// `/metrics` endpoints report measured PUT/GET/etc counts. - pub fn with_default_backend(config: GorillaS3Config) -> Result { - let backend: Arc = Arc::new(S3ObjectStore::new(&config)?); - let counters = global_s3_cost_counters(); - let tracked = S3CostTrackingObjectStore::new(backend, counters); - Ok(Self::new(Arc::new(tracked), config)) - } - - /// Borrow the active config — useful for diagnostics. - pub fn config(&self) -> &GorillaS3Config { - &self.config - } - - /// Render the configured `prefix_template` for one - /// `(metric, hour)` bucket and append `index.json`. - fn index_key(&self, metric: &str, ts_ms: i64) -> String { - let mut key = self.bucket_prefix(metric, ts_ms); - key.push_str("index.json"); - key - } - - /// **mvp/v5**: derive the postings-v1.json key for the same - /// `(metric, hour)` bucket as [`Self::index_key`]. - fn postings_key(&self, metric: &str, ts_ms: i64) -> String { - let mut key = self.bucket_prefix(metric, ts_ms); - key.push_str("postings-v1.json"); - key - } - - /// Shared prefix-rendering helper used by [`Self::index_key`] / - /// [`Self::postings_key`]. Always ends with `/`. - /// - /// Accepts BOTH placeholder vocabularies: - /// - /// * `{year}`/`{month}`/`{day}`/`{hour}` — the backend's - /// long-standing names. - /// * `{YYYY}`/`{MM}`/`{DD}`/`{HH}` — the agent - /// `gorillas3processor`'s naming. - fn bucket_prefix(&self, metric: &str, ts_ms: i64) -> String { - let dt: DateTime = DateTime::::from_timestamp_millis(ts_ms) - .unwrap_or_else(|| DateTime::::from_timestamp(0, 0).unwrap()); - let year = format!("{:04}", dt.year()); - let month = format!("{:02}", dt.month()); - let day = format!("{:02}", dt.day()); - let hour = format!("{:02}", dt.hour()); - let prefix = self - .config - .prefix_template - .replace("{tenant}", &self.config.tenant) - .replace("{metric}", metric) - .replace("{year}", &year) - .replace("{month}", &month) - .replace("{day}", &day) - .replace("{hour}", &hour) - .replace("{YYYY}", &year) - .replace("{MM}", &month) - .replace("{DD}", &day) - .replace("{HH}", &hour); - let mut key = prefix; - if !key.ends_with('/') { - key.push('/'); - } - key - } - - /// Iterate the wall-clock-hour starts (in ms) covered by - /// `[start_ms, end_ms)`. Always emits at least one bucket. - fn hour_starts(start_ms: i64, end_ms: i64) -> Vec { - const HOUR_MS: i64 = 3_600_000; - if end_ms <= start_ms { - let h = (start_ms / HOUR_MS) * HOUR_MS; - return vec![h]; - } - let first = (start_ms / HOUR_MS) * HOUR_MS; - let last = ((end_ms - 1) / HOUR_MS) * HOUR_MS; - let mut out = Vec::new(); - let mut cur = first; - while cur <= last { - out.push(cur); - cur += HOUR_MS; - } - out - } - - /// Fetch + parse one hour's `index.json`. Missing index = empty - /// catalog (the producer may not have flushed yet); transport - /// failure surfaces as `StoreError::Backend`. - async fn fetch_index(&self, metric: &str, hour_ms: i64) -> Result { - let key = self.index_key(metric, hour_ms); - match self.object_store.get_object(&key).await { - Ok(bytes) => IndexFile::read(bytes.as_slice()) - .map_err(|e| StoreError::Malformed(format!("index.json at {key}: {e}"))), - Err(e) if self.object_store.object_missing(&e) => { - debug!(key = %key, "gorilla-s3: index.json missing for hour bucket; skipping"); - Ok(IndexFile::new(0)) - } - Err(e) => Err(e), - } - } -} - -#[async_trait] -impl Store for GorillaS3Store { - async fn scan( - &self, - metric: &str, - start_ms: i64, - end_ms: i64, - ) -> Result, StoreError> { - let chunks = self.list_chunks(metric, start_ms, end_ms).await?; - let mut out = Vec::new(); - for chunk in chunks { - let samples = self.read_chunk(&chunk).await?; - for s in samples { - if s.ts_ms >= start_ms && s.ts_ms < end_ms { - out.push(s); - } - } - } - Ok(out) - } - - async fn list_chunks( - &self, - metric: &str, - start_ms: i64, - end_ms: i64, - ) -> Result, StoreError> { - // Convert the request window to the nanosecond unit the - // index file uses (`IndexEntry.time_range` is `(ns, ns)`, - // mirroring the Go encoder's `time.Time.UnixNano()` source). - let start_ns = (start_ms as i128).saturating_mul(1_000_000) as u64; - let end_ns = if end_ms <= start_ms { - start_ns - } else { - ((end_ms as i128).saturating_mul(1_000_000) - 1).max(0) as u64 - }; - - let mut out = Vec::new(); - for hour_ms in Self::hour_starts(start_ms, end_ms) { - let idx = self.fetch_index(metric, hour_ms).await?; - let bucket_prefix = self.bucket_prefix(metric, hour_ms); - for entry in idx.prune_by_time((start_ns, end_ns)) { - let (entry_start_ms, entry_end_ms) = ( - (entry.time_range.0 / 1_000_000) as i64, - (entry.time_range.1 / 1_000_000) as i64, - ); - // v7: agent-produced index entries carry just the - // chunk's basename (`part-NNNN-MMMM.gor`), not the - // full S3 key. Detect a bare basename (no `/`) and - // prepend the bucket prefix. - let key = if entry.key.contains('/') { - entry.key.clone() - } else { - format!("{}{}", bucket_prefix, entry.key) - }; - out.push(ChunkRef { - key, - metric: metric.to_string(), - time_range_ms: (entry_start_ms, entry_end_ms), - label_hash: entry.label_hash, - sample_count: entry.sample_count, - size_bytes: entry.size_bytes, - }); - } - } - Ok(out) - } - - async fn read_chunk(&self, chunk: &ChunkRef) -> Result, StoreError> { - // Cache hit fast path. - { - let mut guard = self.cache.lock().await; - if let Some(cached) = guard.get(&chunk.key).cloned() { - return Ok((*cached).clone()); - } - } - - let bytes = self.object_store.get_object(&chunk.key).await?; - let samples = decode_block(&bytes) - .map_err(|e| StoreError::Malformed(format!("decode {}: {e}", chunk.key)))?; - - let arc = Arc::new(samples.clone()); - { - let mut guard = self.cache.lock().await; - guard.put(chunk.key.clone(), arc); - } - Ok(samples) - } - - /// **mvp/v5**: postings-aware chunk pruning — delegated to - /// [`super::postings::intersect_per_bucket_postings`] so the - /// per-bucket fetch + intersect logic sits in one place - /// regardless of which `Store` impl owns the postings cache. - async fn list_postings_for( - &self, - metric: &str, - start_ms: i64, - end_ms: i64, - matchers: &[(String, String)], - ) -> Result { - let buckets = Self::hour_starts(start_ms, end_ms); - let mut keys: Vec = Vec::with_capacity(buckets.len()); - for hour_ms in buckets { - keys.push(self.postings_key(metric, hour_ms)); - } - intersect_per_bucket_postings( - self.object_store.as_ref(), - &self.postings_cache, - &keys, - matchers, - ) - .await - } -} - -// ───────────────────────────────────────────────────────────────────── -// Decoding helper — converts a GORILLA1 block to RawSample units -// ───────────────────────────────────────────────────────────────────── - -/// Decode a single in-memory `GORILLA1` block into [`RawSample`]s. -fn decode_block(bytes: &[u8]) -> Result, asap_gorilla::DecodeError> { - let mut decoder = GorillaDecoder::from_reader(bytes)?; - let mut out: Vec = Vec::new(); - while let Some(header) = decoder.header().cloned() { - let labels: BTreeMap = header.labels.iter().cloned().collect(); - for sample in decoder.samples() { - let (ts_ns, value) = sample?; - out.push(RawSample { - ts_ms: (ts_ns / 1_000_000) as i64, - labels: labels.clone(), - value, - }); - } - if !decoder.next_series()? { - break; - } - } - Ok(out) -} - -// ───────────────────────────────────────────────────────────────────── -// In-memory ObjectStore mock — pub(crate) so tests in sibling files -// can exercise the same fixture without a live MinIO. -// ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -#[derive(Default)] -pub(crate) struct InMemoryObjectStore { - inner: Mutex, -} - -#[cfg(test)] -#[derive(Default)] -struct InMemoryState { - objects: HashMap>, - fetch_counts: HashMap, - fail_all: Option, -} - -#[cfg(test)] -impl InMemoryObjectStore { - pub(crate) fn new() -> Self { - Self::default() - } - - pub(crate) async fn put(&self, key: impl Into, body: Vec) { - let mut g = self.inner.lock().await; - g.objects.insert(key.into(), body); - } - - pub(crate) async fn get_count(&self, key: &str) -> usize { - let g = self.inner.lock().await; - g.fetch_counts.get(key).copied().unwrap_or(0) - } - - /// Make every subsequent `get_object` fail with a backend error - /// containing `msg`. Used by the network-error test. - pub(crate) async fn fail_all(&self, msg: impl Into) { - let mut g = self.inner.lock().await; - g.fail_all = Some(msg.into()); - } -} - -#[cfg(test)] -#[async_trait] -impl ObjectStore for InMemoryObjectStore { - async fn get_object(&self, key: &str) -> Result, StoreError> { - let mut g = self.inner.lock().await; - if let Some(msg) = g.fail_all.clone() { - return Err(StoreError::Backend(msg)); - } - *g.fetch_counts.entry(key.to_string()).or_insert(0) += 1; - match g.objects.get(key) { - Some(b) => Ok(b.clone()), - None => Err(StoreError::Backend(format!("get {key}: not found"))), - } - } -} - -// Static `Send` assertion — `GorillaS3Store` must be storable -// behind an `Arc` in the engine wiring. -const _: fn() = || { - fn _assert_send() {} - _assert_send::(); -}; - -// ───────────────────────────────────────────────────────────────────── -// Tests -// ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - use asap_gorilla::{GorillaEncoder, IndexEntry, IndexFile}; - use chrono::TimeZone; - - /// Build a minimal index.json fixture. - fn make_index(entries: Vec) -> Vec { - let mut idx = IndexFile::new(0); - idx.entries = entries; - let mut buf = Vec::new(); - idx.write(&mut buf).unwrap(); - buf - } - - /// Encode a single-series Gorilla block from `(ts_ms, value)` pairs. - fn make_block(metric: &str, labels: &[(&str, &str)], samples: &[(i64, f64)]) -> Vec { - let mut enc = GorillaEncoder::new( - metric.to_string(), - labels - .iter() - .map(|(k, v)| ((*k).to_string(), (*v).to_string())) - .collect(), - ); - for (ts_ms, v) in samples { - enc.append((*ts_ms as u64) * 1_000_000, *v); - } - enc.finalize().unwrap() - } - - fn ms(year: i32, month: u32, day: u32, hour: u32, min: u32, sec: u32) -> i64 { - Utc.with_ymd_and_hms(year, month, day, hour, min, sec) - .unwrap() - .timestamp_millis() - } - - fn cfg() -> GorillaS3Config { - GorillaS3Config { - endpoint: Some("http://mock".to_string()), - bucket: "test-bucket".to_string(), - tenant: "tenant1".to_string(), - prefix_template: "{tenant}/{metric}/{year}/{month}/{day}/{hour}/".to_string(), - region: "us-east-1".to_string(), - access_key_id: None, - secret_access_key: None, - cache_capacity: 4, - use_ssl: false, - } - } - - #[tokio::test] - async fn list_chunks_via_indexfile_prunes_by_time() { - let store = InMemoryObjectStore::new(); - - let h0 = ms(2026, 5, 6, 12, 0, 0); - let metric = "node_cpu_seconds_total"; - - let key_a = "tenant1/node_cpu_seconds_total/2026/05/06/12/part-A.gor".to_string(); - let key_b = "tenant1/node_cpu_seconds_total/2026/05/06/12/part-B.gor".to_string(); - let key_c = "tenant1/node_cpu_seconds_total/2026/05/06/12/part-C.gor".to_string(); - - let entries = vec![ - IndexEntry { - key: key_a.clone(), - time_range: ((h0) as u64 * 1_000_000, (h0 + 999) as u64 * 1_000_000), - sample_count: 10, - label_hash: 0xAAAA, - size_bytes: 100, - object_key: None, - byte_offset: None, - byte_length: None, - }, - IndexEntry { - key: key_b.clone(), - time_range: ( - (h0 + 5_000) as u64 * 1_000_000, - (h0 + 6_000) as u64 * 1_000_000, - ), - sample_count: 11, - label_hash: 0xBBBB, - size_bytes: 110, - object_key: None, - byte_offset: None, - byte_length: None, - }, - IndexEntry { - key: key_c.clone(), - time_range: ( - (h0 + 10_000) as u64 * 1_000_000, - (h0 + 11_000) as u64 * 1_000_000, - ), - sample_count: 12, - label_hash: 0xCCCC, - size_bytes: 120, - object_key: None, - byte_offset: None, - byte_length: None, - }, - ]; - - store - .put( - "tenant1/node_cpu_seconds_total/2026/05/06/12/index.json", - make_index(entries), - ) - .await; - - let cs = GorillaS3Store::new(Arc::new(store), cfg()); - let chunks = cs - .list_chunks(metric, h0 + 5_500, h0 + 5_800) - .await - .unwrap(); - - assert_eq!(chunks.len(), 1); - assert_eq!(chunks[0].key, key_b); - assert_eq!(chunks[0].sample_count, 11); - assert_eq!(chunks[0].label_hash, 0xBBBB); - assert_eq!(chunks[0].metric, metric); - } - - #[tokio::test] - async fn read_chunk_decodes_via_asap_gorilla() { - let store = InMemoryObjectStore::new(); - - let h0 = ms(2026, 5, 6, 12, 0, 0); - let metric = "node_cpu_seconds_total"; - let labels = &[("instance", "i-1"), ("mode", "user")]; - - let block = make_block( - metric, - labels, - &[(h0 + 1_000, 0.5), (h0 + 2_000, 0.7), (h0 + 3_000, 0.7)], - ); - let chunk_key = "tenant1/node_cpu_seconds_total/2026/05/06/12/part-001.gor".to_string(); - - store.put(chunk_key.clone(), block.clone()).await; - store - .put( - "tenant1/node_cpu_seconds_total/2026/05/06/12/index.json", - make_index(vec![IndexEntry { - key: chunk_key.clone(), - time_range: ( - (h0 + 1_000) as u64 * 1_000_000, - (h0 + 3_000) as u64 * 1_000_000, - ), - sample_count: 3, - label_hash: 0x1234, - size_bytes: block.len() as u32, - object_key: None, - byte_offset: None, - byte_length: None, - }]), - ) - .await; - - let cs = GorillaS3Store::new(Arc::new(store), cfg()); - let chunks = cs.list_chunks(metric, h0, h0 + 60_000).await.unwrap(); - assert_eq!(chunks.len(), 1); - - let samples = cs.read_chunk(&chunks[0]).await.unwrap(); - assert_eq!(samples.len(), 3); - assert_eq!(samples[0].ts_ms, h0 + 1_000); - assert_eq!(samples[0].value, 0.5); - assert_eq!(samples[1].ts_ms, h0 + 2_000); - assert_eq!(samples[1].value, 0.7); - assert_eq!(samples[2].ts_ms, h0 + 3_000); - assert_eq!(samples[2].value, 0.7); - assert_eq!( - samples[0].labels.get("instance").map(String::as_str), - Some("i-1") - ); - assert_eq!( - samples[0].labels.get("mode").map(String::as_str), - Some("user") - ); - } - - #[tokio::test] - async fn cache_hit_skips_s3_fetch() { - let store = Arc::new(InMemoryObjectStore::new()); - - let h0 = ms(2026, 5, 6, 12, 0, 0); - let metric = "m"; - let block = make_block(metric, &[], &[(h0 + 1_000, 1.0), (h0 + 2_000, 2.0)]); - let chunk_key = "tenant1/m/2026/05/06/12/part-X.gor".to_string(); - store.put(chunk_key.clone(), block.clone()).await; - store - .put( - "tenant1/m/2026/05/06/12/index.json", - make_index(vec![IndexEntry { - key: chunk_key.clone(), - time_range: ( - (h0 + 1_000) as u64 * 1_000_000, - (h0 + 2_000) as u64 * 1_000_000, - ), - sample_count: 2, - label_hash: 0, - size_bytes: block.len() as u32, - object_key: None, - byte_offset: None, - byte_length: None, - }]), - ) - .await; - - let cs = GorillaS3Store::new(store.clone(), cfg()); - let chunks = cs.list_chunks(metric, h0, h0 + 60_000).await.unwrap(); - assert_eq!(chunks.len(), 1); - - let _ = cs.read_chunk(&chunks[0]).await.unwrap(); - let count_after_first = store.get_count(&chunk_key).await; - let _ = cs.read_chunk(&chunks[0]).await.unwrap(); - let count_after_second = store.get_count(&chunk_key).await; - - assert_eq!(count_after_first, 1); - assert_eq!( - count_after_second, 1, - "second read_chunk must hit cache and skip S3 GET" - ); - } - - #[tokio::test] - async fn lru_eviction_under_pressure() { - let store = Arc::new(InMemoryObjectStore::new()); - let h0 = ms(2026, 5, 6, 12, 0, 0); - - let mut chunk_refs: Vec = Vec::new(); - let mut entries: Vec = Vec::new(); - for i in 0..3i64 { - let block = make_block( - "m", - &[("i", &i.to_string())], - &[ - (h0 + i * 1_000, i as f64), - (h0 + i * 1_000 + 100, i as f64 + 0.5), - ], - ); - let key = format!("tenant1/m/2026/05/06/12/part-{i}.gor"); - store.put(key.clone(), block.clone()).await; - entries.push(IndexEntry { - key: key.clone(), - time_range: ( - ((h0 + i * 1_000) as u64) * 1_000_000, - ((h0 + i * 1_000 + 100) as u64) * 1_000_000, - ), - sample_count: 2, - label_hash: i as u64, - size_bytes: block.len() as u32, - object_key: None, - byte_offset: None, - byte_length: None, - }); - chunk_refs.push(ChunkRef { - key, - metric: "m".to_string(), - time_range_ms: (h0 + i * 1_000, h0 + i * 1_000 + 100), - label_hash: i as u64, - sample_count: 2, - size_bytes: block.len() as u32, - }); - } - store - .put("tenant1/m/2026/05/06/12/index.json", make_index(entries)) - .await; - - let mut config = cfg(); - config.cache_capacity = 2; - let cs = GorillaS3Store::new(store.clone(), config); - - cs.read_chunk(&chunk_refs[0]).await.unwrap(); - cs.read_chunk(&chunk_refs[1]).await.unwrap(); - cs.read_chunk(&chunk_refs[2]).await.unwrap(); // evicts chunk_refs[0] - - let before = store.get_count(&chunk_refs[0].key).await; - cs.read_chunk(&chunk_refs[0]).await.unwrap(); - let after = store.get_count(&chunk_refs[0].key).await; - assert_eq!( - after, - before + 1, - "evicted chunk must trigger a fresh S3 GET" - ); - } - - #[tokio::test] - async fn index_json_corrupted_returns_error() { - let store = InMemoryObjectStore::new(); - let h0 = ms(2026, 5, 6, 12, 0, 0); - store - .put( - "tenant1/m/2026/05/06/12/index.json", - b"this is not json {{{".to_vec(), - ) - .await; - - let cs = GorillaS3Store::new(Arc::new(store), cfg()); - let res = cs.list_chunks("m", h0, h0 + 60_000).await; - match res { - Err(StoreError::Malformed(msg)) => { - assert!(msg.contains("index.json"), "msg should name the key: {msg}") - } - other => panic!("expected Malformed, got {other:?}"), - } - } - - #[tokio::test] - async fn s3_unavailable_returns_error() { - let store = Arc::new(InMemoryObjectStore::new()); - store.fail_all("simulated network outage").await; - let cs = GorillaS3Store::new(store, cfg()); - let h0 = ms(2026, 5, 6, 12, 0, 0); - let res = cs.list_chunks("m", h0, h0 + 60_000).await; - match res { - Err(StoreError::Backend(msg)) => assert!(msg.contains("simulated network outage")), - other => panic!("expected Backend, got {other:?}"), - } - } - - #[tokio::test] - async fn missing_index_is_empty_not_error() { - let store = InMemoryObjectStore::new(); - let cs = GorillaS3Store::new(Arc::new(store), cfg()); - let h0 = ms(2026, 5, 6, 12, 0, 0); - let chunks = cs - .list_chunks("never_written", h0, h0 + 60_000) - .await - .unwrap(); - assert!(chunks.is_empty()); - let samples = cs.scan("never_written", h0, h0 + 60_000).await.unwrap(); - assert!(samples.is_empty()); - } - - #[tokio::test] - async fn scan_filters_to_requested_range() { - let store = Arc::new(InMemoryObjectStore::new()); - let h0 = ms(2026, 5, 6, 12, 0, 0); - let block = make_block("m", &[], &[(h0 + 1_000, 1.0), (h0 + 10_000, 2.0)]); - let key = "tenant1/m/2026/05/06/12/part-Z.gor".to_string(); - store.put(key.clone(), block.clone()).await; - store - .put( - "tenant1/m/2026/05/06/12/index.json", - make_index(vec![IndexEntry { - key: key.clone(), - time_range: ( - (h0 + 1_000) as u64 * 1_000_000, - (h0 + 10_000) as u64 * 1_000_000, - ), - sample_count: 2, - label_hash: 0, - size_bytes: block.len() as u32, - object_key: None, - byte_offset: None, - byte_length: None, - }]), - ) - .await; - - let cs = GorillaS3Store::new(store, cfg()); - let samples = cs.scan("m", h0 + 5_000, h0 + 9_000).await.unwrap(); - assert!(samples.is_empty(), "no sample inside [5_000, 9_000) ms"); - - let samples = cs.scan("m", h0, h0 + 60_000).await.unwrap(); - assert_eq!(samples.len(), 2); - } - - #[tokio::test] - async fn list_chunks_spans_two_hour_buckets() { - let store = InMemoryObjectStore::new(); - let h12 = ms(2026, 5, 6, 12, 0, 0); - let h13 = ms(2026, 5, 6, 13, 0, 0); - let key12 = "tenant1/m/2026/05/06/12/part-1.gor".to_string(); - let key13 = "tenant1/m/2026/05/06/13/part-1.gor".to_string(); - store - .put( - "tenant1/m/2026/05/06/12/index.json", - make_index(vec![IndexEntry { - key: key12.clone(), - time_range: ( - (h12 + 3_500_000) as u64 * 1_000_000, - (h12 + 3_590_000) as u64 * 1_000_000, - ), - sample_count: 1, - label_hash: 0, - size_bytes: 50, - object_key: None, - byte_offset: None, - byte_length: None, - }]), - ) - .await; - store - .put( - "tenant1/m/2026/05/06/13/index.json", - make_index(vec![IndexEntry { - key: key13.clone(), - time_range: ( - (h13 + 1_000) as u64 * 1_000_000, - (h13 + 30_000) as u64 * 1_000_000, - ), - sample_count: 1, - label_hash: 0, - size_bytes: 50, - object_key: None, - byte_offset: None, - byte_length: None, - }]), - ) - .await; - let cs = GorillaS3Store::new(Arc::new(store), cfg()); - let chunks = cs - .list_chunks("m", h12 + 3_500_000, h13 + 30_000) - .await - .unwrap(); - assert_eq!(chunks.len(), 2); - assert_eq!(chunks[0].key, key12); - assert_eq!(chunks[1].key, key13); - } - - #[test] - fn bucket_prefix_supports_long_form_placeholders() { - let mut config = cfg(); - config.prefix_template = "{tenant}/{metric}/{year}/{month}/{day}/{hour}/".to_string(); - let store = InMemoryObjectStore::new(); - let cs = GorillaS3Store::new(Arc::new(store), config); - let key = cs.bucket_prefix("foo", ms(2026, 5, 6, 12, 0, 0)); - assert_eq!(key, "tenant1/foo/2026/05/06/12/"); - } - - #[test] - fn bucket_prefix_supports_agent_side_yyyy_mm_dd_hh_placeholders() { - let mut config = cfg(); - config.prefix_template = "{tenant}/{metric}/{YYYY}/{MM}/{DD}/{HH}/".to_string(); - let store = InMemoryObjectStore::new(); - let cs = GorillaS3Store::new(Arc::new(store), config); - let key = cs.bucket_prefix("http_freshness_probe_archive", ms(2026, 5, 7, 4, 0, 0)); - assert_eq!(key, "tenant1/http_freshness_probe_archive/2026/05/07/04/",); - } - - #[test] - fn bucket_prefix_handles_mixed_long_and_short_placeholders() { - let mut config = cfg(); - config.prefix_template = "{tenant}/{metric}/{year}/{MM}/{DD}/{hour}/".to_string(); - let store = InMemoryObjectStore::new(); - let cs = GorillaS3Store::new(Arc::new(store), config); - let key = cs.bucket_prefix("m", ms(2026, 5, 7, 4, 0, 0)); - assert_eq!(key, "tenant1/m/2026/05/07/04/"); - } - - #[test] - fn from_env_requires_bucket() { - let prev_bucket = std::env::var("ASAP_GORILLA_S3_BUCKET").ok(); - std::env::remove_var("ASAP_GORILLA_S3_BUCKET"); - let res = GorillaS3Config::from_env(); - if let Some(v) = prev_bucket { - std::env::set_var("ASAP_GORILLA_S3_BUCKET", v); - } - match res { - Err(GorillaS3ConfigError::MissingEnv("ASAP_GORILLA_S3_BUCKET")) => {} - other => panic!("expected MissingEnv(BUCKET), got {other:?}"), - } - } -} diff --git a/data_plane/src/storage_engines/gorilla_object_store/tests.rs b/data_plane/src/storage_engines/gorilla_object_store/tests.rs deleted file mode 100644 index 75de404f4..000000000 --- a/data_plane/src/storage_engines/gorilla_object_store/tests.rs +++ /dev/null @@ -1,862 +0,0 @@ -//! Unit tests for the Gorilla query engine. -//! -//! Tests exercise the engine end-to-end via a `MockStore` -//! injected in place of the production `GorillaS3Store`. The -//! mock is intentionally minimal: it owns a `Vec<(ChunkRef, -//! Vec)>` and answers `list_chunks` / `read_chunk` -//! straight off it, with optional latency injection for the -//! timeout test. - -use std::collections::BTreeMap; -use std::sync::Arc; -use std::time::Duration; - -use async_trait::async_trait; -use tokio::time::sleep; - -use crate::query_engines::query_result::QueryResult; -use crate::storage_engines::sketch_db::accuracy::{AccuracyKind, AccuracyProfile}; - -use super::archive_query::{plan_query_at, QueryStatistic}; -use super::store::{ChunkRef, RawSample, Store, StoreError}; -use super::{ - wrap_result, EngineError, ExactExecutor, ExecutionOutcome, GorillaEngineConfig, - GorillaQueryEngine, PostingsHits, DATA_SOURCE_GORILLA_ARCHIVE, -}; - -// ───────────────────────────────────────────────────────────────────── -// Mock cold store -// ───────────────────────────────────────────────────────────────────── - -/// In-process mock that satisfies the [`Store`] trait without -/// any S3 / disk roundtrip. Built once in each test from a list of -/// `(ChunkRef, samples)` pairs. -#[derive(Default)] -struct MockStore { - chunks: Vec<(ChunkRef, Vec)>, - /// If set, every `read_chunk` call sleeps for this duration — - /// used by the timeout test. - read_delay: Option, - /// **mvp/v5**: optional postings table keyed by `(label_name, - /// label_value)`. When `Some`, [`Store::list_postings_for`] - /// answers from this table; when `None`, returns a "missing - /// postings" outcome (driving the fall-back path test). - postings: Option>>, -} - -impl MockStore { - fn new(chunks: Vec<(ChunkRef, Vec)>) -> Self { - Self { - chunks, - read_delay: None, - postings: None, - } - } - - fn with_read_delay(mut self, d: Duration) -> Self { - self.read_delay = Some(d); - self - } - - /// mvp/v5: install a postings table for the - /// `list_postings_for` path. - fn with_postings(mut self, postings: BTreeMap<(String, String), Vec>) -> Self { - self.postings = Some(postings); - self - } -} - -#[async_trait] -impl Store for MockStore { - async fn scan( - &self, - metric: &str, - start_ms: i64, - end_ms: i64, - ) -> Result, StoreError> { - let mut out = Vec::new(); - let chunks = self.list_chunks(metric, start_ms, end_ms).await?; - for c in chunks { - for s in self.read_chunk(&c).await? { - if s.ts_ms >= start_ms && s.ts_ms < end_ms { - out.push(s); - } - } - } - Ok(out) - } - - async fn list_chunks( - &self, - metric: &str, - start_ms: i64, - end_ms: i64, - ) -> Result, StoreError> { - Ok(self - .chunks - .iter() - .filter(|(c, _)| { - c.metric == metric && c.time_range_ms.0 < end_ms && c.time_range_ms.1 >= start_ms - }) - .map(|(c, _)| c.clone()) - .collect()) - } - - async fn read_chunk(&self, chunk: &ChunkRef) -> Result, StoreError> { - if let Some(d) = self.read_delay { - sleep(d).await; - } - for (c, samples) in &self.chunks { - if c.key == chunk.key { - return Ok(samples.clone()); - } - } - Err(StoreError::Backend(format!( - "mock: no such chunk {}", - chunk.key - ))) - } - - async fn list_postings_for( - &self, - _metric: &str, - _start_ms: i64, - _end_ms: i64, - matchers: &[(String, String)], - ) -> Result { - let Some(table) = &self.postings else { - // Mirror "real" missing-postings behaviour: the trait - // says return Unsupported when the backend doesn't - // know how to compute this. The executor treats that - // as fall-through. - return Err(StoreError::Unsupported("list_postings_for")); - }; - let mut hits = PostingsHits { - series_ids: Vec::new(), - buckets_in_range: 1, - buckets_with_postings: 1, - }; - if matchers.is_empty() { - // Union of every series id in the table. - let mut set: std::collections::BTreeSet = std::collections::BTreeSet::new(); - for ids in table.values() { - set.extend(ids.iter().copied()); - } - hits.series_ids = set.into_iter().collect(); - return Ok(hits); - } - let first = table.get(&matchers[0]).cloned().unwrap_or_default(); - let mut acc: std::collections::BTreeSet = first.into_iter().collect(); - for m in &matchers[1..] { - let next = table.get(m).cloned().unwrap_or_default(); - let next_set: std::collections::BTreeSet = next.into_iter().collect(); - acc = acc.intersection(&next_set).copied().collect(); - } - hits.series_ids = acc.into_iter().collect(); - Ok(hits) - } -} - -// ───────────────────────────────────────────────────────────────────── -// Fixture helpers -// ───────────────────────────────────────────────────────────────────── - -const NOW_MS: i64 = 1_715_000_000_000; -const METRIC: &str = "http_requests_total"; - -fn raw(ts_ms: i64, value: f64) -> RawSample { - RawSample { - ts_ms, - labels: BTreeMap::new(), - value, - } -} - -/// One chunk covering `[start, start + n*step]` with a -/// monotonically-increasing value column (`base + i*step_v`). -fn linear_chunk( - key: &str, - start_ms: i64, - step_ms: i64, - n: usize, - base: f64, - step_v: f64, -) -> (ChunkRef, Vec) { - let samples: Vec = (0..n) - .map(|i| raw(start_ms + (i as i64) * step_ms, base + (i as f64) * step_v)) - .collect(); - let last_ts = samples.last().map(|s| s.ts_ms).unwrap_or(start_ms); - let chunk = ChunkRef { - key: key.to_string(), - metric: METRIC.to_string(), - time_range_ms: (start_ms, last_ts + 1), - label_hash: 0, - sample_count: n as u32, - size_bytes: 0, - }; - (chunk, samples) -} - -fn cfg() -> GorillaEngineConfig { - GorillaEngineConfig { - max_buffered_samples: 1_000_000, - query_timeout_secs: 30, - } -} - -fn engine_with(chunks: Vec<(ChunkRef, Vec)>) -> GorillaQueryEngine { - GorillaQueryEngine::new(Arc::new(MockStore::new(chunks)), cfg()) -} - -fn engine_with_config( - chunks: Vec<(ChunkRef, Vec)>, - config: GorillaEngineConfig, -) -> GorillaQueryEngine { - GorillaQueryEngine::new(Arc::new(MockStore::new(chunks)), config) -} - -// ───────────────────────────────────────────────────────────────────── -// Streaming-additive happy paths -// ───────────────────────────────────────────────────────────────────── - -#[tokio::test] -async fn execute_sum_over_time_streaming() { - // 60 samples × value 2.0 = 120.0 - let chunks = vec![linear_chunk("c1", NOW_MS - 60_000, 1_000, 60, 2.0, 0.0)]; - let engine = engine_with(chunks); - let plan = plan_query_at(&format!("sum_over_time({METRIC}[5m])"), NOW_MS).unwrap(); - assert_eq!(plan.statistic, QueryStatistic::SumOverTime); - - let exec = ExactExecutor::new( - Arc::new(MockStore::new(vec![linear_chunk( - "c1", - NOW_MS - 60_000, - 1_000, - 60, - 2.0, - 0.0, - )])), - cfg(), - ); - let outcome = exec.execute_plan(&plan).await.unwrap(); - assert_eq!(outcome.value, 120.0); - assert_eq!(outcome.samples_scanned, 60); - assert_eq!(outcome.chunks_fetched, 1); - - // Also verify via the high-level engine. - let result = engine - .execute_at(&format!("sum_over_time({METRIC}[5m])"), NOW_MS) - .await - .unwrap(); - assert!(matches!(result, QueryResult::Vector(_))); -} - -#[tokio::test] -async fn execute_count_over_time() { - let chunks = vec![linear_chunk("c1", NOW_MS - 30_000, 1_000, 30, 0.0, 0.0)]; - let engine = engine_with(chunks); - let plan = plan_query_at(&format!("count_over_time({METRIC}[1m])"), NOW_MS).unwrap(); - let exec = ExactExecutor::new( - Arc::new(MockStore::new(vec![linear_chunk( - "c1", - NOW_MS - 30_000, - 1_000, - 30, - 0.0, - 0.0, - )])), - cfg(), - ); - let outcome = exec.execute_plan(&plan).await.unwrap(); - assert_eq!(outcome.value, 30.0); - - let result = engine - .execute_at(&format!("count_over_time({METRIC}[1m])"), NOW_MS) - .await - .unwrap(); - if let QueryResult::Vector(iv) = result { - assert_eq!(iv.values[0].value, 30.0); - } else { - panic!("expected Vector"); - } -} - -#[tokio::test] -async fn execute_avg_over_time() { - // 4 samples: 1, 2, 3, 4 → avg = 2.5 - let chunks = vec![linear_chunk("c1", NOW_MS - 4_000, 1_000, 4, 1.0, 1.0)]; - let engine = engine_with(chunks); - let result = engine - .execute_at(&format!("avg_over_time({METRIC}[10s])"), NOW_MS) - .await - .unwrap(); - if let QueryResult::Vector(iv) = result { - assert!((iv.values[0].value - 2.5).abs() < 1e-12); - } else { - panic!("expected Vector"); - } -} - -#[tokio::test] -async fn execute_min_over_time() { - // values 5, 1, 3, 4 → min = 1 - let samples = vec![ - raw(NOW_MS - 4_000, 5.0), - raw(NOW_MS - 3_000, 1.0), - raw(NOW_MS - 2_000, 3.0), - raw(NOW_MS - 1_000, 4.0), - ]; - let chunk = ChunkRef { - key: "c".into(), - metric: METRIC.into(), - time_range_ms: (NOW_MS - 4_000, NOW_MS), - label_hash: 0, - sample_count: 4, - size_bytes: 0, - }; - let engine = engine_with(vec![(chunk, samples)]); - let result = engine - .execute_at(&format!("min_over_time({METRIC}[10s])"), NOW_MS) - .await - .unwrap(); - if let QueryResult::Vector(iv) = result { - assert_eq!(iv.values[0].value, 1.0); - } else { - panic!("expected Vector"); - } -} - -#[tokio::test] -async fn execute_max_over_time() { - let samples = vec![ - raw(NOW_MS - 4_000, 5.0), - raw(NOW_MS - 3_000, 1.0), - raw(NOW_MS - 2_000, 3.0), - raw(NOW_MS - 1_000, 4.0), - ]; - let chunk = ChunkRef { - key: "c".into(), - metric: METRIC.into(), - time_range_ms: (NOW_MS - 4_000, NOW_MS), - label_hash: 0, - sample_count: 4, - size_bytes: 0, - }; - let engine = engine_with(vec![(chunk, samples)]); - let result = engine - .execute_at(&format!("max_over_time({METRIC}[10s])"), NOW_MS) - .await - .unwrap(); - if let QueryResult::Vector(iv) = result { - assert_eq!(iv.values[0].value, 5.0); - } else { - panic!("expected Vector"); - } -} - -#[tokio::test] -async fn execute_last_over_time_v7() { - // v7 / issue #46 ⑥: `last_over_time([])` returns - // the value of the largest-timestamp sample in the window. - // Counter-shaped freshness probes encode `unix_ts_ms` in their - // value; the replay client subtracts (poll_ts - observed_value) - // to get a per-path freshness delta. - // - // Three samples spanning 0..2s with monotonically-increasing - // values 100, 200, 300. The last-stamp sample is at NOW-1s with - // value 300. `last_over_time(...[10s])` must return 300. - let samples = vec![ - raw(NOW_MS - 3_000, 100.0), - raw(NOW_MS - 2_000, 200.0), - raw(NOW_MS - 1_000, 300.0), - ]; - let chunk = ChunkRef { - key: "c".into(), - metric: METRIC.into(), - time_range_ms: (NOW_MS - 3_000, NOW_MS), - label_hash: 0, - sample_count: 3, - size_bytes: 0, - }; - let engine = engine_with(vec![(chunk, samples)]); - let result = engine - .execute_at(&format!("last_over_time({METRIC}[10s])"), NOW_MS) - .await - .unwrap(); - if let QueryResult::Vector(iv) = result { - assert_eq!( - iv.values[0].value, 300.0, - "last_over_time must return the largest-timestamp sample's value", - ); - } else { - panic!("expected Vector"); - } -} - -#[tokio::test] -async fn execute_last_over_time_unordered_samples_picks_largest_ts() { - // Samples arrive with non-monotonic timestamps — the chunk - // claims (start, last_ts+1) but the per-sample observe() must - // still pick the largest ts, not the last-arrived sample. - let samples = vec![ - raw(NOW_MS - 5_000, 50.0), // largest ts is sample[2] - raw(NOW_MS - 8_000, 80.0), // smallest ts but later in vec - raw(NOW_MS - 1_000, 1234.5), // largest ts - raw(NOW_MS - 3_000, 30.0), - ]; - let chunk = ChunkRef { - key: "c".into(), - metric: METRIC.into(), - time_range_ms: (NOW_MS - 8_000, NOW_MS), - label_hash: 0, - sample_count: 4, - size_bytes: 0, - }; - let engine = engine_with(vec![(chunk, samples)]); - let result = engine - .execute_at(&format!("last_over_time({METRIC}[10s])"), NOW_MS) - .await - .unwrap(); - if let QueryResult::Vector(iv) = result { - assert!( - (iv.values[0].value - 1234.5).abs() < 1e-12, - "must return the value of the largest-ts sample, not the last-arrived", - ); - } else { - panic!("expected Vector"); - } -} - -#[tokio::test] -async fn execute_rate_basic() { - // Counter goes from 100 at t=NOW-10s to 200 at t=NOW-1s. - // rate over 10s window = (200 - 100) / 10s = 10.0 - let samples = vec![ - raw(NOW_MS - 10_000, 100.0), - raw(NOW_MS - 5_000, 150.0), - raw(NOW_MS - 1_000, 200.0), - ]; - let chunk = ChunkRef { - key: "c".into(), - metric: METRIC.into(), - time_range_ms: (NOW_MS - 10_000, NOW_MS), - label_hash: 0, - sample_count: 3, - size_bytes: 0, - }; - let engine = engine_with(vec![(chunk, samples)]); - let result = engine - .execute_at(&format!("rate({METRIC}[10s])"), NOW_MS) - .await - .unwrap(); - if let QueryResult::Vector(iv) = result { - assert!((iv.values[0].value - 10.0).abs() < 1e-9); - } else { - panic!("expected Vector"); - } -} - -#[tokio::test] -async fn execute_increase_basic() { - let samples = vec![ - raw(NOW_MS - 10_000, 100.0), - raw(NOW_MS - 5_000, 150.0), - raw(NOW_MS - 1_000, 250.0), - ]; - let chunk = ChunkRef { - key: "c".into(), - metric: METRIC.into(), - time_range_ms: (NOW_MS - 10_000, NOW_MS), - label_hash: 0, - sample_count: 3, - size_bytes: 0, - }; - let engine = engine_with(vec![(chunk, samples)]); - let result = engine - .execute_at(&format!("increase({METRIC}[10s])"), NOW_MS) - .await - .unwrap(); - if let QueryResult::Vector(iv) = result { - assert!((iv.values[0].value - 150.0).abs() < 1e-9); - } else { - panic!("expected Vector"); - } -} - -// ───────────────────────────────────────────────────────────────────── -// Buffered quantile + topk -// ───────────────────────────────────────────────────────────────────── - -#[tokio::test] -async fn execute_quantile_buffered_basic() { - // Values 0..100; q0.99 → index round((100-1)*0.99) = round(98.01) = 98 → value 98. - let mut samples: Vec = (0..100) - .map(|i| raw(NOW_MS - 100_000 + (i as i64) * 1_000, i as f64)) - .collect(); - // Shuffle the value order so the executor must sort. - samples.sort_by_key(|s| s.value as i64); - samples.reverse(); - - let chunk = ChunkRef { - key: "c".into(), - metric: METRIC.into(), - time_range_ms: (NOW_MS - 100_000, NOW_MS), - label_hash: 0, - sample_count: 100, - size_bytes: 0, - }; - let engine = engine_with(vec![(chunk, samples)]); - let result = engine - .execute_at(&format!("quantile_over_time(0.99, {METRIC}[2m])"), NOW_MS) - .await - .unwrap(); - if let QueryResult::Vector(iv) = result { - assert_eq!(iv.values[0].value, 98.0); - } else { - panic!("expected Vector"); - } -} - -#[tokio::test] -async fn execute_quantile_too_many_samples_errors() { - // Generate 100 samples but cap the buffered budget at 5. - let samples: Vec = (0..100) - .map(|i| raw(NOW_MS - 100_000 + (i as i64) * 1_000, i as f64)) - .collect(); - let chunk = ChunkRef { - key: "c".into(), - metric: METRIC.into(), - time_range_ms: (NOW_MS - 100_000, NOW_MS), - label_hash: 0, - sample_count: 100, - size_bytes: 0, - }; - let cfg = GorillaEngineConfig { - max_buffered_samples: 5, - query_timeout_secs: 30, - }; - let engine = engine_with_config(vec![(chunk, samples)], cfg); - let res = engine - .execute_at(&format!("quantile_over_time(0.5, {METRIC}[2m])"), NOW_MS) - .await; - match res { - Err(EngineError::TooManySamples { count, limit }) => { - assert_eq!(limit, 5); - assert!(count > limit); - } - other => panic!("expected TooManySamples, got {other:?}"), - } -} - -#[tokio::test] -async fn execute_topk_basic() { - // Values [1, 2, 3, 10, 20]; topk(2) → 30 - let samples = vec![ - raw(NOW_MS - 5_000, 1.0), - raw(NOW_MS - 4_000, 2.0), - raw(NOW_MS - 3_000, 3.0), - raw(NOW_MS - 2_000, 10.0), - raw(NOW_MS - 1_000, 20.0), - ]; - let chunk = ChunkRef { - key: "c".into(), - metric: METRIC.into(), - time_range_ms: (NOW_MS - 5_000, NOW_MS), - label_hash: 0, - sample_count: 5, - size_bytes: 0, - }; - let engine = engine_with(vec![(chunk, samples)]); - let result = engine - .execute_at(&format!("topk(2, sum_over_time({METRIC}[10s]))"), NOW_MS) - .await - .unwrap(); - if let QueryResult::Vector(iv) = result { - assert_eq!(iv.values[0].value, 30.0); - } else { - panic!("expected Vector"); - } -} - -// ───────────────────────────────────────────────────────────────────── -// Edge cases -// ───────────────────────────────────────────────────────────────────── - -#[tokio::test] -async fn execute_empty_chunks_returns_zero_or_nan() { - let engine = engine_with(Vec::new()); - let sum = engine - .execute_at(&format!("sum_over_time({METRIC}[5m])"), NOW_MS) - .await - .unwrap(); - if let QueryResult::Vector(iv) = sum { - assert!(iv.values[0].value.is_nan(), "sum on empty should be NaN"); - } else { - panic!("expected Vector"); - } - let count = engine - .execute_at(&format!("count_over_time({METRIC}[5m])"), NOW_MS) - .await - .unwrap(); - if let QueryResult::Vector(iv) = count { - assert_eq!(iv.values[0].value, 0.0); - } else { - panic!("expected Vector"); - } -} - -#[tokio::test] -async fn execute_chunks_partially_outside_range_filtered() { - // Chunk has 100 samples spanning [NOW-100s, NOW]; request - // covers the latter half [NOW-50s, NOW] → exactly 50 samples - // contribute. - let samples: Vec = (0..100) - .map(|i| raw(NOW_MS - 100_000 + (i as i64) * 1_000, 1.0)) - .collect(); - let chunk = ChunkRef { - key: "c".into(), - metric: METRIC.into(), - time_range_ms: (NOW_MS - 100_000, NOW_MS), - label_hash: 0, - sample_count: 100, - size_bytes: 0, - }; - let engine = engine_with(vec![(chunk, samples)]); - let result = engine - .execute_at(&format!("count_over_time({METRIC}[50s])"), NOW_MS) - .await - .unwrap(); - if let QueryResult::Vector(iv) = result { - assert_eq!( - iv.values[0].value, 50.0, - "exactly 50 samples should match a 50s window" - ); - } else { - panic!("expected Vector"); - } -} - -// ───────────────────────────────────────────────────────────────────── -// Result wrapping -// ───────────────────────────────────────────────────────────────────── - -#[tokio::test] -async fn result_carries_exact_accuracy_envelope() { - let chunks = vec![linear_chunk("c", NOW_MS - 1_000, 100, 10, 1.0, 0.0)]; - let engine = engine_with(chunks); - let result = engine - .execute_at(&format!("sum_over_time({METRIC}[5s])"), NOW_MS) - .await - .unwrap(); - let env = result - .accuracy() - .expect("Gorilla engine result must carry an accuracy envelope"); - assert_eq!(env.profile.kind, AccuracyKind::Exact); - assert_eq!(env.profile.epsilon, 0.0); - assert_eq!(env.profile.delta, 0.0); - // And the summary string the dashboards parse: - assert_eq!(env.profile.summary(), AccuracyProfile::exact().summary()); -} - -#[tokio::test] -async fn result_includes_data_source_thanos_query() { - // The wrapping fn surfaces the data_source line on - // ExecutionOutcome::info_lines — pin both the marker constant - // and the assembled info strings. - let outcome = ExecutionOutcome { - value: 42.0, - samples_scanned: 7, - chunks_fetched: 2, - chunks_skipped_via_postings: 0, - postings_filtered_series_count: 0, - postings_missing: false, - }; - let infos = outcome.info_lines(); - assert!( - infos.contains(&DATA_SOURCE_GORILLA_ARCHIVE.to_string()), - "infos must include `{DATA_SOURCE_GORILLA_ARCHIVE}`; got {infos:?}" - ); - assert!( - infos.iter().any(|i| i == "samples_scanned: 7"), - "infos must report the scanned-samples count" - ); - assert!( - infos.iter().any(|i| i == "chunks_fetched: 2"), - "infos must report the chunk-fetch count" - ); - - // And via the wrap_result path, the QueryResult itself carries - // the exact-accuracy envelope (data_source line is on the - // info-array which is assembled at the HTTP-driver layer; see - // wrap_result docs). - let plan = plan_query_at(&format!("sum_over_time({METRIC}[5s])"), NOW_MS).unwrap(); - let qr = wrap_result(&plan, outcome.clone()); - let env = qr.accuracy().expect("wrap_result must attach envelope"); - assert_eq!(env.profile.kind, AccuracyKind::Exact); -} - -// ───────────────────────────────────────────────────────────────────── -// Timeout -// ───────────────────────────────────────────────────────────────────── - -#[tokio::test] -async fn engine_respects_config_timeout() { - // 1 chunk + 250 ms read delay; engine timeout = 1 s ceil. We - // configure the timeout to 1s (the floor) and force the chunk - // count up so the cumulative read time > 1 s. - let mut chunks = Vec::new(); - for i in 0..10 { - let chunk = ChunkRef { - key: format!("k-{i}"), - metric: METRIC.into(), - time_range_ms: (NOW_MS - 60_000, NOW_MS), - label_hash: 0, - sample_count: 1, - size_bytes: 0, - }; - chunks.push((chunk, vec![raw(NOW_MS - 1_000, 1.0)])); - } - let mock = MockStore::new(chunks).with_read_delay(Duration::from_millis(250)); - let cfg = GorillaEngineConfig { - max_buffered_samples: 1_000_000, - query_timeout_secs: 1, - }; - let engine = GorillaQueryEngine::new(Arc::new(mock), cfg); - let res = engine - .execute_at(&format!("sum_over_time({METRIC}[5m])"), NOW_MS) - .await; - match res { - Err(EngineError::Timeout(_)) => {} - other => panic!("expected Timeout, got {other:?}"), - } -} - -// ───────────────────────────────────────────────────────────────────── -// mvp/v5 — postings-aware path tests -// ───────────────────────────────────────────────────────────────────── - -/// Build a chunk with explicit `label_hash` so the postings-aware -/// path can prune via `series_id == label_hash`. -fn labeled_chunk( - key: &str, - label_hash: u64, - label_value: &str, - start_ms: i64, - samples: &[(i64, f64)], -) -> (ChunkRef, Vec) { - let last_ts = samples.last().map(|(t, _)| *t).unwrap_or(start_ms); - let chunk = ChunkRef { - key: key.to_string(), - metric: METRIC.to_string(), - time_range_ms: (start_ms, last_ts + 1), - label_hash, - sample_count: samples.len() as u32, - size_bytes: 0, - }; - let mut labels = BTreeMap::new(); - labels.insert("zone".to_string(), label_value.to_string()); - let raw_samples: Vec = samples - .iter() - .map(|(t, v)| RawSample { - ts_ms: *t, - labels: labels.clone(), - value: *v, - }) - .collect(); - (chunk, raw_samples) -} - -#[tokio::test] -async fn postings_aware_path_prunes_chunks() { - // Two chunks: one for zone=a (label_hash=11), one for zone=b - // (label_hash=22). Postings says zone=a → [11]. The engine - // must read only the zone=a chunk. - let (chunk_a, samples_a) = labeled_chunk( - "k-a", - 11, - "a", - NOW_MS - 30_000, - &[(NOW_MS - 1_000, 5.0), (NOW_MS - 500, 5.0)], - ); - let (chunk_b, samples_b) = labeled_chunk( - "k-b", - 22, - "b", - NOW_MS - 30_000, - &[(NOW_MS - 1_000, 99.0), (NOW_MS - 500, 99.0)], - ); - let mut postings: BTreeMap<(String, String), Vec> = BTreeMap::new(); - postings.insert(("zone".to_string(), "a".to_string()), vec![11]); - postings.insert(("zone".to_string(), "b".to_string()), vec![22]); - let mock = - MockStore::new(vec![(chunk_a, samples_a), (chunk_b, samples_b)]).with_postings(postings); - let engine = GorillaQueryEngine::new(Arc::new(mock), cfg()); - let plan = plan_query_at( - &format!(r#"sum_over_time({METRIC}{{zone="a"}}[5m])"#), - NOW_MS, - ) - .unwrap(); - assert_eq!(plan.label_matchers.len(), 1); - let exec = ExactExecutor::new(engine.store_for_tests(), cfg()); - let outcome = exec.execute_plan(&plan).await.unwrap(); - // Only zone=a chunk contributed: 5.0 + 5.0 = 10.0 (NOT 5+5+99+99=208). - assert_eq!(outcome.value, 10.0); - assert_eq!(outcome.chunks_fetched, 1); - assert_eq!(outcome.chunks_skipped_via_postings, 1); - assert_eq!(outcome.postings_filtered_series_count, 1); - assert!(!outcome.postings_missing); -} - -#[tokio::test] -async fn postings_missing_falls_back_to_scan_all() { - // Same chunks, NO postings table → the executor falls through - // to the scan-all path and uses the post-decode label filter - // for correctness. The `postings_missing` flag must be set. - let (chunk_a, samples_a) = - labeled_chunk("k-a", 11, "a", NOW_MS - 30_000, &[(NOW_MS - 1_000, 5.0)]); - let (chunk_b, samples_b) = - labeled_chunk("k-b", 22, "b", NOW_MS - 30_000, &[(NOW_MS - 1_000, 99.0)]); - let mock = MockStore::new(vec![(chunk_a, samples_a), (chunk_b, samples_b)]); - let engine = GorillaQueryEngine::new(Arc::new(mock), cfg()); - let plan = plan_query_at( - &format!(r#"sum_over_time({METRIC}{{zone="a"}}[5m])"#), - NOW_MS, - ) - .unwrap(); - let exec = ExactExecutor::new(engine.store_for_tests(), cfg()); - let outcome = exec.execute_plan(&plan).await.unwrap(); - // Correctness: only zone=a sample (5.0) folded in. The - // post-decode filter does the work. - assert_eq!(outcome.value, 5.0); - // Both chunks were fetched — postings filter no-oped. - assert_eq!(outcome.chunks_fetched, 2); - assert_eq!(outcome.chunks_skipped_via_postings, 0); - assert!( - outcome.postings_missing, - "missing-postings flag must be set" - ); - let infos = outcome.info_lines(); - assert!(infos - .iter() - .any(|i| i == "data_source_quirk: postings_missing")); -} - -#[tokio::test] -async fn postings_path_no_label_predicate_skips_postings_lookup() { - // No label predicate → postings filter is a no-op; the - // postings table is never consulted. Total = 5+99 = 104. - let (chunk_a, samples_a) = - labeled_chunk("k-a", 11, "a", NOW_MS - 30_000, &[(NOW_MS - 1_000, 5.0)]); - let (chunk_b, samples_b) = - labeled_chunk("k-b", 22, "b", NOW_MS - 30_000, &[(NOW_MS - 1_000, 99.0)]); - let mock = MockStore::new(vec![(chunk_a, samples_a), (chunk_b, samples_b)]); - let engine = GorillaQueryEngine::new(Arc::new(mock), cfg()); - let plan = plan_query_at(&format!("sum_over_time({METRIC}[5m])"), NOW_MS).unwrap(); - assert!(plan.label_matchers.is_empty()); - let exec = ExactExecutor::new(engine.store_for_tests(), cfg()); - let outcome = exec.execute_plan(&plan).await.unwrap(); - assert_eq!(outcome.value, 104.0); - assert_eq!(outcome.chunks_fetched, 2); - assert!(!outcome.postings_missing); - assert_eq!(outcome.chunks_skipped_via_postings, 0); -} diff --git a/data_plane/src/storage_engines/mod.rs b/data_plane/src/storage_engines/mod.rs index 8c70b9563..c86282a71 100644 --- a/data_plane/src/storage_engines/mod.rs +++ b/data_plane/src/storage_engines/mod.rs @@ -9,22 +9,19 @@ //! layer (schema registry, schema timeline, backfill types / //! workers / HTTP endpoints) AND the physical storage backend //! (`sketch_db::index`) are co-located under this path. -//! * `gorilla_object_store` — S3/MinIO-backed Gorilla TSDB block -//! store used by the archive tier. +//! +//! The archive tier is served by the +//! [`crate::query_engines::thanos_query_engine::ThanosQueryEngine`] +//! (Path A2). The superseded in-process `gorilla_object_store` +//! (custom GORILLA1 container format) has been deleted. //! //! `SketchStore` is re-exported at the top level //! (`crate::storage_engines::SketchStore`) for call-site stability. -pub mod gorilla_object_store; pub mod sketch_db; pub mod traits; pub mod types; -pub use gorilla_object_store::{ - global_s3_cost_counters, ChunkRef, GorillaEngineConfig, GorillaQueryEngine, GorillaS3Config, - GorillaS3ConfigError, GorillaS3Store, ObjectStore, RawSample, S3CostCounters, S3CostSnapshot, - S3CostTrackingObjectStore, -}; pub use sketch_db::index::{ AccuracyBound, Capability, SidLookup, SketchConfig, SketchEncoding, SketchStore, SketchInstanceMetadata, SketchKindHandle, SketchSampleState, SketchTimeSeries,