From 763c4db5daf3f82f7459dd8839689f69dd4fb680 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 12 May 2026 09:44:58 -0600 Subject: [PATCH 1/7] refactor(data_plane): rename engine crate, finish module flatten on top of #137 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds on top of PR #137's storage/engine naming split. Where #137 stopped at `query-engines/` + kept `data_model/`, `precompute_operators/`, `routing/`, two `controller_client.rs` files, and the `SimpleEngine` type, this PR finishes the reorg: - Crate rename: `asap-query-engine/` → `data_plane/` (folder + cargo package + binary name); the controller sibling is the "control plane" - `query-engines/` → `query_engines/` (snake_case) - Per-engine `_engine` suffix: - `asap_query/` → `asap_query_engine/` - `thanos_query/` → `thanos_query_engine/` - `prometheus/` → `prometheus_query_engine/` - `warm_tier/` promoted from `asap_query_engine/warm_tier/` to a top-level `query_engines/warm_tier/` (it's shared infra, not asap_query-specific) - `data_model/` → `stores/schema/` (storage schema types belong with storage) - `precompute_operators/` → `precompute_engine/operators/` (accumulator impls belong under the engine that orchestrates them) - `routing/` → `query_engines/routing/` (engine dispatcher, scoped to query path) - `drivers/controller_client.rs` + `drivers/query/controller_client.rs` co-located into `drivers/controller_client/{config_fetcher.rs, miss_notifier.rs, mod.rs}` - `stores/promsketch_store/` deleted (was commented out of the public API; not referenced) - `SimpleEngine` struct renamed to `ASAPQueryEngine` (was a type alias on main); compat alias removed Test counts after the reorg match origin/main: - data_plane lib: 804 passed / 2 pre-existing failures (schema_timeline_dispatch_tests, documented in controller_todo §5) / 4 ignored - controller lib: 710/710; bins: 27/27 Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 114 ++--- Cargo.toml | 2 +- asap-query-engine/src/drivers/query/mod.rs | 10 - .../src/stores/promsketch_store/config.rs | 154 ------ .../src/stores/promsketch_store/metrics.rs | 43 -- .../src/stores/promsketch_store/mod.rs | 9 - .../src/stores/promsketch_store/query.rs | 286 ----------- .../src/stores/promsketch_store/series.rs | 180 ------- .../src/stores/promsketch_store/store.rs | 456 ------------------ .../src/stores/promsketch_store/types.rs | 75 --- controller/src/emit/stage_config.rs | 4 +- controller/src/optimizer/cost/wire.rs | 2 +- crates/asap_types/src/streaming_config.rs | 4 +- .../.cargo/config.toml | 0 {asap-query-engine => data_plane}/.gitignore | 0 {asap-query-engine => data_plane}/Cargo.toml | 2 +- {asap-query-engine => data_plane}/Dockerfile | 22 +- {asap-query-engine => data_plane}/LICENSE | 0 .../benches/simple_store_bench.rs | 12 +- .../docker-compose.yml.j2 | 0 .../docs/README.md | 0 .../docs/adding-fallback-backend.md | 0 .../docs/adding-protocol-adapter.md | 0 .../docs/adding-protocol-server.md | 0 .../docs/promsketch-integration.md | 0 .../examples/promql/inference_config.yaml | 0 .../examples/promql/streaming_config.yaml | 0 .../query-engine-rust-cli-compose.yml.j2 | 0 .../rustfmt.toml | 0 .../controller_client/config_fetcher.rs | 0 .../controller_client/miss_notifier.rs | 4 +- .../src/drivers/controller_client/mod.rs | 18 + .../src/drivers/ingest/kafka.rs | 14 +- .../src/drivers/ingest/mod.rs | 0 .../src/drivers/ingest/otel.rs | 20 +- .../src/drivers/ingest/series_resolver.rs | 0 .../src/drivers/mod.rs | 2 +- .../src/drivers/query/adapters/config.rs | 2 +- .../src/drivers/query/adapters/factory.rs | 2 +- .../src/drivers/query/adapters/mod.rs | 0 .../drivers/query/adapters/prometheus_http.rs | 4 +- .../src/drivers/query/adapters/traits.rs | 2 +- .../src/drivers/query/fallback/metrics.rs | 4 +- .../src/drivers/query/fallback/mod.rs | 0 .../src/drivers/query/fallback/prometheus.rs | 0 data_plane/src/drivers/query/mod.rs | 14 + .../src/drivers/query/servers/http.rs | 234 ++++----- .../src/drivers/query/servers/metrics.rs | 0 .../src/drivers/query/servers/mod.rs | 0 {asap-query-engine => data_plane}/src/lib.rs | 12 +- {asap-query-engine => data_plane}/src/main.rs | 90 ++-- .../precompute_engine/accumulator_factory.rs | 6 +- .../src/precompute_engine/config.rs | 0 .../src/precompute_engine/engine.rs | 2 +- .../src/precompute_engine/ingest_handler.rs | 14 +- .../src/precompute_engine/mod.rs | 1 + .../count_min_sketch_accumulator.rs | 8 +- .../count_min_sketch_with_heap_accumulator.rs | 4 +- .../operators}/count_sketch_accumulator.rs | 4 +- .../datasketches_kll_accumulator.rs | 8 +- .../operators}/dd_sketch_accumulator.rs | 4 +- .../delta_set_aggregator_accumulator.rs | 4 +- .../operators}/edge_runtime_adapter.rs | 0 .../operators}/hll_sketch_accumulator.rs | 4 +- .../operators}/hydra_kll_accumulator.rs | 4 +- .../operators}/increase_accumulator.rs | 4 +- .../operators}/min_max_accumulator.rs | 4 +- .../src/precompute_engine/operators}/mod.rs | 0 .../multiple_increase_accumulator.rs | 10 +- .../multiple_min_max_accumulator.rs | 4 +- .../operators}/multiple_sum_accumulator.rs | 4 +- .../operators}/set_aggregator_accumulator.rs | 4 +- .../operators}/sketch_envelope_accumulator.rs | 6 +- .../operators}/sum_accumulator.rs | 4 +- .../src/precompute_engine/output_sink.rs | 2 +- .../precompute_engine_design_doc.md | 0 .../src/precompute_engine/series_buffer.rs | 0 .../src/precompute_engine/series_router.rs | 2 +- .../src/precompute_engine/window_manager.rs | 0 .../src/precompute_engine/worker.rs | 22 +- .../asap_query_engine}/engine.rs | 252 +++++----- .../query_engines/asap_query_engine}/mod.rs | 8 +- .../query_engines/asap_query_engine}/tests.rs | 2 +- .../src/query_engines}/mod.rs | 36 +- .../src/query_engines}/no_data_archive.rs | 6 +- .../prometheus_query_engine}/forward.rs | 32 +- .../prometheus_query_engine}/mod.rs | 2 +- .../src/query_engines}/query_result.rs | 4 +- .../routing/backend_storage_routing.rs | 6 +- .../routing/freshness_probe_cache.rs | 0 .../src/query_engines}/routing/mod.rs | 4 +- .../routing/query_engine_routing.rs | 10 +- .../thanos_query_engine}/forward.rs | 28 +- .../query_engines/thanos_query_engine}/mod.rs | 0 .../src/query_engines}/timeline_dispatch.rs | 2 +- .../src/query_engines}/warm_tier/decoders.rs | 2 +- .../query_engines}/warm_tier/delta_apply.rs | 2 +- .../src/query_engines}/warm_tier/mod.rs | 6 +- .../warm_tier/sketch_reducer.rs | 6 +- .../src/query_engines}/warm_tier/tests.rs | 4 +- .../src/query_engines}/window_merger.rs | 4 +- .../gorilla_object_store/archive_query.rs | 0 .../src/stores/gorilla_object_store/mod.rs | 20 +- .../stores/gorilla_object_store/postings.rs | 0 .../stores/gorilla_object_store/s3_cost.rs | 0 .../src/stores/gorilla_object_store/store.rs | 0 .../src/stores/gorilla_object_store/tests.rs | 2 +- .../src/stores/mod.rs | 33 +- .../src/stores/schema}/aggregation_config.rs | 0 .../stores/schema}/aggregation_reference.rs | 0 .../src/stores/schema}/enums.rs | 0 .../src/stores/schema}/hot_reload_config.rs | 10 +- .../src/stores/schema}/inference_config.rs | 0 .../src/stores/schema}/key_by_label_values.rs | 0 .../src/stores/schema}/measurement.rs | 0 .../src/stores/schema}/mod.rs | 6 +- .../src/stores/schema}/precomputed_output.rs | 24 +- .../src/stores/schema}/promql_schema.rs | 0 .../src/stores/schema}/query_config.rs | 0 .../src/stores/schema}/streaming_config.rs | 0 .../src/stores/schema}/traits.rs | 2 +- .../src/stores/sketch_db/accuracy.rs | 2 +- .../src/stores/sketch_db/backfill.rs | 0 .../stores/sketch_db/backfill_processor.rs | 16 +- .../src/stores/sketch_db/backfill_service.rs | 12 +- .../sketch_db/backfill_window_builder.rs | 2 +- .../src/stores/sketch_db/backfill_worker.rs | 0 .../src/stores/sketch_db/epoch_columnar.rs | 0 .../src/stores/sketch_db/metrics.rs | 0 .../src/stores/sketch_db/mod.rs | 2 +- .../src/stores/sketch_db/prometheus_reader.rs | 0 .../src/stores/sketch_db/raw_sample_reader.rs | 0 .../src/stores/sketch_db/schema.rs | 8 +- .../src/stores/sketch_db/schema_eviction.rs | 6 +- .../simple_map_store/INDEX_DESIGN.md | 0 .../sketch_db/simple_map_store/common.rs | 2 +- .../sketch_db/simple_map_store/global.rs | 2 +- .../simple_map_store/legacy/global.rs | 2 +- .../sketch_db/simple_map_store/legacy/mod.rs | 0 .../simple_map_store/legacy/per_key.rs | 2 +- .../stores/sketch_db/simple_map_store/mod.rs | 6 +- .../sketch_db/simple_map_store/per_key.rs | 2 +- .../simple_map_store/persistence/cache.rs | 0 .../simple_map_store/persistence/config.rs | 0 .../simple_map_store/persistence/flusher.rs | 2 +- .../simple_map_store/persistence/manifest.rs | 0 .../simple_map_store/persistence/mod.rs | 0 .../simple_map_store/persistence/part.rs | 6 +- .../simple_map_store/persistence/recovery.rs | 2 +- .../simple_map_store/persistence/source.rs | 4 +- .../src/stores/sketch_db/sketch_index.rs | 0 .../src/stores/traits.rs | 2 +- .../accuracy_empirical_validation_tests.rs | 0 .../accuracy_in_promql_response_tests.rs | 0 .../src/tests/capability_matching_tests.rs | 24 +- .../tests/capability_miss_http_e2e_tests.rs | 10 +- .../src/tests/mod.rs | 0 .../tests/persist_format_versioning_tests.rs | 4 +- .../tests/persistence_integration_tests.rs | 2 +- .../src/tests/persistence_perf_tests.rs | 6 +- .../src/tests/prometheus_forwarding_tests.rs | 16 +- .../tests/schema_timeline_dispatch_tests.rs | 12 +- .../src/tests/store_correctness_tests.rs | 4 +- .../src/tests/test_utilities/comparison.rs | 4 +- .../tests/test_utilities/engine_factories.rs | 44 +- .../src/tests/test_utilities/mod.rs | 0 .../src/tests/trait_design_tests.rs | 4 +- .../src/utils/file_io.rs | 4 +- .../src/utils/http.rs | 6 +- .../src/utils/mod.rs | 0 .../src/utils/precompute_dumper.rs | 4 +- .../tests/e2e_modified_otlp_sketch_path.rs | 24 +- .../edge_runtime_consumes_precompute_rs.rs | 6 +- .../tests/inference_yaml_pattern_coverage.rs | 44 +- 174 files changed, 773 insertions(+), 1964 deletions(-) delete mode 100644 asap-query-engine/src/drivers/query/mod.rs delete mode 100644 asap-query-engine/src/stores/promsketch_store/config.rs delete mode 100644 asap-query-engine/src/stores/promsketch_store/metrics.rs delete mode 100644 asap-query-engine/src/stores/promsketch_store/mod.rs delete mode 100644 asap-query-engine/src/stores/promsketch_store/query.rs delete mode 100644 asap-query-engine/src/stores/promsketch_store/series.rs delete mode 100644 asap-query-engine/src/stores/promsketch_store/store.rs delete mode 100644 asap-query-engine/src/stores/promsketch_store/types.rs rename {asap-query-engine => data_plane}/.cargo/config.toml (100%) rename {asap-query-engine => data_plane}/.gitignore (100%) rename {asap-query-engine => data_plane}/Cargo.toml (99%) rename {asap-query-engine => data_plane}/Dockerfile (67%) rename {asap-query-engine => data_plane}/LICENSE (100%) rename {asap-query-engine => data_plane}/benches/simple_store_bench.rs (98%) rename {asap-query-engine => data_plane}/docker-compose.yml.j2 (100%) rename {asap-query-engine => data_plane}/docs/README.md (100%) rename {asap-query-engine => data_plane}/docs/adding-fallback-backend.md (100%) rename {asap-query-engine => data_plane}/docs/adding-protocol-adapter.md (100%) rename {asap-query-engine => data_plane}/docs/adding-protocol-server.md (100%) rename {asap-query-engine => data_plane}/docs/promsketch-integration.md (100%) rename {asap-query-engine => data_plane}/examples/promql/inference_config.yaml (100%) rename {asap-query-engine => data_plane}/examples/promql/streaming_config.yaml (100%) rename {asap-query-engine => data_plane}/query-engine-rust-cli-compose.yml.j2 (100%) rename {asap-query-engine => data_plane}/rustfmt.toml (100%) rename asap-query-engine/src/drivers/controller_client.rs => data_plane/src/drivers/controller_client/config_fetcher.rs (100%) rename asap-query-engine/src/drivers/query/controller_client.rs => data_plane/src/drivers/controller_client/miss_notifier.rs (98%) create mode 100644 data_plane/src/drivers/controller_client/mod.rs rename {asap-query-engine => data_plane}/src/drivers/ingest/kafka.rs (96%) rename {asap-query-engine => data_plane}/src/drivers/ingest/mod.rs (100%) rename {asap-query-engine => data_plane}/src/drivers/ingest/otel.rs (99%) rename {asap-query-engine => data_plane}/src/drivers/ingest/series_resolver.rs (100%) rename {asap-query-engine => data_plane}/src/drivers/mod.rs (72%) rename {asap-query-engine => data_plane}/src/drivers/query/adapters/config.rs (96%) rename {asap-query-engine => data_plane}/src/drivers/query/adapters/factory.rs (89%) rename {asap-query-engine => data_plane}/src/drivers/query/adapters/mod.rs (100%) rename {asap-query-engine => data_plane}/src/drivers/query/adapters/prometheus_http.rs (99%) rename {asap-query-engine => data_plane}/src/drivers/query/adapters/traits.rs (99%) rename {asap-query-engine => data_plane}/src/drivers/query/fallback/metrics.rs (94%) rename {asap-query-engine => data_plane}/src/drivers/query/fallback/mod.rs (100%) rename {asap-query-engine => data_plane}/src/drivers/query/fallback/prometheus.rs (100%) create mode 100644 data_plane/src/drivers/query/mod.rs rename {asap-query-engine => data_plane}/src/drivers/query/servers/http.rs (96%) rename {asap-query-engine => data_plane}/src/drivers/query/servers/metrics.rs (100%) rename {asap-query-engine => data_plane}/src/drivers/query/servers/mod.rs (100%) rename {asap-query-engine => data_plane}/src/lib.rs (81%) rename {asap-query-engine => data_plane}/src/main.rs (92%) rename {asap-query-engine => data_plane}/src/precompute_engine/accumulator_factory.rs (99%) rename {asap-query-engine => data_plane}/src/precompute_engine/config.rs (100%) rename {asap-query-engine => data_plane}/src/precompute_engine/engine.rs (99%) rename {asap-query-engine => data_plane}/src/precompute_engine/ingest_handler.rs (96%) rename {asap-query-engine => data_plane}/src/precompute_engine/mod.rs (93%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/count_min_sketch_accumulator.rs (99%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/count_min_sketch_with_heap_accumulator.rs (99%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/count_sketch_accumulator.rs (99%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/datasketches_kll_accumulator.rs (99%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/dd_sketch_accumulator.rs (98%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/delta_set_aggregator_accumulator.rs (99%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/edge_runtime_adapter.rs (100%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/hll_sketch_accumulator.rs (98%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/hydra_kll_accumulator.rs (98%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/increase_accumulator.rs (99%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/min_max_accumulator.rs (99%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/mod.rs (100%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/multiple_increase_accumulator.rs (98%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/multiple_min_max_accumulator.rs (99%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/multiple_sum_accumulator.rs (99%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/set_aggregator_accumulator.rs (99%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/sketch_envelope_accumulator.rs (95%) rename {asap-query-engine/src/precompute_operators => data_plane/src/precompute_engine/operators}/sum_accumulator.rs (99%) rename {asap-query-engine => data_plane}/src/precompute_engine/output_sink.rs (98%) rename {asap-query-engine => data_plane}/src/precompute_engine/precompute_engine_design_doc.md (100%) rename {asap-query-engine => data_plane}/src/precompute_engine/series_buffer.rs (100%) rename {asap-query-engine => data_plane}/src/precompute_engine/series_router.rs (99%) rename {asap-query-engine => data_plane}/src/precompute_engine/window_manager.rs (100%) rename {asap-query-engine => data_plane}/src/precompute_engine/worker.rs (99%) rename {asap-query-engine/src/query-engines/asap_query => data_plane/src/query_engines/asap_query_engine}/engine.rs (96%) rename {asap-query-engine/src/query-engines/asap_query => data_plane/src/query_engines/asap_query_engine}/mod.rs (72%) rename {asap-query-engine/src/query-engines/asap_query => data_plane/src/query_engines/asap_query_engine}/tests.rs (87%) rename {asap-query-engine/src/query-engines => data_plane/src/query_engines}/mod.rs (76%) rename {asap-query-engine/src/query-engines => data_plane/src/query_engines}/no_data_archive.rs (94%) rename {asap-query-engine/src/query-engines/prometheus => data_plane/src/query_engines/prometheus_query_engine}/forward.rs (96%) rename {asap-query-engine/src/query-engines/prometheus => data_plane/src/query_engines/prometheus_query_engine}/mod.rs (93%) rename {asap-query-engine/src/query-engines => data_plane/src/query_engines}/query_result.rs (99%) rename {asap-query-engine/src => data_plane/src/query_engines}/routing/backend_storage_routing.rs (99%) rename {asap-query-engine/src => data_plane/src/query_engines}/routing/freshness_probe_cache.rs (100%) rename {asap-query-engine/src => data_plane/src/query_engines}/routing/mod.rs (91%) rename {asap-query-engine/src => data_plane/src/query_engines}/routing/query_engine_routing.rs (98%) rename {asap-query-engine/src/query-engines/thanos_query => data_plane/src/query_engines/thanos_query_engine}/forward.rs (97%) rename {asap-query-engine/src/query-engines/thanos_query => data_plane/src/query_engines/thanos_query_engine}/mod.rs (100%) rename {asap-query-engine/src/query-engines => data_plane/src/query_engines}/timeline_dispatch.rs (99%) rename {asap-query-engine/src/query-engines/asap_query => data_plane/src/query_engines}/warm_tier/decoders.rs (98%) rename {asap-query-engine/src/query-engines/asap_query => data_plane/src/query_engines}/warm_tier/delta_apply.rs (99%) rename {asap-query-engine/src/query-engines/asap_query => data_plane/src/query_engines}/warm_tier/mod.rs (93%) rename {asap-query-engine/src/query-engines/asap_query => data_plane/src/query_engines}/warm_tier/sketch_reducer.rs (99%) rename {asap-query-engine/src/query-engines/asap_query => data_plane/src/query_engines}/warm_tier/tests.rs (99%) rename {asap-query-engine/src/query-engines => data_plane/src/query_engines}/window_merger.rs (99%) rename {asap-query-engine => data_plane}/src/stores/gorilla_object_store/archive_query.rs (100%) rename {asap-query-engine => data_plane}/src/stores/gorilla_object_store/mod.rs (94%) rename {asap-query-engine => data_plane}/src/stores/gorilla_object_store/postings.rs (100%) rename {asap-query-engine => data_plane}/src/stores/gorilla_object_store/s3_cost.rs (100%) rename {asap-query-engine => data_plane}/src/stores/gorilla_object_store/store.rs (100%) rename {asap-query-engine => data_plane}/src/stores/gorilla_object_store/tests.rs (99%) rename {asap-query-engine => data_plane}/src/stores/mod.rs (62%) rename {asap-query-engine/src/data_model => data_plane/src/stores/schema}/aggregation_config.rs (100%) rename {asap-query-engine/src/data_model => data_plane/src/stores/schema}/aggregation_reference.rs (100%) rename {asap-query-engine/src/data_model => data_plane/src/stores/schema}/enums.rs (100%) rename {asap-query-engine/src/data_model => data_plane/src/stores/schema}/hot_reload_config.rs (96%) rename {asap-query-engine/src/data_model => data_plane/src/stores/schema}/inference_config.rs (100%) rename {asap-query-engine/src/data_model => data_plane/src/stores/schema}/key_by_label_values.rs (100%) rename {asap-query-engine/src/data_model => data_plane/src/stores/schema}/measurement.rs (100%) rename {asap-query-engine/src/data_model => data_plane/src/stores/schema}/mod.rs (82%) rename {asap-query-engine/src/data_model => data_plane/src/stores/schema}/precomputed_output.rs (97%) rename {asap-query-engine/src/data_model => data_plane/src/stores/schema}/promql_schema.rs (100%) rename {asap-query-engine/src/data_model => data_plane/src/stores/schema}/query_config.rs (100%) rename {asap-query-engine/src/data_model => data_plane/src/stores/schema}/streaming_config.rs (100%) rename {asap-query-engine/src/data_model => data_plane/src/stores/schema}/traits.rs (99%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/accuracy.rs (99%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/backfill.rs (100%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/backfill_processor.rs (98%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/backfill_service.rs (98%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/backfill_window_builder.rs (99%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/backfill_worker.rs (100%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/epoch_columnar.rs (100%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/metrics.rs (100%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/mod.rs (98%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/prometheus_reader.rs (100%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/raw_sample_reader.rs (100%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/schema.rs (99%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/schema_eviction.rs (98%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/INDEX_DESIGN.md (100%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/common.rs (99%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/global.rs (99%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/legacy/global.rs (99%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/legacy/mod.rs (100%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/legacy/per_key.rs (99%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/mod.rs (98%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/per_key.rs (99%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/persistence/cache.rs (100%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/persistence/config.rs (100%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/persistence/flusher.rs (99%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/persistence/manifest.rs (100%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/persistence/mod.rs (100%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/persistence/part.rs (99%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/persistence/recovery.rs (99%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/simple_map_store/persistence/source.rs (96%) rename {asap-query-engine => data_plane}/src/stores/sketch_db/sketch_index.rs (100%) rename {asap-query-engine => data_plane}/src/stores/traits.rs (97%) rename {asap-query-engine => data_plane}/src/tests/accuracy_empirical_validation_tests.rs (100%) rename {asap-query-engine => data_plane}/src/tests/accuracy_in_promql_response_tests.rs (100%) rename {asap-query-engine => data_plane}/src/tests/capability_matching_tests.rs (94%) rename {asap-query-engine => data_plane}/src/tests/capability_miss_http_e2e_tests.rs (98%) rename {asap-query-engine => data_plane}/src/tests/mod.rs (100%) rename {asap-query-engine => data_plane}/src/tests/persist_format_versioning_tests.rs (99%) rename {asap-query-engine => data_plane}/src/tests/persistence_integration_tests.rs (96%) rename {asap-query-engine => data_plane}/src/tests/persistence_perf_tests.rs (99%) rename {asap-query-engine => data_plane}/src/tests/prometheus_forwarding_tests.rs (94%) rename {asap-query-engine => data_plane}/src/tests/schema_timeline_dispatch_tests.rs (97%) rename {asap-query-engine => data_plane}/src/tests/store_correctness_tests.rs (99%) rename {asap-query-engine => data_plane}/src/tests/test_utilities/comparison.rs (98%) rename {asap-query-engine => data_plane}/src/tests/test_utilities/engine_factories.rs (95%) rename {asap-query-engine => data_plane}/src/tests/test_utilities/mod.rs (100%) rename {asap-query-engine => data_plane}/src/tests/trait_design_tests.rs (96%) rename {asap-query-engine => data_plane}/src/utils/file_io.rs (97%) rename {asap-query-engine => data_plane}/src/utils/http.rs (98%) rename {asap-query-engine => data_plane}/src/utils/mod.rs (100%) rename {asap-query-engine => data_plane}/src/utils/precompute_dumper.rs (97%) rename {asap-query-engine => data_plane}/tests/e2e_modified_otlp_sketch_path.rs (97%) rename {asap-query-engine => data_plane}/tests/edge_runtime_consumes_precompute_rs.rs (98%) rename {asap-query-engine => data_plane}/tests/inference_yaml_pattern_coverage.rs (93%) diff --git a/Cargo.lock b/Cargo.lock index ad3e359e..65e4e3ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1037,6 +1037,63 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "data_plane" +version = "0.1.0" +dependencies = [ + "anyhow", + "arc-swap", + "arrow", + "asap-gorilla", + "asap-precompute-rs", + "asap_otel_proto", + "asap_sketchlib", + "asap_types", + "async-trait", + "axum", + "base64 0.21.7", + "bincode", + "chrono", + "clap 4.6.1", + "controller", + "crc32fast", + "criterion", + "dashmap", + "flate2", + "form_urlencoded", + "futures", + "hex", + "lazy_static", + "lru", + "memmap2", + "moka", + "prometheus", + "promql-parser 0.5.1", + "promql_utilities", + "prost", + "rdkafka", + "regex", + "reqwest 0.11.27", + "rmp-serde", + "rusqlite", + "rust-s3", + "serde", + "serde_json", + "serde_yaml", + "structopt", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-appender", + "tracing-subscriber", + "urlencoding", + "uuid", + "xxhash-rust", +] + [[package]] name = "deranged" version = "0.5.8" @@ -2808,63 +2865,6 @@ dependencies = [ "cc", ] -[[package]] -name = "query_engine_rust" -version = "0.1.0" -dependencies = [ - "anyhow", - "arc-swap", - "arrow", - "asap-gorilla", - "asap-precompute-rs", - "asap_otel_proto", - "asap_sketchlib", - "asap_types", - "async-trait", - "axum", - "base64 0.21.7", - "bincode", - "chrono", - "clap 4.6.1", - "controller", - "crc32fast", - "criterion", - "dashmap", - "flate2", - "form_urlencoded", - "futures", - "hex", - "lazy_static", - "lru", - "memmap2", - "moka", - "prometheus", - "promql-parser 0.5.1", - "promql_utilities", - "prost", - "rdkafka", - "regex", - "reqwest 0.11.27", - "rmp-serde", - "rusqlite", - "rust-s3", - "serde", - "serde_json", - "serde_yaml", - "structopt", - "tempfile", - "thiserror 1.0.69", - "tokio", - "tokio-stream", - "tonic", - "tracing", - "tracing-appender", - "tracing-subscriber", - "urlencoding", - "uuid", - "xxhash-rust", -] - [[package]] name = "quick-xml" version = "0.38.4" diff --git a/Cargo.toml b/Cargo.toml index ec22820f..273b3d78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = [ "crates/promql_utilities", "crates/asap_otel_proto", "crates/asap_types", - "asap-query-engine", + "data_plane", "controller", ] diff --git a/asap-query-engine/src/drivers/query/mod.rs b/asap-query-engine/src/drivers/query/mod.rs deleted file mode 100644 index 208c6ca6..00000000 --- a/asap-query-engine/src/drivers/query/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub mod adapters; -pub mod controller_client; -pub mod fallback; -pub mod servers; - -// Re-export commonly used types for convenience -pub use adapters::{create_http_adapter, AdapterConfig, HttpProtocolAdapter}; -pub use controller_client::{spawn_capability_miss_notify, ControllerClient, HttpControllerClient}; -pub use fallback::FallbackClient; -pub use servers::{HttpServer, HttpServerConfig}; diff --git a/asap-query-engine/src/stores/promsketch_store/config.rs b/asap-query-engine/src/stores/promsketch_store/config.rs deleted file mode 100644 index 498751b6..00000000 --- a/asap-query-engine/src/stores/promsketch_store/config.rs +++ /dev/null @@ -1,154 +0,0 @@ -use anyhow::{Context, Result}; -use serde::Deserialize; - -/// Configuration for ExponentialHistogram wrapping UnivMon sketches. -#[derive(Clone, Debug, Deserialize)] -pub struct EHUnivConfig { - /// Number of EH buckets (k parameter for ExponentialHistogram). - pub k: usize, - /// Time window size in milliseconds. - pub time_window: u64, -} - -impl Default for EHUnivConfig { - fn default() -> Self { - Self { - k: 50, - time_window: 1_000_000, - } - } -} - -/// Configuration for ExponentialHistogram wrapping KLL sketches. -#[derive(Clone, Debug, Deserialize)] -pub struct EHKLLConfig { - /// Number of EH buckets (k parameter for ExponentialHistogram). - pub k: usize, - /// KLL sketch k parameter (controls accuracy vs memory). - pub kll_k: i32, - /// Time window size in milliseconds. - pub time_window: u64, -} - -impl Default for EHKLLConfig { - fn default() -> Self { - Self { - k: 50, - kll_k: 256, - time_window: 1_000_000, - } - } -} - -/// Configuration for ExponentialHistogram wrapping UniformSampling sketches. -#[derive(Clone, Debug, Deserialize)] -pub struct SamplingConfig { - /// Fraction of data points to sample (0.0 to 1.0). - pub sample_rate: f64, - /// Time window size in milliseconds. - pub time_window: u64, -} - -impl Default for SamplingConfig { - fn default() -> Self { - Self { - sample_rate: 0.2, - time_window: 1_000_000, - } - } -} - -/// Bundled configuration for all sketch types. -#[derive(Clone, Debug, Default, Deserialize)] -pub struct PromSketchConfig { - pub eh_univ: EHUnivConfig, - pub eh_kll: EHKLLConfig, - pub sampling: SamplingConfig, -} - -impl PromSketchConfig { - /// Load a PromSketchConfig from a YAML file. - pub fn from_yaml_file(path: &str) -> Result { - let contents = std::fs::read_to_string(path) - .with_context(|| format!("Failed to read sketch config file: {path}"))?; - let config: PromSketchConfig = serde_yaml::from_str(&contents) - .with_context(|| format!("Failed to parse sketch config YAML from: {path}"))?; - Ok(config) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_default_config_values() { - let config = PromSketchConfig::default(); - - assert_eq!(config.eh_univ.k, 50); - assert_eq!(config.eh_univ.time_window, 1_000_000); - - assert_eq!(config.eh_kll.k, 50); - assert_eq!(config.eh_kll.kll_k, 256); - assert_eq!(config.eh_kll.time_window, 1_000_000); - - assert!((config.sampling.sample_rate - 0.2).abs() < f64::EPSILON); - assert_eq!(config.sampling.time_window, 1_000_000); - } - - #[test] - fn test_yaml_deserialization() { - let yaml = r#" -eh_univ: - k: 100 - time_window: 2000000 -eh_kll: - k: 80 - kll_k: 512 - time_window: 3000000 -sampling: - sample_rate: 0.5 - time_window: 4000000 -"#; - let config: PromSketchConfig = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(config.eh_univ.k, 100); - assert_eq!(config.eh_univ.time_window, 2_000_000); - assert_eq!(config.eh_kll.k, 80); - assert_eq!(config.eh_kll.kll_k, 512); - assert_eq!(config.eh_kll.time_window, 3_000_000); - assert!((config.sampling.sample_rate - 0.5).abs() < f64::EPSILON); - assert_eq!(config.sampling.time_window, 4_000_000); - } - - #[test] - fn test_from_yaml_file() { - use std::io::Write; - use tempfile::NamedTempFile; - - let yaml = r#" -eh_univ: - k: 30 - time_window: 500000 -eh_kll: - k: 40 - kll_k: 128 - time_window: 600000 -sampling: - sample_rate: 0.1 - time_window: 700000 -"#; - let mut tmp = NamedTempFile::new().unwrap(); - write!(tmp, "{}", yaml).unwrap(); - - let config = PromSketchConfig::from_yaml_file(tmp.path().to_str().unwrap()).unwrap(); - assert_eq!(config.eh_univ.k, 30); - assert_eq!(config.eh_kll.kll_k, 128); - assert!((config.sampling.sample_rate - 0.1).abs() < f64::EPSILON); - } - - #[test] - fn test_from_yaml_file_nonexistent() { - let result = PromSketchConfig::from_yaml_file("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/nonexistent/path.yaml"); - assert!(result.is_err()); - } -} diff --git a/asap-query-engine/src/stores/promsketch_store/metrics.rs b/asap-query-engine/src/stores/promsketch_store/metrics.rs deleted file mode 100644 index b9eb9047..00000000 --- a/asap-query-engine/src/stores/promsketch_store/metrics.rs +++ /dev/null @@ -1,43 +0,0 @@ -use lazy_static::lazy_static; -use prometheus::{ - register_counter, register_counter_vec, register_gauge, register_histogram, Counter, - CounterVec, Gauge, Histogram, -}; - -lazy_static! { - /// Number of live series in the PromSketchStore. - pub static ref SERIES_TOTAL: Gauge = - register_gauge!("promsketch_series_total", "Number of live series in store").unwrap(); - - /// Total raw samples successfully inserted. - pub static ref SAMPLES_INGESTED_TOTAL: Counter = - register_counter!("promsketch_samples_ingested_total", "Total raw samples inserted").unwrap(); - - /// Failed sample insertions. - pub static ref INGEST_ERRORS_TOTAL: Counter = - register_counter!("promsketch_ingest_errors_total", "Failed sample insertions").unwrap(); - - /// Time to flush a batch of raw samples. - pub static ref INGEST_BATCH_DURATION: Histogram = - register_histogram!( - "promsketch_ingest_batch_duration_seconds", - "Time to flush a batch of raw samples", - vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0] - ).unwrap(); - - /// Sketch queries executed, labeled by result (hit or miss). - pub static ref SKETCH_QUERIES_TOTAL: CounterVec = - register_counter_vec!( - "promsketch_sketch_queries_total", - "Sketch queries executed", - &["result"] - ).unwrap(); - - /// Sketch query evaluation latency. - pub static ref SKETCH_QUERY_DURATION: Histogram = - register_histogram!( - "promsketch_sketch_query_duration_seconds", - "Sketch query eval latency", - vec![0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0] - ).unwrap(); -} diff --git a/asap-query-engine/src/stores/promsketch_store/mod.rs b/asap-query-engine/src/stores/promsketch_store/mod.rs deleted file mode 100644 index d2c67bf2..00000000 --- a/asap-query-engine/src/stores/promsketch_store/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -// pub mod config; -// pub mod metrics; -// pub mod query; -// pub mod series; -// mod store; -// mod types; - -// pub use store::PromSketchStore; -// pub use types::{is_usampling_function, promsketch_func_map, PromSketchType}; diff --git a/asap-query-engine/src/stores/promsketch_store/query.rs b/asap-query-engine/src/stores/promsketch_store/query.rs deleted file mode 100644 index 2f610eac..00000000 --- a/asap-query-engine/src/stores/promsketch_store/query.rs +++ /dev/null @@ -1,286 +0,0 @@ -use asap_sketchlib::{EHSketchList, DataInput, UniformSampling}; - -use super::series::PromSketchMemSeries; - -/// Evaluate a PromQL aggregation function over sketches for a given time range. -/// -/// # Arguments -/// * `func_name` - PromQL function name (e.g. "quantile_over_time") -/// * `series` - The PromSketchMemSeries containing sketch instances -/// * `args` - Extra argument (e.g. quantile phi value) -/// * `mint` - Start of query time range (milliseconds) -/// * `maxt` - End of query time range (milliseconds) -pub fn eval_function( - func_name: &str, - series: &PromSketchMemSeries, - args: f64, - mint: u64, - maxt: u64, -) -> Result> { - match func_name { - "entropy_over_time" => eval_univmon(series, "entropy", mint, maxt), - "distinct_over_time" => eval_univmon(series, "cardinality", mint, maxt), - "l1_over_time" => eval_univmon(series, "l1", mint, maxt), - "l2_over_time" => eval_univmon(series, "l2", mint, maxt), - "quantile_over_time" => eval_kll_quantile(series, args, mint, maxt), - "min_over_time" => eval_kll_quantile(series, 0.0, mint, maxt), - "max_over_time" => eval_kll_quantile(series, 1.0, mint, maxt), - "avg_over_time" => eval_sampling_stat(series, "avg", mint, maxt), - "count_over_time" => eval_sampling_stat(series, "count", mint, maxt), - "sum_over_time" => eval_sampling_stat(series, "sum", mint, maxt), - "sum2_over_time" => eval_sampling_stat(series, "sum2", mint, maxt), - "stddev_over_time" => eval_sampling_stat(series, "stddev", mint, maxt), - "stdvar_over_time" => eval_sampling_stat(series, "stdvar", mint, maxt), - _ => Err(format!("unsupported function: {}", func_name).into()), - } -} - -/// Evaluate UnivMon-based functions (entropy, cardinality, L1, L2) -/// using the optimized EHUnivOptimized backend. -fn eval_univmon( - series: &PromSketchMemSeries, - stat: &str, - mint: u64, - maxt: u64, -) -> Result> { - let eh = series - .sketch_instances - .eh_univ - .as_ref() - .ok_or("eh_univ not initialized")?; - - let result = eh - .query_interval(mint, maxt) - .ok_or("no buckets cover the requested time range for UnivMon")?; - - match stat { - "entropy" => Ok(result.calc_entropy()), - "cardinality" => Ok(result.calc_card()), - "l1" => Ok(result.calc_l1()), - "l2" => Ok(result.calc_l2()), - _ => Err(format!("unknown univmon stat: {}", stat).into()), - } -} - -/// Evaluate KLL-based functions (quantile, min, max). -fn eval_kll_quantile( - series: &PromSketchMemSeries, - phi: f64, - mint: u64, - maxt: u64, -) -> Result> { - let eh = series - .sketch_instances - .eh_kll - .as_ref() - .ok_or("eh_kll not initialized")?; - - let merged = eh - .query_interval_merge(mint, maxt) - .ok_or("no volumes cover the requested time range for KLL")?; - - merged - .query(&DataInput::F64(phi)) - .map_err(|e| -> Box { e.into() }) -} - -/// Evaluate sampling-based functions (avg, count, sum, sum2, stddev, stdvar). -/// -/// Since asap_sketchlib's `UniformSampling` exposes `samples()` and `total_seen()` -/// but not dedicated query methods like the Go version, we compute statistics -/// from the raw merged samples. -fn eval_sampling_stat( - series: &PromSketchMemSeries, - stat: &str, - mint: u64, - maxt: u64, -) -> Result> { - let eh = series - .sketch_instances - .eh_sampling - .as_ref() - .ok_or("eh_sampling not initialized")?; - - let merged = eh - .query_interval_merge(mint, maxt) - .ok_or("no volumes cover the requested time range for sampling")?; - - let sampler = match &merged { - EHSketchList::UNIFORM(us) => us, - _ => return Err("merged EHSketchList is not UniformSampling".into()), - }; - - compute_sampling_stat(sampler, stat) -} - -/// Compute a statistic from a merged UniformSampling instance. -fn compute_sampling_stat( - sampler: &UniformSampling, - stat: &str, -) -> Result> { - let samples = sampler.samples(); - if samples.is_empty() { - return Err("no samples available".into()); - } - - let n = samples.len() as f64; - let total_seen = sampler.total_seen() as f64; - - match stat { - "count" => { - // Estimate total count from sample rate - Ok(total_seen) - } - "sum" => { - let sample_sum: f64 = samples.iter().sum(); - // Scale up by (total_seen / n_samples) to estimate population sum - Ok(sample_sum * (total_seen / n)) - } - "sum2" => { - let sample_sum2: f64 = samples.iter().map(|x| x * x).sum(); - Ok(sample_sum2 * (total_seen / n)) - } - "avg" => { - let sample_sum: f64 = samples.iter().sum(); - Ok(sample_sum / n) - } - "stddev" => { - let mean = samples.iter().sum::() / n; - let variance = samples.iter().map(|x| (x - mean).powi(2)).sum::() / n; - Ok(variance.sqrt()) - } - "stdvar" => { - let mean = samples.iter().sum::() / n; - let variance = samples.iter().map(|x| (x - mean).powi(2)).sum::() / n; - Ok(variance) - } - _ => Err(format!("unknown sampling stat: {}", stat).into()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::stores::promsketch_store::config::PromSketchConfig; - use crate::stores::promsketch_store::series::PromSketchMemSeries; - use crate::stores::promsketch_store::PromSketchType; - - fn create_test_series_with_kll_data() -> PromSketchMemSeries { - let config = PromSketchConfig::default(); - let mut series = PromSketchMemSeries::new("test".to_string()); - series - .sketch_instances - .ensure_initialized(PromSketchType::EHKLL, &config); - - // Insert values 1..=100 at successive timestamps - for i in 1..=100u64 { - let input = DataInput::F64(i as f64); - if let Some(ref mut eh) = series.sketch_instances.eh_kll { - eh.update(i, &input); - } - } - series - } - - fn create_test_series_with_sampling_data() -> PromSketchMemSeries { - let config = PromSketchConfig::default(); - let mut series = PromSketchMemSeries::new("test".to_string()); - series - .sketch_instances - .ensure_initialized(PromSketchType::USampling, &config); - - for i in 1..=1000u64 { - let input = DataInput::F64(i as f64); - if let Some(ref mut eh) = series.sketch_instances.eh_sampling { - eh.update(i, &input); - } - } - series - } - - fn create_test_series_with_univmon_data() -> PromSketchMemSeries { - let config = PromSketchConfig::default(); - let mut series = PromSketchMemSeries::new("test".to_string()); - series - .sketch_instances - .ensure_initialized(PromSketchType::EHUniv, &config); - - for i in 1..=100u64 { - let input = DataInput::F64(i as f64); - if let Some(ref mut eh) = series.sketch_instances.eh_univ { - eh.update(i, &input, 1); - } - } - series - } - - #[test] - fn test_eval_kll_quantile() { - let series = create_test_series_with_kll_data(); - let result = eval_function("quantile_over_time", &series, 0.5, 1, 100); - assert!(result.is_ok()); - let val = result.unwrap(); - // Median of 1..100 should be around 50 - assert!(val > 30.0 && val < 70.0, "median was {}", val); - } - - #[test] - fn test_eval_min_max() { - let series = create_test_series_with_kll_data(); - - let min_result = eval_function("min_over_time", &series, 0.0, 1, 100); - assert!(min_result.is_ok()); - let min_val = min_result.unwrap(); - assert!(min_val <= 5.0, "min was {}", min_val); - - let max_result = eval_function("max_over_time", &series, 0.0, 1, 100); - assert!(max_result.is_ok()); - let max_val = max_result.unwrap(); - assert!(max_val >= 95.0, "max was {}", max_val); - } - - #[test] - fn test_eval_sampling_avg() { - let series = create_test_series_with_sampling_data(); - let result = eval_function("avg_over_time", &series, 0.0, 1, 1000); - assert!(result.is_ok()); - let val = result.unwrap(); - // avg of 1..1000 should be around 500.5 - assert!( - val > 300.0 && val < 700.0, - "avg was {} (expected ~500.5)", - val - ); - } - - #[test] - fn test_eval_sampling_count() { - let series = create_test_series_with_sampling_data(); - let result = eval_function("count_over_time", &series, 0.0, 1, 1000); - assert!(result.is_ok()); - let val = result.unwrap(); - // total_seen should be 1000 - assert!( - val > 500.0 && val <= 1000.0, - "count was {} (expected ~1000)", - val - ); - } - - #[test] - fn test_eval_univmon_entropy() { - let series = create_test_series_with_univmon_data(); - let result = eval_function("entropy_over_time", &series, 0.0, 1, 100); - assert!(result.is_ok()); - // UnivMon entropy with small data can be 0; verify query dispatches correctly - let val = result.unwrap(); - assert!(val >= 0.0, "entropy was {}", val); - } - - #[test] - fn test_unsupported_function() { - let series = PromSketchMemSeries::new("test".to_string()); - let result = eval_function("nonexistent_func", &series, 0.0, 1, 100); - assert!(result.is_err()); - } -} diff --git a/asap-query-engine/src/stores/promsketch_store/series.rs b/asap-query-engine/src/stores/promsketch_store/series.rs deleted file mode 100644 index c5c26f95..00000000 --- a/asap-query-engine/src/stores/promsketch_store/series.rs +++ /dev/null @@ -1,180 +0,0 @@ -use asap_sketchlib::{ - EHSketchList, EHUnivOptimized, ExponentialHistogram, DataInput, UniformSampling, KLL, -}; - -use super::config::PromSketchConfig; -use super::PromSketchType; - -/// Per-series sketch instances. Each field wraps a different EHSketchList type -/// inside an ExponentialHistogram for time-windowed merging. -pub struct PromSketchInstances { - /// Optimized hybrid EH for UnivMon — entropy, cardinality, L1, L2, distinct. - pub eh_univ: Option, - /// EH wrapping KLL — for quantile, min, max. - pub eh_kll: Option, - /// EH wrapping UniformSampling — for avg, count, sum, stddev, stdvar. - pub eh_sampling: Option, -} - -impl Default for PromSketchInstances { - fn default() -> Self { - Self::new() - } -} - -impl PromSketchInstances { - pub fn new() -> Self { - Self { - eh_univ: None, - eh_kll: None, - eh_sampling: None, - } - } - - /// Lazily initialize the sketch for the given type if not already present. - pub fn ensure_initialized(&mut self, stype: PromSketchType, config: &PromSketchConfig) { - match stype { - PromSketchType::EHUniv => { - if self.eh_univ.is_none() { - self.eh_univ = Some(EHUnivOptimized::with_defaults( - config.eh_univ.k, - config.eh_univ.time_window, - )); - } - } - PromSketchType::EHKLL => { - if self.eh_kll.is_none() { - let chapter = EHSketchList::KLL(KLL::init_kll(config.eh_kll.kll_k)); - self.eh_kll = Some(ExponentialHistogram::new( - config.eh_kll.k, - config.eh_kll.time_window, - chapter, - )); - } - } - PromSketchType::USampling => { - if self.eh_sampling.is_none() { - let chapter = - EHSketchList::UNIFORM(UniformSampling::new(config.sampling.sample_rate)); - self.eh_sampling = Some(ExponentialHistogram::new( - config.eh_kll.k, - config.sampling.time_window, - chapter, - )); - } - } - } - } - - /// Insert a data point into all active sketches. - pub fn insert(&mut self, time: u64, value: f64) { - let input = DataInput::F64(value); - - if let Some(ref mut eh) = self.eh_univ { - // EHUnivOptimized::update(time, key, frequency_count) - eh.update(time, &input, 1); - } - if let Some(ref mut eh) = self.eh_kll { - eh.update(time, &input); - } - if let Some(ref mut eh) = self.eh_sampling { - eh.update(time, &input); - } - } - - /// Check whether the sketch for the given type covers the time range. - pub fn cover(&self, stype: PromSketchType, mint: u64, maxt: u64) -> bool { - match stype { - PromSketchType::EHUniv => self.eh_univ.as_ref().is_some_and(|eh| eh.cover(mint, maxt)), - PromSketchType::EHKLL => self.eh_kll.as_ref().is_some_and(|eh| eh.cover(mint, maxt)), - PromSketchType::USampling => self - .eh_sampling - .as_ref() - .is_some_and(|eh| eh.cover(mint, maxt)), - } - } -} - -/// A single time series with its label string and associated sketch instances. -pub struct PromSketchMemSeries { - pub labels: String, - pub sketch_instances: PromSketchInstances, - /// Earliest timestamp seen for this series (-1 means uninitialized). - pub oldest_timestamp: i64, -} - -impl PromSketchMemSeries { - pub fn new(labels: String) -> Self { - Self { - labels, - sketch_instances: PromSketchInstances::new(), - oldest_timestamp: -1, - } - } - - /// Insert a data point, updating oldest_timestamp tracking. - pub fn insert(&mut self, time: u64, value: f64) { - if self.oldest_timestamp == -1 { - self.oldest_timestamp = time as i64; - } - self.sketch_instances.insert(time, value); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_ensure_initialized_creates_correct_types() { - let config = PromSketchConfig::default(); - let mut instances = PromSketchInstances::new(); - - assert!(instances.eh_univ.is_none()); - assert!(instances.eh_kll.is_none()); - assert!(instances.eh_sampling.is_none()); - - instances.ensure_initialized(PromSketchType::EHUniv, &config); - assert!(instances.eh_univ.is_some()); - assert!(instances.eh_kll.is_none()); - - instances.ensure_initialized(PromSketchType::EHKLL, &config); - assert!(instances.eh_kll.is_some()); - assert!(instances.eh_sampling.is_none()); - - instances.ensure_initialized(PromSketchType::USampling, &config); - assert!(instances.eh_sampling.is_some()); - } - - #[test] - fn test_ensure_initialized_idempotent() { - let config = PromSketchConfig::default(); - let mut instances = PromSketchInstances::new(); - - instances.ensure_initialized(PromSketchType::EHUniv, &config); - let ptr1 = instances.eh_univ.as_ref().unwrap() as *const EHUnivOptimized; - - // Calling again should not replace the instance. - instances.ensure_initialized(PromSketchType::EHUniv, &config); - let ptr2 = instances.eh_univ.as_ref().unwrap() as *const EHUnivOptimized; - assert_eq!(ptr1, ptr2); - } - - #[test] - fn test_mem_series_insert_updates_oldest() { - let mut series = PromSketchMemSeries::new("test_metric".to_string()); - assert_eq!(series.oldest_timestamp, -1); - - let config = PromSketchConfig::default(); - series - .sketch_instances - .ensure_initialized(PromSketchType::EHKLL, &config); - - series.insert(100, 1.0); - assert_eq!(series.oldest_timestamp, 100); - - series.insert(50, 2.0); - // oldest_timestamp should not change once set - assert_eq!(series.oldest_timestamp, 100); - } -} diff --git a/asap-query-engine/src/stores/promsketch_store/store.rs b/asap-query-engine/src/stores/promsketch_store/store.rs deleted file mode 100644 index 3183597a..00000000 --- a/asap-query-engine/src/stores/promsketch_store/store.rs +++ /dev/null @@ -1,456 +0,0 @@ -// use std::sync::atomic::{AtomicU64, Ordering}; -// use std::sync::RwLock; - -// use dashmap::DashMap; - -// use super::config::PromSketchConfig; -// use super::metrics; -// use super::query; -// use super::series::PromSketchMemSeries; -// use super::types::{promsketch_func_map, PromSketchType}; - -// /// Concurrent store for live per-time-series sketch instances. -// /// -// /// This is NOT a `Store` trait implementation — it stores live sketch instances -// /// for direct insert/query, not precomputed aggregation buckets. -// pub struct PromSketchStore { -// /// Concurrent outer map keyed by label string, per-series RwLock. -// series: DashMap>, -// /// Total number of distinct series. -// num_series: AtomicU64, -// /// Sketch configuration shared across all series. -// config: PromSketchConfig, -// } - -// impl PromSketchStore { -// /// Create a new store with the given configuration. -// pub fn new(config: PromSketchConfig) -> Self { -// Self { -// series: DashMap::new(), -// num_series: AtomicU64::new(0), -// config, -// } -// } - -// /// Create a new store with default configuration. -// pub fn with_default_config() -> Self { -// Self::new(PromSketchConfig::default()) -// } - -// /// Return the number of distinct series tracked. -// pub fn num_series(&self) -> u64 { -// self.num_series.load(Ordering::Relaxed) -// } - -// /// Get or create a series entry for the given labels string. -// /// Returns true if a new series was created. -// pub fn get_or_create(&self, labels: &str) -> bool { -// use dashmap::mapref::entry::Entry; -// match self.series.entry(labels.to_string()) { -// Entry::Occupied(_) => false, -// Entry::Vacant(vacant) => { -// vacant.insert(RwLock::new(PromSketchMemSeries::new(labels.to_string()))); -// self.num_series.fetch_add(1, Ordering::Relaxed); -// metrics::SERIES_TOTAL.inc(); -// true -// } -// } -// } - -// /// Initialize all 3 sketch types (EHUniv, EHKLL, USampling) for a series. -// /// -// /// Idempotent — calls `ensure_initialized` which is a no-op if already initialized. -// /// Intended for use by the raw Kafka consumer on first data arrival for a new series. -// pub fn ensure_all_sketches( -// &self, -// labels: &str, -// ) -> Result<(), Box> { -// self.get_or_create(labels); - -// let entry = self -// .series -// .get(labels) -// .ok_or("series disappeared unexpectedly")?; -// let mut series = entry.write().map_err(|e| format!("lock poisoned: {}", e))?; - -// series -// .sketch_instances -// .ensure_initialized(PromSketchType::EHUniv, &self.config); -// series -// .sketch_instances -// .ensure_initialized(PromSketchType::EHKLL, &self.config); -// series -// .sketch_instances -// .ensure_initialized(PromSketchType::USampling, &self.config); - -// Ok(()) -// } - -// /// Initialize sketch instances for a given function on a series. -// /// Creates the series if it doesn't exist, then lazily initializes -// /// whichever sketch types the function requires. -// pub fn new_sketch_cache_instance( -// &self, -// labels: &str, -// func_name: &str, -// ) -> Result<(), Box> { -// let stypes = promsketch_func_map(func_name) -// .ok_or_else(|| format!("unsupported function: {}", func_name))?; - -// self.get_or_create(labels); - -// let entry = self -// .series -// .get(labels) -// .ok_or("series disappeared unexpectedly")?; -// let mut series = entry.write().map_err(|e| format!("lock poisoned: {}", e))?; - -// for &stype in stypes { -// series -// .sketch_instances -// .ensure_initialized(stype, &self.config); -// } - -// Ok(()) -// } - -// /// Initialize sketch instances with a custom config (for overriding time windows, etc.). -// pub fn new_sketch_cache_instance_with_config( -// &self, -// labels: &str, -// func_name: &str, -// config: &PromSketchConfig, -// ) -> Result<(), Box> { -// let stypes = promsketch_func_map(func_name) -// .ok_or_else(|| format!("unsupported function: {}", func_name))?; - -// self.get_or_create(labels); - -// let entry = self -// .series -// .get(labels) -// .ok_or("series disappeared unexpectedly")?; -// let mut series = entry.write().map_err(|e| format!("lock poisoned: {}", e))?; - -// for &stype in stypes { -// series.sketch_instances.ensure_initialized(stype, config); -// } - -// Ok(()) -// } - -// /// Insert a data point into all active sketches for the given series. -// /// No-op if the series or its sketches haven't been initialized. -// pub fn sketch_insert( -// &self, -// labels: &str, -// time: u64, -// value: f64, -// ) -> Result<(), Box> { -// let entry = match self.series.get(labels) { -// Some(e) => e, -// None => return Ok(()), // No series yet — silent no-op like Go version -// }; - -// let mut series = entry.write().map_err(|e| format!("lock poisoned: {}", e))?; -// series.insert(time, value); -// Ok(()) -// } - -// /// Check whether the sketches for a given function cover the requested time range. -// pub fn lookup(&self, labels: &str, func_name: &str, mint: u64, maxt: u64) -> bool { -// let stypes = match promsketch_func_map(func_name) { -// Some(s) => s, -// None => return false, -// }; - -// let entry = match self.series.get(labels) { -// Some(e) => e, -// None => return false, -// }; - -// let series = match entry.read() { -// Ok(s) => s, -// Err(_) => return false, -// }; - -// for &stype in stypes { -// if !series.sketch_instances.cover(stype, mint, maxt) { -// return false; -// } -// } - -// true -// } - -// /// Evaluate a PromQL function on the sketches for a given series and time range. -// pub fn eval( -// &self, -// func_name: &str, -// labels: &str, -// args: f64, -// mint: u64, -// maxt: u64, -// ) -> Result> { -// let entry = self -// .series -// .get(labels) -// .ok_or_else(|| format!("series not found: {}", labels))?; - -// let series = entry.read().map_err(|e| format!("lock poisoned: {}", e))?; - -// query::eval_function(func_name, &series, args, mint, maxt) -// } - -// /// Return the labels of all series whose labels string starts with the given metric name. -// /// If `metric_or_labels` contains `{`, matches exactly; otherwise matches series -// /// whose labels string starts with `metric_or_labels`. -// pub fn matching_series_labels(&self, metric_or_labels: &str) -> Vec { -// let mut matched = Vec::new(); -// for entry in self.series.iter() { -// let key = entry.key(); -// if key == metric_or_labels -// || key.starts_with(&format!("{}{}", metric_or_labels, "{")) -// || key.starts_with(&format!("{},", metric_or_labels)) -// { -// matched.push(key.clone()); -// } -// } -// // If nothing matched by prefix, try exact match (the metric_or_labels IS the full key) -// if matched.is_empty() && self.series.contains_key(metric_or_labels) { -// matched.push(metric_or_labels.to_string()); -// } -// matched -// } - -// /// Evaluate a function across all series whose labels match the given metric/labels key, -// /// returning results keyed by full labels string. -// pub fn eval_matching( -// &self, -// func_name: &str, -// metric_or_labels: &str, -// args: f64, -// mint: u64, -// maxt: u64, -// ) -> Result, Box> { -// let matched_labels = self.matching_series_labels(metric_or_labels); -// let mut results = Vec::new(); -// for labels in matched_labels { -// match self.eval(func_name, &labels, args, mint, maxt) { -// Ok(value) => results.push((labels, value)), -// Err(e) => { -// tracing::debug!("Skipping series {}: {}", labels, e); -// } -// } -// } -// Ok(results) -// } -// } - -// #[cfg(test)] -// mod tests { -// use super::*; - -// #[test] -// fn test_get_or_create() { -// let store = PromSketchStore::with_default_config(); -// assert_eq!(store.num_series(), 0); - -// let created = store.get_or_create("metric{host=\"a\"}"); -// assert!(created); -// assert_eq!(store.num_series(), 1); - -// // Second call should not create -// let created = store.get_or_create("metric{host=\"a\"}"); -// assert!(!created); -// assert_eq!(store.num_series(), 1); - -// // Different labels should create -// let created = store.get_or_create("metric{host=\"b\"}"); -// assert!(created); -// assert_eq!(store.num_series(), 2); -// } - -// #[test] -// fn test_new_sketch_cache_instance() { -// let store = PromSketchStore::with_default_config(); -// let result = store.new_sketch_cache_instance("m1", "quantile_over_time"); -// assert!(result.is_ok()); -// assert_eq!(store.num_series(), 1); - -// // Verify EHKLL was initialized -// let entry = store.series.get("m1").unwrap(); -// let series = entry.read().unwrap(); -// assert!(series.sketch_instances.eh_kll.is_some()); -// assert!(series.sketch_instances.eh_univ.is_none()); -// assert!(series.sketch_instances.eh_sampling.is_none()); -// } - -// #[test] -// fn test_insert_lookup_eval_roundtrip() { -// let store = PromSketchStore::with_default_config(); - -// // Initialize KLL sketches for quantile queries -// store -// .new_sketch_cache_instance("ts1", "quantile_over_time") -// .unwrap(); - -// // Insert data points -// for i in 1..=100u64 { -// store.sketch_insert("ts1", i, i as f64).unwrap(); -// } - -// // Lookup should succeed -// assert!(store.lookup("ts1", "quantile_over_time", 1, 100)); - -// // Eval median -// let result = store.eval("quantile_over_time", "ts1", 0.5, 1, 100); -// assert!(result.is_ok()); -// let val = result.unwrap(); -// assert!(val > 30.0 && val < 70.0, "median was {}", val); -// } - -// #[test] -// fn test_insert_noop_for_unknown_series() { -// let store = PromSketchStore::with_default_config(); -// // Insert into non-existent series should be a no-op -// let result = store.sketch_insert("nonexistent", 1, 1.0); -// assert!(result.is_ok()); -// } - -// #[test] -// fn test_lookup_returns_false_for_missing() { -// let store = PromSketchStore::with_default_config(); -// assert!(!store.lookup("missing", "quantile_over_time", 1, 100)); -// } - -// #[test] -// fn test_multiple_sketch_types_on_same_series() { -// let store = PromSketchStore::with_default_config(); - -// store -// .new_sketch_cache_instance("ts1", "quantile_over_time") -// .unwrap(); -// store -// .new_sketch_cache_instance("ts1", "entropy_over_time") -// .unwrap(); -// store -// .new_sketch_cache_instance("ts1", "avg_over_time") -// .unwrap(); - -// let entry = store.series.get("ts1").unwrap(); -// let series = entry.read().unwrap(); -// assert!(series.sketch_instances.eh_kll.is_some()); -// assert!(series.sketch_instances.eh_univ.is_some()); -// assert!(series.sketch_instances.eh_sampling.is_some()); -// } - -// #[test] -// fn test_concurrent_insert_and_query() { -// use std::sync::Arc; -// use std::thread; - -// let store = Arc::new(PromSketchStore::with_default_config()); - -// // Initialize -// store -// .new_sketch_cache_instance("ts1", "quantile_over_time") -// .unwrap(); - -// let n_threads = 4; -// let n_inserts = 100; - -// // Spawn writer threads -// let mut handles = Vec::new(); -// for t in 0..n_threads { -// let store = Arc::clone(&store); -// handles.push(thread::spawn(move || { -// for i in 0..n_inserts { -// let time = (t * n_inserts + i + 1) as u64; -// store.sketch_insert("ts1", time, time as f64).unwrap(); -// } -// })); -// } - -// // Spawn a reader thread -// let store_read = Arc::clone(&store); -// handles.push(thread::spawn(move || { -// for _ in 0..50 { -// let _ = store_read.lookup("ts1", "quantile_over_time", 1, 400); -// } -// })); - -// for h in handles { -// h.join().unwrap(); -// } - -// // After all inserts, eval should work -// let result = store.eval( -// "quantile_over_time", -// "ts1", -// 0.5, -// 1, -// (n_threads * n_inserts) as u64, -// ); -// assert!(result.is_ok()); -// } - -// #[test] -// fn test_ensure_all_sketches() { -// let store = PromSketchStore::with_default_config(); -// store.ensure_all_sketches("ts_all").unwrap(); - -// assert_eq!(store.num_series(), 1); -// let entry = store.series.get("ts_all").unwrap(); -// let series = entry.read().unwrap(); -// assert!(series.sketch_instances.eh_univ.is_some()); -// assert!(series.sketch_instances.eh_kll.is_some()); -// assert!(series.sketch_instances.eh_sampling.is_some()); -// } - -// #[test] -// fn test_ensure_all_sketches_idempotent() { -// let store = PromSketchStore::with_default_config(); -// store.ensure_all_sketches("ts_idem").unwrap(); -// store.ensure_all_sketches("ts_idem").unwrap(); // second call should be no-op - -// assert_eq!(store.num_series(), 1); -// let entry = store.series.get("ts_idem").unwrap(); -// let series = entry.read().unwrap(); -// assert!(series.sketch_instances.eh_univ.is_some()); -// assert!(series.sketch_instances.eh_kll.is_some()); -// assert!(series.sketch_instances.eh_sampling.is_some()); -// } - -// #[test] -// fn test_auto_init_insert_query_roundtrip() { -// let store = PromSketchStore::with_default_config(); - -// // Auto-init all sketches (as the raw consumer would do) -// store.ensure_all_sketches("ts_rt").unwrap(); - -// // Insert data points -// for i in 1..=100u64 { -// store.sketch_insert("ts_rt", i, i as f64).unwrap(); -// } - -// // Verify quantile (EHKLL) -// assert!(store.lookup("ts_rt", "quantile_over_time", 1, 100)); -// let val = store -// .eval("quantile_over_time", "ts_rt", 0.5, 1, 100) -// .unwrap(); -// assert!(val > 30.0 && val < 70.0, "median was {}", val); - -// // Verify avg (USampling) -// assert!(store.lookup("ts_rt", "avg_over_time", 1, 100)); -// let avg = store.eval("avg_over_time", "ts_rt", 0.0, 1, 100).unwrap(); -// assert!(avg > 30.0 && avg < 70.0, "avg was {}", avg); - -// // Verify entropy (EHUniv) -// assert!(store.lookup("ts_rt", "entropy_over_time", 1, 100)); -// let entropy = store -// .eval("entropy_over_time", "ts_rt", 0.0, 1, 100) -// .unwrap(); -// assert!(entropy >= 0.0, "entropy was {}", entropy); -// } -// } diff --git a/asap-query-engine/src/stores/promsketch_store/types.rs b/asap-query-engine/src/stores/promsketch_store/types.rs deleted file mode 100644 index e663e1b0..00000000 --- a/asap-query-engine/src/stores/promsketch_store/types.rs +++ /dev/null @@ -1,75 +0,0 @@ -/// The sketch types supported by PromSketch. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum PromSketchType { - /// ExponentialHistogram wrapping UnivMon (entropy, cardinality, L1, L2, distinct). - EHUniv, - /// ExponentialHistogram wrapping KLL (quantile, min, max). - EHKLL, - /// ExponentialHistogram wrapping UniformSampling (avg, count, sum, stddev, stdvar). - USampling, -} - -/// Maps a PromQL function name to the sketch types it requires. -pub fn promsketch_func_map(func_name: &str) -> Option<&'static [PromSketchType]> { - match func_name { - "entropy_over_time" => Some(&[PromSketchType::EHUniv]), - "distinct_over_time" => Some(&[PromSketchType::EHUniv]), - "l1_over_time" => Some(&[PromSketchType::EHUniv]), - "l2_over_time" => Some(&[PromSketchType::EHUniv]), - "quantile_over_time" => Some(&[PromSketchType::EHKLL]), - "min_over_time" => Some(&[PromSketchType::EHKLL]), - "max_over_time" => Some(&[PromSketchType::EHKLL]), - "avg_over_time" => Some(&[PromSketchType::USampling]), - "count_over_time" => Some(&[PromSketchType::USampling]), - "sum_over_time" => Some(&[PromSketchType::USampling]), - "sum2_over_time" => Some(&[PromSketchType::USampling]), - "stddev_over_time" => Some(&[PromSketchType::USampling]), - "stdvar_over_time" => Some(&[PromSketchType::USampling]), - _ => None, - } -} - -/// Returns `true` when `func_name` maps to a USampling-backed sketch function. -pub fn is_usampling_function(func_name: &str) -> bool { - matches!(promsketch_func_map(func_name), - Some(types) if types.contains(&PromSketchType::USampling)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_promsketch_func_map_coverage() { - assert_eq!( - promsketch_func_map("entropy_over_time"), - Some([PromSketchType::EHUniv].as_slice()) - ); - assert_eq!( - promsketch_func_map("quantile_over_time"), - Some([PromSketchType::EHKLL].as_slice()) - ); - assert_eq!( - promsketch_func_map("avg_over_time"), - Some([PromSketchType::USampling].as_slice()) - ); - assert!(promsketch_func_map("nonexistent").is_none()); - } - - #[test] - fn test_is_usampling_function() { - // USampling functions - assert!(is_usampling_function("avg_over_time")); - assert!(is_usampling_function("count_over_time")); - assert!(is_usampling_function("sum_over_time")); - assert!(is_usampling_function("sum2_over_time")); - assert!(is_usampling_function("stddev_over_time")); - assert!(is_usampling_function("stdvar_over_time")); - - // Non-USampling functions - assert!(!is_usampling_function("entropy_over_time")); - assert!(!is_usampling_function("quantile_over_time")); - assert!(!is_usampling_function("min_over_time")); - assert!(!is_usampling_function("nonexistent")); - } -} diff --git a/controller/src/emit/stage_config.rs b/controller/src/emit/stage_config.rs index 3c17c67e..23e0d12c 100644 --- a/controller/src/emit/stage_config.rs +++ b/controller/src/emit/stage_config.rs @@ -588,7 +588,7 @@ pub fn emit_backend_storage_routing( } /// Tenant id used when the deploy is single-tenant. Mirrors the -/// backend's `crate::routing::DEFAULT_TENANT` (defined in +/// backend's `crate::query_engines::routing::DEFAULT_TENANT` (defined in /// `ASAPQuery-backend/asap-query-engine/src/routing/backend_storage_routing.rs`) /// — kept as a literal here so the controller doesn't take a build-time /// dependency on the backend crate just for one constant. @@ -2121,7 +2121,7 @@ mod tests { // The archive-only L3 intents (Absent, Present, Delta, Deriv, …) // bind to `SketchExpr::Logical` rather than producing a `BackendAggregation`, // so they correctly stay OUT of the warm-tier StreamingConfig the - // backend's SimpleEngine receives. Phase α wires the archive routing + // backend's ASAPQueryEngine receives. Phase α wires the archive routing // entry separately. This snapshot pins that contract. /// Snapshot: an empty `BackendStageConfig` produces the canonical diff --git a/controller/src/optimizer/cost/wire.rs b/controller/src/optimizer/cost/wire.rs index 7e8d9106..f260e8ec 100644 --- a/controller/src/optimizer/cost/wire.rs +++ b/controller/src/optimizer/cost/wire.rs @@ -7,7 +7,7 @@ //! //! | Mode | Edge action | Wire | Backend role | Accuracy | //! |-----------------------------------|----------------------------|------------------------------|------------------------------|-----------------| -//! | [`BindMode::SketchAtEdge`] | sketch processor at edge | edge → backend OTLP | SimpleEngine over sketch | bounded ε > 0 | +//! | [`BindMode::SketchAtEdge`] | sketch processor at edge | edge → backend OTLP | ASAPQueryEngine over sketch | bounded ε > 0 | //! | [`BindMode::RawAtEdgeSketchAtBackend`] | no sketch processor | edge → backend OTLP (raw) | builds sketches at ingest | bounded ε > 0 | //! | [`BindMode::RawAtEdgePrometheusArchive`] | no sketch processor | edge → Prometheus OTLP HTTP | backend HTTP-forwards queries | exact (ε = 0) | //! diff --git a/crates/asap_types/src/streaming_config.rs b/crates/asap_types/src/streaming_config.rs index ad0353cc..63467a9b 100644 --- a/crates/asap_types/src/streaming_config.rs +++ b/crates/asap_types/src/streaming_config.rs @@ -20,7 +20,7 @@ pub struct StreamingConfig { /// per-metric runtime config. The controller pushes this when planning /// (see `docs/design-gorilla-s3-cold-engine.md` §8); pre-Phase-5 /// configs decode with `#[serde(default)]` to `SketchStore` so - /// existing deploys keep dispatching to `SimpleEngine`. + /// existing deploys keep dispatching to `ASAPQueryEngine`. #[serde(default)] pub storage_backend: StorageBackend, } @@ -162,7 +162,7 @@ mod tests { /// Pre-Phase-5 deploys serialize `StreamingConfig` without the /// `storage_backend` field; deserialize must default to `SketchStore` - /// so the router keeps dispatching to `SimpleEngine` unchanged. + /// so the router keeps dispatching to `ASAPQueryEngine` unchanged. #[test] fn deserialize_legacy_yaml_defaults_to_warm_tier() { let yaml = "{\"aggregation_configs\":{}}"; diff --git a/asap-query-engine/.cargo/config.toml b/data_plane/.cargo/config.toml similarity index 100% rename from asap-query-engine/.cargo/config.toml rename to data_plane/.cargo/config.toml diff --git a/asap-query-engine/.gitignore b/data_plane/.gitignore similarity index 100% rename from asap-query-engine/.gitignore rename to data_plane/.gitignore diff --git a/asap-query-engine/Cargo.toml b/data_plane/Cargo.toml similarity index 99% rename from asap-query-engine/Cargo.toml rename to data_plane/Cargo.toml index 727f52f2..a4f5f07a 100644 --- a/asap-query-engine/Cargo.toml +++ b/data_plane/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "query_engine_rust" +name = "data_plane" version.workspace = true edition.workspace = true diff --git a/asap-query-engine/Dockerfile b/data_plane/Dockerfile similarity index 67% rename from asap-query-engine/Dockerfile rename to data_plane/Dockerfile index 77b5048a..ecc83f8e 100644 --- a/asap-query-engine/Dockerfile +++ b/data_plane/Dockerfile @@ -19,23 +19,23 @@ COPY crates ./crates COPY Cargo.toml ./ COPY Cargo.lock ./ -COPY asap-query-engine/Cargo.toml ./asap-query-engine/ +COPY data_plane/Cargo.toml ./data_plane/ # Create dummy source files so Cargo can resolve all workspace members # All explicit [[bin]] targets in Cargo.toml must have stubs here for the dependency cache layer -RUN mkdir -p asap-query-engine/src/bin \ - && echo "fn main() {}" > asap-query-engine/src/main.rs \ - && echo "fn main() {}" > asap-query-engine/src/bin/test_e2e_precompute.rs \ - && echo "fn main() {}" > asap-query-engine/src/bin/bench_precompute_sketch.rs \ - && echo "fn main() {}" > asap-query-engine/src/bin/e2e_quickstart_resource_test.rs \ - && mkdir -p asap-query-engine/benches && echo "fn main() {}" > asap-query-engine/benches/simple_store_bench.rs +RUN mkdir -p data_plane/src/bin \ + && echo "fn main() {}" > data_plane/src/main.rs \ + && echo "fn main() {}" > data_plane/src/bin/test_e2e_precompute.rs \ + && echo "fn main() {}" > data_plane/src/bin/bench_precompute_sketch.rs \ + && echo "fn main() {}" > data_plane/src/bin/e2e_quickstart_resource_test.rs \ + && mkdir -p data_plane/benches && echo "fn main() {}" > data_plane/benches/simple_store_bench.rs # Build dependencies (this layer will be cached) -WORKDIR /code/asap-query-engine +WORKDIR /code/data_plane RUN cargo build --release && rm -rf src/ # Copy source code -COPY asap-query-engine/src ./src +COPY data_plane/src ./src # Build the actual application RUN touch src/main.rs && cargo build --release @@ -53,7 +53,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # Copy the built binary -COPY --from=builder /code/target/release/query_engine_rust /usr/local/bin/query_engine_rust +COPY --from=builder /code/target/release/data_plane /usr/local/bin/data_plane # Expose the HTTP server port EXPOSE 8088 @@ -62,4 +62,4 @@ EXPOSE 8088 # This allows writing to mounted volumes without permission issues # Use ENTRYPOINT to allow passing command line arguments -ENTRYPOINT ["query_engine_rust"] +ENTRYPOINT ["data_plane"] diff --git a/asap-query-engine/LICENSE b/data_plane/LICENSE similarity index 100% rename from asap-query-engine/LICENSE rename to data_plane/LICENSE diff --git a/asap-query-engine/benches/simple_store_bench.rs b/data_plane/benches/simple_store_bench.rs similarity index 98% rename from asap-query-engine/benches/simple_store_bench.rs rename to data_plane/benches/simple_store_bench.rs index a7969b15..76d786c9 100644 --- a/asap-query-engine/benches/simple_store_bench.rs +++ b/data_plane/benches/simple_store_bench.rs @@ -20,22 +20,22 @@ //! - `kll` — `DatasketchesKLLAccumulator` k=200 (~1 KB sketch, realistic clone cost) //! //! Run with: -//! cargo bench -p query_engine_rust --bench simple_store_bench +//! cargo bench -p data_plane --bench simple_store_bench //! //! Results land in `target/criterion/`. use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use promql_utilities::data_model::KeyByLabelNames; -use query_engine_rust::data_model::{ +use data_plane::stores::schema::{ AggregateCore, AggregationType, CleanupPolicy, KeyByLabelValues, LockStrategy, StreamingConfig, WindowType, }; -use query_engine_rust::precompute_operators::{DatasketchesKLLAccumulator, SumAccumulator}; -use query_engine_rust::stores::sketch_db::simple_map_store::legacy::{ +use data_plane::precompute_engine::operators::{DatasketchesKLLAccumulator, SumAccumulator}; +use data_plane::stores::sketch_db::simple_map_store::legacy::{ LegacySimpleMapStoreGlobal, LegacySimpleMapStorePerKey, }; -use query_engine_rust::stores::Store; -use query_engine_rust::{AggregationConfig, PrecomputedOutput, SimpleMapStore}; +use data_plane::stores::Store; +use data_plane::{AggregationConfig, PrecomputedOutput, SimpleMapStore}; use std::collections::HashMap; use std::sync::{Arc, Barrier}; diff --git a/asap-query-engine/docker-compose.yml.j2 b/data_plane/docker-compose.yml.j2 similarity index 100% rename from asap-query-engine/docker-compose.yml.j2 rename to data_plane/docker-compose.yml.j2 diff --git a/asap-query-engine/docs/README.md b/data_plane/docs/README.md similarity index 100% rename from asap-query-engine/docs/README.md rename to data_plane/docs/README.md diff --git a/asap-query-engine/docs/adding-fallback-backend.md b/data_plane/docs/adding-fallback-backend.md similarity index 100% rename from asap-query-engine/docs/adding-fallback-backend.md rename to data_plane/docs/adding-fallback-backend.md diff --git a/asap-query-engine/docs/adding-protocol-adapter.md b/data_plane/docs/adding-protocol-adapter.md similarity index 100% rename from asap-query-engine/docs/adding-protocol-adapter.md rename to data_plane/docs/adding-protocol-adapter.md diff --git a/asap-query-engine/docs/adding-protocol-server.md b/data_plane/docs/adding-protocol-server.md similarity index 100% rename from asap-query-engine/docs/adding-protocol-server.md rename to data_plane/docs/adding-protocol-server.md diff --git a/asap-query-engine/docs/promsketch-integration.md b/data_plane/docs/promsketch-integration.md similarity index 100% rename from asap-query-engine/docs/promsketch-integration.md rename to data_plane/docs/promsketch-integration.md diff --git a/asap-query-engine/examples/promql/inference_config.yaml b/data_plane/examples/promql/inference_config.yaml similarity index 100% rename from asap-query-engine/examples/promql/inference_config.yaml rename to data_plane/examples/promql/inference_config.yaml diff --git a/asap-query-engine/examples/promql/streaming_config.yaml b/data_plane/examples/promql/streaming_config.yaml similarity index 100% rename from asap-query-engine/examples/promql/streaming_config.yaml rename to data_plane/examples/promql/streaming_config.yaml diff --git a/asap-query-engine/query-engine-rust-cli-compose.yml.j2 b/data_plane/query-engine-rust-cli-compose.yml.j2 similarity index 100% rename from asap-query-engine/query-engine-rust-cli-compose.yml.j2 rename to data_plane/query-engine-rust-cli-compose.yml.j2 diff --git a/asap-query-engine/rustfmt.toml b/data_plane/rustfmt.toml similarity index 100% rename from asap-query-engine/rustfmt.toml rename to data_plane/rustfmt.toml diff --git a/asap-query-engine/src/drivers/controller_client.rs b/data_plane/src/drivers/controller_client/config_fetcher.rs similarity index 100% rename from asap-query-engine/src/drivers/controller_client.rs rename to data_plane/src/drivers/controller_client/config_fetcher.rs diff --git a/asap-query-engine/src/drivers/query/controller_client.rs b/data_plane/src/drivers/controller_client/miss_notifier.rs similarity index 98% rename from asap-query-engine/src/drivers/query/controller_client.rs rename to data_plane/src/drivers/controller_client/miss_notifier.rs index 3d9dcfa9..298f2d07 100644 --- a/asap-query-engine/src/drivers/query/controller_client.rs +++ b/data_plane/src/drivers/controller_client/miss_notifier.rs @@ -1,7 +1,7 @@ //! Client for notifying the DataCollector controller of query-side //! capability misses — the query plane side of PR G. //! -//! When `SimpleEngine` fails to match a query against any stored +//! When `ASAPQueryEngine` fails to match a query against any stored //! aggregation (`find_compatible_aggregation` returns `None`), it //! fires a fire-and-forget notification to the configured controller //! with the `QueryRequirements` that failed to match. The controller @@ -210,7 +210,7 @@ mod tests { assert!(json.contains("\"grouping_labels\":[\"service\"]")); } - /// Mock client that records calls, used by SimpleEngine unit + /// Mock client that records calls, used by ASAPQueryEngine unit /// tests in other modules to verify fire-and-forget wiring /// without needing an HTTP mock server. pub struct MockControllerClient { diff --git a/data_plane/src/drivers/controller_client/mod.rs b/data_plane/src/drivers/controller_client/mod.rs new file mode 100644 index 00000000..7eefe07a --- /dev/null +++ b/data_plane/src/drivers/controller_client/mod.rs @@ -0,0 +1,18 @@ +//! Controller-client surface. +//! +//! Two cooperating submodules — bidirectional plumbing between the +//! data plane and the control plane (controller crate): +//! +//! * [`config_fetcher`] — outbound `GET /api/v1/plan/:metric` polling. +//! Fetches plan config from the controller. Optional; today's +//! binary path does not consume it. +//! * [`miss_notifier`] — outbound capability-miss notifications. The +//! warm-tier engine fires fire-and-forget POSTs here when a query +//! has no compatible stored aggregation, so the controller can +//! generate a new sketch plan and push it back via the streaming +//! config endpoint. + +pub mod config_fetcher; +pub mod miss_notifier; + +pub use miss_notifier::{spawn_capability_miss_notify, ControllerClient, HttpControllerClient}; diff --git a/asap-query-engine/src/drivers/ingest/kafka.rs b/data_plane/src/drivers/ingest/kafka.rs similarity index 96% rename from asap-query-engine/src/drivers/ingest/kafka.rs rename to data_plane/src/drivers/ingest/kafka.rs index d9c3bf89..ce962a9b 100644 --- a/asap-query-engine/src/drivers/ingest/kafka.rs +++ b/data_plane/src/drivers/ingest/kafka.rs @@ -5,10 +5,10 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use tracing::{debug, error, info, warn}; -use crate::data_model::enums::{InputFormat, StreamingEngine}; -use crate::data_model::traits::SerializableToSink; -use crate::data_model::PrecomputedOutput; -use crate::data_model::StreamingConfig; +use crate::stores::schema::enums::{InputFormat, StreamingEngine}; +use crate::stores::schema::traits::SerializableToSink; +use crate::stores::schema::PrecomputedOutput; +use crate::stores::schema::StreamingConfig; use crate::stores::Store; use crate::utils::PrecomputeDumper; @@ -118,7 +118,7 @@ impl KafkaConsumer { // Check if this is an empty DeltaSetAggregator and skip it if let Some(delta_acc) = precompute_accumulator .as_any() - .downcast_ref::() + .downcast_ref::() { if delta_acc.is_empty() { debug!("Skipping empty DeltaSetAggregatorAccumulator"); @@ -182,7 +182,7 @@ impl KafkaConsumer { async fn process_batch( &self, - batch: &mut Vec<(PrecomputedOutput, Box)>, + batch: &mut Vec<(PrecomputedOutput, Box)>, ) -> Result<(), Box> { if batch.is_empty() { return Ok(()); @@ -230,7 +230,7 @@ impl KafkaConsumer { &self, message: &rdkafka::message::BorrowedMessage<'_>, ) -> Result< - Option<(PrecomputedOutput, Box)>, + Option<(PrecomputedOutput, Box)>, Box, > { let message_start_time = Instant::now(); diff --git a/asap-query-engine/src/drivers/ingest/mod.rs b/data_plane/src/drivers/ingest/mod.rs similarity index 100% rename from asap-query-engine/src/drivers/ingest/mod.rs rename to data_plane/src/drivers/ingest/mod.rs diff --git a/asap-query-engine/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs similarity index 99% rename from asap-query-engine/src/drivers/ingest/otel.rs rename to data_plane/src/drivers/ingest/otel.rs index cccea2b4..24653a90 100644 --- a/asap-query-engine/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -24,11 +24,11 @@ use std::collections::HashMap; use std::io::Read; -use crate::data_model::AggregateCore; +use crate::stores::schema::AggregateCore; use crate::precompute_engine::series_router::WorkerMessage; use crate::precompute_engine::IngestState; -use crate::precompute_operators::sketch_envelope_accumulator::SketchEnvelopeAccumulator; -use crate::routing::FreshnessProbeCache; +use crate::precompute_engine::operators::sketch_envelope_accumulator::SketchEnvelopeAccumulator; +use crate::query_engines::routing::FreshnessProbeCache; use asap_otel_proto::tonic::collector::metrics::v1::{ metrics_service_server::MetricsService, ExportMetricsServiceRequest, ExportMetricsServiceResponse, @@ -1207,7 +1207,7 @@ fn decode_modified_otlp_sketch_bytes( encoding: i32, bytes: &[u8], ) -> Result, Box> { - use crate::precompute_operators::{ + use crate::precompute_engine::operators::{ CountMinSketchAccumulator, CountSketchAccumulator, DDSketchAccumulator, DatasketchesKLLAccumulator, HllSketchAccumulator, }; @@ -1238,7 +1238,7 @@ fn decode_modified_otlp_sketch_bytes( // lands those three sketches keep using the backend's // existing per-accumulator decoder. SketchKind::DdSketch => { - use crate::precompute_operators::edge_runtime_adapter::{ + use crate::precompute_engine::operators::edge_runtime_adapter::{ reconstruct_via_runtime, ReconstructedSketch, SketchType as RtSketchType, }; // Prefer the asap-precompute-rs runtime path (envelope- @@ -1262,7 +1262,7 @@ fn decode_modified_otlp_sketch_bytes( } } SketchKind::Kll => { - use crate::precompute_operators::edge_runtime_adapter::{ + use crate::precompute_engine::operators::edge_runtime_adapter::{ reconstruct_via_runtime, ReconstructedSketch, SketchType as RtSketchType, }; // Same envelope-vs-bare-state handling as DDSketch above. @@ -1358,7 +1358,7 @@ pub(crate) fn apply_modified_otlp_delta_bytes( existing: &mut Box, bytes: &[u8], ) -> Result<(), Box> { - use crate::precompute_operators::{ + use crate::precompute_engine::operators::{ CountMinSketchAccumulator, CountSketchAccumulator, DDSketchAccumulator, HllSketchAccumulator, }; @@ -1763,8 +1763,8 @@ fn attributes_to_map( #[cfg(test)] mod dispatcher_tests { use super::*; - use crate::data_model::AggregateCore; - use crate::precompute_operators::{DDSketchAccumulator, HllSketchAccumulator}; + use crate::stores::schema::AggregateCore; + use crate::precompute_engine::operators::{DDSketchAccumulator, HllSketchAccumulator}; use asap_sketchlib::sketches::ddsketch::DdSketch; use asap_sketchlib::sketches::hll::HllVariant; @@ -1888,7 +1888,7 @@ mod dispatcher_tests { #[cfg(test)] mod sid_resolution_tests { use super::*; - use crate::data_model::{HotReloadStreamingConfig, StreamingConfig}; + use crate::stores::schema::{HotReloadStreamingConfig, StreamingConfig}; use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::series_router::SeriesRouter; use crate::stores::sketch_db::SchemaRegistry; diff --git a/asap-query-engine/src/drivers/ingest/series_resolver.rs b/data_plane/src/drivers/ingest/series_resolver.rs similarity index 100% rename from asap-query-engine/src/drivers/ingest/series_resolver.rs rename to data_plane/src/drivers/ingest/series_resolver.rs diff --git a/asap-query-engine/src/drivers/mod.rs b/data_plane/src/drivers/mod.rs similarity index 72% rename from asap-query-engine/src/drivers/mod.rs rename to data_plane/src/drivers/mod.rs index 926e4582..b58936d4 100644 --- a/asap-query-engine/src/drivers/mod.rs +++ b/data_plane/src/drivers/mod.rs @@ -3,6 +3,6 @@ pub mod ingest; pub mod query; // Re-export commonly used types for convenience -pub use controller_client::{ControllerClient, ControllerClientConfig}; +pub use controller_client::{spawn_capability_miss_notify, ControllerClient, HttpControllerClient}; pub use ingest::{KafkaConsumer, KafkaConsumerConfig, OtlpReceiver, OtlpReceiverConfig}; pub use query::{AdapterConfig, HttpServer, HttpServerConfig}; diff --git a/asap-query-engine/src/drivers/query/adapters/config.rs b/data_plane/src/drivers/query/adapters/config.rs similarity index 96% rename from asap-query-engine/src/drivers/query/adapters/config.rs rename to data_plane/src/drivers/query/adapters/config.rs index 3ae5296f..2562c9cd 100644 --- a/asap-query-engine/src/drivers/query/adapters/config.rs +++ b/data_plane/src/drivers/query/adapters/config.rs @@ -1,4 +1,4 @@ -use crate::data_model::enums::{QueryLanguage, QueryProtocol}; +use crate::stores::schema::enums::{QueryLanguage, QueryProtocol}; use crate::drivers::query::fallback::FallbackClient; use std::sync::Arc; diff --git a/asap-query-engine/src/drivers/query/adapters/factory.rs b/data_plane/src/drivers/query/adapters/factory.rs similarity index 89% rename from asap-query-engine/src/drivers/query/adapters/factory.rs rename to data_plane/src/drivers/query/adapters/factory.rs index 5d132e8e..e28aed42 100644 --- a/asap-query-engine/src/drivers/query/adapters/factory.rs +++ b/data_plane/src/drivers/query/adapters/factory.rs @@ -1,7 +1,7 @@ use super::config::AdapterConfig; use super::prometheus_http::PrometheusHttpAdapter; use super::traits::HttpProtocolAdapter; -use crate::data_model::enums::QueryProtocol; +use crate::stores::schema::enums::QueryProtocol; use std::sync::Arc; /// Factory function to create appropriate HTTP adapter based on protocol diff --git a/asap-query-engine/src/drivers/query/adapters/mod.rs b/data_plane/src/drivers/query/adapters/mod.rs similarity index 100% rename from asap-query-engine/src/drivers/query/adapters/mod.rs rename to data_plane/src/drivers/query/adapters/mod.rs diff --git a/asap-query-engine/src/drivers/query/adapters/prometheus_http.rs b/data_plane/src/drivers/query/adapters/prometheus_http.rs similarity index 99% rename from asap-query-engine/src/drivers/query/adapters/prometheus_http.rs rename to data_plane/src/drivers/query/adapters/prometheus_http.rs index dcef608e..0cd98d0b 100644 --- a/asap-query-engine/src/drivers/query/adapters/prometheus_http.rs +++ b/data_plane/src/drivers/query/adapters/prometheus_http.rs @@ -1,6 +1,6 @@ use super::config::AdapterConfig; use super::traits::*; -use crate::engines::QueryResult; +use crate::query_engines::QueryResult; use crate::utils::http::{convert_query_result_to_prometheus, convert_range_result_to_prometheus}; use async_trait::async_trait; use axum::{ @@ -411,7 +411,7 @@ impl HttpProtocolAdapter for PrometheusHttpAdapter { #[cfg(test)] mod tests { use super::*; - use crate::data_model::enums::{QueryLanguage, QueryProtocol}; + use crate::stores::schema::enums::{QueryLanguage, QueryProtocol}; fn create_test_adapter() -> PrometheusHttpAdapter { let config = AdapterConfig::new(QueryProtocol::PrometheusHttp, QueryLanguage::promql, None); diff --git a/asap-query-engine/src/drivers/query/adapters/traits.rs b/data_plane/src/drivers/query/adapters/traits.rs similarity index 99% rename from asap-query-engine/src/drivers/query/adapters/traits.rs rename to data_plane/src/drivers/query/adapters/traits.rs index 44031b62..28b7dba0 100644 --- a/asap-query-engine/src/drivers/query/adapters/traits.rs +++ b/data_plane/src/drivers/query/adapters/traits.rs @@ -9,7 +9,7 @@ use promql_utilities::data_model::KeyByLabelNames; use serde_json::Value; use std::collections::HashMap; -use crate::engines::QueryResult; +use crate::query_engines::QueryResult; /// Parsed query request data ready for engine processing #[derive(Debug, Clone)] diff --git a/asap-query-engine/src/drivers/query/fallback/metrics.rs b/data_plane/src/drivers/query/fallback/metrics.rs similarity index 94% rename from asap-query-engine/src/drivers/query/fallback/metrics.rs rename to data_plane/src/drivers/query/fallback/metrics.rs index 27e25aa1..e1d21502 100644 --- a/asap-query-engine/src/drivers/query/fallback/metrics.rs +++ b/data_plane/src/drivers/query/fallback/metrics.rs @@ -6,7 +6,7 @@ //! pattern: `lazy_static!` registration, one counter per signal, //! keyed by query shape so the dashboard can split by metric. //! -//! * **Hot** = the `SimpleEngine` handled the query from live +//! * **Hot** = the `ASAPQueryEngine` handled the query from live //! sketch-backed state. //! * **Cold** = the query was answered from the Gorilla archive //! tier ([`crate::stores::gorilla_object_store::GorillaQueryEngine`]). @@ -24,7 +24,7 @@ use prometheus::{register_counter_vec, CounterVec}; lazy_static! { /// Queries served by the hot (sketch) path, keyed by /// `(metric, shape)`. Incremented once per successful - /// `SimpleEngine::handle_query` that returned `Some(_)`. + /// `ASAPQueryEngine::handle_query` that returned `Some(_)`. pub static ref QUERIES_HOT_TOTAL: CounterVec = register_counter_vec!( "queryengine_hot_queries_total", "Queries answered from the sketch (hot) path, keyed by metric + query shape", diff --git a/asap-query-engine/src/drivers/query/fallback/mod.rs b/data_plane/src/drivers/query/fallback/mod.rs similarity index 100% rename from asap-query-engine/src/drivers/query/fallback/mod.rs rename to data_plane/src/drivers/query/fallback/mod.rs diff --git a/asap-query-engine/src/drivers/query/fallback/prometheus.rs b/data_plane/src/drivers/query/fallback/prometheus.rs similarity index 100% rename from asap-query-engine/src/drivers/query/fallback/prometheus.rs rename to data_plane/src/drivers/query/fallback/prometheus.rs diff --git a/data_plane/src/drivers/query/mod.rs b/data_plane/src/drivers/query/mod.rs new file mode 100644 index 00000000..c776451a --- /dev/null +++ b/data_plane/src/drivers/query/mod.rs @@ -0,0 +1,14 @@ +pub mod adapters; +pub mod fallback; +pub mod servers; + +// Re-export commonly used types for convenience +pub use adapters::{create_http_adapter, AdapterConfig, HttpProtocolAdapter}; +pub use fallback::FallbackClient; +pub use servers::{HttpServer, HttpServerConfig}; + +// `controller_client` moved up one level in the 2026-05 reorg +// (drivers/query/controller_client.rs → drivers/controller_client/miss_notifier.rs). +// Existing import sites that say `drivers::query::controller_client::X` get +// a redirect alias so the cutover doesn't touch every caller in one PR. +pub use crate::drivers::controller_client; diff --git a/asap-query-engine/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs similarity index 96% rename from asap-query-engine/src/drivers/query/servers/http.rs rename to data_plane/src/drivers/query/servers/http.rs index a3519c3d..cb2aa277 100644 --- a/asap-query-engine/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -16,8 +16,8 @@ use tracing::{debug, info, warn}; use crate::drivers::query::adapters::{create_http_adapter, AdapterConfig, HttpProtocolAdapter}; use crate::drivers::query::servers::metrics as srv_metrics; -use crate::engines::ASAPQueryEngine; -use crate::routing::{EngineRouter, EngineRouterError, FreshnessProbeCache, QueryEngine}; +use crate::query_engines::ASAPQueryEngine; +use crate::query_engines::routing::{EngineRouter, EngineRouterError, FreshnessProbeCache, QueryEngine}; use crate::stores::Store; use asap_types::{AccuracyTarget, StorageBackend}; use promql_utilities::query_logics::enums::Statistic; @@ -125,7 +125,7 @@ pub const ENGINE_OVERRIDE_QUERY_PARAM: &str = "engine"; pub const TENANT_HEADER: &str = "X-ASAP-Tenant"; /// Extract the tenant id from the per-request `X-ASAP-Tenant` -/// header, or fall back to [`crate::routing::DEFAULT_TENANT`] when +/// header, or fall back to [`crate::query_engines::routing::DEFAULT_TENANT`] when /// the header is missing / empty / not valid UTF-8. Used by the /// instant-query and range-query handlers to scope per-tenant /// routing-table lookup. @@ -135,7 +135,7 @@ fn extract_tenant(headers: &axum::http::HeaderMap) -> String { .and_then(|v| v.to_str().ok()) .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) - .unwrap_or_else(|| crate::routing::DEFAULT_TENANT.to_string()) + .unwrap_or_else(|| crate::query_engines::routing::DEFAULT_TENANT.to_string()) } #[derive(Debug, Clone)] @@ -150,20 +150,20 @@ pub struct HttpServer { config: HttpServerConfig, query_engine: Arc, /// Phase-5/6 capability router. Built from `query_engine` at - /// construction time (`SimpleEngine` registered as the warm-tier + /// construction time (`ASAPQueryEngine` registered as the warm-tier /// `QueryEngine`) and extended via [`Self::with_query_engine`] — /// e.g. to plug in a `GorillaQueryEngine` for the cold archive /// tier. Instant-query dispatch consults this for metrics whose /// `StreamingConfig::storage_backend()` is anything other than /// `SketchStore`; warm-tier queries still take the direct - /// `SimpleEngine::handle_query` path so they keep the + /// `ASAPQueryEngine::handle_query` path so they keep the /// `KeyByLabelNames` Prometheus needs to populate the `metric` /// map. See `docs/design-gorilla-s3-cold-engine.md` §8. query_router: Arc, store: Arc, /// Hot-reloadable `StreamingConfig` source. `None` when hot-reload /// is not wired up by the caller (unit tests, legacy binaries). - hot_reload_config: Option, + hot_reload_config: Option, /// Per-metric storage-backend routing table consulted by the HTTP /// instant-query handler at request time. When `Some(..)` and the /// query parses, the handler extracts the metric name from the @@ -183,7 +183,7 @@ pub struct HttpServer { /// existing read path snapshots the wrapper once per request /// (`handle.snapshot().lookup_with_shape(...)`); swap is observed /// by the next request without restart. - backend_storage_routing: Option, + backend_storage_routing: Option, /// Per-`agg_id` schema registry (sketch DB §6). `None` when the /// caller hasn't wired the precompute engine into the HTTP /// server — in that case the `POST /api/v1/streaming-config` @@ -228,9 +228,9 @@ struct AppState { store: Arc, adapter: Arc, fallback: Option>, - hot_reload_config: Option, + hot_reload_config: Option, /// See [`HttpServer::backend_storage_routing`]. - backend_storage_routing: Option, + backend_storage_routing: Option, /// Per-`agg_id` schema registry (sketch DB §6). Phase 2b wires /// `POST /api/v1/streaming-config` to call `schemas.reconcile()` /// on every swap so schema lifecycle transitions happen @@ -276,7 +276,7 @@ impl HttpServer { /// Plug an additional [`QueryEngine`] into the capability router. /// Used by the binary to register `GorillaQueryEngine` (cold - /// archive) alongside the `SimpleEngine` registered by `new`. + /// archive) alongside the `ASAPQueryEngine` registered by `new`. /// Engines are keyed by their `data_source_id`; calling this with /// an engine whose id collides with an already-registered one /// replaces the previous registration (matches `EngineRouter`'s @@ -308,7 +308,7 @@ impl HttpServer { /// endpoints return `503 Service Unavailable`. pub fn with_hot_reload_config( mut self, - handle: crate::data_model::HotReloadStreamingConfig, + handle: crate::stores::schema::HotReloadStreamingConfig, ) -> Self { self.hot_reload_config = Some(handle); self @@ -333,10 +333,10 @@ impl HttpServer { /// `StreamingConfig::storage_backend()`. pub fn with_backend_storage_routing( mut self, - routing: Arc, + routing: Arc, ) -> Self { self.backend_storage_routing = Some( - crate::routing::HotReloadBackendStorageRouting::from_arc(routing), + crate::query_engines::routing::HotReloadBackendStorageRouting::from_arc(routing), ); self } @@ -349,7 +349,7 @@ impl HttpServer { /// that don't. pub fn with_hot_reload_backend_storage_routing( mut self, - handle: crate::routing::HotReloadBackendStorageRouting, + handle: crate::query_engines::routing::HotReloadBackendStorageRouting, ) -> Self { self.backend_storage_routing = Some(handle); self @@ -699,7 +699,7 @@ async fn process_query_request( // deploys ride this path; it always lands on `SketchStore` // unless the YAML was hand-patched. // (c) Default — `SketchStore`. Keeps the direct - // `SimpleEngine::handle_query` path so the response carries + // `ASAPQueryEngine::handle_query` path so the response carries // the `KeyByLabelNames` the Prometheus adapter needs to // populate the `metric` map. // @@ -740,7 +740,7 @@ async fn process_query_request( /// /// v7: when the routing table has multi-target rows for the metric, /// the parsed AST is also classified via -/// [`crate::data_model::classify_query_shape`] and the lookup picks +/// [`crate::stores::schema::classify_query_shape`] and the lookup picks /// the target whose `applies_to_query_shape` matches. v6.1 /// single-target metrics keep their original semantics — every shape /// resolves to the one configured backend. @@ -757,7 +757,7 @@ fn resolve_metric_storage(state: &AppState, query: &str, tenant: &str) -> Storag match promql_parser::parser::parse(query) { Ok(expr) => { if let Some(metric_name) = first_metric_name(&expr) { - let shape = crate::data_model::classify_query_shape(&expr); + let shape = crate::stores::schema::classify_query_shape(&expr); let backend = routing.lookup_with_shape(&metric_name, shape); debug!( "resolve_metric_storage: routing-table hit for tenant={} metric={} shape={:?} → {:?}", @@ -828,12 +828,12 @@ async fn try_answer_freshness_probe( start_time: Instant, ) -> Option { use crate::drivers::query::adapters::QueryExecutionResult; - use crate::engines::query_result::{InstantVectorElement, QueryResult}; + use crate::query_engines::query_result::{InstantVectorElement, QueryResult}; use promql_utilities::data_model::KeyByLabelNames; let cache = state.probe_cache.as_ref()?; let (metric, range_ms) = parse_last_over_time_probe(&parsed_request.query)?; - if !crate::routing::is_freshness_probe(&metric) { + if !crate::query_engines::routing::is_freshness_probe(&metric) { return None; } @@ -844,7 +844,7 @@ async fn try_answer_freshness_probe( let now_ms = if parsed_request.time > 0.0 { (parsed_request.time * 1_000.0) as i64 } else { - crate::routing::freshness_probe_now_ms() + crate::query_engines::routing::freshness_probe_now_ms() }; let sample = cache.lookup(&metric, now_ms, range_ms)?; @@ -858,7 +858,7 @@ async fn try_answer_freshness_probe( ); let element = - InstantVectorElement::new(crate::data_model::KeyByLabelValues::new(), sample.value); + InstantVectorElement::new(crate::stores::schema::KeyByLabelValues::new(), sample.value); // The instant-vector timestamp is unix milliseconds — match the // adapter's expectations downstream (the Prometheus adapter // divides by 1000 to render the wire `value: [, ...]`). @@ -926,7 +926,7 @@ fn parse_last_over_time_probe(query: &str) -> Option<(String, i64)> { Some((metric, range_ms)) } -/// Direct `SimpleEngine::handle_query` dispatch — preserves the +/// Direct `ASAPQueryEngine::handle_query` dispatch — preserves the /// `KeyByLabelNames` the Prometheus adapter needs to fill in the /// `metric` map. Used for warm-tier metrics (the default) so the /// response surface is byte-identical to the pre-router path. Adds a @@ -1010,7 +1010,7 @@ async fn process_via_simple_engine( // see which tier the request was dispatched against — // the routing decision happened, the metric just had no // compatible aggregation. Mirrors the - // SimpleEngine-as-router-engine path where a + // ASAPQueryEngine-as-router-engine path where a // `EngineError::CapabilityMiss` response is still tagged. match state.adapter.format_unsupported_query_response().await { Ok(response) => { @@ -1046,7 +1046,7 @@ async fn process_via_named_engine( data_source_id: &str, ) -> Response { use crate::drivers::query::adapters::QueryExecutionResult; - use crate::engines::EngineError; + use crate::query_engines::EngineError; let query_start_time = Instant::now(); debug!( @@ -1180,7 +1180,7 @@ async fn process_via_router( metric_storage: StorageBackend, ) -> Response { use crate::drivers::query::adapters::QueryExecutionResult; - use crate::engines::EngineError; + use crate::query_engines::EngineError; let query_start_time = Instant::now(); debug!( @@ -1211,7 +1211,7 @@ async fn process_via_router( // The Phase-4 Gorilla MVP returns a scalar with empty // labels; trait dispatch loses the `KeyByLabelNames` shape - // SimpleEngine carries. Default to an empty `KeyByLabelNames` + // ASAPQueryEngine carries. Default to an empty `KeyByLabelNames` // — the Prometheus adapter renders an empty `metric: {}`, // which is a valid Prometheus shape (every label is just // unset) and matches `wrap_result`'s Phase-4 contract. @@ -1806,8 +1806,8 @@ async fn handle_range_query_post(State(state): State, body: Bytes) -> #[cfg(test)] mod tests { use super::*; - use crate::data_model::{HotReloadStreamingConfig, InferenceConfig, StreamingConfig}; - use crate::engines::SimpleEngine; + use crate::stores::schema::{HotReloadStreamingConfig, InferenceConfig, StreamingConfig}; + use crate::query_engines::ASAPQueryEngine; use crate::stores::sketch_db::simple_map_store::SimpleMapStore; use reqwest::Client; use std::sync::Arc; @@ -1831,21 +1831,21 @@ mod tests { }; let inference_config = InferenceConfig::new( - crate::data_model::QueryLanguage::promql, - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::QueryLanguage::promql, + crate::stores::schema::CleanupPolicy::NoCleanup, ); let streaming_config = Arc::new(StreamingConfig::default()); let store = Arc::new(SimpleMapStore::new( streaming_config.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); - let query_engine = Arc::new(SimpleEngine::new( + let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), // None, inference_config, streaming_config.clone(), 15000, - crate::data_model::QueryLanguage::promql, + crate::stores::schema::QueryLanguage::promql, )); let mut server = HttpServer::new(config, query_engine, store); @@ -2088,20 +2088,20 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::data_model::QueryLanguage::promql, - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::QueryLanguage::promql, + crate::stores::schema::CleanupPolicy::NoCleanup, ); let streaming_config = Arc::new(StreamingConfig::default()); let store = Arc::new(SimpleMapStore::new( streaming_config.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); - let query_engine = Arc::new(SimpleEngine::new( + let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_config.clone(), 15000, - crate::data_model::QueryLanguage::promql, + crate::stores::schema::QueryLanguage::promql, )); let server = HttpServer::new(config, query_engine, store) .with_hot_reload_config(hot_reload) @@ -2571,20 +2571,20 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::data_model::QueryLanguage::promql, - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::QueryLanguage::promql, + crate::stores::schema::CleanupPolicy::NoCleanup, ); let streaming_config = Arc::new(StreamingConfig::default()); let store = Arc::new(SimpleMapStore::new( streaming_config.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); - let query_engine = Arc::new(SimpleEngine::new( + let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_config.clone(), 15000, - crate::data_model::QueryLanguage::promql, + crate::stores::schema::QueryLanguage::promql, )); let schemas = { use asap_types::aggregation_config::AggregationConfig; @@ -2890,8 +2890,8 @@ aggregations: // carries a `data_source: ` info-line so dashboards / e2e // tests can byte-compare which engine answered. - use crate::engines::{EngineError, QueryResult}; - use crate::routing::{EngineCapabilities, QueryEngine}; + use crate::query_engines::{EngineError, QueryResult}; + use crate::query_engines::routing::{EngineCapabilities, QueryEngine}; use async_trait::async_trait; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -2965,8 +2965,8 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::data_model::QueryLanguage::promql, - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::QueryLanguage::promql, + crate::stores::schema::CleanupPolicy::NoCleanup, ); // Pin `storage_backend` on the streaming config so the http // dispatcher reads it back through the hot-reload handle. @@ -2976,14 +2976,14 @@ aggregations: let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); let store = Arc::new(SimpleMapStore::new( streaming_arc.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); - let query_engine = Arc::new(SimpleEngine::new( + let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_arc, 15000, - crate::data_model::QueryLanguage::promql, + crate::stores::schema::QueryLanguage::promql, )); let mut server = HttpServer::new(config, query_engine, store).with_hot_reload_config(hot_reload); @@ -3007,7 +3007,7 @@ aggregations: /// `setup_test_server_with_router` helper above which mocks the /// resolution by pinning `streaming_cfg.storage_backend` directly. async fn setup_test_server_with_routing_table( - routing: crate::data_model::BackendStorageRouting, + routing: crate::stores::schema::BackendStorageRouting, extra_engines: Vec>, ) -> u16 { let adapter_config = @@ -3018,8 +3018,8 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::data_model::QueryLanguage::promql, - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::QueryLanguage::promql, + crate::stores::schema::CleanupPolicy::NoCleanup, ); // Streaming-config stays on the default `SketchStore` axis // — exactly what the production deploy looks like (the YAML @@ -3030,14 +3030,14 @@ aggregations: let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); let store = Arc::new(SimpleMapStore::new( streaming_arc.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); - let query_engine = Arc::new(SimpleEngine::new( + let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_arc, 15000, - crate::data_model::QueryLanguage::promql, + crate::stores::schema::QueryLanguage::promql, )); let mut server = HttpServer::new(config, query_engine, store) .with_hot_reload_config(hot_reload) @@ -3053,17 +3053,17 @@ aggregations: /// Build a server whose `EngineRouter` has zero registered /// engines. We can't reach this through the public API - /// (`HttpServer::new` always registers `SimpleEngine`), so the + /// (`HttpServer::new` always registers `ASAPQueryEngine`), so the /// helper drops in a router by hand via the same builder /// surface — but registers nothing, then asks the router-path /// dispatch to route an archive metric. Used by the /// `503 NoEngineRegistered` test. async fn setup_test_server_with_empty_router(metric_storage_backend: StorageBackend) -> u16 { - // `HttpServer::new` always registers SimpleEngine for the + // `HttpServer::new` always registers ASAPQueryEngine for the // warm tier. To force `NoEngineRegistered` we point the // metric at a backend whose data_source_id doesn't match // any registered engine — since `HttpServer::new` only - // registers SimpleEngine (asap_query), routing a + // registers ASAPQueryEngine (asap_query), routing a // `GorillaObjectStore`-only metric trips the empty path // (compatible_storage_backends = [GorillaObjectStore], no // engine registered for that id). @@ -3088,7 +3088,7 @@ aggregations: #[tokio::test] async fn http_routes_warm_tier_metric_to_simple_engine() { // Default (no hot-reload) → `SketchStore`. The handler - // takes the direct `SimpleEngine::handle_query` path; the + // takes the direct `ASAPQueryEngine::handle_query` path; the // response's `infos` array carries `data_source: asap_query` // so callers can byte-compare which engine answered. let server_port = @@ -3113,7 +3113,7 @@ aggregations: async fn http_routes_archive_metric_to_gorilla_engine() { // Pin `storage_backend = GorillaObjectStore` and register a // `MockQueryEngine` under that id. The handler must dispatch - // through the router (not SimpleEngine) and the response's + // through the router (not ASAPQueryEngine) and the response's // `infos` array must carry `data_source: thanos_query`. let (gorilla, gorilla_calls) = MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); @@ -3152,7 +3152,7 @@ aggregations: // SketchStore` (per the `#[serde(default)]` on the // field — see `streaming_config.rs`). A server set up // without a hot-reload handle still infers warm-tier and - // takes the SimpleEngine direct path. Back-compat for + // takes the ASAPQueryEngine direct path. Back-compat for // pre-Phase-5 deploys whose YAML doesn't include the new // `storage_backend` key. let server_port = setup_test_server().await; // No hot-reload handle attached. @@ -3175,7 +3175,7 @@ aggregations: #[tokio::test] async fn http_returns_503_when_no_engines_registered() { // Pin `storage_backend = GorillaObjectStore` but register no - // archive engine (only `SimpleEngine` is registered under + // archive engine (only `ASAPQueryEngine` is registered under // `asap_query`). The router walks // `compatible_storage_backends = [GorillaObjectStore]` and // bails out with `NoEngineRegistered`, which the HTTP layer @@ -3334,7 +3334,7 @@ aggregations: "http_requests_total".to_string(), StorageBackend::GorillaObjectStore, ); - let routing = crate::data_model::BackendStorageRouting::new_from_single_targets( + let routing = crate::stores::schema::BackendStorageRouting::new_from_single_targets( StorageBackend::SketchStore, metrics, ); @@ -3377,7 +3377,7 @@ aggregations: "http_requests_total".to_string(), StorageBackend::GorillaObjectStore, ); - let routing = crate::data_model::BackendStorageRouting::new_from_single_targets( + let routing = crate::stores::schema::BackendStorageRouting::new_from_single_targets( StorageBackend::SketchStore, metrics, ); @@ -3407,7 +3407,7 @@ aggregations: // top-level `default: thanos_query` — every metric must // route through the router. Pins the §8 "all-metrics-archive" // deploy mode. - let routing = crate::data_model::BackendStorageRouting::new_from_single_targets( + let routing = crate::stores::schema::BackendStorageRouting::new_from_single_targets( StorageBackend::GorillaObjectStore, std::collections::HashMap::new(), ); @@ -3445,7 +3445,7 @@ aggregations: #[tokio::test] async fn http_v7_dual_routing_count_lands_on_archive() { - use crate::data_model::{BackendStorageRouting, QueryShape, RoutingTarget}; + use crate::stores::schema::{BackendStorageRouting, QueryShape, RoutingTarget}; let mut metrics = std::collections::HashMap::new(); metrics.insert( "http_requests_total".to_string(), @@ -3491,7 +3491,7 @@ aggregations: #[tokio::test] async fn http_v7_dual_routing_quantile_stays_on_warm_tier() { - use crate::data_model::{BackendStorageRouting, QueryShape, RoutingTarget}; + use crate::stores::schema::{BackendStorageRouting, QueryShape, RoutingTarget}; let mut metrics = std::collections::HashMap::new(); metrics.insert( "http_requests_total".to_string(), @@ -3525,7 +3525,7 @@ aggregations: .await .expect("Failed to send request"); // Warm tier path returns 2xx with `data_source: asap_query` - // (the SimpleEngine returns None for this unconfigured + // (the ASAPQueryEngine returns None for this unconfigured // metric, but the handler still annotates the wire response // with the warm-tier source). assert!( @@ -3561,7 +3561,7 @@ aggregations: /// shape-classifier and dispatches to the explicitly named engine. #[tokio::test] async fn http_engine_override_header_routes_to_named_engine() { - use crate::data_model::{BackendStorageRouting, QueryShape, RoutingTarget}; + use crate::stores::schema::{BackendStorageRouting, QueryShape, RoutingTarget}; let (gorilla, gorilla_calls) = MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); @@ -3619,7 +3619,7 @@ aggregations: /// backwards-compatible. #[tokio::test] async fn http_engine_override_missing_uses_default_routing() { - use crate::data_model::{BackendStorageRouting, QueryShape, RoutingTarget}; + use crate::stores::schema::{BackendStorageRouting, QueryShape, RoutingTarget}; let (gorilla, gorilla_calls) = MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); @@ -3766,9 +3766,9 @@ aggregations: /// install an empty hot-reload routing handle, hold the handle so /// the test can introspect the swap result. async fn setup_test_server_for_storage_routing( - ) -> (u16, crate::routing::HotReloadBackendStorageRouting) { - use crate::data_model::{HotReloadStreamingConfig, StreamingConfig}; - use crate::routing::HotReloadBackendStorageRouting; + ) -> (u16, crate::query_engines::routing::HotReloadBackendStorageRouting) { + use crate::stores::schema::{HotReloadStreamingConfig, StreamingConfig}; + use crate::query_engines::routing::HotReloadBackendStorageRouting; let adapter_config = AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); @@ -3778,22 +3778,22 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::data_model::QueryLanguage::promql, - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::QueryLanguage::promql, + crate::stores::schema::CleanupPolicy::NoCleanup, ); let streaming_cfg = StreamingConfig::default(); let streaming_arc = Arc::new(streaming_cfg); let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); let store = Arc::new(SimpleMapStore::new( streaming_arc.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); - let query_engine = Arc::new(SimpleEngine::new( + let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_arc, 15000, - crate::data_model::QueryLanguage::promql, + crate::stores::schema::QueryLanguage::promql, )); let routing_handle = HotReloadBackendStorageRouting::empty(); let server = HttpServer::new(config, query_engine, store) @@ -3854,7 +3854,7 @@ aggregations: // what the response advertised. let snap = handle.snapshot(); assert_eq!(snap.len(), 1); - let live_hash = crate::routing::routing_table_hash(snap.as_ref()); + let live_hash = crate::query_engines::routing::routing_table_hash(snap.as_ref()); assert_eq!(live_hash, returned_hash, "live hash must match advertised"); } @@ -3904,7 +3904,7 @@ aggregations: let (port, handle) = setup_test_server_for_storage_routing().await; // Pre-load the table. let new = - crate::data_model::BackendStorageRouting::from_json_payload(&fixture_routing_json()) + crate::stores::schema::BackendStorageRouting::from_json_payload(&fixture_routing_json()) .expect("parse"); handle.swap(new); @@ -3920,7 +3920,7 @@ aggregations: assert_eq!(body["default_engine"], "asap_query"); assert_eq!(body["metrics_count"], 1); let snap_hash = body["table_hash"].as_str().unwrap(); - let live_hash = crate::routing::routing_table_hash(handle.snapshot().as_ref()); + let live_hash = crate::query_engines::routing::routing_table_hash(handle.snapshot().as_ref()); assert_eq!(snap_hash, live_hash); } @@ -3950,7 +3950,7 @@ aggregations: assert_eq!(body["metrics_count"], 1); // Default tenant table is unchanged (still empty). - let snap_default = handle.snapshot_for_tenant(crate::routing::DEFAULT_TENANT); + let snap_default = handle.snapshot_for_tenant(crate::query_engines::routing::DEFAULT_TENANT); assert_eq!(snap_default.len(), 0); // Tenant-a table has the new entry. let snap_a = handle.snapshot_for_tenant("tenant-a"); @@ -3981,7 +3981,7 @@ aggregations: let snap_b = handle.snapshot_for_tenant("tenant-b"); assert_eq!(snap_b.len(), 1); // Default tenant is still empty. - let snap_default = handle.snapshot_for_tenant(crate::routing::DEFAULT_TENANT); + let snap_default = handle.snapshot_for_tenant(crate::query_engines::routing::DEFAULT_TENANT); assert_eq!(snap_default.len(), 0); } @@ -4003,8 +4003,8 @@ aggregations: .expect("send ok"); assert!(resp.status().is_success()); let body: serde_json::Value = resp.json().await.unwrap(); - assert_eq!(body["tenant"], crate::routing::DEFAULT_TENANT); - let snap_default = handle.snapshot_for_tenant(crate::routing::DEFAULT_TENANT); + assert_eq!(body["tenant"], crate::query_engines::routing::DEFAULT_TENANT); + let snap_default = handle.snapshot_for_tenant(crate::query_engines::routing::DEFAULT_TENANT); assert_eq!(snap_default.len(), 1); } @@ -4074,13 +4074,13 @@ aggregations: // the very next read. let snap = handle.snapshot(); assert_eq!( - snap.lookup_with_shape("http_requests_total", crate::data_model::QueryShape::Count,), + snap.lookup_with_shape("http_requests_total", crate::stores::schema::QueryShape::Count,), StorageBackend::GorillaObjectStore, ); assert_eq!( snap.lookup_with_shape( "http_requests_total", - crate::data_model::QueryShape::Quantile, + crate::stores::schema::QueryShape::Quantile, ), StorageBackend::SketchStore, ); @@ -4101,10 +4101,10 @@ aggregations: #[tokio::test] async fn http_archive_metric_forwards_to_thanos_query() { - use crate::engines::thanos_query::forward::test_support::{ + use crate::query_engines::thanos_query_engine::forward::test_support::{ spawn_mock_thanos, CANNED_VECTOR_BODY, }; - use crate::engines::thanos_query::{ThanosQueryConfig, ThanosQueryEngine}; + use crate::query_engines::thanos_query_engine::{ThanosQueryConfig, ThanosQueryEngine}; let (mock_url, _mock_handle) = spawn_mock_thanos(CANNED_VECTOR_BODY).await; let cfg = ThanosQueryConfig { @@ -4150,8 +4150,8 @@ aggregations: #[tokio::test] async fn http_thanos_unreachable_returns_503_with_quirk() { - use crate::engines::thanos_query::forward::test_support::spawn_mock_thanos_503; - use crate::engines::thanos_query::{ThanosQueryConfig, ThanosQueryEngine}; + use crate::query_engines::thanos_query_engine::forward::test_support::spawn_mock_thanos_503; + use crate::query_engines::thanos_query_engine::{ThanosQueryConfig, ThanosQueryEngine}; let (mock_url, _mock_handle) = spawn_mock_thanos_503().await; let cfg = ThanosQueryConfig { @@ -4199,10 +4199,10 @@ aggregations: // even when the metric's storage axis would otherwise route // to the warm tier. Path A2's accuracy reducer relies on // this for apples-to-apples comparison runs. - use crate::engines::thanos_query::forward::test_support::{ + use crate::query_engines::thanos_query_engine::forward::test_support::{ spawn_mock_thanos, CANNED_VECTOR_BODY, }; - use crate::engines::thanos_query::{ + use crate::query_engines::thanos_query_engine::{ ThanosQueryConfig, ThanosQueryEngine, DATA_SOURCE_THANOS_QUERY_ID, }; @@ -4249,8 +4249,8 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::data_model::QueryLanguage::promql, - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::QueryLanguage::promql, + crate::stores::schema::CleanupPolicy::NoCleanup, ); let streaming_cfg = StreamingConfig::with_storage_backend(Default::default(), metric_storage_backend); @@ -4258,14 +4258,14 @@ aggregations: let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); let store = Arc::new(SimpleMapStore::new( streaming_arc.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); - let query_engine = Arc::new(SimpleEngine::new( + let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_arc, 15000, - crate::data_model::QueryLanguage::promql, + crate::stores::schema::QueryLanguage::promql, )); let mut server = HttpServer::new(config, query_engine, store).with_hot_reload_config(hot_reload); @@ -4298,7 +4298,7 @@ aggregations: /// hitting `/api/v1/query`. The router holds no cold-archive /// engine; the freshness probe short-circuit must answer /// without ever consulting the cold tier. - async fn setup_test_server_with_probe_cache() -> (u16, Arc) + async fn setup_test_server_with_probe_cache() -> (u16, Arc) { let adapter_config = AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); @@ -4308,22 +4308,22 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::data_model::QueryLanguage::promql, - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::QueryLanguage::promql, + crate::stores::schema::CleanupPolicy::NoCleanup, ); let streaming_arc = Arc::new(StreamingConfig::default()); let store = Arc::new(SimpleMapStore::new( streaming_arc.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); - let query_engine = Arc::new(SimpleEngine::new( + let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_arc, 15000, - crate::data_model::QueryLanguage::promql, + crate::stores::schema::QueryLanguage::promql, )); - let cache = Arc::new(crate::routing::FreshnessProbeCache::new()); + let cache = Arc::new(crate::query_engines::routing::FreshnessProbeCache::new()); let server = HttpServer::new(config, query_engine, store).with_probe_cache(cache.clone()); let port = server .start_test_server() @@ -4339,7 +4339,7 @@ aggregations: // Synthetic sample: probe encodes its emission unix_ms as the // counter value (matches `deploy/fake-exporter/probes.go`). // Record the sample at "now" so the 10 s lookback hits. - let now_ms = crate::routing::freshness_probe_now_ms(); + let now_ms = crate::query_engines::routing::freshness_probe_now_ms(); let probe_value_ms = now_ms - 50; // sample emitted 50 ms ago cache.record( "http_freshness_probe_warm", @@ -4386,12 +4386,12 @@ aggregations: // Sample is older than the lookback window — the cache lookup // returns None and the handler falls through to the normal // routing path. The default routing landed on - // `SketchStore`, which the test's empty `SimpleEngine` + // `SketchStore`, which the test's empty `ASAPQueryEngine` // can't answer, so the response is a structured error or an // empty-result success — anything but a crash. The test // pins the no-crash contract; the exact error surface is // covered by the routing-table tests. - let now_ms = crate::routing::freshness_probe_now_ms(); + let now_ms = crate::query_engines::routing::freshness_probe_now_ms(); let stale_ts = now_ms - 60_000; // 60 s old, outside [now-10s, now] cache.record("http_freshness_probe_warm", stale_ts, stale_ts as f64); @@ -4403,7 +4403,7 @@ aggregations: .await .expect("Failed to send request"); // The response either succeeds with an empty vector (cache - // miss → fall through → SimpleEngine no-data) or returns a + // miss → fall through → ASAPQueryEngine no-data) or returns a // 4xx/5xx with a structured error. Either is fine as long as // the handler did not panic. let body: serde_json::Value = resp.json().await.expect("Failed to parse JSON"); @@ -4461,7 +4461,7 @@ aggregations: #[tokio::test] async fn freshness_probe_short_circuit_ignores_non_probe_metrics() { let (port, cache) = setup_test_server_with_probe_cache().await; - let now_ms = crate::routing::freshness_probe_now_ms(); + let now_ms = crate::query_engines::routing::freshness_probe_now_ms(); cache.record("http_freshness_probe_warm", now_ms, now_ms as f64); // Different metric — must NOT be served from the cache (the @@ -4713,7 +4713,7 @@ async fn handle_store_metrics(State(state): State) -> axum::response:: // atomically swap via ArcSwap. // // Phase 1 scope: the swap only takes effect for new readers that -// snapshot after the swap. `SimpleEngine`, the ingest router, and +// snapshot after the swap. `ASAPQueryEngine`, the ingest router, and // in-flight precompute workers all hold startup snapshots today and // ignore the swap until they are rebuilt — see the module doc on // `HotReloadStreamingConfig` for the full contract. Tests POST a new @@ -4870,7 +4870,7 @@ async fn handle_get_storage_routing( "tenant": tenant, "default_engine": snap.default_backend().data_source_id(), "metrics_count": snap.len(), - "table_hash": crate::routing::routing_table_hash(snap.as_ref()), + "table_hash": crate::query_engines::routing::routing_table_hash(snap.as_ref()), "tenants": handle.tenant_ids(), }); (StatusCode::OK, axum::Json(body)).into_response() @@ -4930,7 +4930,7 @@ async fn handle_post_storage_routing( return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); } }; - let new_table = match crate::data_model::BackendStorageRouting::from_json_payload(&json_value) { + let new_table = match crate::stores::schema::BackendStorageRouting::from_json_payload(&json_value) { Ok(t) => t, Err(e) => { let body = serde_json::json!({ @@ -4947,7 +4947,7 @@ async fn handle_post_storage_routing( // body left the tenant field implicit — it's the controller's // primary signal for "which tenant am I pushing for". let tenant_from_body = new_table.tenant().to_string(); - let tenant = if tenant_from_body == crate::routing::DEFAULT_TENANT { + let tenant = if tenant_from_body == crate::query_engines::routing::DEFAULT_TENANT { // Body left it implicit; honour the header. extract_tenant(&headers) } else { @@ -4955,7 +4955,7 @@ async fn handle_post_storage_routing( }; let entries = new_table.len(); - let hash = crate::routing::routing_table_hash(&new_table); + let hash = crate::query_engines::routing::routing_table_hash(&new_table); let _old = handle.swap_tenant(&tenant, new_table); info!( tenant = %tenant, diff --git a/asap-query-engine/src/drivers/query/servers/metrics.rs b/data_plane/src/drivers/query/servers/metrics.rs similarity index 100% rename from asap-query-engine/src/drivers/query/servers/metrics.rs rename to data_plane/src/drivers/query/servers/metrics.rs diff --git a/asap-query-engine/src/drivers/query/servers/mod.rs b/data_plane/src/drivers/query/servers/mod.rs similarity index 100% rename from asap-query-engine/src/drivers/query/servers/mod.rs rename to data_plane/src/drivers/query/servers/mod.rs diff --git a/asap-query-engine/src/lib.rs b/data_plane/src/lib.rs similarity index 81% rename from asap-query-engine/src/lib.rs rename to data_plane/src/lib.rs index c8f16773..9d314a9e 100644 --- a/asap-query-engine/src/lib.rs +++ b/data_plane/src/lib.rs @@ -1,10 +1,6 @@ -pub mod data_model; pub mod drivers; -#[path = "query-engines/mod.rs"] -pub mod engines; pub mod precompute_engine; -pub mod precompute_operators; -pub mod routing; +pub mod query_engines; pub mod stores; #[cfg(test)] @@ -12,20 +8,20 @@ pub mod tests; pub mod utils; // Re-export commonly used types to avoid glob import conflicts -pub use data_model::{ +pub use stores::schema::{ AccumulatorFactory, AggregateCore, AggregationConfig, InferenceConfig, KeyByLabelValues, Measurement, MergeableAccumulator, MultipleSubpopulationAggregate, MultipleSubpopulationAggregateFactory, PrecomputedOutput, PromQLSchema, QueryConfig, SerializableToSink, SingleSubpopulationAggregate, SingleSubpopulationAggregateFactory, }; -pub use precompute_operators::{ +pub use precompute_engine::operators::{ IncreaseAccumulator, MinMaxAccumulator, MultipleSumAccumulator, SumAccumulator, }; pub use stores::{SimpleMapStore, Store, StoreResult}; -pub use engines::{ASAPQueryEngine, InstantVector, QueryResult, SimpleEngine}; +pub use query_engines::{ASAPQueryEngine, InstantVector, QueryResult}; pub use drivers::{ HttpServer, HttpServerConfig, KafkaConsumer, KafkaConsumerConfig, OtlpReceiver, diff --git a/asap-query-engine/src/main.rs b/data_plane/src/main.rs similarity index 92% rename from asap-query-engine/src/main.rs rename to data_plane/src/main.rs index 66b32ab2..3b247f6d 100644 --- a/asap-query-engine/src/main.rs +++ b/data_plane/src/main.rs @@ -11,23 +11,23 @@ // this same backend process — there is no longer a separate // `asap-controller` container in `mvp-multinode/run_demo.sh`. use clap::Parser; -use query_engine_rust::data_model::QueryLanguage; +use data_plane::stores::schema::QueryLanguage; use std::fs; use std::sync::Arc; use tokio::signal; use tracing::{error, info, warn}; -use query_engine_rust::data_model::enums::{ +use data_plane::stores::schema::enums::{ CleanupPolicy, InputFormat, LockStrategy, StreamingEngine, }; -use query_engine_rust::data_model::InferenceConfig; -use query_engine_rust::drivers::AdapterConfig; -use query_engine_rust::precompute_engine::config::LateDataPolicy; -use query_engine_rust::precompute_engine::PrecomputeWorkerDiagnostics; -use query_engine_rust::utils::file_io::{read_inference_config, read_streaming_config}; -use query_engine_rust::{ +use data_plane::stores::schema::InferenceConfig; +use data_plane::drivers::AdapterConfig; +use data_plane::precompute_engine::config::LateDataPolicy; +use data_plane::precompute_engine::PrecomputeWorkerDiagnostics; +use data_plane::utils::file_io::{read_inference_config, read_streaming_config}; +use data_plane::{ HttpServer, HttpServerConfig, KafkaConsumer, KafkaConsumerConfig, OtlpReceiver, - OtlpReceiverConfig, PrecomputeEngine, PrecomputeEngineConfig, Result, SimpleEngine, + OtlpReceiverConfig, PrecomputeEngine, PrecomputeEngineConfig, Result, ASAPQueryEngine, SimpleMapStore, StoreOutputSink, }; @@ -64,7 +64,7 @@ struct Args { streaming_engine: StreamingEngine, /// Prometheus scrape interval (seconds). Default 30 matches - /// the e2e harness's 30s window. SimpleEngine uses this as the + /// the e2e harness's 30s window. ASAPQueryEngine uses this as the /// instant-query lookback window — for tumbling-window /// aggregations it must be ≥ the window size in /// `streaming-config`. @@ -91,7 +91,7 @@ struct Args { prometheus_server: String, /// DataCollector controller endpoint for capability-miss - /// notifications (PR G). When set, `SimpleEngine` fires a + /// notifications (PR G). When set, `ASAPQueryEngine` fires a /// fire-and-forget POST to this URL every time a query can't /// find a compatible stored aggregation, so the controller /// can generate a new sketch plan. When unset (default), @@ -281,7 +281,7 @@ struct Args { /// Path to the per-metric backend storage routing YAML /// (`{metric_name: storage_backend}` map). Loaded at startup and /// consulted by the HTTP query handler on every PromQL request to - /// pick the right engine (`SimpleEngine` for warm-tier sketches, + /// pick the right engine (`ASAPQueryEngine` for warm-tier sketches, /// `GorillaQueryEngine` for the cold archive, etc.). Without /// this flag the handler falls back to the streaming-config /// single axis (always `SketchStore`) and the EngineRouter is @@ -357,19 +357,19 @@ async fn main() -> Result<()> { // Wrap the streaming config in a hot-reload handle so the HTTP // server's `/api/v1/streaming-config` endpoints can swap it at // runtime (PR E phase 1). Existing consumers downstream - // (SimpleEngine, PrecomputeEngine, Store) still take their + // (ASAPQueryEngine, PrecomputeEngine, Store) still take their // startup snapshot; hot-reload currently only affects the // control-plane GET/POST endpoint. Phase 2 will extend the swap // to query execution and ingest routing. let hot_reload_config = - query_engine_rust::data_model::HotReloadStreamingConfig::from_arc(streaming_config.clone()); + data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config.clone()); // Setup store (equivalent to Python's SimpleMapStore()) // Get cleanup policy from inference config let cleanup_policy = inference_config.cleanup_policy; info!("Using cleanup policy: {:?}", cleanup_policy); let store = if args.persistence_enabled { - use query_engine_rust::stores::sketch_db::simple_map_store::persistence::SimpleMapStorePersistenceConfig; + use data_plane::stores::sketch_db::simple_map_store::persistence::SimpleMapStorePersistenceConfig; let disk_path = args .persistence_dir .clone() @@ -434,25 +434,25 @@ async fn main() -> Result<()> { // SeriesIdResolver + SketchIndex once. The OTLP receive path // (sid resolution + unknown_series_ids stamping; SketchIndex // .append_sample on every modified-OTLP sketch DP) AND the - // SimpleEngine query path (SketchIndex.classify / query_range + // ASAPQueryEngine query path (SketchIndex.classify / query_range // for warm-tier reads) hold clones of these Arcs. Allocated - // here before BOTH the SimpleEngine and the precompute engine + // here before BOTH the ASAPQueryEngine and the precompute engine // are constructed so both can be wired with a single canonical // instance — even when precompute is disabled, the engine still // needs the index for the Phase 6 archive failover trigger. let series_resolver = - Arc::new(query_engine_rust::drivers::ingest::series_resolver::SeriesIdResolver::new()); + Arc::new(data_plane::drivers::ingest::series_resolver::SeriesIdResolver::new()); let sketch_index = - Arc::new(query_engine_rust::stores::sketch_db::sketch_index::SketchIndex::new()); + Arc::new(data_plane::stores::sketch_db::sketch_index::SketchIndex::new()); - // Setup query engine. SimpleEngine shares the same + // Setup query engine. ASAPQueryEngine shares the same // HotReloadStreamingConfig handle as the HTTP server, so a POST // to /api/v1/streaming-config is observable by the next query - // (PR E phase 2). Without sharing the handle, SimpleEngine + // (PR E phase 2). Without sharing the handle, ASAPQueryEngine // would take a one-time snapshot at construction and ignore // subsequent swaps. let mut engine = { - let mut engine = SimpleEngine::new_with_hot_reload( + let mut engine = ASAPQueryEngine::new_with_hot_reload( store.clone(), inference_config, hot_reload_config.clone(), @@ -471,9 +471,9 @@ async fn main() -> Result<()> { controller_endpoint ); let client: Arc< - dyn query_engine_rust::drivers::query::controller_client::ControllerClient, + dyn data_plane::drivers::query::controller_client::ControllerClient, > = Arc::new( - query_engine_rust::drivers::query::controller_client::HttpControllerClient::new( + data_plane::drivers::query::controller_client::HttpControllerClient::new( controller_endpoint.clone(), ), ); @@ -616,7 +616,7 @@ async fn main() -> Result<()> { // 10 s lookback window empty). Allocated unconditionally — non- // probe traffic doesn't touch the cache, so the cost is one // `RwLock` of three entries for the whole demo run. - let probe_cache = Arc::new(query_engine_rust::routing::FreshnessProbeCache::new()); + let probe_cache = Arc::new(data_plane::query_engines::routing::FreshnessProbeCache::new()); let otel_handle = if args.enable_otel_ingest { let otel_config = OtlpReceiverConfig { @@ -697,7 +697,7 @@ async fn main() -> Result<()> { // dev / standalone — the YAML supplies the bootstrap, controller // pushes overwrite it. let bootstrap_routing = if let Some(routing_path) = args.backend_storage_routing.as_deref() { - match query_engine_rust::data_model::BackendStorageRouting::from_yaml_file(routing_path) { + match data_plane::stores::schema::BackendStorageRouting::from_yaml_file(routing_path) { Ok(routing) => { info!( "Loaded backend-storage-routing from {:?}: default={:?}, entries={}", @@ -712,14 +712,14 @@ async fn main() -> Result<()> { "Failed to load backend-storage-routing from {:?}: {} — installing an empty routing table; the controller's first POST /api/v1/storage_routing push will fill it", routing_path, e, ); - query_engine_rust::data_model::BackendStorageRouting::empty() + data_plane::stores::schema::BackendStorageRouting::empty() } } } else { info!( "--backend-storage-routing not set — installing an empty routing table; the controller's first POST /api/v1/storage_routing push will fill it", ); - query_engine_rust::data_model::BackendStorageRouting::empty() + data_plane::stores::schema::BackendStorageRouting::empty() }; server = server.with_backend_storage_routing(Arc::new(bootstrap_routing)); @@ -746,9 +746,9 @@ async fn main() -> Result<()> { // original fail-loud behaviour can opt back in by setting // `ASAP_REQUIRE_ARCHIVE_ENGINE=1`. let mut archive_registered = false; - match query_engine_rust::engines::thanos_query::thanos_engine_from_env() { + match data_plane::query_engines::thanos_query_engine::thanos_engine_from_env() { Ok(Some(thanos)) => { - use query_engine_rust::routing::QueryEngine; + use data_plane::query_engines::routing::QueryEngine; info!( upstream = thanos.base_url(), "Path A2: registering ThanosQueryEngine for the archive tier (data_source_id=thanos_query); legacy in-process GorillaQueryEngine skipped", @@ -757,14 +757,14 @@ async fn main() -> Result<()> { server = server.with_archive_query_engine(thanos_arc); archive_registered = true; } - Ok(None) => match query_engine_rust::stores::gorilla_object_store::GorillaS3Config::from_env() { + Ok(None) => match data_plane::stores::gorilla_object_store::GorillaS3Config::from_env() { Ok(s3_cfg) => { - match query_engine_rust::stores::gorilla_object_store::GorillaS3Store::with_default_backend( + match data_plane::stores::gorilla_object_store::GorillaS3Store::with_default_backend( s3_cfg, ) { Ok(store) => { - use query_engine_rust::stores::{GorillaEngineConfig, GorillaQueryEngine}; - use query_engine_rust::routing::QueryEngine; + use data_plane::stores::{GorillaEngineConfig, GorillaQueryEngine}; + use data_plane::query_engines::routing::QueryEngine; let gorilla = Arc::new(GorillaQueryEngine::with_gorilla_s3( Arc::new(store), GorillaEngineConfig::default(), @@ -808,8 +808,8 @@ async fn main() -> Result<()> { "ASAP_REQUIRE_ARCHIVE_ENGINE=1 set and no archive engine configured — cold queries will return 503 NoEngineRegistered", ); } else { - use query_engine_rust::engines::NoDataArchiveEngine; - use query_engine_rust::routing::QueryEngine; + use data_plane::query_engines::NoDataArchiveEngine; + use data_plane::query_engines::routing::QueryEngine; info!( "Registering NoDataArchiveEngine stub on the archive slot (canonical data_source_id=thanos_query); set ASAP_REQUIRE_ARCHIVE_ENGINE=1 to disable", ); @@ -836,9 +836,9 @@ async fn main() -> Result<()> { // memory-only and restart wipes job history. let backfill_registry = Arc::new(match args.backfill_persist_path.as_ref() { Some(path) => { - query_engine_rust::stores::sketch_db::BackfillRegistry::load_or_new(path.clone()) + data_plane::stores::sketch_db::BackfillRegistry::load_or_new(path.clone()) } - None => query_engine_rust::stores::sketch_db::BackfillRegistry::new(), + None => data_plane::stores::sketch_db::BackfillRegistry::new(), }); server = server.with_backfill_registry(backfill_registry.clone()); @@ -856,13 +856,13 @@ async fn main() -> Result<()> { precompute_ingest_state.as_ref(), ) { let schemas = ingest_state.schemas.clone(); - let service = query_engine_rust::stores::sketch_db::BackfillService::new( + let service = data_plane::stores::sketch_db::BackfillService::new( backfill_registry.clone(), schemas, store.clone(), hot_reload_config.clone(), - query_engine_rust::stores::sketch_db::default_reader_factory(), - query_engine_rust::stores::sketch_db::BackfillServiceConfig::default(), + data_plane::stores::sketch_db::default_reader_factory(), + data_plane::stores::sketch_db::BackfillServiceConfig::default(), ); info!( "Spawning BackfillService drain loop (reader factory: default — Prometheus sources wired, S3/OtherSketch fail fast)" @@ -894,15 +894,15 @@ async fn main() -> Result<()> { args.persistence_delete_older_than_secs, )) }; - query_engine_rust::stores::sketch_db::warn_if_retention_inverted( + data_plane::stores::sketch_db::warn_if_retention_inverted( data_retention_opt, ingest_state.schemas.retirement_retention(), ); - let svc = query_engine_rust::stores::sketch_db::SchemaEvictionService::new( + let svc = data_plane::stores::sketch_db::SchemaEvictionService::new( ingest_state.schemas.clone(), backfill_registry.clone(), store.clone(), - query_engine_rust::stores::sketch_db::SchemaEvictionConfig { + data_plane::stores::sketch_db::SchemaEvictionConfig { poll_interval: std::time::Duration::from_secs(args.schema_eviction_poll_secs), dry_run: args.schema_eviction_dry_run, }, @@ -1065,7 +1065,7 @@ fn setup_logging( #[cfg(test)] mod tests { - use query_engine_rust::drivers::AdapterConfig; + use data_plane::drivers::AdapterConfig; // Step-1 of the JSONL deprecation refactor deleted the // §5.2 `ColdFallback` adapter and the diff --git a/asap-query-engine/src/precompute_engine/accumulator_factory.rs b/data_plane/src/precompute_engine/accumulator_factory.rs similarity index 99% rename from asap-query-engine/src/precompute_engine/accumulator_factory.rs rename to data_plane/src/precompute_engine/accumulator_factory.rs index 840bb088..da7ec56f 100644 --- a/asap-query-engine/src/precompute_engine/accumulator_factory.rs +++ b/data_plane/src/precompute_engine/accumulator_factory.rs @@ -1,5 +1,5 @@ -use crate::data_model::{AggregateCore, AggregationType, KeyByLabelValues, Measurement}; -use crate::precompute_operators::{ +use crate::stores::schema::{AggregateCore, AggregationType, KeyByLabelValues, Measurement}; +use crate::precompute_engine::operators::{ CountMinSketchAccumulator, DDSketchAccumulator, DatasketchesKLLAccumulator, HydraKllSketchAccumulator, IncreaseAccumulator, MinMaxAccumulator, MultipleIncreaseAccumulator, MultipleMinMaxAccumulator, MultipleSumAccumulator, SumAccumulator, @@ -934,7 +934,7 @@ mod tests { let acc = updater.snapshot_accumulator(); let kll = acc .as_any() - .downcast_ref::() + .downcast_ref::() .expect("should be KLL"); assert_eq!(kll.inner.k, 50, "k should be 50 from capital-K param"); } diff --git a/asap-query-engine/src/precompute_engine/config.rs b/data_plane/src/precompute_engine/config.rs similarity index 100% rename from asap-query-engine/src/precompute_engine/config.rs rename to data_plane/src/precompute_engine/config.rs diff --git a/asap-query-engine/src/precompute_engine/engine.rs b/data_plane/src/precompute_engine/engine.rs similarity index 99% rename from asap-query-engine/src/precompute_engine/engine.rs rename to data_plane/src/precompute_engine/engine.rs index 0b497a60..5a971cae 100644 --- a/asap-query-engine/src/precompute_engine/engine.rs +++ b/data_plane/src/precompute_engine/engine.rs @@ -1,4 +1,4 @@ -use crate::data_model::HotReloadStreamingConfig; +use crate::stores::schema::HotReloadStreamingConfig; use crate::precompute_engine::config::PrecomputeEngineConfig; use crate::precompute_engine::ingest_handler::IngestState; use crate::precompute_engine::output_sink::OutputSink; diff --git a/asap-query-engine/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs similarity index 96% rename from asap-query-engine/src/precompute_engine/ingest_handler.rs rename to data_plane/src/precompute_engine/ingest_handler.rs index 595cf934..87cce418 100644 --- a/asap-query-engine/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -1,4 +1,4 @@ -use crate::data_model::HotReloadStreamingConfig; +use crate::stores::schema::HotReloadStreamingConfig; use crate::precompute_engine::series_router::SeriesRouter; use crate::precompute_engine::worker::parse_labels_from_series_key; use crate::stores::sketch_db::SchemaRegistry; @@ -52,7 +52,7 @@ pub struct IngestState { /// follow-up will add TTL-based eviction keyed by last-seen /// timestamp so long-running deployments don't leak memory /// on retired series. - pub sketch_snapshots: dashmap::DashMap>, + pub sketch_snapshots: dashmap::DashMap>, /// Phase 4 — centralized series_id resolver. Shared across the OTLP /// receive path (sid resolution + `unknown_series_ids` population) and /// the `ResolveSeriesIDs` RPC (eager batch resolution from the agent's @@ -62,7 +62,7 @@ pub struct IngestState { /// Phase 5 — two-level sketch warm tier (instance metadata + /// per-sid columnar state). Populated by the OTLP ingest path on /// every modified-OTLP first-class sketch DataPoint; queried by - /// the `SimpleEngine` query path (warm-tier hit / ghost / unknown + /// the `ASAPQueryEngine` query path (warm-tier hit / ghost / unknown /// classification drives the Phase 6 archive failover). pub sketch_index: Arc, } @@ -76,7 +76,7 @@ impl IngestState { /// Returns the shared `Arc` — no cloning of /// individual AggregationConfig objects, just an atomic refcount /// increment (~5ns). - pub fn config_snapshot(&self) -> Arc { + pub fn config_snapshot(&self) -> Arc { self.hot_reload_config.snapshot() } } @@ -123,7 +123,7 @@ fn extract_group_key(series_key: &str, config: &AggregationConfig) -> String { #[cfg(test)] mod tests { use super::*; - use crate::data_model::StreamingConfig; + use crate::stores::schema::StreamingConfig; use crate::precompute_engine::series_router::SeriesRouter; use crate::stores::sketch_db::SchemaRegistry; use asap_types::aggregation_config::AggregationConfig; @@ -168,7 +168,7 @@ mod tests { let mut map = std::collections::HashMap::new(); map.insert(agg_id, make_config(agg_id, metric)); let streaming = StreamingConfig::new(map); - let hot_reload = crate::data_model::HotReloadStreamingConfig::new(streaming.clone()); + let hot_reload = crate::stores::schema::HotReloadStreamingConfig::new(streaming.clone()); let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); @@ -235,7 +235,7 @@ mod tests { #[tokio::test] async fn delta_path_reconstitutes_cumulative_state() { use crate::drivers::ingest::otel::{apply_modified_otlp_delta_bytes, SketchKind}; - use crate::precompute_operators::DDSketchAccumulator; + use crate::precompute_engine::operators::DDSketchAccumulator; use asap_otel_proto::sketchlib::v1::{DdSketchBucketDelta, DdSketchDelta as PbDelta}; use asap_sketchlib::sketches::ddsketch::DdSketch; use prost::Message; diff --git a/asap-query-engine/src/precompute_engine/mod.rs b/data_plane/src/precompute_engine/mod.rs similarity index 93% rename from asap-query-engine/src/precompute_engine/mod.rs rename to data_plane/src/precompute_engine/mod.rs index 275bc93e..e97f7ecd 100644 --- a/asap-query-engine/src/precompute_engine/mod.rs +++ b/data_plane/src/precompute_engine/mod.rs @@ -2,6 +2,7 @@ pub mod accumulator_factory; pub mod config; mod engine; pub mod ingest_handler; +pub mod operators; pub mod output_sink; pub mod series_buffer; pub mod series_router; diff --git a/asap-query-engine/src/precompute_operators/count_min_sketch_accumulator.rs b/data_plane/src/precompute_engine/operators/count_min_sketch_accumulator.rs similarity index 99% rename from asap-query-engine/src/precompute_operators/count_min_sketch_accumulator.rs rename to data_plane/src/precompute_engine/operators/count_min_sketch_accumulator.rs index a8d61440..22bf9c09 100644 --- a/asap-query-engine/src/precompute_operators/count_min_sketch_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/count_min_sketch_accumulator.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, }; @@ -265,7 +265,7 @@ impl CountMinSketchAccumulator { /// Merge multiple accumulators efficiently without cloning all of them. pub fn merge_multiple( - accumulators: &[Box], + accumulators: &[Box], ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); @@ -382,7 +382,7 @@ impl AggregateCore for CountMinSketchAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::data_model::MultipleSubpopulationAggregate; + use crate::stores::schema::MultipleSubpopulationAggregate; use promql_utilities::query_logics::enums::Statistic; // Key-provided path: route to MultipleSubpopulationAggregate::query @@ -686,7 +686,7 @@ mod tests { let boxed_accs: Vec> = vec![Box::new(cms1), Box::new(cms2)]; assert!(CountMinSketchAccumulator::merge_multiple(&boxed_accs).is_err()); - use crate::precompute_operators::sum_accumulator::SumAccumulator; + use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; let cms = CountMinSketchAccumulator::new(2, 3); let sum = SumAccumulator::new(); let mixed_accs: Vec> = vec![Box::new(cms), Box::new(sum)]; diff --git a/asap-query-engine/src/precompute_operators/count_min_sketch_with_heap_accumulator.rs b/data_plane/src/precompute_engine/operators/count_min_sketch_with_heap_accumulator.rs similarity index 99% rename from asap-query-engine/src/precompute_operators/count_min_sketch_with_heap_accumulator.rs rename to data_plane/src/precompute_engine/operators/count_min_sketch_with_heap_accumulator.rs index 337df073..47d0329b 100644 --- a/asap-query-engine/src/precompute_operators/count_min_sketch_with_heap_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/count_min_sketch_with_heap_accumulator.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, }; @@ -187,7 +187,7 @@ impl AggregateCore for CountMinSketchWithHeapAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::data_model::MultipleSubpopulationAggregate; + use crate::stores::schema::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for CountMinSketchWithHeapAccumulator")?; diff --git a/asap-query-engine/src/precompute_operators/count_sketch_accumulator.rs b/data_plane/src/precompute_engine/operators/count_sketch_accumulator.rs similarity index 99% rename from asap-query-engine/src/precompute_operators/count_sketch_accumulator.rs rename to data_plane/src/precompute_engine/operators/count_sketch_accumulator.rs index 4881044a..dae59a30 100644 --- a/asap-query-engine/src/precompute_operators/count_sketch_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/count_sketch_accumulator.rs @@ -19,7 +19,7 @@ //! The wire format carries the matrix losslessly, so the merge + store //! round-trip works end-to-end without that richer query surface. -use crate::data_model::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; +use crate::stores::schema::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; use asap_sketchlib::sketches::countsketch::{CountSketch, CountSketchDelta}; use serde_json::Value; use std::collections::HashMap; @@ -505,7 +505,7 @@ mod tests { #[test] fn test_aggregate_core_merge_wrong_type_rejects() { - use crate::precompute_operators::count_min_sketch_accumulator::CountMinSketchAccumulator; + use crate::precompute_engine::operators::count_min_sketch_accumulator::CountMinSketchAccumulator; let cs = CountSketchAccumulator::new(2, 3); let cms = CountMinSketchAccumulator::new(2, 3); let result = cs.merge_with(&cms); diff --git a/asap-query-engine/src/precompute_operators/datasketches_kll_accumulator.rs b/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs similarity index 99% rename from asap-query-engine/src/precompute_operators/datasketches_kll_accumulator.rs rename to data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs index d4abf23a..458be54f 100644 --- a/asap-query-engine/src/precompute_operators/datasketches_kll_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, AuxStats, MergeableAccumulator, SerializableToSink, SingleSubpopulationAggregate, }; @@ -155,7 +155,7 @@ impl DatasketchesKLLAccumulator { /// Merge multiple accumulators efficiently without cloning all of them. pub fn merge_multiple( - accumulators: &[Box], + accumulators: &[Box], ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); @@ -315,7 +315,7 @@ impl AggregateCore for DatasketchesKLLAccumulator { _key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::data_model::SingleSubpopulationAggregate; + use crate::stores::schema::SingleSubpopulationAggregate; self.query(statistic, Some(query_kwargs)) } } @@ -560,7 +560,7 @@ mod tests { let boxed_accs: Vec> = vec![Box::new(kll1), Box::new(kll2)]; assert!(DatasketchesKLLAccumulator::merge_multiple(&boxed_accs).is_err()); - use crate::precompute_operators::sum_accumulator::SumAccumulator; + use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; let kll = DatasketchesKLLAccumulator::new(200); let sum = SumAccumulator::new(); let mixed_accs: Vec> = vec![Box::new(kll), Box::new(sum)]; diff --git a/asap-query-engine/src/precompute_operators/dd_sketch_accumulator.rs b/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs similarity index 98% rename from asap-query-engine/src/precompute_operators/dd_sketch_accumulator.rs rename to data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs index f0203bc4..904fd9d4 100644 --- a/asap-query-engine/src/precompute_operators/dd_sketch_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs @@ -11,7 +11,7 @@ //! offset, and aggregates losslessly, so the merge + store round-trip //! works end-to-end without that richer query surface. -use crate::data_model::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; +use crate::stores::schema::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; use asap_sketchlib::sketches::ddsketch::{DdSketch, DdSketchDelta}; use serde_json::Value; use std::collections::HashMap; @@ -345,7 +345,7 @@ mod tests { #[test] fn test_aggregate_core_merge_wrong_type_rejects() { - use crate::precompute_operators::count_sketch_accumulator::CountSketchAccumulator; + use crate::precompute_engine::operators::count_sketch_accumulator::CountSketchAccumulator; let dd = DDSketchAccumulator::new(0.01); let cs = CountSketchAccumulator::new(2, 3); assert!(dd.merge_with(&cs).is_err()); diff --git a/asap-query-engine/src/precompute_operators/delta_set_aggregator_accumulator.rs b/data_plane/src/precompute_engine/operators/delta_set_aggregator_accumulator.rs similarity index 99% rename from asap-query-engine/src/precompute_operators/delta_set_aggregator_accumulator.rs rename to data_plane/src/precompute_engine/operators/delta_set_aggregator_accumulator.rs index 18360be4..aa44fb15 100644 --- a/asap-query-engine/src/precompute_operators/delta_set_aggregator_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/delta_set_aggregator_accumulator.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, }; @@ -268,7 +268,7 @@ impl AggregateCore for DeltaSetAggregatorAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::data_model::MultipleSubpopulationAggregate; + use crate::stores::schema::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for DeltaSetAggregatorAccumulator")?; diff --git a/asap-query-engine/src/precompute_operators/edge_runtime_adapter.rs b/data_plane/src/precompute_engine/operators/edge_runtime_adapter.rs similarity index 100% rename from asap-query-engine/src/precompute_operators/edge_runtime_adapter.rs rename to data_plane/src/precompute_engine/operators/edge_runtime_adapter.rs diff --git a/asap-query-engine/src/precompute_operators/hll_sketch_accumulator.rs b/data_plane/src/precompute_engine/operators/hll_sketch_accumulator.rs similarity index 98% rename from asap-query-engine/src/precompute_operators/hll_sketch_accumulator.rs rename to data_plane/src/precompute_engine/operators/hll_sketch_accumulator.rs index aa5cf48a..7777c5fc 100644 --- a/asap-query-engine/src/precompute_operators/hll_sketch_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/hll_sketch_accumulator.rs @@ -11,7 +11,7 @@ //! registers + variant + HIP accumulators losslessly, so the merge + //! store round-trip works end-to-end without that richer query surface. -use crate::data_model::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; +use crate::stores::schema::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; use asap_sketchlib::sketches::hll::{HllSketch, HllSketchDelta, HllVariant}; use serde_json::Value; use std::collections::HashMap; @@ -425,7 +425,7 @@ mod tests { #[test] fn test_aggregate_core_merge_wrong_type_rejects() { - use crate::precompute_operators::count_sketch_accumulator::CountSketchAccumulator; + use crate::precompute_engine::operators::count_sketch_accumulator::CountSketchAccumulator; let hll = HllSketchAccumulator::new(HllVariant::Regular, 2); let cs = CountSketchAccumulator::new(2, 3); assert!(hll.merge_with(&cs).is_err()); diff --git a/asap-query-engine/src/precompute_operators/hydra_kll_accumulator.rs b/data_plane/src/precompute_engine/operators/hydra_kll_accumulator.rs similarity index 98% rename from asap-query-engine/src/precompute_operators/hydra_kll_accumulator.rs rename to data_plane/src/precompute_engine/operators/hydra_kll_accumulator.rs index 38b827ad..02bc2f8a 100644 --- a/asap-query-engine/src/precompute_operators/hydra_kll_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/hydra_kll_accumulator.rs @@ -1,5 +1,5 @@ use crate::{ - data_model::{ + stores::schema::{ AggregateCore, AggregationType, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, }, @@ -136,7 +136,7 @@ impl AggregateCore for HydraKllSketchAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::data_model::MultipleSubpopulationAggregate; + use crate::stores::schema::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for HydraKllSketchAccumulator")?; diff --git a/asap-query-engine/src/precompute_operators/increase_accumulator.rs b/data_plane/src/precompute_engine/operators/increase_accumulator.rs similarity index 99% rename from asap-query-engine/src/precompute_operators/increase_accumulator.rs rename to data_plane/src/precompute_engine/operators/increase_accumulator.rs index a2a88a0d..78669cec 100644 --- a/asap-query-engine/src/precompute_operators/increase_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/increase_accumulator.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, Measurement, MergeableAccumulator, SerializableToSink, SingleSubpopulationAggregate, SingleSubpopulationAggregateFactory, }; @@ -264,7 +264,7 @@ impl AggregateCore for IncreaseAccumulator { _key: &Option, _query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::data_model::SingleSubpopulationAggregate; + use crate::stores::schema::SingleSubpopulationAggregate; self.query(statistic, None) } } diff --git a/asap-query-engine/src/precompute_operators/min_max_accumulator.rs b/data_plane/src/precompute_engine/operators/min_max_accumulator.rs similarity index 99% rename from asap-query-engine/src/precompute_operators/min_max_accumulator.rs rename to data_plane/src/precompute_engine/operators/min_max_accumulator.rs index 957e9c80..4b6ff999 100644 --- a/asap-query-engine/src/precompute_operators/min_max_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/min_max_accumulator.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, AuxStats, MergeableAccumulator, SerializableToSink, SingleSubpopulationAggregate, SingleSubpopulationAggregateFactory, }; @@ -225,7 +225,7 @@ impl AggregateCore for MinMaxAccumulator { _key: &Option, _query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::data_model::SingleSubpopulationAggregate; + use crate::stores::schema::SingleSubpopulationAggregate; self.query(statistic, None) } } diff --git a/asap-query-engine/src/precompute_operators/mod.rs b/data_plane/src/precompute_engine/operators/mod.rs similarity index 100% rename from asap-query-engine/src/precompute_operators/mod.rs rename to data_plane/src/precompute_engine/operators/mod.rs diff --git a/asap-query-engine/src/precompute_operators/multiple_increase_accumulator.rs b/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs similarity index 98% rename from asap-query-engine/src/precompute_operators/multiple_increase_accumulator.rs rename to data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs index ee417d01..56ded8ff 100644 --- a/asap-query-engine/src/precompute_operators/multiple_increase_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs @@ -1,13 +1,13 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, SingleSubpopulationAggregate, }; -use crate::precompute_operators::IncreaseAccumulator; +use crate::precompute_engine::operators::IncreaseAccumulator; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::HashMap; -use crate::data_model::Measurement; +use crate::stores::schema::Measurement; use promql_utilities::query_logics::enums::Statistic; /// Accumulator that maintains separate increase accumulators for multiple keys @@ -303,7 +303,7 @@ impl AggregateCore for MultipleIncreaseAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::data_model::MultipleSubpopulationAggregate; + use crate::stores::schema::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for MultipleIncreaseAccumulator")?; @@ -367,7 +367,7 @@ impl MergeableAccumulator for MultipleIncreaseAccum #[cfg(test)] mod tests { use super::*; - use crate::data_model::Measurement; + use crate::stores::schema::Measurement; fn create_test_increase_accumulator(start_val: f64, end_val: f64) -> IncreaseAccumulator { IncreaseAccumulator::new( diff --git a/asap-query-engine/src/precompute_operators/multiple_min_max_accumulator.rs b/data_plane/src/precompute_engine/operators/multiple_min_max_accumulator.rs similarity index 99% rename from asap-query-engine/src/precompute_operators/multiple_min_max_accumulator.rs rename to data_plane/src/precompute_engine/operators/multiple_min_max_accumulator.rs index 90f0391e..a4c90845 100644 --- a/asap-query-engine/src/precompute_operators/multiple_min_max_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/multiple_min_max_accumulator.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, }; @@ -259,7 +259,7 @@ impl AggregateCore for MultipleMinMaxAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::data_model::MultipleSubpopulationAggregate; + use crate::stores::schema::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for MultipleMinMaxAccumulator")?; diff --git a/asap-query-engine/src/precompute_operators/multiple_sum_accumulator.rs b/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs similarity index 99% rename from asap-query-engine/src/precompute_operators/multiple_sum_accumulator.rs rename to data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs index 5f565356..d4e3de5c 100644 --- a/asap-query-engine/src/precompute_operators/multiple_sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, MultipleSubpopulationAggregateFactory, SerializableToSink, }; @@ -251,7 +251,7 @@ impl AggregateCore for MultipleSumAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::data_model::MultipleSubpopulationAggregate; + use crate::stores::schema::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for MultipleSumAccumulator")?; diff --git a/asap-query-engine/src/precompute_operators/set_aggregator_accumulator.rs b/data_plane/src/precompute_engine/operators/set_aggregator_accumulator.rs similarity index 99% rename from asap-query-engine/src/precompute_operators/set_aggregator_accumulator.rs rename to data_plane/src/precompute_engine/operators/set_aggregator_accumulator.rs index 73745220..9a045e62 100644 --- a/asap-query-engine/src/precompute_operators/set_aggregator_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/set_aggregator_accumulator.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, }; @@ -201,7 +201,7 @@ impl AggregateCore for SetAggregatorAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::data_model::MultipleSubpopulationAggregate; + use crate::stores::schema::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for SetAggregatorAccumulator")?; diff --git a/asap-query-engine/src/precompute_operators/sketch_envelope_accumulator.rs b/data_plane/src/precompute_engine/operators/sketch_envelope_accumulator.rs similarity index 95% rename from asap-query-engine/src/precompute_operators/sketch_envelope_accumulator.rs rename to data_plane/src/precompute_engine/operators/sketch_envelope_accumulator.rs index 07fbf575..c1527a69 100644 --- a/asap-query-engine/src/precompute_operators/sketch_envelope_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/sketch_envelope_accumulator.rs @@ -5,7 +5,7 @@ //! (via `SketchEnvelope::decode`) only when merge or query operations need //! the inner sketch type. -use crate::data_model::{AggregateCore, KeyByLabelValues, SerializableToSink}; +use crate::stores::schema::{AggregateCore, KeyByLabelValues, SerializableToSink}; use asap_sketchlib::proto::sketchlib::{sketch_envelope, SketchEnvelope}; use prost::Message; use serde_json::Value; @@ -139,7 +139,7 @@ impl AggregateCore for SketchEnvelopeAccumulator { } } -impl crate::data_model::MultipleSubpopulationAggregate for SketchEnvelopeAccumulator { +impl crate::stores::schema::MultipleSubpopulationAggregate for SketchEnvelopeAccumulator { fn query( &self, _statistic: Statistic, @@ -152,7 +152,7 @@ impl crate::data_model::MultipleSubpopulationAggregate for SketchEnvelopeAccumul ) } - fn clone_boxed(&self) -> Box { + fn clone_boxed(&self) -> Box { Box::new(self.clone()) } } diff --git a/asap-query-engine/src/precompute_operators/sum_accumulator.rs b/data_plane/src/precompute_engine/operators/sum_accumulator.rs similarity index 99% rename from asap-query-engine/src/precompute_operators/sum_accumulator.rs rename to data_plane/src/precompute_engine/operators/sum_accumulator.rs index 696b66d9..ce65d278 100644 --- a/asap-query-engine/src/precompute_operators/sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/sum_accumulator.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, AuxStats, MergeableAccumulator, SerializableToSink, SingleSubpopulationAggregate, SingleSubpopulationAggregateFactory, }; @@ -146,7 +146,7 @@ impl AggregateCore for SumAccumulator { _key: &Option, _query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::data_model::SingleSubpopulationAggregate; + use crate::stores::schema::SingleSubpopulationAggregate; self.query(statistic, None) } } diff --git a/asap-query-engine/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs similarity index 98% rename from asap-query-engine/src/precompute_engine/output_sink.rs rename to data_plane/src/precompute_engine/output_sink.rs index d1bc2bca..2137feef 100644 --- a/asap-query-engine/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -1,4 +1,4 @@ -use crate::data_model::{AggregateCore, PrecomputedOutput}; +use crate::stores::schema::{AggregateCore, PrecomputedOutput}; use crate::stores::Store; use std::sync::{Arc, Mutex}; use tracing::debug_span; diff --git a/asap-query-engine/src/precompute_engine/precompute_engine_design_doc.md b/data_plane/src/precompute_engine/precompute_engine_design_doc.md similarity index 100% rename from asap-query-engine/src/precompute_engine/precompute_engine_design_doc.md rename to data_plane/src/precompute_engine/precompute_engine_design_doc.md diff --git a/asap-query-engine/src/precompute_engine/series_buffer.rs b/data_plane/src/precompute_engine/series_buffer.rs similarity index 100% rename from asap-query-engine/src/precompute_engine/series_buffer.rs rename to data_plane/src/precompute_engine/series_buffer.rs diff --git a/asap-query-engine/src/precompute_engine/series_router.rs b/data_plane/src/precompute_engine/series_router.rs similarity index 99% rename from asap-query-engine/src/precompute_engine/series_router.rs rename to data_plane/src/precompute_engine/series_router.rs index 39936c16..6f536837 100644 --- a/asap-query-engine/src/precompute_engine/series_router.rs +++ b/data_plane/src/precompute_engine/series_router.rs @@ -1,4 +1,4 @@ -use crate::data_model::AggregateCore; +use crate::stores::schema::AggregateCore; use futures::future::try_join_all; use std::collections::HashMap; use std::fmt; diff --git a/asap-query-engine/src/precompute_engine/window_manager.rs b/data_plane/src/precompute_engine/window_manager.rs similarity index 100% rename from asap-query-engine/src/precompute_engine/window_manager.rs rename to data_plane/src/precompute_engine/window_manager.rs diff --git a/asap-query-engine/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs similarity index 99% rename from asap-query-engine/src/precompute_engine/worker.rs rename to data_plane/src/precompute_engine/worker.rs index df0d7672..3c0c23a1 100644 --- a/asap-query-engine/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, HotReloadStreamingConfig, KeyByLabelValues, PrecomputedOutput, }; use crate::precompute_engine::accumulator_factory::{ @@ -8,7 +8,7 @@ use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::output_sink::OutputSink; use crate::precompute_engine::series_router::WorkerMessage; use crate::precompute_engine::window_manager::WindowManager; -use crate::precompute_operators::sum_accumulator::SumAccumulator; +use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; use asap_types::aggregation_config::AggregationConfig; use std::collections::{BTreeMap, HashMap}; use std::sync::atomic::{AtomicI64, AtomicUsize, Ordering}; @@ -1070,12 +1070,12 @@ mod tests { // Helpers // ----------------------------------------------------------------------- - use crate::data_model::StreamingConfig; + use crate::stores::schema::StreamingConfig; use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::output_sink::CapturingOutputSink; - use crate::precompute_operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; - use crate::precompute_operators::multiple_sum_accumulator::MultipleSumAccumulator; - use crate::precompute_operators::sum_accumulator::SumAccumulator; + use crate::precompute_engine::operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; + use crate::precompute_engine::operators::multiple_sum_accumulator::MultipleSumAccumulator; + use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; use asap_sketchlib::sketches::kll::KllSketch; use asap_types::enums::{AggregationType, WindowType}; @@ -1176,8 +1176,8 @@ mod tests { /// callsite. fn make_hot_reload( configs: HashMap, - ) -> crate::data_model::HotReloadStreamingConfig { - crate::data_model::HotReloadStreamingConfig::new(crate::data_model::StreamingConfig::new( + ) -> crate::stores::schema::HotReloadStreamingConfig { + crate::stores::schema::HotReloadStreamingConfig::new(crate::stores::schema::StreamingConfig::new( configs, )) } @@ -2348,7 +2348,7 @@ aggregations: // OTLP ingest dispatch builds via `decode_modified_otlp_sketch_bytes`. // ----------------------------------------------------------------------- - use crate::precompute_operators::DDSketchAccumulator; + use crate::precompute_engine::operators::DDSketchAccumulator; use asap_sketchlib::sketches::ddsketch::DdSketch; /// Build a fresh DDSketch holding `vals` so each test has a real, @@ -2451,7 +2451,7 @@ aggregations: /// queried metric / agg_id. #[test] fn test_sketch_ingest_persists_and_query_returns_non_empty() { - use crate::data_model::{CleanupPolicy, StreamingConfig}; + use crate::stores::schema::{CleanupPolicy, StreamingConfig}; use crate::precompute_engine::output_sink::StoreOutputSink; use crate::stores::sketch_db::simple_map_store::per_key::SimpleMapStorePerKey; use crate::stores::Store; @@ -2487,7 +2487,7 @@ aggregations: 0, rx, sink, - crate::data_model::HotReloadStreamingConfig::new(StreamingConfig::new(configs_map)), + crate::stores::schema::HotReloadStreamingConfig::new(StreamingConfig::new(configs_map)), WorkerRuntimeConfig { max_buffer_per_series: 10_000, allowed_lateness_ms: 0, diff --git a/asap-query-engine/src/query-engines/asap_query/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs similarity index 96% rename from asap-query-engine/src/query-engines/asap_query/engine.rs rename to data_plane/src/query_engines/asap_query_engine/engine.rs index 36795b16..7e92fc76 100644 --- a/asap-query-engine/src/query-engines/asap_query/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -1,8 +1,8 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregationIdInfo, InferenceConfig, KeyByLabelValues, QueryConfig, QueryLanguage, SchemaConfig, StreamingConfig, }; -use crate::engines::query_result::{InstantVectorElement, QueryResult, RangeVectorElement}; +use crate::query_engines::query_result::{InstantVectorElement, QueryResult, RangeVectorElement}; // use crate::stores::promsketch_store::{ // self, is_usampling_function, metrics as ps_metrics, PromSketchStore, // }; @@ -249,7 +249,7 @@ pub struct RangeQueryExecutionContext { // } /// Simple query engine for processing PromQL-like queries against precomputed data -pub struct SimpleEngine { +pub struct ASAPQueryEngine { store: Arc, // promsketch_store: Option>, inference_config: InferenceConfig, @@ -260,9 +260,9 @@ pub struct SimpleEngine { /// **next** query without restarting the binary (PR E phase 2). /// Clones of `HotReloadStreamingConfig` share the same /// underlying `ArcSwap`, so when `main.rs` hands the same handle - /// to both `SimpleEngine` and `HttpServer::with_hot_reload_config`, + /// to both `ASAPQueryEngine` and `HttpServer::with_hot_reload_config`, /// a POST is immediately visible to the next query. - streaming_config_source: crate::data_model::HotReloadStreamingConfig, + streaming_config_source: crate::stores::schema::HotReloadStreamingConfig, prometheus_scrape_interval: u64, controller_patterns: HashMap>, query_language: QueryLanguage, @@ -274,7 +274,7 @@ pub struct SimpleEngine { controller_client: Option>, /// Per-`agg_id` schema registry used for §7 schema-timeline /// dispatch (`docs/design-sketch-db.md`). The combiner lives in - /// [`crate::engines::timeline_dispatch`] and the lookup primitive + /// [`crate::query_engines::timeline_dispatch`] and the lookup primitive /// is exposed on [`crate::stores::sketch_db::SchemaRegistry`]; /// the engine consults the registry on every query to resolve /// which agg_id owns each sub-range of the query's time window. @@ -303,17 +303,11 @@ pub struct SimpleEngine { /// When `None` (no archive engine wired), the engine returns the /// warm answer as-is; the existing `EngineRouter` failover handles /// the rest of the routing matrix. - archive_engine: Option>, + archive_engine: Option>, } -/// Public production name for the warm-tier sketch query engine. -/// -/// `SimpleEngine` remains as a compatibility alias in existing tests -/// and downstream code, but new code should use `ASAPQueryEngine`. -pub type ASAPQueryEngine = SimpleEngine; - -impl SimpleEngine { - /// Construct a `SimpleEngine` with a static `Arc`. +impl ASAPQueryEngine { + /// Construct a `ASAPQueryEngine` with a static `Arc`. /// Wraps the config in a fresh `HotReloadStreamingConfig` internally /// — callers that need to share the hot-reload handle with the HTTP /// server should use `new_with_hot_reload` instead so a POST to @@ -328,7 +322,7 @@ impl SimpleEngine { prometheus_scrape_interval: u64, query_language: QueryLanguage, ) -> Self { - let hot_reload = crate::data_model::HotReloadStreamingConfig::from_arc(streaming_config); + let hot_reload = crate::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config); Self::new_with_hot_reload( store, inference_config, @@ -338,14 +332,14 @@ impl SimpleEngine { ) } - /// Construct a `SimpleEngine` that shares a `HotReloadStreamingConfig` + /// Construct a `ASAPQueryEngine` that shares a `HotReloadStreamingConfig` /// handle with another holder (typically the HTTP server). This is /// the constructor `main.rs` should call so `POST /api/v1/streaming-config` /// is observable by the next query. pub fn new_with_hot_reload( store: Arc, inference_config: InferenceConfig, - streaming_config_source: crate::data_model::HotReloadStreamingConfig, + streaming_config_source: crate::stores::schema::HotReloadStreamingConfig, prometheus_scrape_interval: u64, query_language: QueryLanguage, ) -> Self { @@ -502,7 +496,7 @@ impl SimpleEngine { /// When `None`, the engine returns whatever the warm tier covers. pub fn with_archive_engine( mut self, - archive: Arc, + archive: Arc, ) -> Self { self.archive_engine = Some(archive); self @@ -526,7 +520,7 @@ impl SimpleEngine { /// is stable for the caller's lifetime — a concurrent swap /// produces a new `Arc` and leaves the one returned here alone. /// - /// Internal read sites inside `SimpleEngine` bind this once per + /// Internal read sites inside `ASAPQueryEngine` bind this once per /// logical unit of work (typically per query-handler invocation /// or per helper call) and use the local Arc for the duration, /// so references into the underlying `StreamingConfig` stay @@ -720,15 +714,15 @@ impl SimpleEngine { // shapes are eligible for this resolver. Any other shape // falls through unchanged. let ast = promql_parser::parser::parse(query).ok()?; - let shape = crate::routing::classify_query_shape(&ast); + let shape = crate::query_engines::routing::classify_query_shape(&ast); let suffixes: &[&str] = match shape { - crate::routing::QueryShape::Quantile => &["_quantile"], + crate::query_engines::routing::QueryShape::Quantile => &["_quantile"], // `count(metric)` against an HLL-backed agg is the // cardinality readout — see `compatible_agg_types(Count)` // and `HllSketchAccumulator::query_statistic`. Capture it // here so the wire-side `_hll` rename is invisible to // user PromQL. - crate::routing::QueryShape::Count => &["_hll"], + crate::query_engines::routing::QueryShape::Count => &["_hll"], _ => return None, }; @@ -2358,7 +2352,7 @@ impl SimpleEngine { /// the dispatch builds a context targeting that segment's /// `agg_id`, executes it against the clipped segment range, /// collects the scalar, and combines across segments via - /// [`crate::engines::timeline_dispatch::combine_statistic`]. + /// [`crate::query_engines::timeline_dispatch::combine_statistic`]. /// /// Returns `None` when the query can't be parsed / pattern-matched, /// or when `forced_agg_id` isn't in the current `StreamingConfig`. @@ -2531,7 +2525,7 @@ impl SimpleEngine { query: &str, time: f64, ) -> Option<(KeyByLabelNames, QueryResult)> { - use crate::engines::timeline_dispatch::{combine_statistic, CombinedResult, SegmentValue}; + use crate::query_engines::timeline_dispatch::{combine_statistic, CombinedResult, SegmentValue}; use crate::stores::sketch_db::{TimelineCoverage, TimelineSegment}; // Phase 1: shared pipeline with the default path — parse, @@ -2767,7 +2761,7 @@ impl SimpleEngine { precomputed_outputs_map: &TimestampedBucketsMap, do_merge: bool, aggregation_type: AggregationType, - ) -> HashMap, Box> { + ) -> HashMap, Box> { #[cfg(feature = "extra_debugging")] let start_time = Instant::now(); #[cfg(feature = "extra_debugging")] @@ -2833,8 +2827,8 @@ impl SimpleEngine { /// This follows the Python merge_accumulators approach fn merge_accumulators( &self, - accumulators: &[Box], - ) -> Box { + accumulators: &[Box], + ) -> Box { if accumulators.is_empty() { panic!("No accumulators to merge"); } @@ -2845,7 +2839,7 @@ impl SimpleEngine { // Try to use optimized batch merge for KLL accumulators if accumulators[0].get_accumulator_type() == AggregationType::DatasketchesKLL { - use crate::precompute_operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; + use crate::precompute_engine::operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; match DatasketchesKLLAccumulator::merge_multiple(accumulators) { Ok(merged) => return Box::new(merged), @@ -2861,7 +2855,7 @@ impl SimpleEngine { // Try to use optimized batch merge for CountMinSketch accumulators if accumulators[0].get_accumulator_type() == AggregationType::CountMinSketch { - use crate::precompute_operators::count_min_sketch_accumulator::CountMinSketchAccumulator; + use crate::precompute_engine::operators::count_min_sketch_accumulator::CountMinSketchAccumulator; match CountMinSketchAccumulator::merge_multiple(accumulators) { Ok(merged) => return Box::new(merged), @@ -3336,9 +3330,9 @@ impl SimpleEngine { fn execute_range_query_pipeline( &self, context: &RangeQueryExecutionContext, - ) -> Result, String> { - use crate::engines::query_result::RangeVectorElement; - use crate::engines::window_merger::create_window_merger; + ) -> Result, String> { + use crate::query_engines::query_result::RangeVectorElement; + use crate::query_engines::window_merger::create_window_merger; // Step 1: Fetch all data needed for the entire range let all_data = self.execute_store_query(&context.base.store_plan.values_query)?; @@ -3507,10 +3501,10 @@ impl SimpleEngine { // to the next compatible backend. // --------------------------------------------------------------------------- -/// Adapt a [`crate::engines::asap_query::warm_tier::WarmTierResult`] to the engine's +/// Adapt a [`crate::query_engines::warm_tier::WarmTierResult`] to the engine's /// existing `QueryResult` shape. The reducer hands back per-series /// time-stamped scalars; we materialize them as a -/// `QueryResult::Matrix` whose [`crate::engines::query_result::RangeVectorElement`]s +/// `QueryResult::Matrix` whose [`crate::query_engines::query_result::RangeVectorElement`]s /// each map onto one (label-values, samples) entry. /// /// `now_ms` is unused for the matrix variant (each sample carries its @@ -3522,15 +3516,15 @@ impl SimpleEngine { /// the warm value (warm is approximate but more recent); samples /// outside that window come from the archive answer. For /// labels-not-present-in-warm series the archive series is taken in -/// full. Used by `SimpleEngine`'s hybrid-stitch path when the +/// full. Used by `ASAPQueryEngine`'s hybrid-stitch path when the /// warm-tier reducer reports `coverage` narrower than the request. fn stitch_warm_and_archive( - warm: crate::engines::query_result::QueryResult, - archive: crate::engines::query_result::QueryResult, + warm: crate::query_engines::query_result::QueryResult, + archive: crate::query_engines::query_result::QueryResult, cov_lo: u64, cov_hi: u64, -) -> crate::engines::query_result::QueryResult { - use crate::engines::query_result::{QueryResult, RangeVectorElement, Sample}; +) -> crate::query_engines::query_result::QueryResult { + use crate::query_engines::query_result::{QueryResult, RangeVectorElement, Sample}; use std::collections::BTreeMap; let warm_matrix = match &warm { @@ -3581,11 +3575,11 @@ fn stitch_warm_and_archive( } fn warm_tier_result_to_query_result( - result: crate::engines::asap_query::warm_tier::WarmTierResult, + result: crate::query_engines::warm_tier::WarmTierResult, _now_ms: u64, -) -> crate::engines::query_result::QueryResult { - use crate::data_model::KeyByLabelValues; - use crate::engines::query_result::{QueryResult, RangeVectorElement}; +) -> crate::query_engines::query_result::QueryResult { + use crate::stores::schema::KeyByLabelValues; + use crate::query_engines::query_result::{QueryResult, RangeVectorElement}; let mut elements: Vec = Vec::with_capacity(result.series.len()); for (label_values, samples) in result.series { @@ -3621,11 +3615,11 @@ fn warm_tier_result_to_query_result( } #[async_trait::async_trait] -impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { +impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQueryEngine { async fn execute( &self, query: &str, - ) -> Result { + ) -> Result { // Phase 9 controller-unification (2026-05) — the warm-tier // hook is now a thin driver around the controller's // `analyze_promql_for_warm_tier`. The analyzer is the single @@ -3663,7 +3657,7 @@ impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { // Branch 1 — the controller analyzer rejects the shape. if let Some(reason) = &analysis.unsupported { - return Err(crate::engines::EngineError::capability_miss( + return Err(crate::query_engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!( "SketchStore analyzer rejected `{query}`: {reason:?} — \ @@ -3676,7 +3670,7 @@ impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { // caught this; analyzer guarantees `unsupported.is_some()` // when `candidates.is_empty()` but we keep the // belt-and-braces miss-path for safety. - return Err(crate::engines::EngineError::capability_miss( + return Err(crate::query_engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!( "SketchStore analyzer produced no warm-tier candidates for \ @@ -3701,21 +3695,21 @@ impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { // for instant-vector candidates (range_seconds == 0). const DEFAULT_LOOKBACK_MS: u64 = 5 * 60 * 1000; - let reducer = crate::engines::asap_query::warm_tier::SketchReducer::new(idx); + let reducer = crate::query_engines::warm_tier::SketchReducer::new(idx); // Multi-candidate aggregation is deferred (single-result // shapes today). On the first reducer error we surface // CapabilityMiss; on Ok we keep the result for the // hybrid-stitch path below. (When more than one // candidate is supported, a follow-up will fold // per-candidate WarmTierResults.) - let mut combined_result: Option = + let mut combined_result: Option = None; let mut combined_t0: u64 = u64::MAX; for candidate in &analysis.candidates { let sids = idx.instances_matching(&candidate.metric_name, &candidate.group_by_keys); if sids.is_empty() { - return Err(crate::engines::EngineError::capability_miss( + return Err(crate::query_engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!( "SketchStore has no instance for metric `{}` \ @@ -3741,7 +3735,7 @@ impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { crate::stores::sketch_db::sketch_index::SidLookup::Hit => {} crate::stores::sketch_db::sketch_index::SidLookup::Ghost | crate::stores::sketch_db::sketch_index::SidLookup::Unknown => { - return Err(crate::engines::EngineError::capability_miss( + return Err(crate::query_engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!( "SketchStore ghost/unknown sid {sid} for metric \ @@ -3760,7 +3754,7 @@ impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { } } if hit_sids.is_empty() { - return Err(crate::engines::EngineError::capability_miss( + return Err(crate::query_engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!( "SketchStore has no sid satisfying capability \ @@ -3789,11 +3783,11 @@ impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { ) { Ok(r) => r, Err( - crate::engines::asap_query::warm_tier::WarmTierError::UnsupportedFunction( + crate::query_engines::warm_tier::WarmTierError::UnsupportedFunction( name, ), ) => { - return Err(crate::engines::EngineError::capability_miss( + return Err(crate::query_engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!( "SketchStore reducer does not support function `{name}` \ @@ -3801,11 +3795,11 @@ impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { ), )); } - Err(crate::engines::asap_query::warm_tier::WarmTierError::UnsupportedCapability { + Err(crate::query_engines::warm_tier::WarmTierError::UnsupportedCapability { function, capability, }) => { - return Err(crate::engines::EngineError::capability_miss( + return Err(crate::query_engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!( "SketchStore reducer cannot answer `{function}` against \ @@ -3813,12 +3807,12 @@ impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { ), )); } - Err(crate::engines::asap_query::warm_tier::WarmTierError::DeserializeFailure { + Err(crate::query_engines::warm_tier::WarmTierError::DeserializeFailure { sid, encoding, reason, }) => { - return Err(crate::engines::EngineError::capability_miss( + return Err(crate::query_engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!( "SketchStore reducer failed to decode sketch for sid \ @@ -3827,10 +3821,10 @@ impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { ), )); } - Err(crate::engines::asap_query::warm_tier::WarmTierError::NoData { + Err(crate::query_engines::warm_tier::WarmTierError::NoData { metric_name: m, }) => { - return Err(crate::engines::EngineError::capability_miss( + return Err(crate::query_engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!( "SketchStore reducer found no samples for metric \ @@ -3838,11 +3832,11 @@ impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { ), )); } - Err(crate::engines::asap_query::warm_tier::WarmTierError::MissingHeap { + Err(crate::query_engines::warm_tier::WarmTierError::MissingHeap { sid, sketch_kind, }) => { - return Err(crate::engines::EngineError::capability_miss( + return Err(crate::query_engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!( "SketchStore reducer cannot enumerate top-k for sid \ @@ -3891,15 +3885,15 @@ impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { .unwrap_or(0.0); match self.handle_query(query.to_string(), now_ms) { Some((_labels, result)) => Ok(result), - None => Err(crate::engines::EngineError::capability_miss( + None => Err(crate::query_engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!("ASAPQueryEngine has no compatible aggregation for `{query}`"), )), } } - fn capabilities(&self) -> crate::routing::query_engine_routing::EngineCapabilities { - crate::routing::query_engine_routing::EngineCapabilities { + fn capabilities(&self) -> crate::query_engines::routing::query_engine_routing::EngineCapabilities { + crate::query_engines::routing::query_engine_routing::EngineCapabilities { data_source_id: asap_types::StorageBackend::SketchStore.data_source_id(), storage_backend: asap_types::StorageBackend::SketchStore, // Warm-tier sketches are O(sketch-size); call it 16 MiB ceiling @@ -3911,8 +3905,8 @@ impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { #[cfg(test)] mod range_query_tests { - use crate::data_model::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; - use crate::engines::window_merger::NaiveMerger; + use crate::stores::schema::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; + use crate::query_engines::window_merger::NaiveMerger; use serde_json::Value; use std::any::Any; @@ -3999,7 +3993,7 @@ mod range_query_tests { end_ms: u64, step_ms: u64, ) -> Vec<(u64, f64, u64)> { - use crate::engines::window_merger::WindowMerger; + use crate::query_engines::window_merger::WindowMerger; let mut results = Vec::new(); @@ -4064,7 +4058,7 @@ mod range_query_tests { step_ms: u64, expected_bucket_count: usize, ) -> Vec<(u64, f64, u64)> { - use crate::engines::window_merger::WindowMerger; + use crate::query_engines::window_merger::WindowMerger; let mut results = Vec::new(); @@ -4383,7 +4377,7 @@ mod range_query_tests { end_ms: u64, step_ms: u64, ) -> Vec<(u64, f64, u64)> { - use crate::engines::window_merger::WindowMerger; + use crate::query_engines::window_merger::WindowMerger; use std::collections::HashMap; let mut results = Vec::new(); @@ -4686,8 +4680,8 @@ mod range_query_tests { #[cfg(test)] mod sketch_query_tests { - // use crate::data_model::{CleanupPolicy, InferenceConfig, QueryLanguage, StreamingConfig}; - // use crate::engines::asap_query::engine::SimpleEngine; + // use crate::stores::schema::{CleanupPolicy, InferenceConfig, QueryLanguage, StreamingConfig}; + // use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; // use crate::stores::promsketch_store::PromSketchStore; // use crate::stores::{Store, TimestampedBucketsMap}; // use std::collections::HashMap; @@ -4717,16 +4711,16 @@ mod sketch_query_tests { // } // fn insert_precomputed_output( // &self, - // _: crate::data_model::PrecomputedOutput, - // _: Box, + // _: crate::stores::schema::PrecomputedOutput, + // _: Box, // ) -> Result<(), Box> { // panic!("NoOpStore should not be called for sketch queries"); // } // fn insert_precomputed_output_batch( // &self, // _: Vec<( - // crate::data_model::PrecomputedOutput, - // Box, + // crate::stores::schema::PrecomputedOutput, + // Box, // )>, // ) -> Result<(), Box> { // panic!("NoOpStore should not be called for sketch queries"); @@ -4743,7 +4737,7 @@ mod sketch_query_tests { // /// Helper: create an engine with a populated PromSketchStore. // /// Inserts data points 1..=100 into a series with labels = `series_key`. - // fn engine_with_sketch_data(series_key: &str) -> SimpleEngine { + // fn engine_with_sketch_data(series_key: &str) -> ASAPQueryEngine { // let ps = Arc::new(PromSketchStore::with_default_config()); // ps.ensure_all_sketches(series_key).unwrap(); // for i in 1..=100u64 { @@ -4754,7 +4748,7 @@ mod sketch_query_tests { // InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::NoCleanup); // let streaming_config = Arc::new(StreamingConfig::default()); - // SimpleEngine::new( + // ASAPQueryEngine::new( // Arc::new(NoOpStore), // Some(ps), // inference_config, @@ -4774,7 +4768,7 @@ mod sketch_query_tests { // assert!(result.is_some(), "entropy_over_time should return a result"); // let (labels, qr) = result.unwrap(); // assert!(!labels.labels.is_empty()); - // if let crate::engines::query_result::QueryResult::Vector(iv) = qr { + // if let crate::query_engines::query_result::QueryResult::Vector(iv) = qr { // assert!(!iv.values.is_empty(), "should have at least one result"); // let val = iv.values[0].value; // assert!(val >= 0.0, "entropy should be non-negative, got {}", val); @@ -4793,7 +4787,7 @@ mod sketch_query_tests { // "quantile_over_time should return a result" // ); // let (_labels, qr) = result.unwrap(); - // if let crate::engines::query_result::QueryResult::Vector(iv) = qr { + // if let crate::query_engines::query_result::QueryResult::Vector(iv) = qr { // assert!(!iv.values.is_empty()); // let val = iv.values[0].value; // // Median of 1..100 should be roughly 50 @@ -4813,7 +4807,7 @@ mod sketch_query_tests { // let result = engine.handle_query_promql("avg_over_time(cpu[100s])".into(), 0.1); // assert!(result.is_some(), "avg_over_time should return a result"); // let (_labels, qr) = result.unwrap(); - // if let crate::engines::query_result::QueryResult::Vector(iv) = qr { + // if let crate::query_engines::query_result::QueryResult::Vector(iv) = qr { // assert!(!iv.values.is_empty()); // let val = iv.values[0].value; // // avg of 1..100 = 50.5 @@ -4829,7 +4823,7 @@ mod sketch_query_tests { // let inference_config = // InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::NoCleanup); // let streaming_config = Arc::new(StreamingConfig::default()); - // let engine = SimpleEngine::new( + // let engine = ASAPQueryEngine::new( // Arc::new(NoOpStore), // None, // inference_config, @@ -4876,7 +4870,7 @@ mod sketch_query_tests { // "sketch range query should return a result" // ); // let (_labels, qr) = result.unwrap(); - // if let crate::engines::query_result::QueryResult::Matrix(rv) = qr { + // if let crate::query_engines::query_result::QueryResult::Matrix(rv) = qr { // assert!(!rv.values.is_empty(), "should have at least one series"); // let samples = &rv.values[0].samples; // assert!( @@ -4901,7 +4895,7 @@ mod sketch_query_tests { // let inference_config = // InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::NoCleanup); // let streaming_config = Arc::new(StreamingConfig::default()); - // let engine = SimpleEngine::new( + // let engine = ASAPQueryEngine::new( // Arc::new(NoOpStore), // None, // inference_config, @@ -4931,15 +4925,15 @@ mod sketch_query_tests { #[cfg(test)] mod hot_reload_phase2_tests { use super::*; - use crate::data_model::{ + use crate::stores::schema::{ AggregationType, CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, QueryLanguage, StreamingConfig, WindowType, }; use crate::stores::sketch_db::simple_map_store::SimpleMapStore; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; - fn dummy_agg(id: u64, metric: &str) -> crate::data_model::AggregationConfig { - crate::data_model::AggregationConfig::new( + fn dummy_agg(id: u64, metric: &str) -> crate::stores::schema::AggregationConfig { + crate::stores::schema::AggregationConfig::new( id, AggregationType::Sum, String::new(), @@ -4966,7 +4960,7 @@ mod hot_reload_phase2_tests { StreamingConfig::new(map) } - fn build_engine(handle: HotReloadStreamingConfig) -> SimpleEngine { + fn build_engine(handle: HotReloadStreamingConfig) -> ASAPQueryEngine { let streaming_config = Arc::new(StreamingConfig::default()); let store = Arc::new(SimpleMapStore::new( streaming_config, @@ -4974,7 +4968,7 @@ mod hot_reload_phase2_tests { )); let inference_config = InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::NoCleanup); - SimpleEngine::new_with_hot_reload( + ASAPQueryEngine::new_with_hot_reload( store, inference_config, handle, @@ -5031,7 +5025,7 @@ mod hot_reload_phase2_tests { #[test] fn legacy_new_constructor_is_independent_of_external_handle() { - // The legacy `SimpleEngine::new` path wraps the provided + // The legacy `ASAPQueryEngine::new` path wraps the provided // Arc in a FRESH HotReloadStreamingConfig // internally, so external swaps must NOT leak in. This is // the behavior tests and binaries that don't own a shared @@ -5045,7 +5039,7 @@ mod hot_reload_phase2_tests { )); let inference_config = InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::NoCleanup); - let engine = SimpleEngine::new( + let engine = ASAPQueryEngine::new( store, inference_config, streaming_config, @@ -5074,7 +5068,7 @@ mod hot_reload_phase2_tests { // The minimum-viable integration test for the full miss → notify → // plan-push → next-query-hit loop. Covers every seam landed in PR #10 // (HotReloadStreamingConfig endpoint), PR #11 (fire-and-forget -// capability-miss notification), PR #12 (SimpleEngine per-query +// capability-miss notification), PR #12 (ASAPQueryEngine per-query // re-snapshot), and mirrors the DataCollector controller side from // DataCollector PR #156 via an in-process mock client. // @@ -5082,7 +5076,7 @@ mod hot_reload_phase2_tests { // binaries. The mock controller is an in-process closure that directly // swaps the `HotReloadStreamingConfig` handle. This is deliberate — // each component is tested on its own in other suites, and the seams -// between them (`SimpleEngine` field types, the shared `ArcSwap`, +// between them (`ASAPQueryEngine` field types, the shared `ArcSwap`, // the `spawn_capability_miss_notify` helper) are what this test // validates. // @@ -5092,7 +5086,7 @@ mod hot_reload_phase2_tests { #[cfg(test)] mod e2e_feedback_loop_tests { use super::*; - use crate::data_model::{ + use crate::stores::schema::{ AggregationType, CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, QueryLanguage, StreamingConfig, WindowType, }; @@ -5105,8 +5099,8 @@ mod e2e_feedback_loop_tests { use std::sync::Mutex; use std::time::{Duration, Instant}; - fn agg_for_metric(id: u64, metric: &str) -> crate::data_model::AggregationConfig { - crate::data_model::AggregationConfig::new( + fn agg_for_metric(id: u64, metric: &str) -> crate::stores::schema::AggregationConfig { + crate::stores::schema::AggregationConfig::new( id, AggregationType::Sum, String::new(), @@ -5191,7 +5185,7 @@ mod e2e_feedback_loop_tests { /// to end: /// /// 1. Start with an empty `HotReloadStreamingConfig`. - /// 2. Build a `SimpleEngine` wired to the handle (PR #12) and + /// 2. Build a `ASAPQueryEngine` wired to the handle (PR #12) and /// to an in-process mock controller client (PR #11 + /// DC #156 mirror). /// 3. Observe initial snapshot: empty. @@ -5206,7 +5200,7 @@ mod e2e_feedback_loop_tests { /// This test simulates, inside a single process, exactly what a /// real backend ↔ controller deployment does across HTTP. The /// observable contract is: once the controller acts on a miss, - /// the next `SimpleEngine` query snapshot reflects the new + /// the next `ASAPQueryEngine` query snapshot reflects the new /// plan. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn capability_miss_feedback_loop_closes() { @@ -5230,14 +5224,14 @@ mod e2e_feedback_loop_tests { streaming_config_with(&req.metric, id) })); - // 3. Build SimpleEngine with the handle and mock controller. + // 3. Build ASAPQueryEngine with the handle and mock controller. let store = Arc::new(SimpleMapStore::new( Arc::new(StreamingConfig::default()), CleanupPolicy::NoCleanup, )); let inference_config = InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::NoCleanup); - let engine = SimpleEngine::new_with_hot_reload( + let engine = ASAPQueryEngine::new_with_hot_reload( store, inference_config, hot_reload.clone(), @@ -5357,7 +5351,7 @@ mod e2e_feedback_loop_tests { )); let inference_config = InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::NoCleanup); - let engine = SimpleEngine::new_with_hot_reload( + let engine = ASAPQueryEngine::new_with_hot_reload( store, inference_config, hot_reload.clone(), @@ -5431,7 +5425,7 @@ mod e2e_feedback_loop_tests { #[cfg(test)] mod aux_pushdown_tests { use super::*; - use crate::precompute_operators::{ + use crate::precompute_engine::operators::{ min_max_accumulator::MinMaxAccumulator, sum_accumulator::SumAccumulator, }; use promql_utilities::query_logics::enums::Statistic; @@ -5445,7 +5439,7 @@ mod aux_pushdown_tests { query_calls: Arc, } - impl crate::data_model::SerializableToSink for SpyAccumulator { + impl crate::stores::schema::SerializableToSink for SpyAccumulator { fn serialize_to_bytes(&self) -> Vec { Vec::new() } @@ -5492,16 +5486,16 @@ mod aux_pushdown_tests { self.query_calls.fetch_add(1, Ordering::Relaxed); Ok(-1.0) // sentinel: fast path should not return this } - fn aux_stats(&self) -> crate::data_model::AuxStats { - crate::data_model::AuxStats { + fn aux_stats(&self) -> crate::stores::schema::AuxStats { + crate::stores::schema::AuxStats { sum: Some(self.inner_sum), - ..crate::data_model::AuxStats::empty() + ..crate::stores::schema::AuxStats::empty() } } } - fn make_engine() -> SimpleEngine { - use crate::data_model::{ + fn make_engine() -> ASAPQueryEngine { + use crate::stores::schema::{ CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, PromQLSchema, QueryLanguage, SchemaConfig, StreamingConfig, }; @@ -5517,7 +5511,7 @@ mod aux_pushdown_tests { let sc = Arc::new(StreamingConfig::new(HashMap::new())); let hr = HotReloadStreamingConfig::from_arc(sc.clone()); let store = Arc::new(SimpleMapStore::new(sc, CleanupPolicy::NoCleanup)); - SimpleEngine::new_with_hot_reload(store, ic, hr, 60, QueryLanguage::promql) + ASAPQueryEngine::new_with_hot_reload(store, ic, hr, 60, QueryLanguage::promql) } #[test] @@ -5621,7 +5615,7 @@ mod aux_pushdown_tests { #[cfg(test)] mod forced_agg_id_tests { use super::*; - use crate::precompute_operators::sum_accumulator::SumAccumulator; + use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; use crate::tests::test_utilities::engine_factories::create_engine_single_pop; /// Sanity: the refactored auto-resolve path still produces a @@ -5761,7 +5755,7 @@ mod forced_agg_id_tests { #[cfg(test)] mod sketch_alias_resolver_tests { use super::*; - use crate::data_model::{ + use crate::stores::schema::{ AggregationConfig, CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, PromQLSchema, QueryLanguage, SchemaConfig, StreamingConfig, WindowType, }; @@ -5790,11 +5784,11 @@ mod sketch_alias_resolver_tests { ) } - /// Build a SimpleEngine whose streaming-config holds the supplied + /// Build a ASAPQueryEngine whose streaming-config holds the supplied /// (metric, agg_type) pairs and whose schema is empty (matches the /// production warm-tier deploy where the controller drives the /// label set). - fn engine_with(metrics: &[(&str, AggregationType)]) -> SimpleEngine { + fn engine_with(metrics: &[(&str, AggregationType)]) -> ASAPQueryEngine { let mut configs = HashMap::new(); for (i, (m, t)) in metrics.iter().enumerate() { configs.insert((i + 1) as u64, agg_for((i + 1) as u64, m, *t)); @@ -5810,7 +5804,7 @@ mod sketch_alias_resolver_tests { cleanup_policy: CleanupPolicy::NoCleanup, }; let hot_reload = HotReloadStreamingConfig::from_arc(Arc::new(streaming_config)); - SimpleEngine::new_with_hot_reload( + ASAPQueryEngine::new_with_hot_reload( store, inference_config, hot_reload, @@ -5932,7 +5926,7 @@ mod sketch_alias_resolver_tests { #[cfg(test)] mod hll_count_query_tests { use super::*; - use crate::precompute_operators::HllSketchAccumulator; + use crate::precompute_engine::operators::HllSketchAccumulator; use crate::tests::test_utilities::engine_factories::create_engine_single_pop; use asap_sketchlib::sketches::hll::HllVariant; @@ -6018,7 +6012,7 @@ mod hll_count_query_tests { #[test] fn capability_matching_resolves_count_to_hll() { - // End-to-end through the SimpleEngine: register an HLL agg for + // End-to-end through the ASAPQueryEngine: register an HLL agg for // `unique_users_per_min_hll`, run the `count(...)` query // through `build_query_execution_context_promql`, and assert // the resolved agg is the HLL one. Regression guard for the @@ -6057,7 +6051,7 @@ mod hll_count_query_tests { #[cfg(test)] mod kll_quantile_query_tests { use super::*; - use crate::precompute_operators::DatasketchesKLLAccumulator; + use crate::precompute_engine::operators::DatasketchesKLLAccumulator; use crate::tests::test_utilities::engine_factories::create_engine_single_pop; #[test] @@ -6105,7 +6099,7 @@ mod kll_quantile_query_tests { #[cfg(test)] mod cms_rate_capability_tests { use super::*; - use crate::precompute_operators::CountMinSketchAccumulator; + use crate::precompute_engine::operators::CountMinSketchAccumulator; use crate::tests::test_utilities::engine_factories::create_engine_single_pop; #[test] @@ -6152,9 +6146,9 @@ mod cms_rate_capability_tests { #[cfg(test)] mod warm_tier_classify_tests { use super::*; - use crate::data_model::{CleanupPolicy, HotReloadStreamingConfig, InferenceConfig}; - use crate::engines::EngineError; - use crate::routing::query_engine_routing::QueryEngine as _; + use crate::stores::schema::{CleanupPolicy, HotReloadStreamingConfig, InferenceConfig}; + use crate::query_engines::EngineError; + use crate::query_engines::routing::query_engine_routing::QueryEngine as _; use crate::stores::sketch_db::simple_map_store::SimpleMapStore; use crate::stores::sketch_db::sketch_index::{ AccuracyBound, Capability, SketchConfig, SketchIndex, SketchInstanceMetadata, @@ -6162,23 +6156,23 @@ mod warm_tier_classify_tests { }; use std::collections::{BTreeMap, BTreeSet}; - fn build_engine_with_index(idx: Arc) -> SimpleEngine { - let streaming_config = Arc::new(crate::data_model::StreamingConfig::default()); + fn build_engine_with_index(idx: Arc) -> ASAPQueryEngine { + let streaming_config = Arc::new(crate::stores::schema::StreamingConfig::default()); let store = Arc::new(SimpleMapStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); let hot_reload = HotReloadStreamingConfig::from_arc(streaming_config); let inference_config = InferenceConfig::new( - crate::data_model::QueryLanguage::promql, + crate::stores::schema::QueryLanguage::promql, CleanupPolicy::NoCleanup, ); - SimpleEngine::new_with_hot_reload( + ASAPQueryEngine::new_with_hot_reload( store, inference_config, hot_reload, 15000, - crate::data_model::QueryLanguage::promql, + crate::stores::schema::QueryLanguage::promql, ) .with_sketch_index(idx) } @@ -6303,8 +6297,8 @@ mod warm_tier_classify_tests { #[cfg(test)] mod hybrid_stitch_tests { use super::stitch_warm_and_archive; - use crate::data_model::KeyByLabelValues; - use crate::engines::query_result::{QueryResult, RangeVectorElement, Sample}; + use crate::stores::schema::KeyByLabelValues; + use crate::query_engines::query_result::{QueryResult, RangeVectorElement, Sample}; fn matrix_with_samples(label: &str, samples: Vec<(u64, f64)>) -> QueryResult { let labels = KeyByLabelValues::new_with_labels(vec![label.to_string()]); diff --git a/asap-query-engine/src/query-engines/asap_query/mod.rs b/data_plane/src/query_engines/asap_query_engine/mod.rs similarity index 72% rename from asap-query-engine/src/query-engines/asap_query/mod.rs rename to data_plane/src/query_engines/asap_query_engine/mod.rs index 8b8bf79f..900ad0fe 100644 --- a/asap-query-engine/src/query-engines/asap_query/mod.rs +++ b/data_plane/src/query_engines/asap_query_engine/mod.rs @@ -8,19 +8,13 @@ //! a capability miss (router falls through, which after Step-1 of //! the JSONL deprecation means the archive tier or a hard 404 — //! the JSONL leg has been deleted). -//! -//! `SimpleEngine` remains as a compatibility type alias in the -//! implementation, but new code should refer to this engine as -//! `ASAPQueryEngine`. pub mod engine; -pub mod warm_tier; #[cfg(test)] pub mod tests; pub use engine::{ - ASAPQueryEngine, QueryExecutionContext, QueryMetadata, QueryTimestamps, SimpleEngine, + ASAPQueryEngine, QueryExecutionContext, QueryMetadata, QueryTimestamps, StoreQueryParams, StoreQueryPlan, }; -pub use warm_tier::{SketchReducer, WarmTierError, WarmTierResult}; diff --git a/asap-query-engine/src/query-engines/asap_query/tests.rs b/data_plane/src/query_engines/asap_query_engine/tests.rs similarity index 87% rename from asap-query-engine/src/query-engines/asap_query/tests.rs rename to data_plane/src/query_engines/asap_query_engine/tests.rs index 7bc593f1..1262a029 100644 --- a/asap-query-engine/src/query-engines/asap_query/tests.rs +++ b/data_plane/src/query_engines/asap_query_engine/tests.rs @@ -5,7 +5,7 @@ //! engine's tests live inline in [`super::engine`] (~6 distinct //! `#[cfg(test)] mod tests { ... }` blocks, each pinning a //! specific dispatch axis). They are exercised under -//! `crate::engines::asap_query::engine::tests` rather than this file +//! `crate::query_engines::asap_query_engine::engine::tests` rather than this file //! to preserve `git blame` continuity across the move. //! //! Step-2 (Prometheus-block format + Thanos store-gateway) can diff --git a/asap-query-engine/src/query-engines/mod.rs b/data_plane/src/query_engines/mod.rs similarity index 76% rename from asap-query-engine/src/query-engines/mod.rs rename to data_plane/src/query_engines/mod.rs index f8ad6915..c1625212 100644 --- a/asap-query-engine/src/query-engines/mod.rs +++ b/data_plane/src/query_engines/mod.rs @@ -1,8 +1,8 @@ //! Query engines. //! //! The public query-engine surface is intentionally small: -//! [`asap_query`] answers from ASAP's sketch store, and -//! [`thanos_query`] forwards exact/archive queries to `thanos-query`. +//! [`asap_query_engine`] answers from ASAP's sketch store, and +//! [`thanos_query_engine`] forwards exact/archive queries to `thanos-query`. //! Gorilla object storage lives under [`crate::stores::gorilla_object_store`] //! because it is a storage implementation detail, not a public query-engine //! family. @@ -11,28 +11,34 @@ //! //! Engines + the shared error envelope: //! -//! * [`asap_query::ASAPQueryEngine`] — warm-tier sketch query engine. -//! * [`thanos_query::ThanosQueryEngine`] — archive-tier query engine. -//! * [`prometheus::PrometheusForwardEngine`] — HTTP-forwarder to a -//! Prometheus `/api/v1/query` endpoint, registered under the +//! * [`asap_query_engine::ASAPQueryEngine`] — warm-tier sketch query +//! engine. +//! * [`thanos_query_engine::ThanosQueryEngine`] — archive-tier query +//! engine. +//! * [`prometheus_query_engine::PrometheusForwardEngine`] — HTTP-forwarder +//! to a Prometheus `/api/v1/query` endpoint, registered under the //! `prometheus_remote` engine id when //! `ASAP_PROMETHEUS_QUERY_URL` is set (Phase ε.2). //! * [`EngineError`] — the trait-level error envelope every -//! `crate::routing::QueryEngine` impl returns. +//! `crate::query_engines::routing::QueryEngine` impl returns. -pub mod asap_query; +pub mod asap_query_engine; pub mod no_data_archive; -pub mod prometheus; +pub mod prometheus_query_engine; pub mod query_result; -pub mod thanos_query; +pub mod routing; +pub mod thanos_query_engine; pub mod timeline_dispatch; +pub mod warm_tier; pub mod window_merger; -pub use asap_query::{ASAPQueryEngine, SimpleEngine}; +pub use asap_query_engine::ASAPQueryEngine; pub use no_data_archive::{NoDataArchiveEngine, DATA_SOURCE_ID_NO_DATA_ARCHIVE}; -pub use prometheus::{PrometheusForwardConfig, PrometheusForwardEngine, PrometheusForwardError}; +pub use prometheus_query_engine::{ + PrometheusForwardConfig, PrometheusForwardEngine, PrometheusForwardError, +}; pub use query_result::{InstantVector, QueryResult, RangeVector, RangeVectorElement, Sample}; -pub use thanos_query::{ +pub use thanos_query_engine::{ thanos_engine_from_env, ThanosQueryConfig, ThanosQueryEngine, ThanosQueryError, ASAP_THANOS_QUERY_URL_ENV, DATA_SOURCE_THANOS_QUERY_ID, DATA_SOURCE_THANOS_QUERY_INFO, DEFAULT_THANOS_QUERY_URL, QUIRK_THANOS_UNREACHABLE, @@ -51,7 +57,7 @@ pub use window_merger::{create_window_merger, NaiveMerger, WindowMerger}; use thiserror::Error; -/// Top-level error returned by any [`crate::routing::QueryEngine`] impl. +/// Top-level error returned by any [`crate::query_engines::routing::QueryEngine`] impl. /// /// Concrete engines convert their internal error types into this envelope. /// The router uses the variant to decide whether a failover is sensible @@ -60,7 +66,7 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum EngineError { /// The engine has no aggregation that can answer this query. Mirrors - /// `SimpleEngine::handle_query` returning `None`. The router treats + /// `ASAPQueryEngine::handle_query` returning `None`. The router treats /// this as a "hard miss" and falls through to the next backend in /// the `compatible_storage_backends` list. After Step-1 of the /// JSONL deprecation, the surviving failovers are warm-tier diff --git a/asap-query-engine/src/query-engines/no_data_archive.rs b/data_plane/src/query_engines/no_data_archive.rs similarity index 94% rename from asap-query-engine/src/query-engines/no_data_archive.rs rename to data_plane/src/query_engines/no_data_archive.rs index 402e9193..0dd50429 100644 --- a/asap-query-engine/src/query-engines/no_data_archive.rs +++ b/data_plane/src/query_engines/no_data_archive.rs @@ -6,7 +6,7 @@ //! When a deploy is configured with the warm-tier sketch path only and //! has neither `ASAP_THANOS_QUERY_URL` nor `ASAP_GORILLA_S3_*` env //! vars set, no archive engine is registered on the -//! [`crate::routing::EngineRouter`]. Cold queries (queries the +//! [`crate::query_engines::routing::EngineRouter`]. Cold queries (queries the //! per-metric routing table sends to `thanos_query`) then surface as //! `503 NoEngineRegistered` from the HTTP handler. //! @@ -29,8 +29,8 @@ use tracing::info; use asap_types::StorageBackend; -use crate::engines::{EngineError, QueryResult}; -use crate::routing::{EngineCapabilities, QueryEngine}; +use crate::query_engines::{EngineError, QueryResult}; +use crate::query_engines::routing::{EngineCapabilities, QueryEngine}; /// Stable engine id for the no-data fallback. Reported in /// `data_source_id` on the wire response for cold queries when the diff --git a/asap-query-engine/src/query-engines/prometheus/forward.rs b/data_plane/src/query_engines/prometheus_query_engine/forward.rs similarity index 96% rename from asap-query-engine/src/query-engines/prometheus/forward.rs rename to data_plane/src/query_engines/prometheus_query_engine/forward.rs index 3973054c..e77c5d7f 100644 --- a/asap-query-engine/src/query-engines/prometheus/forward.rs +++ b/data_plane/src/query_engines/prometheus_query_engine/forward.rs @@ -17,17 +17,17 @@ //! startup: //! //! * **Phase ε.2 mode** (env set) — `PrometheusForwardEngine` is -//! registered in the [`crate::routing::EngineRouter`] under id +//! registered in the [`crate::query_engines::routing::EngineRouter`] under id //! `prometheus_remote`. Routing-table entries that target this //! engine POST to `${ASAP_PROMETHEUS_QUERY_URL}/api/v1/query` and //! the answer is wrapped in ASAP's standard -//! [`crate::engines::QueryResult`] shape. +//! [`crate::query_engines::QueryResult`] shape. //! * **Off** (env unset) — engine is not registered. Routing-table //! entries that reference `prometheus_remote` surface a //! `NoEngineRegistered` 503 from the HTTP handler — the correct //! fail-loud behaviour for a misconfigured deploy. //! -//! This is a near-mirror of [`crate::engines::thanos_query::forward`] +//! This is a near-mirror of [`crate::query_engines::thanos_query_engine::forward`] //! (the Step-2.3 archive forwarder), pointed at Prometheus's standard //! `/api/v1/query` endpoint instead of a `thanos-query` sidecar. The //! two engines coexist: `thanos_query` answers archive-tier queries @@ -43,9 +43,9 @@ use serde::Deserialize; use serde_json::Value; use tracing::{debug, warn}; -use crate::data_model::KeyByLabelValues; -use crate::engines::query_result::{InstantVectorElement, QueryResult, RangeVectorElement}; -use crate::routing::query_engine_routing::{EngineCapabilities, QueryEngine}; +use crate::stores::schema::KeyByLabelValues; +use crate::query_engines::query_result::{InstantVectorElement, QueryResult, RangeVectorElement}; +use crate::query_engines::routing::query_engine_routing::{EngineCapabilities, QueryEngine}; use crate::stores::sketch_db::accuracy::{AccuracyEnvelope, AccuracyProfile}; // --------------------------------------------------------------------------- @@ -148,7 +148,7 @@ impl PrometheusForwardConfig { /// [`QueryResult`] shape. /// /// Implements the [`QueryEngine`] trait so the -/// [`crate::routing::EngineRouter`] can hold it as `Arc`. Reports `data_source_id = "prometheus_remote"` /// and (for the compatibility-list dispatch path) `storage_backend = /// StorageBackend::PrometheusRemote` — Mode 3 gives the @@ -261,7 +261,7 @@ impl PrometheusForwardEngine { #[async_trait] impl QueryEngine for PrometheusForwardEngine { - async fn execute(&self, query: &str) -> Result { + async fn execute(&self, query: &str) -> Result { match self.query(query).await { Ok(result) => Ok(result), Err(PrometheusForwardError::Unreachable(reason)) => { @@ -274,25 +274,25 @@ impl QueryEngine for PrometheusForwardEngine { error = %reason, "prometheus-forward: upstream unreachable", ); - Err(crate::engines::EngineError::backend( + Err(crate::query_engines::EngineError::backend( DATA_SOURCE_PROMETHEUS_REMOTE_ID, format!("prometheus_unreachable: {reason}"), )) } Err(PrometheusForwardError::BadQuery { status, body }) => { - Err(crate::engines::EngineError::capability_miss( + Err(crate::query_engines::EngineError::capability_miss( DATA_SOURCE_PROMETHEUS_REMOTE_ID, format!("prometheus rejected query (status {status}): {body}"), )) } Err(PrometheusForwardError::ParseError(msg)) => { - Err(crate::engines::EngineError::backend( + Err(crate::query_engines::EngineError::backend( DATA_SOURCE_PROMETHEUS_REMOTE_ID, format!("prometheus response parse error: {msg}"), )) } Err(PrometheusForwardError::ConfigInvalid(msg)) => { - Err(crate::engines::EngineError::backend( + Err(crate::query_engines::EngineError::backend( DATA_SOURCE_PROMETHEUS_REMOTE_ID, format!("prometheus client misconfigured: {msg}"), )) @@ -480,7 +480,7 @@ fn labels_from_metric(metric: &Value) -> KeyByLabelValues { // --------------------------------------------------------------------------- /// Failure modes of the HTTP-forwarder. The [`QueryEngine`] impl -/// folds these into the trait-level [`crate::engines::EngineError`] +/// folds these into the trait-level [`crate::query_engines::EngineError`] /// envelope; the public `query` method returns the richer surface /// for tests and direct callers. #[derive(Debug, thiserror::Error)] @@ -666,8 +666,8 @@ mod tests { spawn_mock_prometheus, spawn_mock_prometheus_503, CANNED_VECTOR_BODY, ENV_LOCK, }; use super::*; - use crate::engines::query_result::QueryResult; - use crate::routing::query_engine_routing::{EngineRouter, QueryEngine as RouterQueryEngine}; + use crate::query_engines::query_result::QueryResult; + use crate::query_engines::routing::query_engine_routing::{EngineRouter, QueryEngine as RouterQueryEngine}; use std::sync::Arc; fn config_for(url: &str) -> PrometheusForwardConfig { @@ -742,7 +742,7 @@ mod tests { // it into a 503 with the `prometheus_unreachable` quirk infos. let trait_path = RouterQueryEngine::execute(&engine, "up").await; match trait_path { - Err(crate::engines::EngineError::Backend { engine_id, message }) => { + Err(crate::query_engines::EngineError::Backend { engine_id, message }) => { assert_eq!(engine_id, DATA_SOURCE_PROMETHEUS_REMOTE_ID); assert!( message.contains("prometheus_unreachable"), diff --git a/asap-query-engine/src/query-engines/prometheus/mod.rs b/data_plane/src/query_engines/prometheus_query_engine/mod.rs similarity index 93% rename from asap-query-engine/src/query-engines/prometheus/mod.rs rename to data_plane/src/query_engines/prometheus_query_engine/mod.rs index cd66551e..d37af189 100644 --- a/asap-query-engine/src/query-engines/prometheus/mod.rs +++ b/data_plane/src/query_engines/prometheus_query_engine/mod.rs @@ -12,7 +12,7 @@ //! 503 from the HTTP handler (the correct fail-loud behaviour for a //! misconfigured deploy). //! -//! Sibling of [`crate::engines::thanos_query::forward`] (the +//! Sibling of [`crate::query_engines::thanos_query_engine::forward`] (the //! Step-2.3 archive forwarder); the two engines coexist in the //! router under different ids and answer different routing-table //! entries. diff --git a/asap-query-engine/src/query-engines/query_result.rs b/data_plane/src/query_engines/query_result.rs similarity index 99% rename from asap-query-engine/src/query-engines/query_result.rs rename to data_plane/src/query_engines/query_result.rs index ea4d04c4..a1b6cc42 100644 --- a/asap-query-engine/src/query-engines/query_result.rs +++ b/data_plane/src/query_engines/query_result.rs @@ -1,4 +1,4 @@ -use crate::data_model::KeyByLabelValues; +use crate::stores::schema::KeyByLabelValues; use crate::stores::sketch_db::AccuracyEnvelope; use serde::{Deserialize, Serialize}; @@ -126,7 +126,7 @@ pub struct InstantVector { /// Prometheus's top-level `warnings` field. Empty for /// single-schema queries; populated by the schema-timeline /// dispatcher when one or more segments produced a - /// [`crate::engines::timeline_dispatch::CombinedResult::Partial`] + /// [`crate::query_engines::timeline_dispatch::CombinedResult::Partial`] /// (non-combinable statistic, purged coverage, or agg_id /// missing from the current config). #[serde(default, skip_serializing_if = "Vec::is_empty")] diff --git a/asap-query-engine/src/routing/backend_storage_routing.rs b/data_plane/src/query_engines/routing/backend_storage_routing.rs similarity index 99% rename from asap-query-engine/src/routing/backend_storage_routing.rs rename to data_plane/src/query_engines/routing/backend_storage_routing.rs index 41fb363a..db759f21 100644 --- a/asap-query-engine/src/routing/backend_storage_routing.rs +++ b/data_plane/src/query_engines/routing/backend_storage_routing.rs @@ -10,7 +10,7 @@ //! `precompute_engine` binary loading `backend-streaming.yaml`) the //! field decodes via `Self::new(...)` which always defaults to //! `SketchStore`, so the handler always took the -//! `SimpleEngine`-direct-dispatch branch and the `EngineRouter` was +//! `ASAPQueryEngine`-direct-dispatch branch and the `EngineRouter` was //! effectively bypassed for every query — the `data_source: //! thanos_query` info-line never landed on cold-archive responses //! even when the chunks were on disk in MinIO. @@ -409,7 +409,7 @@ pub struct BackendStorageRouting { impl BackendStorageRouting { /// Build an empty router — every metric resolves to /// `SketchStore`. Equivalent to "no routing config at all" and - /// preserves pre-Phase-5 dispatch (`SimpleEngine` direct path). + /// preserves pre-Phase-5 dispatch (`ASAPQueryEngine` direct path). /// Scoped to the [`DEFAULT_TENANT`] tenant. pub fn empty() -> Self { Self { @@ -892,7 +892,7 @@ pub fn routing_table_hash(table: &BackendStorageRouting) -> String { // --------------------------------------------------------------------------- /// Per-tenant atomic-swap wrapper around `BackendStorageRouting`, -/// mirroring [`crate::data_model::HotReloadStreamingConfig`]. Lets the +/// mirroring [`crate::stores::schema::HotReloadStreamingConfig`]. Lets the /// `POST /api/v1/storage_routing` HTTP handler swap one tenant's table /// at runtime without restarting the backend or touching any other /// tenant's table. Cloneable; clones share the underlying `ArcSwap` so diff --git a/asap-query-engine/src/routing/freshness_probe_cache.rs b/data_plane/src/query_engines/routing/freshness_probe_cache.rs similarity index 100% rename from asap-query-engine/src/routing/freshness_probe_cache.rs rename to data_plane/src/query_engines/routing/freshness_probe_cache.rs diff --git a/asap-query-engine/src/routing/mod.rs b/data_plane/src/query_engines/routing/mod.rs similarity index 91% rename from asap-query-engine/src/routing/mod.rs rename to data_plane/src/query_engines/routing/mod.rs index 798e2993..f4b49862 100644 --- a/asap-query-engine/src/routing/mod.rs +++ b/data_plane/src/query_engines/routing/mod.rs @@ -2,7 +2,7 @@ //! //! This module is the dispatch boundary between the HTTP query //! handler and the tier-co-located engines (warm sketch tier in -//! [`crate::engines::asap_query`], archive tier in +//! [`crate::query_engines::asap_query_engine`], archive tier in //! [`crate::stores::gorilla_object_store`]). Two cooperating pieces: //! //! * [`backend_storage_routing`] — config loader + multi-target @@ -19,7 +19,7 @@ //! Step-1 of the JSONL deprecation refactor lifted these out of //! `data_model/backend_storage_routing.rs` and `query-engines/router.rs` //! into this dedicated `routing/` directory so the HTTP handler's -//! dispatch surface is a single import (`use crate::routing::*`) +//! dispatch surface is a single import (`use crate::query_engines::routing::*`) //! instead of straddling two unrelated module trees. pub mod backend_storage_routing; diff --git a/asap-query-engine/src/routing/query_engine_routing.rs b/data_plane/src/query_engines/routing/query_engine_routing.rs similarity index 98% rename from asap-query-engine/src/routing/query_engine_routing.rs rename to data_plane/src/query_engines/routing/query_engine_routing.rs index b25ef805..78d921e7 100644 --- a/asap-query-engine/src/routing/query_engine_routing.rs +++ b/data_plane/src/query_engines/routing/query_engine_routing.rs @@ -23,13 +23,13 @@ use tracing::{debug, warn}; use asap_types::{compatible_storage_backends, AccuracyTarget, StorageBackend}; use promql_utilities::query_logics::enums::Statistic; -use crate::engines::{EngineError, QueryResult}; +use crate::query_engines::{EngineError, QueryResult}; // --------------------------------------------------------------------------- // `QueryEngine` trait — the abstraction the router holds. // // The trait is intentionally narrow: a single `execute(&str)` method (so it -// integrates with both `SimpleEngine::handle_query` and +// integrates with both `ASAPQueryEngine::handle_query` and // `GorillaQueryEngine::execute` without forcing either side to refactor its // public surface), plus a `capabilities()` accessor the router consults at // registration time. @@ -62,7 +62,7 @@ pub struct EngineCapabilities { /// /// `execute` takes the query as `&str` (matching `GorillaQueryEngine`'s /// existing surface) and returns a wire-ready [`QueryResult`]. Internal -/// engine signatures (e.g. `SimpleEngine::handle_query`'s `Option<...>`) +/// engine signatures (e.g. `ASAPQueryEngine::handle_query`'s `Option<...>`) /// are translated by the impl so callers can program against the trait. #[async_trait] pub trait QueryEngine: Send + Sync { @@ -245,12 +245,12 @@ impl EngineRouter { #[cfg(test)] mod tests { use super::*; - use crate::engines::query_result::QueryResult; + use crate::query_engines::query_result::QueryResult; use std::sync::atomic::{AtomicUsize, Ordering}; /// Stub engine that records call counts and returns either a canned /// vector result or a configured error. Keeps the router tests - /// hermetic — no SimpleEngine / GorillaQueryEngine wire-up needed. + /// hermetic — no ASAPQueryEngine / GorillaQueryEngine wire-up needed. struct StubEngine { caps: EngineCapabilities, calls: Arc, diff --git a/asap-query-engine/src/query-engines/thanos_query/forward.rs b/data_plane/src/query_engines/thanos_query_engine/forward.rs similarity index 97% rename from asap-query-engine/src/query-engines/thanos_query/forward.rs rename to data_plane/src/query_engines/thanos_query_engine/forward.rs index ba9bfb9f..1ca9d757 100644 --- a/asap-query-engine/src/query-engines/thanos_query/forward.rs +++ b/data_plane/src/query_engines/thanos_query_engine/forward.rs @@ -12,10 +12,10 @@ //! startup: //! //! * **Path A2 mode** (env set) — `ThanosQueryEngine` is -//! registered in the [`crate::routing::EngineRouter`]. Archive +//! registered in the [`crate::query_engines::routing::EngineRouter`]. Archive //! queries POST to `${ASAP_THANOS_QUERY_URL}/api/v1/query` and //! the answer is wrapped in ASAP's standard -//! [`crate::engines::QueryResult`] shape. +//! [`crate::query_engines::QueryResult`] shape. //! * **Legacy mode** (env unset) — the in-process //! [`crate::stores::gorilla_object_store::GorillaQueryEngine`] //! handles archive queries from the per-hour Gorilla chunks that @@ -39,9 +39,9 @@ use serde::Deserialize; use serde_json::Value; use tracing::{debug, warn}; -use crate::data_model::KeyByLabelValues; -use crate::engines::query_result::{InstantVectorElement, QueryResult, RangeVectorElement}; -use crate::routing::query_engine_routing::{EngineCapabilities, QueryEngine}; +use crate::stores::schema::KeyByLabelValues; +use crate::query_engines::query_result::{InstantVectorElement, QueryResult, RangeVectorElement}; +use crate::query_engines::routing::query_engine_routing::{EngineCapabilities, QueryEngine}; use crate::stores::sketch_db::accuracy::{AccuracyEnvelope, AccuracyProfile}; // --------------------------------------------------------------------------- @@ -146,7 +146,7 @@ impl ThanosQueryConfig { /// [`QueryResult`] shape. /// /// Implements the [`QueryEngine`] trait so the -/// [`crate::routing::EngineRouter`] can hold it as `Arc`. Reports `data_source_id = /// "thanos_query"` and (for the compatibility-list dispatch path) /// `storage_backend = StorageBackend::GorillaObjectStore` — Path A2 @@ -268,7 +268,7 @@ impl ThanosQueryEngine { #[async_trait] impl QueryEngine for ThanosQueryEngine { - async fn execute(&self, query: &str) -> Result { + async fn execute(&self, query: &str) -> Result { match self.query(query).await { Ok(result) => Ok(result), Err(ThanosQueryError::Unreachable(reason)) => { @@ -283,22 +283,22 @@ impl QueryEngine for ThanosQueryEngine { error = %reason, "thanos-forward: upstream unreachable", ); - Err(crate::engines::EngineError::backend( + Err(crate::query_engines::EngineError::backend( self.data_source_id, format!("thanos_unreachable: {reason}"), )) } Err(ThanosQueryError::BadQuery { status, body }) => { - Err(crate::engines::EngineError::capability_miss( + Err(crate::query_engines::EngineError::capability_miss( self.data_source_id, format!("thanos rejected query (status {status}): {body}"), )) } - Err(ThanosQueryError::ParseError(msg)) => Err(crate::engines::EngineError::backend( + Err(ThanosQueryError::ParseError(msg)) => Err(crate::query_engines::EngineError::backend( self.data_source_id, format!("thanos response parse error: {msg}"), )), - Err(ThanosQueryError::ConfigInvalid(msg)) => Err(crate::engines::EngineError::backend( + Err(ThanosQueryError::ConfigInvalid(msg)) => Err(crate::query_engines::EngineError::backend( self.data_source_id, format!("thanos client misconfigured: {msg}"), )), @@ -500,7 +500,7 @@ fn labels_from_metric(metric: &Value) -> KeyByLabelValues { // --------------------------------------------------------------------------- /// Failure modes of the HTTP-forwarder. The [`QueryEngine`] impl -/// folds these into the trait-level [`crate::engines::EngineError`] +/// folds these into the trait-level [`crate::query_engines::EngineError`] /// envelope; the public `query` method returns the richer surface /// for tests and direct callers. #[derive(Debug, thiserror::Error)] @@ -684,7 +684,7 @@ mod tests { spawn_mock_thanos, spawn_mock_thanos_503, CANNED_VECTOR_BODY, ENV_LOCK, }; use super::*; - use crate::engines::query_result::QueryResult; + use crate::query_engines::query_result::QueryResult; fn config_for(url: &str) -> ThanosQueryConfig { ThanosQueryConfig { @@ -755,7 +755,7 @@ mod tests { // 503 with the `thanos_unreachable` quirk infos. let trait_path = QueryEngine::execute(&engine, "up").await; match trait_path { - Err(crate::engines::EngineError::Backend { engine_id, message }) => { + Err(crate::query_engines::EngineError::Backend { engine_id, message }) => { assert_eq!(engine_id, DATA_SOURCE_THANOS_QUERY_ID); assert!( message.contains("thanos_unreachable"), diff --git a/asap-query-engine/src/query-engines/thanos_query/mod.rs b/data_plane/src/query_engines/thanos_query_engine/mod.rs similarity index 100% rename from asap-query-engine/src/query-engines/thanos_query/mod.rs rename to data_plane/src/query_engines/thanos_query_engine/mod.rs diff --git a/asap-query-engine/src/query-engines/timeline_dispatch.rs b/data_plane/src/query_engines/timeline_dispatch.rs similarity index 99% rename from asap-query-engine/src/query-engines/timeline_dispatch.rs rename to data_plane/src/query_engines/timeline_dispatch.rs index a8e6baa0..3b636d48 100644 --- a/asap-query-engine/src/query-engines/timeline_dispatch.rs +++ b/data_plane/src/query_engines/timeline_dispatch.rs @@ -18,7 +18,7 @@ //! that before calling here). //! //! The query engine wires this primitive into -//! `SimpleEngine::try_handle_query_promql_via_timeline`, which +//! `ASAPQueryEngine::try_handle_query_promql_via_timeline`, which //! runs the per-segment evaluation loop and feeds the scalars back //! through `combine_statistic`. //! diff --git a/asap-query-engine/src/query-engines/asap_query/warm_tier/decoders.rs b/data_plane/src/query_engines/warm_tier/decoders.rs similarity index 98% rename from asap-query-engine/src/query-engines/asap_query/warm_tier/decoders.rs rename to data_plane/src/query_engines/warm_tier/decoders.rs index 97a005d3..81e0fd08 100644 --- a/asap-query-engine/src/query-engines/asap_query/warm_tier/decoders.rs +++ b/data_plane/src/query_engines/warm_tier/decoders.rs @@ -1,7 +1,7 @@ //! Per-sketch-kind decoder helpers — out-of-line wrappers around //! `asap_sketchlib` deserialize / proto-decode paths. //! -//! Lifted from the inline closures in [`crate::engines::asap_query::warm_tier::sketch_reducer`] +//! Lifted from the inline closures in [`crate::query_engines::warm_tier::sketch_reducer`] //! once the reducer started decoding CMS / CountSketch / CMS-with-heap //! payloads in addition to DDSketch / KLL / HLL. The CMS / CountSketch //! / CMS-with-heap decoders mirror diff --git a/asap-query-engine/src/query-engines/asap_query/warm_tier/delta_apply.rs b/data_plane/src/query_engines/warm_tier/delta_apply.rs similarity index 99% rename from asap-query-engine/src/query-engines/asap_query/warm_tier/delta_apply.rs rename to data_plane/src/query_engines/warm_tier/delta_apply.rs index b7b09f0a..16f4da89 100644 --- a/asap-query-engine/src/query-engines/asap_query/warm_tier/delta_apply.rs +++ b/data_plane/src/query_engines/warm_tier/delta_apply.rs @@ -9,7 +9,7 @@ //! format ships a sparse-but-mergeable sketch fragment. //! //! Two reducer modes, picked by the PromQL function name in -//! [`crate::engines::asap_query::warm_tier::sketch_reducer`]: +//! [`crate::query_engines::warm_tier::sketch_reducer`]: //! //! * **per-window** (`quantile`, `histogram_quantile`, //! `cardinality_estimate`): emit one scalar per window. A `Full` diff --git a/asap-query-engine/src/query-engines/asap_query/warm_tier/mod.rs b/data_plane/src/query_engines/warm_tier/mod.rs similarity index 93% rename from asap-query-engine/src/query-engines/asap_query/warm_tier/mod.rs rename to data_plane/src/query_engines/warm_tier/mod.rs index 2e6087aa..83b1f29a 100644 --- a/asap-query-engine/src/query-engines/asap_query/warm_tier/mod.rs +++ b/data_plane/src/query_engines/warm_tier/mod.rs @@ -1,7 +1,7 @@ //! Warm-tier sketch query evaluator (Phase 5 follow-up to PR #122). //! //! PR #122 wired the warm-tier classification hook in -//! [`crate::engines::asap_query::engine::SimpleEngine`]'s +//! [`crate::query_engines::asap_query_engine::engine::ASAPQueryEngine`]'s //! `QueryEngine::execute` adapter: parse the PromQL, extract //! `(metric_name, label_keys)`, look up candidate sids via //! [`crate::stores::sketch_db::sketch_index::SketchIndex::instances_matching`], @@ -34,7 +34,7 @@ //! from "decode failure" (defensive — also fall over) and //! "no data in window" (router falls over). //! * [`WarmTierResult`] — per-series timestamped scalar samples -//! matching the shape of [`crate::engines::query_result::QueryResult::Matrix`]. +//! matching the shape of [`crate::query_engines::query_result::QueryResult::Matrix`]. //! //! ## Controller unification (PromQL-shape recognition) //! @@ -62,7 +62,7 @@ //! per-window vs cumulative modes (selected by function name). //! * **TODO 3**: Hybrid warm+archive stitch. [`WarmTierResult::coverage`] //! reports the actual `(min_window_start_ms, max_window_end_ms)` the -//! reducer covered so `SimpleEngine` can stitch the missing prefix / +//! reducer covered so `ASAPQueryEngine` can stitch the missing prefix / //! suffix from the archive engine. pub mod decoders; diff --git a/asap-query-engine/src/query-engines/asap_query/warm_tier/sketch_reducer.rs b/data_plane/src/query_engines/warm_tier/sketch_reducer.rs similarity index 99% rename from asap-query-engine/src/query-engines/asap_query/warm_tier/sketch_reducer.rs rename to data_plane/src/query_engines/warm_tier/sketch_reducer.rs index 019453e9..7ea77520 100644 --- a/asap-query-engine/src/query-engines/asap_query/warm_tier/sketch_reducer.rs +++ b/data_plane/src/query_engines/warm_tier/sketch_reducer.rs @@ -57,11 +57,11 @@ use asap_sketchlib::sketches::ddsketch::DdSketch; use asap_sketchlib::sketches::hll::HllSketch; use asap_sketchlib::sketches::kll::KllSketch; -use crate::engines::asap_query::warm_tier::decoders::{ +use crate::query_engines::warm_tier::decoders::{ decode_cms_from_msgpack, decode_cms_from_proto, decode_cms_with_heap_from_msgpack, decode_cs_from_msgpack, decode_cs_from_proto, }; -use crate::engines::asap_query::warm_tier::delta_apply::{ +use crate::query_engines::warm_tier::delta_apply::{ cumulative_evaluate, per_window_evaluate, DeltaSketchKind, }; use crate::stores::sketch_db::sketch_index::{ @@ -154,7 +154,7 @@ impl std::error::Error for WarmTierError {} /// /// `coverage` is the actual `(min_window_start_ms, max_window_end_ms)` /// the reducer covered. `None` when the reducer didn't observe any -/// in-range window (defensive default). The caller (`SimpleEngine`) +/// in-range window (defensive default). The caller (`ASAPQueryEngine`) /// compares `coverage` against the requested `[t0, t1]` and, on a /// partial hit (`cov_lo > t0 || cov_hi < t1`), falls over to archive /// for the missing range and stitches the two answers. See TODO 3 in diff --git a/asap-query-engine/src/query-engines/asap_query/warm_tier/tests.rs b/data_plane/src/query_engines/warm_tier/tests.rs similarity index 99% rename from asap-query-engine/src/query-engines/asap_query/warm_tier/tests.rs rename to data_plane/src/query_engines/warm_tier/tests.rs index 18725566..06d65b32 100644 --- a/asap-query-engine/src/query-engines/asap_query/warm_tier/tests.rs +++ b/data_plane/src/query_engines/warm_tier/tests.rs @@ -15,7 +15,7 @@ use std::collections::{BTreeMap, BTreeSet}; use asap_sketchlib::sketches::ddsketch::DdSketch; use asap_sketchlib::sketches::hll::{HllSketch, HllVariant}; -use crate::engines::asap_query::warm_tier::{SketchReducer, WarmTierError}; +use crate::query_engines::warm_tier::{SketchReducer, WarmTierError}; use crate::stores::sketch_db::sketch_index::{ AccuracyBound, Capability, SketchConfig, SketchEncoding, SketchIndex, SketchInstanceMetadata, SketchKindHandle, SketchSampleState, @@ -696,7 +696,7 @@ fn hll_cumulative_full_plus_one_delta() { // --------------------------------------------------------------------------- // TODO-3 tests — hybrid warm + archive stitch via `WarmTierResult.coverage`. // -// We don't drive the full SimpleEngine here (that would require +// We don't drive the full ASAPQueryEngine here (that would require // constructing the whole streaming-config plumbing). Instead we exercise // the `stitch_warm_and_archive` helper directly via a small wrapper // test in `engines::asap_query::tests` would be ideal — but to keep this diff --git a/asap-query-engine/src/query-engines/window_merger.rs b/data_plane/src/query_engines/window_merger.rs similarity index 99% rename from asap-query-engine/src/query-engines/window_merger.rs rename to data_plane/src/query_engines/window_merger.rs index e206f0b1..3a2baa6a 100644 --- a/asap-query-engine/src/query-engines/window_merger.rs +++ b/data_plane/src/query_engines/window_merger.rs @@ -8,7 +8,7 @@ //! - `IncrementalMerger`: Add/subtract for subtractable accumulators (future) //! - `SwagMerger`: Two-stack queue for non-subtractable accumulators (future) -use crate::data_model::{AggregateCore, AggregationType}; +use crate::stores::schema::{AggregateCore, AggregationType}; /// Trait for merging buckets in a sliding window /// @@ -107,7 +107,7 @@ pub fn create_window_merger(_accumulator_type: AggregationType) -> Box QueryResult { // Result timestamp is the right edge of the requested range — - // mirrors `SimpleEngine`'s convention for instant-vector queries + // mirrors `ASAPQueryEngine`'s convention for instant-vector queries // against a closed time window. let result_ts = plan.time_range_ms.1.max(0) as u64; @@ -326,23 +326,23 @@ impl ExecutionOutcome { // --------------------------------------------------------------------------- #[async_trait::async_trait] -impl crate::routing::query_engine_routing::QueryEngine for GorillaQueryEngine { - async fn execute(&self, query: &str) -> Result { +impl crate::query_engines::routing::query_engine_routing::QueryEngine for GorillaQueryEngine { + async fn execute(&self, query: &str) -> Result { match GorillaQueryEngine::execute(self, query).await { Ok(result) => Ok(result), - Err(EngineError::Plan(msg)) => Err(crate::engines::EngineError::capability_miss( + Err(EngineError::Plan(msg)) => Err(crate::query_engines::EngineError::capability_miss( asap_types::StorageBackend::GorillaObjectStore.data_source_id(), msg, )), - Err(other) => Err(crate::engines::EngineError::backend( + Err(other) => Err(crate::query_engines::EngineError::backend( asap_types::StorageBackend::GorillaObjectStore.data_source_id(), other, )), } } - fn capabilities(&self) -> crate::routing::query_engine_routing::EngineCapabilities { - crate::routing::query_engine_routing::EngineCapabilities { + fn capabilities(&self) -> crate::query_engines::routing::query_engine_routing::EngineCapabilities { + crate::query_engines::routing::query_engine_routing::EngineCapabilities { data_source_id: asap_types::StorageBackend::GorillaObjectStore.data_source_id(), storage_backend: asap_types::StorageBackend::GorillaObjectStore, // The buffered-aggregate budget gives a natural ceiling: each diff --git a/asap-query-engine/src/stores/gorilla_object_store/postings.rs b/data_plane/src/stores/gorilla_object_store/postings.rs similarity index 100% rename from asap-query-engine/src/stores/gorilla_object_store/postings.rs rename to data_plane/src/stores/gorilla_object_store/postings.rs diff --git a/asap-query-engine/src/stores/gorilla_object_store/s3_cost.rs b/data_plane/src/stores/gorilla_object_store/s3_cost.rs similarity index 100% rename from asap-query-engine/src/stores/gorilla_object_store/s3_cost.rs rename to data_plane/src/stores/gorilla_object_store/s3_cost.rs diff --git a/asap-query-engine/src/stores/gorilla_object_store/store.rs b/data_plane/src/stores/gorilla_object_store/store.rs similarity index 100% rename from asap-query-engine/src/stores/gorilla_object_store/store.rs rename to data_plane/src/stores/gorilla_object_store/store.rs diff --git a/asap-query-engine/src/stores/gorilla_object_store/tests.rs b/data_plane/src/stores/gorilla_object_store/tests.rs similarity index 99% rename from asap-query-engine/src/stores/gorilla_object_store/tests.rs rename to data_plane/src/stores/gorilla_object_store/tests.rs index d0485f01..92502b8e 100644 --- a/asap-query-engine/src/stores/gorilla_object_store/tests.rs +++ b/data_plane/src/stores/gorilla_object_store/tests.rs @@ -14,7 +14,7 @@ use std::time::Duration; use async_trait::async_trait; use tokio::time::sleep; -use crate::engines::query_result::QueryResult; +use crate::query_engines::query_result::QueryResult; use crate::stores::sketch_db::accuracy::{AccuracyKind, AccuracyProfile}; use super::archive_query::{plan_query_at, QueryStatistic}; diff --git a/asap-query-engine/src/stores/mod.rs b/data_plane/src/stores/mod.rs similarity index 62% rename from asap-query-engine/src/stores/mod.rs rename to data_plane/src/stores/mod.rs index 80b6b7ac..1f157116 100644 --- a/asap-query-engine/src/stores/mod.rs +++ b/data_plane/src/stores/mod.rs @@ -1,35 +1,34 @@ -//! Store layer — the sketch DB. -//! -//! This module houses the sketch DB's physical + logical layers: +//! Store layer. //! //! * `traits` — the `Store` trait every concrete store implements. -//! * `sketch_db` — the top-level sketch DB module. Logical +//! * `schema` — storage schemas + per-aggregation config / hot-reload +//! config / measurement / precomputed-output types (was the +//! top-level `data_model/` module before the 2026-05 data_plane +//! reorg). +//! * `sketch_db` — the in-memory + persisted sketch DB. Logical //! layer (schema registry, schema timeline, backfill types / //! workers / HTTP endpoints) AND the physical storage backend //! (`sketch_db::simple_map_store`) are co-located under this -//! path so the project's identity is unambiguous. -//! * `promsketch_store` — legacy alternative store, currently -//! commented out of the public API. Kept for reference. +//! path. +//! * `gorilla_object_store` — S3/MinIO-backed Gorilla TSDB block +//! store used by the archive tier. //! //! `SimpleMapStore` is re-exported at the top level -//! (`crate::stores::SimpleMapStore`) for call-site stability: -//! callers should not care whether it lives under `sketch_db` or -//! at the `stores` top level. +//! (`crate::stores::SimpleMapStore`) for call-site stability. -pub mod promsketch_store; pub mod gorilla_object_store; +pub mod schema; pub mod sketch_db; pub mod traits; -// pub use promsketch_store::PromSketchStore; +pub use gorilla_object_store::{ + global_s3_cost_counters, ChunkRef, GorillaEngineConfig, GorillaQueryEngine, GorillaS3Config, + GorillaS3ConfigError, GorillaS3Store, ObjectStore, RawSample, S3CostCounters, S3CostSnapshot, + S3CostTrackingObjectStore, +}; pub use sketch_db::sketch_index::{ AccuracyBound, Capability, SidLookup, SketchConfig, SketchEncoding, SketchIndex, SketchInstanceMetadata, SketchKindHandle, SketchSampleState, SketchTimeSeries, }; pub use sketch_db::{AggSchema, AggStatus, SchemaRegistry, SimpleMapStore}; pub use traits::*; -pub use gorilla_object_store::{ - global_s3_cost_counters, ChunkRef, GorillaEngineConfig, GorillaQueryEngine, GorillaS3Config, - GorillaS3ConfigError, GorillaS3Store, ObjectStore, RawSample, S3CostCounters, S3CostSnapshot, - S3CostTrackingObjectStore, -}; diff --git a/asap-query-engine/src/data_model/aggregation_config.rs b/data_plane/src/stores/schema/aggregation_config.rs similarity index 100% rename from asap-query-engine/src/data_model/aggregation_config.rs rename to data_plane/src/stores/schema/aggregation_config.rs diff --git a/asap-query-engine/src/data_model/aggregation_reference.rs b/data_plane/src/stores/schema/aggregation_reference.rs similarity index 100% rename from asap-query-engine/src/data_model/aggregation_reference.rs rename to data_plane/src/stores/schema/aggregation_reference.rs diff --git a/asap-query-engine/src/data_model/enums.rs b/data_plane/src/stores/schema/enums.rs similarity index 100% rename from asap-query-engine/src/data_model/enums.rs rename to data_plane/src/stores/schema/enums.rs diff --git a/asap-query-engine/src/data_model/hot_reload_config.rs b/data_plane/src/stores/schema/hot_reload_config.rs similarity index 96% rename from asap-query-engine/src/data_model/hot_reload_config.rs rename to data_plane/src/stores/schema/hot_reload_config.rs index a5d903fb..c95cd7d4 100644 --- a/asap-query-engine/src/data_model/hot_reload_config.rs +++ b/data_plane/src/stores/schema/hot_reload_config.rs @@ -14,7 +14,7 @@ //! * **Writes** — atomic via `ArcSwap::store`. Lock-free; readers that //! hold a stale snapshot finish their work with the old config and //! drop it when the last reference goes out of scope. -//! * **SimpleEngine** — re-snapshots per query +//! * **ASAPQueryEngine** — re-snapshots per query //! (`streaming_config_snapshot()`). New aggregations are //! query-matchable immediately after the swap lands. //! * **IngestState** — re-snapshots per ingest batch @@ -52,8 +52,8 @@ //! retention elapses, at which point the persistence layer's //! time-based TTL sweep drops the corresponding parts. //! 5. Query semantics during the transition: -//! - Before the swap: `SimpleEngine` matches against agg_id 1. -//! - After the swap: `SimpleEngine` matches against agg_id 17. +//! - Before the swap: `ASAPQueryEngine` matches against agg_id 1. +//! - After the swap: `ASAPQueryEngine` matches against agg_id 17. //! Historical data in the store under agg_id 1 is not joined //! into the answer; the new sketch warms up from zero. //! - Callers that need query continuity across parameter changes @@ -79,7 +79,7 @@ use std::sync::Arc; use arc_swap::ArcSwap; -use crate::data_model::StreamingConfig; +use crate::stores::schema::StreamingConfig; /// Thin wrapper around `ArcSwap` with ergonomic /// snapshot + swap helpers. Cloneable; clones share the same @@ -137,7 +137,7 @@ impl std::fmt::Debug for HotReloadStreamingConfig { #[cfg(test)] mod tests { use super::*; - use crate::data_model::AggregationConfig; + use crate::stores::schema::AggregationConfig; use asap_types::enums::{AggregationType, WindowType}; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; use std::collections::HashMap; diff --git a/asap-query-engine/src/data_model/inference_config.rs b/data_plane/src/stores/schema/inference_config.rs similarity index 100% rename from asap-query-engine/src/data_model/inference_config.rs rename to data_plane/src/stores/schema/inference_config.rs diff --git a/asap-query-engine/src/data_model/key_by_label_values.rs b/data_plane/src/stores/schema/key_by_label_values.rs similarity index 100% rename from asap-query-engine/src/data_model/key_by_label_values.rs rename to data_plane/src/stores/schema/key_by_label_values.rs diff --git a/asap-query-engine/src/data_model/measurement.rs b/data_plane/src/stores/schema/measurement.rs similarity index 100% rename from asap-query-engine/src/data_model/measurement.rs rename to data_plane/src/stores/schema/measurement.rs diff --git a/asap-query-engine/src/data_model/mod.rs b/data_plane/src/stores/schema/mod.rs similarity index 82% rename from asap-query-engine/src/data_model/mod.rs rename to data_plane/src/stores/schema/mod.rs index ebdacf8f..ef1939ec 100644 --- a/asap-query-engine/src/data_model/mod.rs +++ b/data_plane/src/stores/schema/mod.rs @@ -25,11 +25,11 @@ pub use streaming_config::*; pub use traits::*; // Step-1 of the JSONL deprecation refactor moved -// `backend_storage_routing` into the new `crate::routing` module +// `backend_storage_routing` into the new `crate::query_engines::routing` module // alongside the engine router. Re-export here to keep -// `crate::data_model::BackendStorageRouting` compiling for any +// `crate::stores::schema::BackendStorageRouting` compiling for any // transitive caller that hasn't been migrated yet. -pub use crate::routing::{ +pub use crate::query_engines::routing::{ classify_query_shape, BackendStorageRouting, HotReloadBackendStorageRouting, QueryShape, RoutingTarget, }; diff --git a/asap-query-engine/src/data_model/precomputed_output.rs b/data_plane/src/stores/schema/precomputed_output.rs similarity index 97% rename from asap-query-engine/src/data_model/precomputed_output.rs rename to data_plane/src/stores/schema/precomputed_output.rs index 8aa0a19e..040336bb 100644 --- a/asap-query-engine/src/data_model/precomputed_output.rs +++ b/data_plane/src/stores/schema/precomputed_output.rs @@ -4,8 +4,8 @@ use serde::{Deserialize, Serialize}; use std::io::Read as _; use tracing::error; -use crate::data_model::traits::SerializableToSink; -use crate::data_model::{AggregationType, KeyByLabelValues, StreamingConfig}; +use crate::stores::schema::traits::SerializableToSink; +use crate::stores::schema::{AggregationType, KeyByLabelValues, StreamingConfig}; /// §5.1 provenance tag on every precompute record: did this window /// come from live ingest or was it materialised by a backfill job? @@ -102,7 +102,7 @@ impl PrecomputedOutput { // /// Serialize PrecomputedOutput with precompute data to match Python JSON format // pub fn serialize_to_json_with_precompute( // &self, - // precompute: &dyn crate::data_model::AggregateCore, + // precompute: &dyn crate::stores::schema::AggregateCore, // ) -> serde_json::Value { // serde_json::json!({ // // "config": self.config.serialize_to_json(), @@ -195,7 +195,7 @@ impl PrecomputedOutput { // streaming_config: &HashMap, streaming_config: &StreamingConfig, ) -> Result< - (Self, Box), + (Self, Box), Box, > { let aggregation_id = data @@ -295,7 +295,7 @@ impl PrecomputedOutput { // pub fn deserialize_from_json_with_precompute( // data: &serde_json::Value, // ) -> Result< - // (Self, Box), + // (Self, Box), // Box, // > { // debug!("Deserializing PrecomputedOutput with precompute from JSON: {data}"); @@ -324,7 +324,7 @@ impl PrecomputedOutput { // data: &[u8], // aggregation_type: &str, // ) -> Result< - // (Self, Box), + // (Self, Box), // Box, // > { // // First get the metadata and precompute bytes @@ -346,9 +346,9 @@ impl PrecomputedOutput { // fn create_precompute_from_json( // precompute_type: &str, // data: &serde_json::Value, - // ) -> Result, Box> + // ) -> Result, Box> // { - // use crate::precompute_operators::*; + // use crate::precompute_engine::operators::*; // match precompute_type { // "Sum" | "sum" => { @@ -416,9 +416,9 @@ impl PrecomputedOutput { fn create_precompute_from_bytes( precompute_type: AggregationType, buffer: &[u8], - ) -> Result, Box> + ) -> Result, Box> { - use crate::precompute_operators::*; + use crate::precompute_engine::operators::*; match precompute_type { AggregationType::Sum => { @@ -560,7 +560,7 @@ impl SerializableToSink for PrecomputedOutput { // #[test] // fn test_precomputed_output_json_serialization_with_precompute() { // // Test Issue 9: PrecomputedOutput JSON serialization alignment with Python behavior -// use crate::precompute_operators::SumAccumulator; +// use crate::precompute_engine::operators::SumAccumulator; // use std::collections::BTreeMap; // let labels = KeyByLabelNames::from_names(vec!["instance".to_string()]); @@ -618,7 +618,7 @@ impl SerializableToSink for PrecomputedOutput { // #[test] // fn test_precomputed_output_byte_serialization_with_precompute() { // // Test Issue 9: PrecomputedOutput byte serialization alignment with Python behavior -// use crate::precompute_operators::SumAccumulator; +// use crate::precompute_engine::operators::SumAccumulator; // let labels = KeyByLabelNames::from_names(vec!["instance".to_string()]); // let empty_labels = KeyByLabelNames::new(vec![]); diff --git a/asap-query-engine/src/data_model/promql_schema.rs b/data_plane/src/stores/schema/promql_schema.rs similarity index 100% rename from asap-query-engine/src/data_model/promql_schema.rs rename to data_plane/src/stores/schema/promql_schema.rs diff --git a/asap-query-engine/src/data_model/query_config.rs b/data_plane/src/stores/schema/query_config.rs similarity index 100% rename from asap-query-engine/src/data_model/query_config.rs rename to data_plane/src/stores/schema/query_config.rs diff --git a/asap-query-engine/src/data_model/streaming_config.rs b/data_plane/src/stores/schema/streaming_config.rs similarity index 100% rename from asap-query-engine/src/data_model/streaming_config.rs rename to data_plane/src/stores/schema/streaming_config.rs diff --git a/asap-query-engine/src/data_model/traits.rs b/data_plane/src/stores/schema/traits.rs similarity index 99% rename from asap-query-engine/src/data_model/traits.rs rename to data_plane/src/stores/schema/traits.rs index 68278aed..854a8ccb 100644 --- a/asap-query-engine/src/data_model/traits.rs +++ b/data_plane/src/stores/schema/traits.rs @@ -1,4 +1,4 @@ -use crate::data_model::KeyByLabelValues; +use crate::stores::schema::KeyByLabelValues; use serde_json::Value; use std::collections::HashMap; diff --git a/asap-query-engine/src/stores/sketch_db/accuracy.rs b/data_plane/src/stores/sketch_db/accuracy.rs similarity index 99% rename from asap-query-engine/src/stores/sketch_db/accuracy.rs rename to data_plane/src/stores/sketch_db/accuracy.rs index 2942370e..bf68fe86 100644 --- a/asap-query-engine/src/stores/sketch_db/accuracy.rs +++ b/data_plane/src/stores/sketch_db/accuracy.rs @@ -82,7 +82,7 @@ pub enum AccuracyKind { /// Theoretical accuracy bound for an [`AggSchema`](super::AggSchema). /// Attached to every schema; surfaced via HTTP endpoints and -/// (in a follow-up) the `QueryResult` that SimpleEngine returns. +/// (in a follow-up) the `QueryResult` that ASAPQueryEngine returns. #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] pub struct AccuracyProfile { pub epsilon: f64, diff --git a/asap-query-engine/src/stores/sketch_db/backfill.rs b/data_plane/src/stores/sketch_db/backfill.rs similarity index 100% rename from asap-query-engine/src/stores/sketch_db/backfill.rs rename to data_plane/src/stores/sketch_db/backfill.rs diff --git a/asap-query-engine/src/stores/sketch_db/backfill_processor.rs b/data_plane/src/stores/sketch_db/backfill_processor.rs similarity index 98% rename from asap-query-engine/src/stores/sketch_db/backfill_processor.rs rename to data_plane/src/stores/sketch_db/backfill_processor.rs index 574c21bd..176753da 100644 --- a/asap-query-engine/src/stores/sketch_db/backfill_processor.rs +++ b/data_plane/src/stores/sketch_db/backfill_processor.rs @@ -69,7 +69,7 @@ use std::sync::Arc; use async_trait::async_trait; use tracing::{debug, warn}; -use crate::data_model::{AggregateCore, HotReloadStreamingConfig, KeyByLabelValues}; +use crate::stores::schema::{AggregateCore, HotReloadStreamingConfig, KeyByLabelValues}; use crate::precompute_engine::worker::parse_labels_from_series_key; use crate::stores::traits::Store; use asap_types::aggregation_config::AggregationConfig; @@ -205,7 +205,7 @@ impl WindowProcessor for BackfillWindowProcessor { return Ok(()); } - let mut batch: Vec<(crate::data_model::PrecomputedOutput, Box)> = + let mut batch: Vec<(crate::stores::schema::PrecomputedOutput, Box)> = Vec::with_capacity(by_group.len()); for (group_key, group_samples) in by_group { @@ -220,7 +220,7 @@ impl WindowProcessor for BackfillWindowProcessor { } else { Some(build_group_key_label_values(&group_key)) }; - let output = crate::data_model::PrecomputedOutput::new_backfilled( + let output = crate::stores::schema::PrecomputedOutput::new_backfilled( window_range.0, window_range.1, key, @@ -260,7 +260,7 @@ impl WindowProcessor for BackfillWindowProcessor { #[cfg(test)] mod tests { use super::*; - use crate::data_model::StreamingConfig; + use crate::stores::schema::StreamingConfig; use crate::stores::sketch_db::backfill::BackfillSource; use crate::stores::sketch_db::backfill_worker::BackfillWorker; use crate::stores::sketch_db::raw_sample_reader::{LabelFilter, MockRawSampleReader}; @@ -310,7 +310,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SimpleMapStore::new( streaming.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( @@ -360,7 +360,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SimpleMapStore::new( streaming.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( @@ -387,7 +387,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SimpleMapStore::new( streaming.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( @@ -413,7 +413,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SimpleMapStore::new( streaming.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( diff --git a/asap-query-engine/src/stores/sketch_db/backfill_service.rs b/data_plane/src/stores/sketch_db/backfill_service.rs similarity index 98% rename from asap-query-engine/src/stores/sketch_db/backfill_service.rs rename to data_plane/src/stores/sketch_db/backfill_service.rs index b090c80a..cf2d315e 100644 --- a/asap-query-engine/src/stores/sketch_db/backfill_service.rs +++ b/data_plane/src/stores/sketch_db/backfill_service.rs @@ -50,7 +50,7 @@ use std::time::Duration; use tokio::sync::oneshot; use tracing::{debug, info, warn}; -use crate::data_model::HotReloadStreamingConfig; +use crate::stores::schema::HotReloadStreamingConfig; use crate::stores::sketch_db::backfill::{BackfillRegistry, BackfillSource, BackfillStatus}; use crate::stores::sketch_db::backfill_processor::BackfillWindowProcessor; use crate::stores::sketch_db::backfill_worker::BackfillWorker; @@ -293,7 +293,7 @@ pub fn default_reader_factory() -> ReaderFactory { #[cfg(test)] mod tests { use super::*; - use crate::data_model::StreamingConfig; + use crate::stores::schema::StreamingConfig; use crate::stores::sketch_db::raw_sample_reader::{MockRawSampleReader, RawSample}; use crate::stores::sketch_db::simple_map_store::SimpleMapStore; use asap_types::aggregation_config::AggregationConfig; @@ -360,7 +360,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SimpleMapStore::new( streaming.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); @@ -413,7 +413,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SimpleMapStore::new( streaming.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); @@ -453,7 +453,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SimpleMapStore::new( streaming.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); @@ -522,7 +522,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SimpleMapStore::new( streaming.clone(), - crate::data_model::CleanupPolicy::NoCleanup, + crate::stores::schema::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); diff --git a/asap-query-engine/src/stores/sketch_db/backfill_window_builder.rs b/data_plane/src/stores/sketch_db/backfill_window_builder.rs similarity index 99% rename from asap-query-engine/src/stores/sketch_db/backfill_window_builder.rs rename to data_plane/src/stores/sketch_db/backfill_window_builder.rs index f8a2b2f4..4b22aea5 100644 --- a/asap-query-engine/src/stores/sketch_db/backfill_window_builder.rs +++ b/data_plane/src/stores/sketch_db/backfill_window_builder.rs @@ -47,7 +47,7 @@ //! MultipleSubpopulation (update_keyed) dispatch — mirrors //! `worker::apply_sample`. -use crate::data_model::{AggregateCore, KeyByLabelValues}; +use crate::stores::schema::{AggregateCore, KeyByLabelValues}; use crate::precompute_engine::accumulator_factory::{ create_accumulator_updater, AccumulatorUpdater, }; diff --git a/asap-query-engine/src/stores/sketch_db/backfill_worker.rs b/data_plane/src/stores/sketch_db/backfill_worker.rs similarity index 100% rename from asap-query-engine/src/stores/sketch_db/backfill_worker.rs rename to data_plane/src/stores/sketch_db/backfill_worker.rs diff --git a/asap-query-engine/src/stores/sketch_db/epoch_columnar.rs b/data_plane/src/stores/sketch_db/epoch_columnar.rs similarity index 100% rename from asap-query-engine/src/stores/sketch_db/epoch_columnar.rs rename to data_plane/src/stores/sketch_db/epoch_columnar.rs diff --git a/asap-query-engine/src/stores/sketch_db/metrics.rs b/data_plane/src/stores/sketch_db/metrics.rs similarity index 100% rename from asap-query-engine/src/stores/sketch_db/metrics.rs rename to data_plane/src/stores/sketch_db/metrics.rs diff --git a/asap-query-engine/src/stores/sketch_db/mod.rs b/data_plane/src/stores/sketch_db/mod.rs similarity index 98% rename from asap-query-engine/src/stores/sketch_db/mod.rs rename to data_plane/src/stores/sketch_db/mod.rs index a33319cd..057b0cb5 100644 --- a/asap-query-engine/src/stores/sketch_db/mod.rs +++ b/data_plane/src/stores/sketch_db/mod.rs @@ -16,7 +16,7 @@ //! the `POST /api/v1/streaming-config` swap handler to drive //! reconciliation explicitly; 2c added on-disk persistence; 3a added //! the §7 schema-timeline read API; 3b added the cross-schema -//! combiner (`crate::engines::timeline_dispatch`). +//! combiner (`crate::query_engines::timeline_dispatch`). //! //! * `backfill` — §10 of the design. `BackfillJob` lifecycle types //! (`BackfillSource`, `BackfillStatus`, `Coverage`) plus an diff --git a/asap-query-engine/src/stores/sketch_db/prometheus_reader.rs b/data_plane/src/stores/sketch_db/prometheus_reader.rs similarity index 100% rename from asap-query-engine/src/stores/sketch_db/prometheus_reader.rs rename to data_plane/src/stores/sketch_db/prometheus_reader.rs diff --git a/asap-query-engine/src/stores/sketch_db/raw_sample_reader.rs b/data_plane/src/stores/sketch_db/raw_sample_reader.rs similarity index 100% rename from asap-query-engine/src/stores/sketch_db/raw_sample_reader.rs rename to data_plane/src/stores/sketch_db/raw_sample_reader.rs diff --git a/asap-query-engine/src/stores/sketch_db/schema.rs b/data_plane/src/stores/sketch_db/schema.rs similarity index 99% rename from asap-query-engine/src/stores/sketch_db/schema.rs rename to data_plane/src/stores/sketch_db/schema.rs index 7386cf9f..49d57b84 100644 --- a/asap-query-engine/src/stores/sketch_db/schema.rs +++ b/data_plane/src/stores/sketch_db/schema.rs @@ -43,8 +43,8 @@ //! state — no separate index to keep consistent. //! //! The query engine stitches per-segment scalars into a single -//! result via the combiner in `crate::engines::timeline_dispatch`, -//! wired through `SimpleEngine::try_handle_query_promql_via_timeline`. +//! result via the combiner in `crate::query_engines::timeline_dispatch`, +//! wired through `ASAPQueryEngine::try_handle_query_promql_via_timeline`. //! //! **On-disk schema persistence:** //! @@ -64,7 +64,7 @@ //! will be filled in once the sketch types' theoretical bounds are //! vendored. //! * `combine_statistic()` and `PartialResult` for cross-segment -//! result stitching — see `crate::engines::timeline_dispatch`. +//! result stitching — see `crate::query_engines::timeline_dispatch`. //! * Compaction policy that reads `AggStatus` to throttle as expiry //! approaches — §9.2 of the design. @@ -77,7 +77,7 @@ use asap_types::aggregation_config::AggregationConfig; use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; -use crate::data_model::StreamingConfig; +use crate::stores::schema::StreamingConfig; /// Lifecycle state of an `aggregation_id`. Derived from the /// `AggSchema`'s timestamps and the current wall clock — never stored diff --git a/asap-query-engine/src/stores/sketch_db/schema_eviction.rs b/data_plane/src/stores/sketch_db/schema_eviction.rs similarity index 98% rename from asap-query-engine/src/stores/sketch_db/schema_eviction.rs rename to data_plane/src/stores/sketch_db/schema_eviction.rs index 64424610..b688c3fc 100644 --- a/asap-query-engine/src/stores/sketch_db/schema_eviction.rs +++ b/data_plane/src/stores/sketch_db/schema_eviction.rs @@ -267,8 +267,8 @@ pub fn warn_if_retention_inverted( #[cfg(test)] mod tests { use super::*; - use crate::data_model::{AggregationType, CleanupPolicy, LockStrategy, StreamingConfig}; - use crate::precompute_operators::SumAccumulator; + use crate::stores::schema::{AggregationType, CleanupPolicy, LockStrategy, StreamingConfig}; + use crate::precompute_engine::operators::SumAccumulator; use crate::stores::sketch_db::{backfill::BackfillSource, simple_map_store::SimpleMapStore}; use asap_types::aggregation_config::AggregationConfig; use asap_types::enums::WindowType; @@ -308,7 +308,7 @@ mod tests { fn write_one(store: &SimpleMapStore, agg_id: u64, ts: u64) { let acc = SumAccumulator::with_sum(1.0); - let output = crate::data_model::PrecomputedOutput::new(ts, ts + 1000, None, agg_id); + let output = crate::stores::schema::PrecomputedOutput::new(ts, ts + 1000, None, agg_id); store .insert_precomputed_output(output, Box::new(acc)) .unwrap(); diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/INDEX_DESIGN.md b/data_plane/src/stores/sketch_db/simple_map_store/INDEX_DESIGN.md similarity index 100% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/INDEX_DESIGN.md rename to data_plane/src/stores/sketch_db/simple_map_store/INDEX_DESIGN.md diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/common.rs b/data_plane/src/stores/sketch_db/simple_map_store/common.rs similarity index 99% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/common.rs rename to data_plane/src/stores/sketch_db/simple_map_store/common.rs index 68109c65..a6dadd37 100644 --- a/asap-query-engine/src/stores/sketch_db/simple_map_store/common.rs +++ b/data_plane/src/stores/sketch_db/simple_map_store/common.rs @@ -1,4 +1,4 @@ -use crate::data_model::{AggregateCore, KeyByLabelValues}; +use crate::stores::schema::{AggregateCore, KeyByLabelValues}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/global.rs b/data_plane/src/stores/sketch_db/simple_map_store/global.rs similarity index 99% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/global.rs rename to data_plane/src/stores/sketch_db/simple_map_store/global.rs index ac551e00..9edc1332 100644 --- a/asap-query-engine/src/stores/sketch_db/simple_map_store/global.rs +++ b/data_plane/src/stores/sketch_db/simple_map_store/global.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, CleanupPolicy, PrecomputedOutput, StreamingConfig, }; use crate::stores::sketch_db::simple_map_store::common::{ diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/legacy/global.rs b/data_plane/src/stores/sketch_db/simple_map_store/legacy/global.rs similarity index 99% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/legacy/global.rs rename to data_plane/src/stores/sketch_db/simple_map_store/legacy/global.rs index 476c2665..86badb53 100644 --- a/asap-query-engine/src/stores/sketch_db/simple_map_store/legacy/global.rs +++ b/data_plane/src/stores/sketch_db/simple_map_store/legacy/global.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, CleanupPolicy, KeyByLabelValues, PrecomputedOutput, StreamingConfig, }; diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/legacy/mod.rs b/data_plane/src/stores/sketch_db/simple_map_store/legacy/mod.rs similarity index 100% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/legacy/mod.rs rename to data_plane/src/stores/sketch_db/simple_map_store/legacy/mod.rs diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/legacy/per_key.rs b/data_plane/src/stores/sketch_db/simple_map_store/legacy/per_key.rs similarity index 99% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/legacy/per_key.rs rename to data_plane/src/stores/sketch_db/simple_map_store/legacy/per_key.rs index b7807e1c..c82144e7 100644 --- a/asap-query-engine/src/stores/sketch_db/simple_map_store/legacy/per_key.rs +++ b/data_plane/src/stores/sketch_db/simple_map_store/legacy/per_key.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, CleanupPolicy, KeyByLabelValues, PrecomputedOutput, StreamingConfig, }; diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/mod.rs b/data_plane/src/stores/sketch_db/simple_map_store/mod.rs similarity index 98% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/mod.rs rename to data_plane/src/stores/sketch_db/simple_map_store/mod.rs index 3319a991..afddf6cd 100644 --- a/asap-query-engine/src/stores/sketch_db/simple_map_store/mod.rs +++ b/data_plane/src/stores/sketch_db/simple_map_store/mod.rs @@ -4,7 +4,7 @@ pub mod legacy; pub mod per_key; pub mod persistence; -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, CleanupPolicy, LockStrategy, PrecomputedOutput, StreamingConfig, }; use crate::stores::{Store, StoreResult, TimestampedBucketsMap}; @@ -173,8 +173,8 @@ impl Store for SimpleMapStore { #[cfg(test)] mod drop_agg_id_tests { use super::*; - use crate::data_model::AggregationType; - use crate::precompute_operators::SumAccumulator; + use crate::stores::schema::AggregationType; + use crate::precompute_engine::operators::SumAccumulator; use asap_types::aggregation_config::AggregationConfig; use asap_types::enums::WindowType; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/per_key.rs b/data_plane/src/stores/sketch_db/simple_map_store/per_key.rs similarity index 99% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/per_key.rs rename to data_plane/src/stores/sketch_db/simple_map_store/per_key.rs index b6a21d67..27d6fdd9 100644 --- a/asap-query-engine/src/stores/sketch_db/simple_map_store/per_key.rs +++ b/data_plane/src/stores/sketch_db/simple_map_store/per_key.rs @@ -1,4 +1,4 @@ -use crate::data_model::{ +use crate::stores::schema::{ AggregateCore, AggregationType, CleanupPolicy, KeyByLabelValues, PrecomputedOutput, StreamingConfig, }; diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/cache.rs b/data_plane/src/stores/sketch_db/simple_map_store/persistence/cache.rs similarity index 100% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/cache.rs rename to data_plane/src/stores/sketch_db/simple_map_store/persistence/cache.rs diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/config.rs b/data_plane/src/stores/sketch_db/simple_map_store/persistence/config.rs similarity index 100% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/config.rs rename to data_plane/src/stores/sketch_db/simple_map_store/persistence/config.rs diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/flusher.rs b/data_plane/src/stores/sketch_db/simple_map_store/persistence/flusher.rs similarity index 99% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/flusher.rs rename to data_plane/src/stores/sketch_db/simple_map_store/persistence/flusher.rs index bd5ec636..f789690a 100644 --- a/asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/flusher.rs +++ b/data_plane/src/stores/sketch_db/simple_map_store/persistence/flusher.rs @@ -424,7 +424,7 @@ fn now_ms() -> u64 { #[cfg(test)] mod tests { use super::*; - use crate::data_model::KeyByLabelValues; + use crate::stores::schema::KeyByLabelValues; use crate::stores::sketch_db::simple_map_store::persistence::source::{ EpochSnapshot, EpochSnapshotEntry, }; diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/manifest.rs b/data_plane/src/stores/sketch_db/simple_map_store/persistence/manifest.rs similarity index 100% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/manifest.rs rename to data_plane/src/stores/sketch_db/simple_map_store/persistence/manifest.rs diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/mod.rs b/data_plane/src/stores/sketch_db/simple_map_store/persistence/mod.rs similarity index 100% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/mod.rs rename to data_plane/src/stores/sketch_db/simple_map_store/persistence/mod.rs diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/part.rs b/data_plane/src/stores/sketch_db/simple_map_store/persistence/part.rs similarity index 99% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/part.rs rename to data_plane/src/stores/sketch_db/simple_map_store/persistence/part.rs index ab655591..acf5a457 100644 --- a/asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/part.rs +++ b/data_plane/src/stores/sketch_db/simple_map_store/persistence/part.rs @@ -99,7 +99,7 @@ pub struct SnapshotEntry { pub agg_id: u64, pub start_ts: u64, pub end_ts: u64, - pub label: Option, + pub label: Option, pub sketch_type_name: String, pub sketch_bytes: Vec, } @@ -559,7 +559,7 @@ impl PartReader { None } else { Some( - crate::data_model::KeyByLabelValues::deserialize_from_bytes(label_bytes) + crate::stores::schema::KeyByLabelValues::deserialize_from_bytes(label_bytes) .map_err(|e| PersistError::Format(format!("label decode: {}", e)))?, ) }; @@ -593,7 +593,7 @@ fn map_file(path: &Path) -> PersistResult { #[cfg(test)] mod tests { use super::*; - use crate::data_model::KeyByLabelValues; + use crate::stores::schema::KeyByLabelValues; use crate::stores::sketch_db::simple_map_store::persistence::source::EpochSnapshotEntry; use tempfile::TempDir; diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/recovery.rs b/data_plane/src/stores/sketch_db/simple_map_store/persistence/recovery.rs similarity index 99% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/recovery.rs rename to data_plane/src/stores/sketch_db/simple_map_store/persistence/recovery.rs index 0c7684cf..79a8b2df 100644 --- a/asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/recovery.rs +++ b/data_plane/src/stores/sketch_db/simple_map_store/persistence/recovery.rs @@ -117,7 +117,7 @@ pub fn recover(disk_path: &Path) -> PersistResult<(Manifest, RecoveryReport)> { #[cfg(test)] mod tests { use super::*; - use crate::data_model::KeyByLabelValues; + use crate::stores::schema::KeyByLabelValues; use crate::stores::sketch_db::simple_map_store::persistence::part::{ part_dir_path, PartWriter, }; diff --git a/asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/source.rs b/data_plane/src/stores/sketch_db/simple_map_store/persistence/source.rs similarity index 96% rename from asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/source.rs rename to data_plane/src/stores/sketch_db/simple_map_store/persistence/source.rs index 2d234e84..ed095004 100644 --- a/asap-query-engine/src/stores/sketch_db/simple_map_store/persistence/source.rs +++ b/data_plane/src/stores/sketch_db/simple_map_store/persistence/source.rs @@ -2,7 +2,7 @@ //! epochs. Decouples `flusher.rs` from `SimpleMapStorePerKey` so the //! flusher can be unit-tested against a fake source. -use crate::data_model::KeyByLabelValues; +use crate::stores::schema::KeyByLabelValues; use super::PersistResult; @@ -29,7 +29,7 @@ pub struct SealedEpochRef { /// The `entries` are ready to write to disk: labels are already resolved /// to `Option` (no intern-table lookup needed) and the /// sketch bytes are already in the Arroyo/MessagePack format used by -/// `crate::engines::physical::accumulator_serde::deserialize_accumulator`. +/// `crate::query_engines::physical::accumulator_serde::deserialize_accumulator`. #[derive(Debug, Clone)] pub struct EpochSnapshot { pub agg_id: u64, diff --git a/asap-query-engine/src/stores/sketch_db/sketch_index.rs b/data_plane/src/stores/sketch_db/sketch_index.rs similarity index 100% rename from asap-query-engine/src/stores/sketch_db/sketch_index.rs rename to data_plane/src/stores/sketch_db/sketch_index.rs diff --git a/asap-query-engine/src/stores/traits.rs b/data_plane/src/stores/traits.rs similarity index 97% rename from asap-query-engine/src/stores/traits.rs rename to data_plane/src/stores/traits.rs index dbd1de2e..815afe7b 100644 --- a/asap-query-engine/src/stores/traits.rs +++ b/data_plane/src/stores/traits.rs @@ -1,4 +1,4 @@ -use crate::data_model::{AggregateCore, KeyByLabelValues, PrecomputedOutput}; +use crate::stores::schema::{AggregateCore, KeyByLabelValues, PrecomputedOutput}; use std::collections::HashMap; use std::sync::Arc; diff --git a/asap-query-engine/src/tests/accuracy_empirical_validation_tests.rs b/data_plane/src/tests/accuracy_empirical_validation_tests.rs similarity index 100% rename from asap-query-engine/src/tests/accuracy_empirical_validation_tests.rs rename to data_plane/src/tests/accuracy_empirical_validation_tests.rs diff --git a/asap-query-engine/src/tests/accuracy_in_promql_response_tests.rs b/data_plane/src/tests/accuracy_in_promql_response_tests.rs similarity index 100% rename from asap-query-engine/src/tests/accuracy_in_promql_response_tests.rs rename to data_plane/src/tests/accuracy_in_promql_response_tests.rs diff --git a/asap-query-engine/src/tests/capability_matching_tests.rs b/data_plane/src/tests/capability_matching_tests.rs similarity index 94% rename from asap-query-engine/src/tests/capability_matching_tests.rs rename to data_plane/src/tests/capability_matching_tests.rs index fe641c01..5472ddcb 100644 --- a/asap-query-engine/src/tests/capability_matching_tests.rs +++ b/data_plane/src/tests/capability_matching_tests.rs @@ -4,16 +4,16 @@ //! the engine falls back to searching StreamingConfig by capability, and that //! the existing query_config path still takes priority when an entry is present. -use crate::data_model::{ +use crate::stores::schema::{ AggregationConfig, AggregationReference, AggregationType, CleanupPolicy, InferenceConfig, PrecomputedOutput, PromQLSchema, QueryConfig, QueryLanguage, SchemaConfig, StreamingConfig, WindowType, }; -use crate::engines::asap_query::engine::SimpleEngine; -use crate::precompute_operators::count_min_sketch_accumulator::CountMinSketchAccumulator; -use crate::precompute_operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; -use crate::precompute_operators::delta_set_aggregator_accumulator::DeltaSetAggregatorAccumulator; -use crate::precompute_operators::sum_accumulator::SumAccumulator; +use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; +use crate::precompute_engine::operators::count_min_sketch_accumulator::CountMinSketchAccumulator; +use crate::precompute_engine::operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; +use crate::precompute_engine::operators::delta_set_aggregator_accumulator::DeltaSetAggregatorAccumulator; +use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; use crate::stores::sketch_db::simple_map_store::SimpleMapStore; use crate::stores::traits::Store; use promql_utilities::data_model::KeyByLabelNames; @@ -55,13 +55,13 @@ fn make_agg_config( } } -/// Build a `SimpleEngine` with an explicit list of `AggregationConfig`s and no query_configs. +/// Build a `ASAPQueryEngine` with an explicit list of `AggregationConfig`s and no query_configs. /// Data is inserted at timestamp 1_000_000 with a window covering [1_000_000 - window_ms, 1_000_000]. fn engine_no_query_configs( metric: &str, schema_labels: &[&str], agg_configs: Vec, -) -> SimpleEngine { +) -> ASAPQueryEngine { let mut agg_map = HashMap::new(); for c in &agg_configs { agg_map.insert(c.aggregation_id, c.clone()); @@ -109,7 +109,7 @@ fn engine_no_query_configs( cleanup_policy: CleanupPolicy::NoCleanup, }; - SimpleEngine::new( + ASAPQueryEngine::new( store, inference_config, streaming_config, @@ -118,13 +118,13 @@ fn engine_no_query_configs( ) } -/// Build a `SimpleEngine` with both a query_config entry AND a streaming aggregation. +/// Build a `ASAPQueryEngine` with both a query_config entry AND a streaming aggregation. fn engine_with_query_config( metric: &str, schema_labels: &[&str], agg_config: AggregationConfig, promql_query: &str, -) -> SimpleEngine { +) -> ASAPQueryEngine { let agg_id = agg_config.aggregation_id; let mut agg_map = HashMap::new(); agg_map.insert(agg_id, agg_config.clone()); @@ -157,7 +157,7 @@ fn engine_with_query_config( cleanup_policy: CleanupPolicy::NoCleanup, }; - SimpleEngine::new( + ASAPQueryEngine::new( store, inference_config, streaming_config, diff --git a/asap-query-engine/src/tests/capability_miss_http_e2e_tests.rs b/data_plane/src/tests/capability_miss_http_e2e_tests.rs similarity index 98% rename from asap-query-engine/src/tests/capability_miss_http_e2e_tests.rs rename to data_plane/src/tests/capability_miss_http_e2e_tests.rs index a032e5af..d4e7ada9 100644 --- a/asap-query-engine/src/tests/capability_miss_http_e2e_tests.rs +++ b/data_plane/src/tests/capability_miss_http_e2e_tests.rs @@ -11,7 +11,7 @@ //! backend HTTP server //! │ 1. PromQL instant query (capability miss) //! ▼ -//! SimpleEngine.find_compatible_aggregation_with_miss_notify +//! ASAPQueryEngine.find_compatible_aggregation_with_miss_notify //! │ 2. fire-and-forget HttpControllerClient POST //! ▼ //! mock controller HTTP server (this file) @@ -37,13 +37,13 @@ //! (plan-arrival + idempotency on repeat query). #[cfg(test)] -use crate::data_model::{ +use crate::stores::schema::{ CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, QueryLanguage, StreamingConfig, }; use crate::drivers::query::adapters::AdapterConfig; use crate::drivers::query::controller_client::{ControllerClient, HttpControllerClient}; use crate::drivers::query::servers::http::{HttpServer, HttpServerConfig}; -use crate::engines::SimpleEngine; +use crate::query_engines::ASAPQueryEngine; use crate::stores::sketch_db::simple_map_store::SimpleMapStore; use axum::{extract::State, routing::post, Router}; use reqwest::Client; @@ -158,7 +158,7 @@ async fn start_backend(controller_url: String, hot_reload: HotReloadStreamingCon )); let inference_config = InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::NoCleanup); let engine = Arc::new( - SimpleEngine::new_with_hot_reload( + ASAPQueryEngine::new_with_hot_reload( store.clone(), inference_config, hot_reload.clone(), @@ -173,7 +173,7 @@ async fn start_backend(controller_url: String, hot_reload: HotReloadStreamingCon // No fallback — we want engine-miss to be visible to the // test and stay out of the hot-vs-cold routing question. let adapter_config = AdapterConfig::new( - crate::data_model::enums::QueryProtocol::PrometheusHttp, + crate::stores::schema::enums::QueryProtocol::PrometheusHttp, QueryLanguage::promql, None, ); diff --git a/asap-query-engine/src/tests/mod.rs b/data_plane/src/tests/mod.rs similarity index 100% rename from asap-query-engine/src/tests/mod.rs rename to data_plane/src/tests/mod.rs diff --git a/asap-query-engine/src/tests/persist_format_versioning_tests.rs b/data_plane/src/tests/persist_format_versioning_tests.rs similarity index 99% rename from asap-query-engine/src/tests/persist_format_versioning_tests.rs rename to data_plane/src/tests/persist_format_versioning_tests.rs index 9844e9db..829add33 100644 --- a/asap-query-engine/src/tests/persist_format_versioning_tests.rs +++ b/data_plane/src/tests/persist_format_versioning_tests.rs @@ -41,7 +41,7 @@ fn tmpdir() -> tempfile::TempDir { mod schema { use super::*; - use crate::data_model::StreamingConfig; + use crate::stores::schema::StreamingConfig; use asap_types::aggregation_config::AggregationConfig; use asap_types::enums::{AggregationType, WindowType}; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; @@ -453,7 +453,7 @@ mod v2_forward_compat { /// config's schemas — no leakage from the future-version blob. #[test] fn schema_v1_with_future_version_falls_back_and_rewrites_clean() { - use crate::data_model::StreamingConfig; + use crate::stores::schema::StreamingConfig; use asap_types::aggregation_config::AggregationConfig; use asap_types::enums::{AggregationType, WindowType}; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; diff --git a/asap-query-engine/src/tests/persistence_integration_tests.rs b/data_plane/src/tests/persistence_integration_tests.rs similarity index 96% rename from asap-query-engine/src/tests/persistence_integration_tests.rs rename to data_plane/src/tests/persistence_integration_tests.rs index 36e50852..91b16f68 100644 --- a/asap-query-engine/src/tests/persistence_integration_tests.rs +++ b/data_plane/src/tests/persistence_integration_tests.rs @@ -14,7 +14,7 @@ use promql_utilities::data_model::KeyByLabelNames; use tempfile::TempDir; use std::time::Duration; -use crate::data_model::{AggregationType, CleanupPolicy, StreamingConfig, WindowType}; +use crate::stores::schema::{AggregationType, CleanupPolicy, StreamingConfig, WindowType}; use crate::stores::sketch_db::simple_map_store::per_key::SimpleMapStorePerKey; use crate::stores::sketch_db::simple_map_store::persistence::SimpleMapStorePersistenceConfig; use crate::AggregationConfig; diff --git a/asap-query-engine/src/tests/persistence_perf_tests.rs b/data_plane/src/tests/persistence_perf_tests.rs similarity index 99% rename from asap-query-engine/src/tests/persistence_perf_tests.rs rename to data_plane/src/tests/persistence_perf_tests.rs index 13753168..31905729 100644 --- a/asap-query-engine/src/tests/persistence_perf_tests.rs +++ b/data_plane/src/tests/persistence_perf_tests.rs @@ -4,7 +4,7 @@ //! `cargo test` run. Exercise them with: //! //! ```text -//! cargo test --release -p query_engine_rust --lib \ +//! cargo test --release -p data_plane --lib \ //! tests::persistence_perf_tests -- --ignored --nocapture //! ``` //! @@ -35,10 +35,10 @@ use std::time::{Duration, Instant}; use promql_utilities::data_model::KeyByLabelNames; use tempfile::TempDir; -use crate::data_model::{ +use crate::stores::schema::{ AggregationType, CleanupPolicy, PrecomputedOutput, StreamingConfig, WindowType, }; -use crate::precompute_operators::SumAccumulator; +use crate::precompute_engine::operators::SumAccumulator; use crate::stores::sketch_db::simple_map_store::per_key::SimpleMapStorePerKey; use crate::stores::sketch_db::simple_map_store::persistence::SimpleMapStorePersistenceConfig; use crate::stores::Store; diff --git a/asap-query-engine/src/tests/prometheus_forwarding_tests.rs b/data_plane/src/tests/prometheus_forwarding_tests.rs similarity index 94% rename from asap-query-engine/src/tests/prometheus_forwarding_tests.rs rename to data_plane/src/tests/prometheus_forwarding_tests.rs index 1796d63d..bca7af2f 100644 --- a/asap-query-engine/src/tests/prometheus_forwarding_tests.rs +++ b/data_plane/src/tests/prometheus_forwarding_tests.rs @@ -1,8 +1,8 @@ #[cfg(test)] -use crate::data_model::{CleanupPolicy, InferenceConfig, QueryLanguage, StreamingConfig}; +use crate::stores::schema::{CleanupPolicy, InferenceConfig, QueryLanguage, StreamingConfig}; use crate::drivers::query::adapters::AdapterConfig; use crate::drivers::query::servers::http::{HttpServer, HttpServerConfig}; -use crate::engines::SimpleEngine; +use crate::query_engines::ASAPQueryEngine; use crate::stores::sketch_db::simple_map_store::SimpleMapStore; use reqwest::Client; use serde_json::Value; @@ -79,13 +79,13 @@ async fn setup_test_server(prometheus_port: u16) -> (HttpServer, u16) { streaming_config.clone(), CleanupPolicy::NoCleanup, )); - let query_engine = Arc::new(SimpleEngine::new( + let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), // None, inference_config, streaming_config.clone(), 15000, // 15s scrape interval - crate::data_model::QueryLanguage::promql, + crate::stores::schema::QueryLanguage::promql, )); let server = HttpServer::new(config, query_engine, store); @@ -173,13 +173,13 @@ async fn test_forwarding_disabled() { CleanupPolicy::NoCleanup, )); - let query_engine = Arc::new(SimpleEngine::new( + let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), // None, inference_config, streaming_config.clone(), 15000, // 15s scrape interval - crate::data_model::QueryLanguage::promql, + crate::stores::schema::QueryLanguage::promql, )); let server = HttpServer::new(config, query_engine, store); @@ -229,13 +229,13 @@ async fn test_prometheus_server_unreachable() { CleanupPolicy::NoCleanup, )); - let query_engine = Arc::new(SimpleEngine::new( + let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), // None, inference_config, streaming_config.clone(), 15000, // 15s scrape interval - crate::data_model::QueryLanguage::promql, + crate::stores::schema::QueryLanguage::promql, )); let server = HttpServer::new(config, query_engine, store); diff --git a/asap-query-engine/src/tests/schema_timeline_dispatch_tests.rs b/data_plane/src/tests/schema_timeline_dispatch_tests.rs similarity index 97% rename from asap-query-engine/src/tests/schema_timeline_dispatch_tests.rs rename to data_plane/src/tests/schema_timeline_dispatch_tests.rs index cefea2e3..9eaf3a7e 100644 --- a/asap-query-engine/src/tests/schema_timeline_dispatch_tests.rs +++ b/data_plane/src/tests/schema_timeline_dispatch_tests.rs @@ -2,7 +2,7 @@ //! //! Exercises the full path from a PromQL query → schema registry //! lookup → per-segment store query → `combine_statistic` → -//! Prometheus `warnings`, on a real `SimpleEngine` + +//! Prometheus `warnings`, on a real `ASAPQueryEngine` + //! `SimpleMapStore` + `SchemaRegistry` with two agg_ids for the //! same metric and a reconfigure boundary inside the query range. //! @@ -26,12 +26,12 @@ use asap_types::promql_schema::PromQLSchema; use asap_types::query_config::QueryConfig; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; -use crate::data_model::{ +use crate::stores::schema::{ CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, KeyByLabelValues, PrecomputedOutput, QueryLanguage, SchemaConfig, StreamingConfig, }; -use crate::engines::{QueryResult, SimpleEngine}; -use crate::precompute_operators::sum_accumulator::SumAccumulator; +use crate::query_engines::{QueryResult, ASAPQueryEngine}; +use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; use crate::stores::sketch_db::simple_map_store::SimpleMapStore; use crate::stores::sketch_db::{AggSchema, SchemaRegistry}; use crate::stores::Store; @@ -105,7 +105,7 @@ fn build_engine( schemas: Arc, store: Arc, query_for_agg_id: u64, -) -> SimpleEngine { +) -> ASAPQueryEngine { let mut inference_config = InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::NoCleanup); let promql_schema = PromQLSchema::new().add_metric( @@ -120,7 +120,7 @@ fn build_engine( .add_aggregation(AggregationReference::new(query_for_agg_id, None))]; let hot_reload = HotReloadStreamingConfig::from_arc(streaming_config); - SimpleEngine::new_with_hot_reload( + ASAPQueryEngine::new_with_hot_reload( store, inference_config, hot_reload, diff --git a/asap-query-engine/src/tests/store_correctness_tests.rs b/data_plane/src/tests/store_correctness_tests.rs similarity index 99% rename from asap-query-engine/src/tests/store_correctness_tests.rs rename to data_plane/src/tests/store_correctness_tests.rs index ef9425bb..100f9b70 100644 --- a/asap-query-engine/src/tests/store_correctness_tests.rs +++ b/data_plane/src/tests/store_correctness_tests.rs @@ -27,11 +27,11 @@ //! | `contract_per_key` | `LockStrategy::PerKey` (reference impl) | //! | `contract_global` | `LockStrategy::Global` | -use crate::data_model::{ +use crate::stores::schema::{ AggregationType, CleanupPolicy, KeyByLabelValues, LockStrategy, Measurement, SerializableToSink, StreamingConfig, WindowType, }; -use crate::precompute_operators::{ +use crate::precompute_engine::operators::{ CountMinSketchAccumulator, CountMinSketchWithHeapAccumulator, DatasketchesKLLAccumulator, DeltaSetAggregatorAccumulator, HydraKllSketchAccumulator, IncreaseAccumulator, MinMaxAccumulator, MultipleMinMaxAccumulator, MultipleSumAccumulator, SetAggregatorAccumulator, diff --git a/asap-query-engine/src/tests/test_utilities/comparison.rs b/data_plane/src/tests/test_utilities/comparison.rs similarity index 98% rename from asap-query-engine/src/tests/test_utilities/comparison.rs rename to data_plane/src/tests/test_utilities/comparison.rs index 6d781eeb..1363d727 100644 --- a/asap-query-engine/src/tests/test_utilities/comparison.rs +++ b/data_plane/src/tests/test_utilities/comparison.rs @@ -2,8 +2,8 @@ //! //! Provides assertion helpers for deep equality checking of query execution contexts. -use crate::data_model::{AggregationIdInfo, AggregationType}; -use crate::engines::asap_query::engine::{ +use crate::stores::schema::{AggregationIdInfo, AggregationType}; +use crate::query_engines::asap_query_engine::engine::{ QueryExecutionContext, QueryMetadata, StoreQueryParams, StoreQueryPlan, }; use promql_utilities::data_model::KeyByLabelNames; diff --git a/asap-query-engine/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs similarity index 95% rename from asap-query-engine/src/tests/test_utilities/engine_factories.rs rename to data_plane/src/tests/test_utilities/engine_factories.rs index d10ef011..b83d5226 100644 --- a/asap-query-engine/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -1,17 +1,17 @@ //! Engine factory helpers for integration tests //! -//! Provides reusable construction helpers for SimpleEngine + SimpleMapStore +//! Provides reusable construction helpers for ASAPQueryEngine + SimpleMapStore //! populated with various accumulator types. Unlike TestConfigBuilder which //! hardcodes "SumAccumulator", these helpers build AggregationConfig with //! the correct aggregation_type string. -use crate::data_model::{ +use crate::stores::schema::{ AggregationConfig, AggregationReference, AggregationType, CleanupPolicy, InferenceConfig, KeyByLabelValues, PrecomputedOutput, PromQLSchema, QueryConfig, QueryLanguage, SchemaConfig, StreamingConfig, WindowType, }; -use crate::engines::query_result::InstantVectorElement; -use crate::engines::asap_query::engine::SimpleEngine; +use crate::query_engines::query_result::InstantVectorElement; +use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; use crate::stores::sketch_db::simple_map_store::SimpleMapStore; use crate::stores::Store; use crate::AggregateCore; @@ -22,7 +22,7 @@ use std::sync::Arc; /// Data to insert into a store: (label_values, accumulator) pub type AccumulatorData = Vec<(Option>, Box)>; -/// Creates a SimpleEngine with a single aggregation populated with given data. +/// Creates a ASAPQueryEngine with a single aggregation populated with given data. /// /// # Arguments /// * `metric` - Metric name @@ -36,7 +36,7 @@ pub fn create_engine_single_pop( grouping_labels: Vec<&str>, data: AccumulatorData, promql_query: &str, -) -> SimpleEngine { +) -> ASAPQueryEngine { create_engine_single_pop_with_aggregated( metric, aggregation_type, @@ -47,7 +47,7 @@ pub fn create_engine_single_pop( ) } -/// Creates a SimpleEngine with aggregated labels (sub-key labels within the accumulator). +/// Creates a ASAPQueryEngine with aggregated labels (sub-key labels within the accumulator). /// /// Use for self-keyed multi-population accumulators (Multiple* types) where /// `aggregated_labels` are the labels that key the accumulator internally @@ -59,7 +59,7 @@ pub fn create_engine_single_pop_with_aggregated( aggregated_labels: Vec<&str>, data: AccumulatorData, promql_query: &str, -) -> SimpleEngine { +) -> ASAPQueryEngine { let grouping_label_strings: Vec = grouping_labels.iter().map(|s| s.to_string()).collect(); let aggregated_label_strings: Vec = @@ -124,7 +124,7 @@ pub fn create_engine_single_pop_with_aggregated( cleanup_policy: CleanupPolicy::NoCleanup, }; - SimpleEngine::new( + ASAPQueryEngine::new( store, // None, inference_config, @@ -134,7 +134,7 @@ pub fn create_engine_single_pop_with_aggregated( ) } -/// Creates a SimpleEngine with dual-input (separate value and keys aggregations). +/// Creates a ASAPQueryEngine with dual-input (separate value and keys aggregations). /// /// # Arguments /// * `metric` - Metric name @@ -155,7 +155,7 @@ pub fn create_engine_dual_input( value_data: AccumulatorData, keys_data: AccumulatorData, promql_query: &str, -) -> SimpleEngine { +) -> ASAPQueryEngine { let grouping_label_strings: Vec = grouping_labels.iter().map(|s| s.to_string()).collect(); let aggregated_label_strings: Vec = @@ -253,7 +253,7 @@ pub fn create_engine_dual_input( cleanup_policy: CleanupPolicy::NoCleanup, }; - SimpleEngine::new( + ASAPQueryEngine::new( store, // None, inference_config, @@ -263,7 +263,7 @@ pub fn create_engine_dual_input( ) } -/// Creates a SimpleEngine with two independent metrics, each with their own +/// Creates a ASAPQueryEngine with two independent metrics, each with their own /// aggregation config and query_config. /// /// agg_id=1 → metric_a, agg_id=2 → metric_b. @@ -280,7 +280,7 @@ pub fn create_engine_two_metrics( grouping_labels_b: Vec<&str>, data_b: AccumulatorData, query_b: &str, -) -> SimpleEngine { +) -> ASAPQueryEngine { let labels_a: Vec = grouping_labels_a.iter().map(|s| s.to_string()).collect(); let labels_b: Vec = grouping_labels_b.iter().map(|s| s.to_string()).collect(); @@ -368,7 +368,7 @@ pub fn create_engine_two_metrics( cleanup_policy: CleanupPolicy::NoCleanup, }; - SimpleEngine::new( + ASAPQueryEngine::new( store, inference_config, streaming_config, @@ -377,7 +377,7 @@ pub fn create_engine_two_metrics( ) } -/// Creates a SimpleEngine with three independent metrics, each with their own +/// Creates a ASAPQueryEngine with three independent metrics, each with their own /// aggregation config and query_config. /// /// agg_id=1 → metric_a, agg_id=2 → metric_b, agg_id=3 → metric_c. @@ -398,7 +398,7 @@ pub fn create_engine_three_metrics( grouping_labels_c: Vec<&str>, data_c: AccumulatorData, query_c: &str, -) -> SimpleEngine { +) -> ASAPQueryEngine { let labels_a: Vec = grouping_labels_a.iter().map(|s| s.to_string()).collect(); let labels_b: Vec = grouping_labels_b.iter().map(|s| s.to_string()).collect(); let labels_c: Vec = grouping_labels_c.iter().map(|s| s.to_string()).collect(); @@ -472,7 +472,7 @@ pub fn create_engine_three_metrics( cleanup_policy: CleanupPolicy::NoCleanup, }; - SimpleEngine::new( + ASAPQueryEngine::new( store, inference_config, streaming_config, @@ -489,7 +489,7 @@ pub fn create_engine_multi_timestamp( grouping_labels: Vec<&str>, data: Vec<(u64, Option>, Box)>, promql_query: &str, -) -> SimpleEngine { +) -> ASAPQueryEngine { let grouping_label_strings: Vec = grouping_labels.iter().map(|s| s.to_string()).collect(); @@ -546,7 +546,7 @@ pub fn create_engine_multi_timestamp( cleanup_policy: CleanupPolicy::NoCleanup, }; - SimpleEngine::new( + ASAPQueryEngine::new( store, // None, inference_config, @@ -570,7 +570,7 @@ pub fn create_engine_multi_timestamp_with_window( promql_query: &str, window_size: u64, window_type: WindowType, -) -> SimpleEngine { +) -> ASAPQueryEngine { let grouping_label_strings: Vec = grouping_labels.iter().map(|s| s.to_string()).collect(); @@ -627,7 +627,7 @@ pub fn create_engine_multi_timestamp_with_window( cleanup_policy: CleanupPolicy::NoCleanup, }; - SimpleEngine::new( + ASAPQueryEngine::new( store, // None, inference_config, diff --git a/asap-query-engine/src/tests/test_utilities/mod.rs b/data_plane/src/tests/test_utilities/mod.rs similarity index 100% rename from asap-query-engine/src/tests/test_utilities/mod.rs rename to data_plane/src/tests/test_utilities/mod.rs diff --git a/asap-query-engine/src/tests/trait_design_tests.rs b/data_plane/src/tests/trait_design_tests.rs similarity index 96% rename from asap-query-engine/src/tests/trait_design_tests.rs rename to data_plane/src/tests/trait_design_tests.rs index 8d84cc4a..b07da6c2 100644 --- a/asap-query-engine/src/tests/trait_design_tests.rs +++ b/data_plane/src/tests/trait_design_tests.rs @@ -1,8 +1,8 @@ #[cfg(test)] -use crate::data_model::{ +use crate::stores::schema::{ KeyByLabelValues, MultipleSubpopulationAggregate, SingleSubpopulationAggregate, }; -use crate::precompute_operators::{MultipleSumAccumulator, SumAccumulator}; +use crate::precompute_engine::operators::{MultipleSumAccumulator, SumAccumulator}; use promql_utilities::Statistic; #[test] diff --git a/asap-query-engine/src/utils/file_io.rs b/data_plane/src/utils/file_io.rs similarity index 97% rename from asap-query-engine/src/utils/file_io.rs rename to data_plane/src/utils/file_io.rs index 6fb950bb..916bc961 100644 --- a/asap-query-engine/src/utils/file_io.rs +++ b/data_plane/src/utils/file_io.rs @@ -1,4 +1,4 @@ -use crate::data_model::{InferenceConfig, QueryLanguage, StreamingConfig}; +use crate::stores::schema::{InferenceConfig, QueryLanguage, StreamingConfig}; // use crate::stores::promsketch_store::config::PromSketchConfig; use anyhow::{Context, Result}; @@ -37,7 +37,7 @@ pub fn read_streaming_config( #[cfg(test)] mod tests { use super::*; - use crate::data_model::QueryLanguage; + use crate::stores::schema::QueryLanguage; use std::io::Write; use tempfile::NamedTempFile; diff --git a/asap-query-engine/src/utils/http.rs b/data_plane/src/utils/http.rs similarity index 98% rename from asap-query-engine/src/utils/http.rs rename to data_plane/src/utils/http.rs index cadf82c9..02789b10 100644 --- a/asap-query-engine/src/utils/http.rs +++ b/data_plane/src/utils/http.rs @@ -2,7 +2,7 @@ use promql_utilities::KeyByLabelNames; use serde_json::{json, Value}; use std::collections::HashMap; -use crate::engines::QueryResult; +use crate::query_engines::QueryResult; // /// Prometheus-compatible response structure // #[derive(Debug, serde::Serialize, serde::Deserialize)] @@ -229,8 +229,8 @@ pub fn convert_range_result_to_prometheus( #[cfg(test)] mod tests { use super::*; - use crate::data_model::KeyByLabelValues; - use crate::engines::query_result::{InstantVectorElement, RangeVectorElement}; + use crate::stores::schema::KeyByLabelValues; + use crate::query_engines::query_result::{InstantVectorElement, RangeVectorElement}; fn create_test_labels() -> KeyByLabelValues { KeyByLabelValues::new_with_labels(vec!["host1".to_string(), "job1".to_string()]) diff --git a/asap-query-engine/src/utils/mod.rs b/data_plane/src/utils/mod.rs similarity index 100% rename from asap-query-engine/src/utils/mod.rs rename to data_plane/src/utils/mod.rs diff --git a/asap-query-engine/src/utils/precompute_dumper.rs b/data_plane/src/utils/precompute_dumper.rs similarity index 97% rename from asap-query-engine/src/utils/precompute_dumper.rs rename to data_plane/src/utils/precompute_dumper.rs index 060c53c0..dfb29f31 100644 --- a/asap-query-engine/src/utils/precompute_dumper.rs +++ b/data_plane/src/utils/precompute_dumper.rs @@ -1,4 +1,4 @@ -use crate::data_model::{AggregateCore, PrecomputedOutput}; +use crate::stores::schema::{AggregateCore, PrecomputedOutput}; use serde::Serialize; use std::fs::{create_dir_all, File}; use std::io::{BufWriter, Write}; @@ -131,7 +131,7 @@ impl Drop for PrecomputeDumper { #[cfg(test)] mod tests { use super::*; - use crate::precompute_operators::SumAccumulator; + use crate::precompute_engine::operators::SumAccumulator; use tempfile::TempDir; #[test] diff --git a/asap-query-engine/tests/e2e_modified_otlp_sketch_path.rs b/data_plane/tests/e2e_modified_otlp_sketch_path.rs similarity index 97% rename from asap-query-engine/tests/e2e_modified_otlp_sketch_path.rs rename to data_plane/tests/e2e_modified_otlp_sketch_path.rs index 63cae47d..c2f9fc76 100644 --- a/asap-query-engine/tests/e2e_modified_otlp_sketch_path.rs +++ b/data_plane/tests/e2e_modified_otlp_sketch_path.rs @@ -42,12 +42,12 @@ use prost::Message; use std::collections::HashMap; use std::sync::Arc; -use query_engine_rust::data_model::StreamingConfig; -use query_engine_rust::drivers::ingest::{OtlpReceiver, OtlpReceiverConfig}; -use query_engine_rust::precompute_engine::config::{LateDataPolicy, PrecomputeEngineConfig}; -use query_engine_rust::precompute_engine::output_sink::CapturingOutputSink; -use query_engine_rust::precompute_engine::PrecomputeEngine; -use query_engine_rust::precompute_operators::{ +use data_plane::stores::schema::StreamingConfig; +use data_plane::drivers::ingest::{OtlpReceiver, OtlpReceiverConfig}; +use data_plane::precompute_engine::config::{LateDataPolicy, PrecomputeEngineConfig}; +use data_plane::precompute_engine::output_sink::CapturingOutputSink; +use data_plane::precompute_engine::PrecomputeEngine; +use data_plane::precompute_engine::operators::{ CountMinSketchAccumulator, CountSketchAccumulator, DDSketchAccumulator, DatasketchesKLLAccumulator, HllSketchAccumulator, }; @@ -223,7 +223,7 @@ async fn e2e_count_min_sketch_modified_otlp_path() { let sink = Arc::new(CapturingOutputSink::new()); let engine = PrecomputeEngine::new( engine_config(), - query_engine_rust::data_model::HotReloadStreamingConfig::from_arc(streaming_config), + data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config), sink.clone(), ); let ingest_state = engine.ingest_state(); @@ -452,7 +452,7 @@ async fn e2e_count_sketch_modified_otlp_path() { let sink = Arc::new(CapturingOutputSink::new()); let engine = PrecomputeEngine::new( engine_config(), - query_engine_rust::data_model::HotReloadStreamingConfig::from_arc(streaming_config), + data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config), sink.clone(), ); let ingest_state = engine.ingest_state(); @@ -650,7 +650,7 @@ async fn e2e_kll_sketch_modified_otlp_path() { let sink = Arc::new(CapturingOutputSink::new()); let engine = PrecomputeEngine::new( engine_config(), - query_engine_rust::data_model::HotReloadStreamingConfig::from_arc(streaming_config), + data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config), sink.clone(), ); let ingest_state = engine.ingest_state(); @@ -838,7 +838,7 @@ async fn e2e_dd_sketch_modified_otlp_path() { let sink = Arc::new(CapturingOutputSink::new()); let engine = PrecomputeEngine::new( engine_config(), - query_engine_rust::data_model::HotReloadStreamingConfig::from_arc(streaming_config), + data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config), sink.clone(), ); let ingest_state = engine.ingest_state(); @@ -1013,7 +1013,7 @@ async fn e2e_hll_sketch_modified_otlp_path() { let sink = Arc::new(CapturingOutputSink::new()); let engine = PrecomputeEngine::new( engine_config(), - query_engine_rust::data_model::HotReloadStreamingConfig::from_arc(streaming_config), + data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config), sink.clone(), ); let ingest_state = engine.ingest_state(); @@ -1162,7 +1162,7 @@ async fn e2e_count_min_sketch_msgpack_modified_otlp_path() { let sink = Arc::new(CapturingOutputSink::new()); let engine = PrecomputeEngine::new( engine_config(), - query_engine_rust::data_model::HotReloadStreamingConfig::from_arc(streaming_config), + data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config), sink.clone(), ); let ingest_state = engine.ingest_state(); diff --git a/asap-query-engine/tests/edge_runtime_consumes_precompute_rs.rs b/data_plane/tests/edge_runtime_consumes_precompute_rs.rs similarity index 98% rename from asap-query-engine/tests/edge_runtime_consumes_precompute_rs.rs rename to data_plane/tests/edge_runtime_consumes_precompute_rs.rs index 32d0261b..ccc7620c 100644 --- a/asap-query-engine/tests/edge_runtime_consumes_precompute_rs.rs +++ b/data_plane/tests/edge_runtime_consumes_precompute_rs.rs @@ -23,8 +23,8 @@ use asap_precompute_rs::sketches::{ }; use asap_precompute_rs::Sketch; -use query_engine_rust::data_model::AggregateCore; -use query_engine_rust::precompute_operators::edge_runtime_adapter::{ +use data_plane::stores::schema::AggregateCore; +use data_plane::precompute_engine::operators::edge_runtime_adapter::{ encode_ddsketch_envelope, reconstruct_via_runtime, snapshot_ddsketch_via_runtime, unwrap_envelope_state, ReconstructedSketch, SketchType, }; @@ -100,7 +100,7 @@ fn ddsketch_envelope_structural_assertions() { /// query-side surface. #[test] fn ddsketch_envelope_ends_up_in_backend_accumulator() { - use query_engine_rust::precompute_operators::DDSketchAccumulator; + use data_plane::precompute_engine::operators::DDSketchAccumulator; let mut w = DDSketchWrapper::new(0.01); for i in 1..=100 { diff --git a/asap-query-engine/tests/inference_yaml_pattern_coverage.rs b/data_plane/tests/inference_yaml_pattern_coverage.rs similarity index 93% rename from asap-query-engine/tests/inference_yaml_pattern_coverage.rs rename to data_plane/tests/inference_yaml_pattern_coverage.rs index b8628474..b8625a31 100644 --- a/asap-query-engine/tests/inference_yaml_pattern_coverage.rs +++ b/data_plane/tests/inference_yaml_pattern_coverage.rs @@ -25,19 +25,19 @@ fn init_test_tracing() { let _ = tracing_subscriber::fmt::try_init(); } -use query_engine_rust::data_model::{ +use data_plane::stores::schema::{ AggregationConfig, AggregationReference, AggregationType, CleanupPolicy, InferenceConfig, KeyByLabelValues, PrecomputedOutput, PromQLSchema, QueryConfig, QueryLanguage, SchemaConfig, StreamingConfig, WindowType, }; -use query_engine_rust::engines::SimpleEngine; -use query_engine_rust::precompute_operators::{ +use data_plane::engines::ASAPQueryEngine; +use data_plane::precompute_engine::operators::{ DDSketchAccumulator, DatasketchesKLLAccumulator, IncreaseAccumulator, SumAccumulator, }; -use query_engine_rust::stores::SimpleMapStore; -use query_engine_rust::stores::Store; -use query_engine_rust::utils::file_io::read_inference_config; -use query_engine_rust::AggregateCore; +use data_plane::stores::SimpleMapStore; +use data_plane::stores::Store; +use data_plane::utils::file_io::read_inference_config; +use data_plane::AggregateCore; const PROMQL_YAML: &str = "examples/promql/inference_config.yaml"; @@ -125,7 +125,7 @@ fn build_engine( window_size: u64, acc: Box, promql_query: &str, -) -> SimpleEngine { +) -> ASAPQueryEngine { let schema_label_strs: Vec = schema_labels.iter().map(|s| s.to_string()).collect(); let grouping_label_strs: Vec = grouping_labels.iter().map(|s| s.to_string()).collect(); @@ -193,7 +193,7 @@ fn build_engine( cleanup_policy: CleanupPolicy::NoCleanup, }; - SimpleEngine::new( + ASAPQueryEngine::new( store, inference_config, streaming_config, @@ -248,7 +248,7 @@ fn quantile_over_time_multi_phi_routes_through_warm_tier() { .expect("warm tier should answer p50 quantile_over_time"); let (_, qr) = result; let elements = match qr { - query_engine_rust::engines::QueryResult::Vector(iv) => iv.values, + data_plane::engines::QueryResult::Vector(iv) => iv.values, other => panic!("expected vector, got {other:?}"), }; assert!(!elements.is_empty(), "expected non-empty p50 result"); @@ -281,7 +281,7 @@ fn quantile_over_time_wider_range_routes_through_warm_tier() { .expect("warm tier should answer [5m] quantile_over_time"); let (_, qr) = result; let elements = match qr { - query_engine_rust::engines::QueryResult::Vector(iv) => iv.values, + data_plane::engines::QueryResult::Vector(iv) => iv.values, other => panic!("expected vector, got {other:?}"), }; assert!(!elements.is_empty(), "expected non-empty [5m] p99 result"); @@ -293,9 +293,9 @@ fn rate_routes_to_increase_accumulator_warm_tier() { // delta, so the YAML's `rate(fake_metric[…])` entries are paired // with this accumulator type at runtime. We exercise that pairing. let acc = IncreaseAccumulator::new( - query_engine_rust::Measurement::new(0.0), + data_plane::Measurement::new(0.0), 0, - query_engine_rust::Measurement::new(100.0), + data_plane::Measurement::new(100.0), 60_000, ); let engine = build_engine( @@ -313,7 +313,7 @@ fn rate_routes_to_increase_accumulator_warm_tier() { .expect("warm tier should answer rate(...[1m])"); let (_, qr) = result; let elements = match qr { - query_engine_rust::engines::QueryResult::Vector(iv) => iv.values, + data_plane::engines::QueryResult::Vector(iv) => iv.values, other => panic!("expected vector, got {other:?}"), }; assert!(!elements.is_empty(), "rate result should not be empty"); @@ -322,9 +322,9 @@ fn rate_routes_to_increase_accumulator_warm_tier() { #[test] fn increase_routes_to_increase_accumulator_warm_tier() { let acc = IncreaseAccumulator::new( - query_engine_rust::Measurement::new(5.0), + data_plane::Measurement::new(5.0), 0, - query_engine_rust::Measurement::new(25.0), + data_plane::Measurement::new(25.0), 60_000, ); let engine = build_engine( @@ -342,7 +342,7 @@ fn increase_routes_to_increase_accumulator_warm_tier() { .expect("warm tier should answer increase(...[1m])"); let (_, qr) = result; let elements = match qr { - query_engine_rust::engines::QueryResult::Vector(iv) => iv.values, + data_plane::engines::QueryResult::Vector(iv) => iv.values, other => panic!("expected vector, got {other:?}"), }; assert!(!elements.is_empty(), "increase result should not be empty"); @@ -368,7 +368,7 @@ fn sum_over_time_wider_range_routes_through_warm_tier() { .expect("warm tier should answer sum_over_time(...[2m])"); let (_, qr) = result; let elements = match qr { - query_engine_rust::engines::QueryResult::Vector(iv) => iv.values, + data_plane::engines::QueryResult::Vector(iv) => iv.values, other => panic!("expected vector, got {other:?}"), }; assert!( @@ -398,7 +398,7 @@ fn count_over_time_routes_through_warm_tier() { .expect("warm tier should answer count_over_time(...[1m])"); let (_, qr) = result; let elements = match qr { - query_engine_rust::engines::QueryResult::Vector(iv) => iv.values, + data_plane::engines::QueryResult::Vector(iv) => iv.values, other => panic!("expected vector, got {other:?}"), }; assert!( @@ -425,7 +425,7 @@ fn spatial_sum_routes_through_warm_tier() { .expect("warm tier should answer sum(metric)"); let (_, qr) = result; let elements = match qr { - query_engine_rust::engines::QueryResult::Vector(iv) => iv.values, + data_plane::engines::QueryResult::Vector(iv) => iv.values, other => panic!("expected vector, got {other:?}"), }; assert!(!elements.is_empty(), "sum() result should not be empty"); @@ -477,7 +477,7 @@ fn canonical_mvp_demo_quantile_over_time_resolves_via_capability_matching() { ); let (_, qr) = result; let elements = match qr { - query_engine_rust::engines::QueryResult::Vector(iv) => iv.values, + data_plane::engines::QueryResult::Vector(iv) => iv.values, other => panic!("expected vector, got {other:?}"), }; assert!( @@ -514,7 +514,7 @@ fn spatial_multi_quantile_routes_through_warm_tier() { .expect("warm tier should answer quantile by(...) (0.5, ...)"); let (_, qr) = result; let elements = match qr { - query_engine_rust::engines::QueryResult::Vector(iv) => iv.values, + data_plane::engines::QueryResult::Vector(iv) => iv.values, other => panic!("expected vector, got {other:?}"), }; assert!( From 4464fa3a03352420d0fde10f3914b31f7bcb46b7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 12 May 2026 09:51:34 -0600 Subject: [PATCH 2/7] =?UTF-8?q?refactor(data=5Fplane):=20delete=20simple?= =?UTF-8?q?=5Fmap=5Fstore/legacy=20+=20rename=20simple=5Fmap=5Fstore=20?= =?UTF-8?q?=E2=86=92=20sketch=5Fstore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `legacy/` submodule under `simple_map_store/` was only referenced by `benches/simple_store_bench.rs`, which profiles the legacy lock-strategy implementations (`LegacySimpleMapStoreGlobal` / `LegacySimpleMapStorePerKey`). No lib / bin / test path consumes either; remove both as a unit: - Delete `data_plane/src/stores/sketch_db/simple_map_store/legacy/` - Delete `data_plane/benches/simple_store_bench.rs` - Drop the matching `[[bench]]` entry from `data_plane/Cargo.toml` - Drop `pub mod legacy;` from `simple_map_store/mod.rs` Rename the surviving physical store to a clearer name: - Folder: `stores/sketch_db/simple_map_store/` → `stores/sketch_db/sketch_store/` - Types: `SimpleMapStore` → `SketchStore`, `SimpleMapStoreGlobal` → `SketchStoreGlobal`, `SimpleMapStorePerKey` → `SketchStorePerKey`, `SimpleMapStorePersistenceConfig` → `SketchStorePersistenceConfig` Test counts unchanged from origin/main: - data_plane lib: 804 passed / 2 pre-existing failures / 4 ignored - controller lib: 710/710; bins: 27/27 Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/asap_types/src/capability_matching.rs | 2 +- data_plane/Cargo.toml | 6 +- data_plane/benches/simple_store_bench.rs | 858 ------------------ data_plane/src/drivers/ingest/otel.rs | 6 +- data_plane/src/drivers/query/servers/http.rs | 26 +- data_plane/src/lib.rs | 2 +- data_plane/src/main.rs | 26 +- data_plane/src/precompute_engine/worker.rs | 4 +- .../query_engines/asap_query_engine/engine.rs | 24 +- .../query_engines/asap_query_engine/mod.rs | 2 +- data_plane/src/stores/mod.rs | 8 +- data_plane/src/stores/schema/traits.rs | 2 +- data_plane/src/stores/sketch_db/backfill.rs | 4 +- .../stores/sketch_db/backfill_processor.rs | 10 +- .../src/stores/sketch_db/backfill_service.rs | 10 +- .../src/stores/sketch_db/epoch_columnar.rs | 4 +- data_plane/src/stores/sketch_db/mod.rs | 6 +- data_plane/src/stores/sketch_db/schema.rs | 6 +- .../src/stores/sketch_db/schema_eviction.rs | 16 +- .../simple_map_store/legacy/global.rs | 550 ----------- .../sketch_db/simple_map_store/legacy/mod.rs | 5 - .../simple_map_store/legacy/per_key.rs | 639 ------------- .../src/stores/sketch_db/sketch_index.rs | 2 +- .../INDEX_DESIGN.md | 0 .../common.rs | 0 .../global.rs | 12 +- .../{simple_map_store => sketch_store}/mod.rs | 65 +- .../per_key.rs | 28 +- .../persistence/cache.rs | 0 .../persistence/config.rs | 6 +- .../persistence/flusher.rs | 12 +- .../persistence/manifest.rs | 0 .../persistence/mod.rs | 8 +- .../persistence/part.rs | 2 +- .../persistence/recovery.rs | 6 +- .../persistence/source.rs | 4 +- .../src/tests/capability_matching_tests.rs | 6 +- .../tests/capability_miss_http_e2e_tests.rs | 4 +- .../tests/persist_format_versioning_tests.rs | 14 +- .../tests/persistence_integration_tests.rs | 14 +- .../src/tests/persistence_perf_tests.rs | 24 +- .../src/tests/prometheus_forwarding_tests.rs | 8 +- .../tests/schema_timeline_dispatch_tests.rs | 12 +- .../src/tests/store_correctness_tests.rs | 12 +- .../tests/test_utilities/engine_factories.rs | 16 +- .../tests/inference_yaml_pattern_coverage.rs | 4 +- 46 files changed, 209 insertions(+), 2266 deletions(-) delete mode 100644 data_plane/benches/simple_store_bench.rs delete mode 100644 data_plane/src/stores/sketch_db/simple_map_store/legacy/global.rs delete mode 100644 data_plane/src/stores/sketch_db/simple_map_store/legacy/mod.rs delete mode 100644 data_plane/src/stores/sketch_db/simple_map_store/legacy/per_key.rs rename data_plane/src/stores/sketch_db/{simple_map_store => sketch_store}/INDEX_DESIGN.md (100%) rename data_plane/src/stores/sketch_db/{simple_map_store => sketch_store}/common.rs (100%) rename data_plane/src/stores/sketch_db/{simple_map_store => sketch_store}/global.rs (99%) rename data_plane/src/stores/sketch_db/{simple_map_store => sketch_store}/mod.rs (81%) rename data_plane/src/stores/sketch_db/{simple_map_store => sketch_store}/per_key.rs (97%) rename data_plane/src/stores/sketch_db/{simple_map_store => sketch_store}/persistence/cache.rs (100%) rename data_plane/src/stores/sketch_db/{simple_map_store => sketch_store}/persistence/config.rs (95%) rename data_plane/src/stores/sketch_db/{simple_map_store => sketch_store}/persistence/flusher.rs (98%) rename data_plane/src/stores/sketch_db/{simple_map_store => sketch_store}/persistence/manifest.rs (100%) rename data_plane/src/stores/sketch_db/{simple_map_store => sketch_store}/persistence/mod.rs (87%) rename data_plane/src/stores/sketch_db/{simple_map_store => sketch_store}/persistence/part.rs (99%) rename data_plane/src/stores/sketch_db/{simple_map_store => sketch_store}/persistence/recovery.rs (96%) rename data_plane/src/stores/sketch_db/{simple_map_store => sketch_store}/persistence/source.rs (96%) diff --git a/crates/asap_types/src/capability_matching.rs b/crates/asap_types/src/capability_matching.rs index 26a2e7d1..986d1407 100644 --- a/crates/asap_types/src/capability_matching.rs +++ b/crates/asap_types/src/capability_matching.rs @@ -42,7 +42,7 @@ pub const CANONICAL_QUERY_ENGINE_IDS: &[&str] = &[ENGINE_ID_ASAP_QUERY, ENGINE_I #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] #[serde(rename_all = "snake_case")] pub enum StorageBackend { - /// Warm-tier sketch DB (today's `SimpleMapStore` + accumulators). + /// Warm-tier sketch DB (today's `SketchStore` + accumulators). /// Served by `ASAPQueryEngine`. Default for unconfigured metrics. #[default] SketchStore, diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index a4f5f07a..d6e03734 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -80,7 +80,7 @@ asap_sketchlib = { git = "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/ProjectASAP/asap_sketchlib", branch # `ASAPQuery-backend` (sibling directories under `~/repos/`) before # `cargo build`. The path is relative so any matching layout works. asap-precompute-rs = { path = "../../ASAPCollector/asap-precompute-rs" } -# Persistence layer (SimpleMapStore parts / manifest / Tier-2 cache) +# Persistence layer (SketchStore parts / manifest / Tier-2 cache) moka = { version = "0.12", features = ["sync"] } memmap2 = "0.9" crc32fast = "1.4" @@ -108,10 +108,6 @@ lru = "0.12" tempfile = "3.20.0" criterion = { version = "0.5", features = ["html_reports"] } -[[bench]] -name = "simple_store_bench" -harness = false - [features] #default = ["lock_profiling", "extra_debugging"] default = [] diff --git a/data_plane/benches/simple_store_bench.rs b/data_plane/benches/simple_store_bench.rs deleted file mode 100644 index 76d786c9..00000000 --- a/data_plane/benches/simple_store_bench.rs +++ /dev/null @@ -1,858 +0,0 @@ -//! Benchmarks for `LegacySimpleMapStore` — insert, range query, exact query, -//! store-analyze, and concurrent reads. -//! -//! These benchmarks profile the legacy store implementation -//! (`LegacySimpleMapStoreGlobal` / `LegacySimpleMapStorePerKey`) and provide -//! concrete measurements of algorithm complexity for: -//! -//! | Operation | Expected complexity | -//! |------------------------------------|--------------------------| -//! | `insert_precomputed_output_batch` | O(B) | -//! | `query_precomputed_output` (range) | O(W·log W + k) | -//! | `query_precomputed_output_exact` | O(1) HashMap lookup | -//! | `get_earliest_timestamp` (analyze) | O(A) — scan agg-id map | -//! | concurrent reads (n threads) | serialised by write lock | -//! -//! where B = batch size, W = stored windows, k = result entries, A = agg IDs. -//! -//! Two accumulator types are benchmarked: -//! - `sum` — `SumAccumulator` (trivial f64, ~0 clone cost, baseline) -//! - `kll` — `DatasketchesKLLAccumulator` k=200 (~1 KB sketch, realistic clone cost) -//! -//! Run with: -//! cargo bench -p data_plane --bench simple_store_bench -//! -//! Results land in `target/criterion/`. - -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; -use promql_utilities::data_model::KeyByLabelNames; -use data_plane::stores::schema::{ - AggregateCore, AggregationType, CleanupPolicy, KeyByLabelValues, LockStrategy, StreamingConfig, - WindowType, -}; -use data_plane::precompute_engine::operators::{DatasketchesKLLAccumulator, SumAccumulator}; -use data_plane::stores::sketch_db::simple_map_store::legacy::{ - LegacySimpleMapStoreGlobal, LegacySimpleMapStorePerKey, -}; -use data_plane::stores::Store; -use data_plane::{AggregationConfig, PrecomputedOutput, SimpleMapStore}; -use std::collections::HashMap; -use std::sync::{Arc, Barrier}; - -#[derive(Clone, Copy)] -enum StoreKind { - LegacyPerKey, - LegacyGlobal, - CurrentPerKey, - CurrentGlobal, -} - -impl StoreKind { - const ALL: [Self; 4] = [ - Self::LegacyPerKey, - Self::LegacyGlobal, - Self::CurrentPerKey, - Self::CurrentGlobal, - ]; - - fn slug(self) -> &'static str { - match self { - Self::LegacyPerKey => "legacy/per_key", - Self::LegacyGlobal => "legacy/global", - Self::CurrentPerKey => "current/per_key", - Self::CurrentGlobal => "current/global", - } - } - - fn build(self, config: Arc, cleanup_policy: CleanupPolicy) -> Arc { - match self { - Self::LegacyPerKey => Arc::new(LegacySimpleMapStorePerKey::new(config, cleanup_policy)), - Self::LegacyGlobal => Arc::new(LegacySimpleMapStoreGlobal::new(config, cleanup_policy)), - Self::CurrentPerKey => Arc::new(SimpleMapStore::new_with_strategy( - config, - cleanup_policy, - LockStrategy::PerKey, - )), - Self::CurrentGlobal => Arc::new(SimpleMapStore::new_with_strategy( - config, - cleanup_policy, - LockStrategy::Global, - )), - } - } -} - -#[derive(Clone, Copy)] -enum AccumulatorKind { - Sum, - Kll, -} - -impl AccumulatorKind { - const ALL: [Self; 2] = [Self::Sum, Self::Kll]; - - fn slug(self) -> &'static str { - match self { - Self::Sum => "sum", - Self::Kll => "kll", - } - } - - fn aggregation_type(self) -> AggregationType { - match self { - Self::Sum => AggregationType::Sum, - Self::Kll => AggregationType::DatasketchesKLL, - } - } - - fn build(self, value: f64) -> Box { - match self { - Self::Sum => Box::new(SumAccumulator::with_sum(value)), - Self::Kll => { - let mut acc = DatasketchesKLLAccumulator::new(200); - for v in 0..20 { - acc.update(v as f64 * (value + 1.0)); - } - Box::new(acc) - } - } - } -} - -fn make_agg_config( - agg_id: u64, - aggregation_type: AggregationType, - metric: &str, - num_aggregates_to_retain: Option, - read_count_threshold: Option, -) -> AggregationConfig { - AggregationConfig::new( - agg_id, - aggregation_type, - "".to_string(), - HashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - "".to_string(), - 60, // window_size (seconds) - 60, // slide_interval (seconds) - WindowType::Tumbling, // window_type - "".to_string(), // spatial_filter - metric.to_string(), - num_aggregates_to_retain, - read_count_threshold, - None, // table_name - None, // value_column - ) -} - -fn make_streaming_config( - agg_ids: &[u64], - accumulator_kind: AccumulatorKind, - metric: &str, - num_aggregates_to_retain: Option, - read_count_threshold: Option, -) -> Arc { - let configs = agg_ids - .iter() - .copied() - .map(|agg_id| { - ( - agg_id, - make_agg_config( - agg_id, - accumulator_kind.aggregation_type(), - metric, - num_aggregates_to_retain, - read_count_threshold, - ), - ) - }) - .collect(); - Arc::new(StreamingConfig::new(configs)) -} - -fn make_batch( - count: usize, - agg_id: u64, - base_ts: u64, - window_ms: u64, - accumulator_kind: AccumulatorKind, -) -> Vec<(PrecomputedOutput, Box)> { - (0..count as u64) - .map(|i| { - let start = base_ts + i * window_ms; - let end = start + window_ms; - ( - PrecomputedOutput::new(start, end, None, agg_id), - accumulator_kind.build(i as f64), - ) - }) - .collect() -} - -fn insert_labelled_entry( - store: &dyn Store, - start: u64, - end: u64, - label: &str, - agg_id: u64, - accumulator_kind: AccumulatorKind, - value: f64, -) { - let output = PrecomputedOutput::new( - start, - end, - Some(KeyByLabelValues::new_with_labels(vec![label.to_string()])), - agg_id, - ); - store - .insert_precomputed_output(output, accumulator_kind.build(value)) - .unwrap(); -} - -fn populate_store_labelled( - store: &dyn Store, - time_ranges: usize, - labels: usize, - agg_id: u64, - accumulator_kind: AccumulatorKind, -) { - for i in 0..time_ranges { - let start = i as u64 * 1_000; - let end = start + 1_000; - for j in 0..labels { - insert_labelled_entry( - store, - start, - end, - &format!("host-{j}"), - agg_id, - accumulator_kind, - 1.0, - ); - } - } -} - -fn populate_store_batch( - store: &dyn Store, - num_windows: usize, - agg_id: u64, - accumulator_kind: AccumulatorKind, -) { - let batch = make_batch(num_windows, agg_id, 1_000_000, 60_000, accumulator_kind); - store.insert_precomputed_output_batch(batch).unwrap(); -} - -fn build_populated_store( - kind: StoreKind, - accumulator_kind: AccumulatorKind, - time_ranges: usize, - labels: usize, -) -> Arc { - let config = make_streaming_config(&[1], accumulator_kind, "test_metric", None, None); - let store = kind.build(config, CleanupPolicy::NoCleanup); - populate_store_labelled(store.as_ref(), time_ranges, labels, 1, accumulator_kind); - store -} - -fn bench_insert_batch_size(c: &mut Criterion) { - let mut group = c.benchmark_group("insert/batch_size"); - - for &batch_size in &[100usize, 1_000, 5_000, 10_000] { - group.throughput(Throughput::Elements(batch_size as u64)); - - for kind in StoreKind::ALL { - for accumulator_kind in AccumulatorKind::ALL { - let id = format!("{}/{}", kind.slug(), accumulator_kind.slug()); - group.bench_with_input(BenchmarkId::new(id, batch_size), &batch_size, |b, &n| { - b.iter_batched( - || { - let config = make_streaming_config( - &[1], - accumulator_kind, - "cpu_usage", - None, - None, - ); - ( - kind.build(config, CleanupPolicy::NoCleanup), - make_batch(n, 1, 1_000_000, 60_000, accumulator_kind), - ) - }, - |(store, batch)| { - store.insert_precomputed_output_batch(batch).unwrap(); - }, - criterion::BatchSize::SmallInput, - ); - }); - } - } - } - - group.finish(); -} - -fn bench_insert_num_agg_ids(c: &mut Criterion) { - let mut group = c.benchmark_group("insert/num_agg_ids"); - const TOTAL_ITEMS: usize = 1_000; - - for &num_ids in &[1usize, 10, 50, 200] { - group.throughput(Throughput::Elements(TOTAL_ITEMS as u64)); - - for kind in StoreKind::ALL { - group.bench_with_input(BenchmarkId::new(kind.slug(), num_ids), &num_ids, |b, &n| { - b.iter_batched( - || { - let agg_ids: Vec = (1..=n as u64).collect(); - let config = make_streaming_config( - &agg_ids, - AccumulatorKind::Sum, - "cpu_usage", - None, - None, - ); - let store = kind.build(config, CleanupPolicy::NoCleanup); - let per_id = TOTAL_ITEMS / n; - let mut batch = Vec::with_capacity(per_id * n); - for agg_id in agg_ids { - batch.extend(make_batch( - per_id, - agg_id, - 1_000_000, - 60_000, - AccumulatorKind::Sum, - )); - } - (store, batch) - }, - |(store, batch)| { - store.insert_precomputed_output_batch(batch).unwrap(); - }, - criterion::BatchSize::SmallInput, - ); - }); - } - } - - group.finish(); -} - -fn bench_query_range_store_size(c: &mut Criterion) { - let mut group = c.benchmark_group("query/range_store_size"); - - for &num_windows in &[500usize, 1_000, 5_000, 10_000] { - for kind in StoreKind::ALL { - for accumulator_kind in AccumulatorKind::ALL { - let store = { - let config = - make_streaming_config(&[1], accumulator_kind, "cpu_usage", None, None); - let store = kind.build(config, CleanupPolicy::NoCleanup); - populate_store_batch(store.as_ref(), num_windows, 1, accumulator_kind); - store - }; - let id = format!("{}/{}", kind.slug(), accumulator_kind.slug()); - let query_start = 1_000_000u64; - let query_end = query_start + num_windows as u64 * 60_000; - - group.bench_with_input(BenchmarkId::new(id, num_windows), &num_windows, |b, _| { - b.iter(|| { - black_box( - store - .query_precomputed_output("cpu_usage", 1, query_start, query_end) - .unwrap(), - ) - }); - }); - } - } - } - - group.finish(); -} - -fn bench_query_exact_store_size(c: &mut Criterion) { - let mut group = c.benchmark_group("query/exact_store_size"); - - for &num_windows in &[500usize, 1_000, 5_000, 10_000] { - for kind in StoreKind::ALL { - let store = { - let config = - make_streaming_config(&[1], AccumulatorKind::Sum, "cpu_usage", None, None); - let store = kind.build(config, CleanupPolicy::NoCleanup); - populate_store_batch(store.as_ref(), num_windows, 1, AccumulatorKind::Sum); - store - }; - let exact_start = 1_000_000u64 + (num_windows as u64 - 1) * 60_000; - let exact_end = exact_start + 60_000; - - group.bench_with_input( - BenchmarkId::new(kind.slug(), num_windows), - &num_windows, - |b, _| { - b.iter(|| { - black_box( - store - .query_precomputed_output_exact( - "cpu_usage", - 1, - exact_start, - exact_end, - ) - .unwrap(), - ) - }); - }, - ); - } - } - - group.finish(); -} - -fn bench_store_analyze(c: &mut Criterion) { - let mut group = c.benchmark_group("store_analyze/num_agg_ids"); - - for &num_ids in &[10usize, 100, 500, 1_000] { - let agg_ids: Vec = (1..=num_ids as u64).collect(); - for kind in StoreKind::ALL { - let config = - make_streaming_config(&agg_ids, AccumulatorKind::Sum, "cpu_usage", None, None); - let store = kind.build(config, CleanupPolicy::NoCleanup); - for agg_id in 1..=num_ids as u64 { - let output = PrecomputedOutput::new(1_000_000, 1_060_000, None, agg_id); - store - .insert_precomputed_output(output, AccumulatorKind::Sum.build(1.0)) - .unwrap(); - } - - group.bench_with_input(BenchmarkId::new(kind.slug(), num_ids), &num_ids, |b, _| { - b.iter(|| black_box(store.get_earliest_timestamp_per_aggregation_id().unwrap())); - }); - } - } - - group.finish(); -} - -fn bench_concurrent_reads(c: &mut Criterion) { - let mut group = c.benchmark_group("concurrent_reads/thread_count"); - let num_windows = 5_000usize; - let query_start = 1_000_000u64; - let query_end = query_start + num_windows as u64 * 60_000; - - for kind in StoreKind::ALL { - let config = make_streaming_config(&[1], AccumulatorKind::Sum, "cpu_usage", None, None); - let store = kind.build(config, CleanupPolicy::NoCleanup); - populate_store_batch(store.as_ref(), num_windows, 1, AccumulatorKind::Sum); - - for &num_threads in &[1usize, 2, 4, 8] { - group.bench_with_input( - BenchmarkId::new(kind.slug(), num_threads), - &num_threads, - |b, &n| { - let store = store.clone(); - b.iter(|| { - let handles: Vec<_> = (0..n) - .map(|_| { - let store = store.clone(); - std::thread::spawn(move || { - store - .query_precomputed_output( - "cpu_usage", - 1, - query_start, - query_end, - ) - .unwrap() - }) - }) - .collect(); - for handle in handles { - black_box(handle.join().unwrap()); - } - }); - }, - ); - } - } - - group.finish(); -} - -fn bench_concurrent_writes(c: &mut Criterion) { - let mut group = c.benchmark_group("concurrent_writes/thread_count"); - let labels = 10usize; - let entries_per_thread = 500usize; - let time_ranges_per_thread = entries_per_thread / labels; - - for kind in StoreKind::ALL { - for &num_threads in &[1usize, 2, 4, 8] { - group.bench_with_input( - BenchmarkId::new(kind.slug(), num_threads), - &num_threads, - |b, &n| { - b.iter(|| { - let config = make_streaming_config( - &[1], - AccumulatorKind::Sum, - "test_metric", - None, - None, - ); - let store = kind.build(config, CleanupPolicy::NoCleanup); - let barrier = Arc::new(Barrier::new(n)); - std::thread::scope(|scope| { - for t in 0..n { - let store = store.clone(); - let barrier = barrier.clone(); - scope.spawn(move || { - barrier.wait(); - for i in 0..time_ranges_per_thread { - let start = i as u64 * 1_000; - let end = start + 1_000; - for j in 0..labels { - insert_labelled_entry( - store.as_ref(), - start, - end, - &format!("thread-{t}-host-{j}"), - 1, - AccumulatorKind::Sum, - 1.0, - ); - } - } - }); - } - }); - black_box(store); - }); - }, - ); - } - } - - group.finish(); -} - -fn bench_concurrent_mixed_read_write(c: &mut Criterion) { - let mut group = c.benchmark_group("concurrent_mixed_rw/config"); - let writers = 2usize; - let readers = 2usize; - let labels = 10usize; - let time_ranges = 1_000usize; - let total_threads = writers + readers; - - for kind in StoreKind::ALL { - let store = build_populated_store(kind, AccumulatorKind::Sum, time_ranges, labels); - let query_end = time_ranges as u64 * 1_000 / 10; - - group.bench_function(kind.slug(), |b| { - let store = store.clone(); - b.iter(|| { - let barrier = Arc::new(Barrier::new(total_threads)); - std::thread::scope(|scope| { - for writer_id in 0..writers { - let store = store.clone(); - let barrier = barrier.clone(); - scope.spawn(move || { - barrier.wait(); - for offset in 0..50usize { - let start = (time_ranges + writer_id * 50 + offset) as u64 * 1_000; - let end = start + 1_000; - for label_id in 0..labels { - insert_labelled_entry( - store.as_ref(), - start, - end, - &format!("mixed-{writer_id}-host-{label_id}"), - 1, - AccumulatorKind::Sum, - 1.0, - ); - } - } - }); - } - - for _ in 0..readers { - let store = store.clone(); - let barrier = barrier.clone(); - scope.spawn(move || { - barrier.wait(); - for _ in 0..20 { - black_box( - store - .query_precomputed_output("test_metric", 1, 0, query_end) - .unwrap(), - ); - } - }); - } - }); - }); - }); - } - - group.finish(); -} - -fn bench_cleanup_overhead(c: &mut Criterion) { - let mut group = c.benchmark_group("cleanup_overhead"); - let labels = 5usize; - - for kind in StoreKind::ALL { - group.bench_function(format!("{}/no_cleanup", kind.slug()), |b| { - b.iter(|| { - let config = - make_streaming_config(&[1], AccumulatorKind::Sum, "test_metric", None, None); - let store = kind.build(config, CleanupPolicy::NoCleanup); - populate_store_labelled(store.as_ref(), 200, labels, 1, AccumulatorKind::Sum); - black_box(store); - }); - }); - - group.bench_function(format!("{}/circular_buffer", kind.slug()), |b| { - b.iter(|| { - let config = make_streaming_config( - &[1], - AccumulatorKind::Sum, - "test_metric", - Some(50), - None, - ); - let store = kind.build(config, CleanupPolicy::CircularBuffer); - populate_store_labelled(store.as_ref(), 200, labels, 1, AccumulatorKind::Sum); - black_box(store); - }); - }); - - group.bench_function(format!("{}/read_based", kind.slug()), |b| { - b.iter(|| { - let config = - make_streaming_config(&[1], AccumulatorKind::Sum, "test_metric", None, Some(2)); - let store = kind.build(config, CleanupPolicy::ReadBased); - populate_store_labelled(store.as_ref(), 100, labels, 1, AccumulatorKind::Sum); - - for _ in 0..2 { - black_box( - store - .query_precomputed_output("test_metric", 1, 0, 100_000) - .unwrap(), - ); - } - - for i in 100..200usize { - let start = i as u64 * 1_000; - let end = start + 1_000; - for j in 0..labels { - insert_labelled_entry( - store.as_ref(), - start, - end, - &format!("host-{j}"), - 1, - AccumulatorKind::Sum, - 1.0, - ); - } - } - - black_box(store); - }); - }); - } - - group.finish(); -} - -fn bench_query_patterns(c: &mut Criterion) { - let mut group = c.benchmark_group("query_patterns"); - let time_ranges = 1_000usize; - let labels = 10usize; - let total_time = time_ranges as u64 * 1_000; - - for kind in StoreKind::ALL { - let store = build_populated_store(kind, AccumulatorKind::Sum, time_ranges, labels); - - for (name, start, end) in [ - ("full_scan", 0, total_time), - ("wide_50pct", 0, total_time / 2), - ("narrow_1pct", 0, total_time / 100), - ("miss", total_time + 1_000_000, total_time + 1_001_000), - ] { - group.bench_function(format!("{}/{}", kind.slug(), name), |b| { - let store = store.clone(); - b.iter(|| { - black_box( - store - .query_precomputed_output("test_metric", 1, start, end) - .unwrap(), - ); - }); - }); - } - } - - group.finish(); -} - -fn bench_high_label_cardinality(c: &mut Criterion) { - let mut group = c.benchmark_group("high_label_cardinality"); - let time_ranges = 20usize; - - for &label_count in &[10usize, 100, 500, 1_000] { - for kind in StoreKind::ALL { - group.bench_with_input( - BenchmarkId::new(format!("{}/insert", kind.slug()), label_count), - &label_count, - |b, &lc| { - b.iter(|| { - let store = - build_populated_store(kind, AccumulatorKind::Sum, time_ranges, lc); - black_box(store); - }); - }, - ); - - let store = build_populated_store(kind, AccumulatorKind::Sum, time_ranges, label_count); - let query_end = time_ranges as u64 * 1_000; - group.bench_with_input( - BenchmarkId::new(format!("{}/query", kind.slug()), label_count), - &label_count, - |b, _| { - let store = store.clone(); - b.iter(|| { - black_box( - store - .query_precomputed_output("test_metric", 1, 0, query_end) - .unwrap(), - ); - }); - }, - ); - } - } - - group.finish(); -} - -fn bench_multi_agg_id(c: &mut Criterion) { - let mut group = c.benchmark_group("multi_agg_id"); - let agg_ids: Vec = (1..=10).collect(); - let time_ranges = 100usize; - let labels = 5usize; - - for kind in StoreKind::ALL { - group.bench_function(format!("{}/insert_10_agg_ids", kind.slug()), |b| { - b.iter(|| { - let config = make_streaming_config( - &agg_ids, - AccumulatorKind::Sum, - "test_metric", - None, - None, - ); - let store = kind.build(config, CleanupPolicy::NoCleanup); - for &agg_id in &agg_ids { - populate_store_labelled( - store.as_ref(), - time_ranges, - labels, - agg_id, - AccumulatorKind::Sum, - ); - } - black_box(store); - }); - }); - - let config = - make_streaming_config(&agg_ids, AccumulatorKind::Sum, "test_metric", None, None); - let store = kind.build(config, CleanupPolicy::NoCleanup); - for &agg_id in &agg_ids { - populate_store_labelled( - store.as_ref(), - time_ranges, - labels, - agg_id, - AccumulatorKind::Sum, - ); - } - let query_end = time_ranges as u64 * 1_000; - - group.bench_function(format!("{}/query_hot_cold", kind.slug()), |b| { - let store = store.clone(); - let mut query_idx = 0u64; - b.iter(|| { - let agg_id = if query_idx % 5 < 4 { - (query_idx % 2) + 1 - } else { - (query_idx % 8) + 3 - }; - query_idx += 1; - black_box( - store - .query_precomputed_output("test_metric", agg_id, 0, query_end) - .unwrap(), - ); - }); - }); - - group.bench_function(format!("{}/concurrent_hot_cold", kind.slug()), |b| { - let store = store.clone(); - b.iter(|| { - let barrier = Arc::new(Barrier::new(4)); - std::thread::scope(|scope| { - for t in 0..4usize { - let store = store.clone(); - let barrier = barrier.clone(); - scope.spawn(move || { - barrier.wait(); - for q in 0..50usize { - let idx = (t * 50 + q) as u64; - let agg_id = if idx % 5 < 4 { - (idx % 2) + 1 - } else { - (idx % 8) + 3 - }; - black_box( - store - .query_precomputed_output( - "test_metric", - agg_id, - 0, - query_end, - ) - .unwrap(), - ); - } - }); - } - }); - }); - }); - } - - group.finish(); -} - -criterion_group!( - benches, - bench_insert_batch_size, - bench_insert_num_agg_ids, - bench_query_range_store_size, - bench_query_exact_store_size, - bench_store_analyze, - bench_concurrent_reads, - bench_concurrent_writes, - bench_concurrent_mixed_read_write, - bench_cleanup_overhead, - bench_query_patterns, - bench_high_label_cardinality, - bench_multi_agg_id, -); -criterion_main!(benches); diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 24653a90..9322c661 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -5,7 +5,7 @@ //! engine via [`OtlpReceiver::with_ingest_state`] — routes both raw metric //! points and pre-built sketches through the precompute engine's worker //! pool. The precompute engine then performs window-aligned aggregation -//! per `StreamingConfig` and writes results to `SimpleMapStore`. +//! per `StreamingConfig` and writes results to `SketchStore`. //! //! Architectural flow: //! ```text @@ -13,7 +13,7 @@ //! → OTLP gRPC/HTTP (this receiver) //! → precompute engine ingest router //! → workers (per (agg_id, group_key) panes) -//! → StoreOutputSink → SimpleMapStore +//! → StoreOutputSink → SketchStore //! → query engine //! ``` //! @@ -1057,7 +1057,7 @@ async fn route_modified_otlp_sketches_to_precompute( // legacy router push stays in tandem until the // query path's warm-tier reducer is wired // end-to-end and the streaming-config / - // SimpleMapStore call sites can be deleted. + // SketchStore call sites can be deleted. messages.push(WorkerMessage::AccumulatorInput { agg_id: config.aggregation_id, group_key, diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index cb2aa277..8a151c62 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -195,7 +195,7 @@ pub struct HttpServer { /// HTTP endpoints stay `Queued` and are visible via the list /// endpoint — useful shadow-mode testing before workers exist. backfill: Option>, - /// SimpleMapStore data-retention horizon in millis, mirroring + /// SketchStore data-retention horizon in millis, mirroring /// `--persistence-delete-older-than-secs` at the CLI. Used by the /// `POST /api/v1/db/backfill` handler to gate job creation via /// `BackfillRegistry::create_checked` (§10.5 Method B). `None` @@ -381,7 +381,7 @@ impl HttpServer { self } - /// Declare the SimpleMapStore data-retention horizon (the value of + /// Declare the SketchStore data-retention horizon (the value of /// `--persistence-delete-older-than-secs` * 1000). When set, the /// `POST /api/v1/db/backfill` handler runs `create_checked` with /// this bound, so jobs that would write windows older than the @@ -1808,7 +1808,7 @@ mod tests { use super::*; use crate::stores::schema::{HotReloadStreamingConfig, InferenceConfig, StreamingConfig}; use crate::query_engines::ASAPQueryEngine; - use crate::stores::sketch_db::simple_map_store::SimpleMapStore; + use crate::stores::sketch_db::sketch_store::SketchStore; use reqwest::Client; use std::sync::Arc; @@ -1835,7 +1835,7 @@ mod tests { crate::stores::schema::CleanupPolicy::NoCleanup, ); let streaming_config = Arc::new(StreamingConfig::default()); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -2092,7 +2092,7 @@ aggregations: crate::stores::schema::CleanupPolicy::NoCleanup, ); let streaming_config = Arc::new(StreamingConfig::default()); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -2575,7 +2575,7 @@ aggregations: crate::stores::schema::CleanupPolicy::NoCleanup, ); let streaming_config = Arc::new(StreamingConfig::default()); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -2974,7 +2974,7 @@ aggregations: StreamingConfig::with_storage_backend(Default::default(), metric_storage_backend); let streaming_arc = Arc::new(streaming_cfg); let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_arc.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -3028,7 +3028,7 @@ aggregations: let streaming_cfg = StreamingConfig::default(); let streaming_arc = Arc::new(streaming_cfg); let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_arc.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -3784,7 +3784,7 @@ aggregations: let streaming_cfg = StreamingConfig::default(); let streaming_arc = Arc::new(streaming_cfg); let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_arc.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -4256,7 +4256,7 @@ aggregations: StreamingConfig::with_storage_backend(Default::default(), metric_storage_backend); let streaming_arc = Arc::new(streaming_cfg); let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_arc.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -4312,7 +4312,7 @@ aggregations: crate::stores::schema::CleanupPolicy::NoCleanup, ); let streaming_arc = Arc::new(StreamingConfig::default()); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_arc.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -4588,7 +4588,7 @@ struct PrecomputeJobRequest { /// The controller creates PrecomputeJobs when a query's upper sub-tree /// (e.g., TopK or a histogram-quantile-shaped Aggregate{Quantile(φ)}) /// requires evaluation on merged sketches. -/// This endpoint receives that job and runs it against the SimpleMapStore. +/// This endpoint receives that job and runs it against the SketchStore. async fn handle_precompute_job( State(state): State, axum::Json(req): axum::Json, @@ -5269,7 +5269,7 @@ fn service_unavailable_no_backfill() -> axum::response::Response { /// * `agg_id` must be known to the schema registry → 404 on miss. /// * `end_ms` must not extend past the agg's `created_at_ms` (no /// race against live ingest) → 409 on overlap. -/// * `start_ms` must be within the SimpleMapStore data-retention +/// * `start_ms` must be within the SketchStore data-retention /// window when one is configured (Method B) → 409 on stale range. /// /// 400 on malformed body / inverted range; 503 when no registry or diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index 9d314a9e..8d01ecbf 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -19,7 +19,7 @@ pub use precompute_engine::operators::{ IncreaseAccumulator, MinMaxAccumulator, MultipleSumAccumulator, SumAccumulator, }; -pub use stores::{SimpleMapStore, Store, StoreResult}; +pub use stores::{SketchStore, Store, StoreResult}; pub use query_engines::{ASAPQueryEngine, InstantVector, QueryResult}; diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 3b247f6d..67ad3fa7 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -28,7 +28,7 @@ use data_plane::utils::file_io::{read_inference_config, read_streaming_config}; use data_plane::{ HttpServer, HttpServerConfig, KafkaConsumer, KafkaConsumerConfig, OtlpReceiver, OtlpReceiverConfig, PrecomputeEngine, PrecomputeEngineConfig, Result, ASAPQueryEngine, - SimpleMapStore, StoreOutputSink, + SketchStore, StoreOutputSink, }; #[derive(Parser, Debug)] @@ -145,7 +145,7 @@ struct Args { #[arg(long, value_enum, default_value = "promql")] query_language: QueryLanguage, - /// Lock strategy for SimpleMapStore: "global" for single mutex, + /// Lock strategy for SketchStore: "global" for single mutex, /// "per-key" for fine-grained locking. Default `per-key`. #[arg(long, value_enum, default_value = "per-key")] lock_strategy: LockStrategy, @@ -236,14 +236,14 @@ struct Args { #[arg(long)] schema_eviction_dry_run: bool, - // ---- SimpleMapStore persistence ---- + // ---- SketchStore persistence ---- // // When --persistence-enabled is set, the store is constructed via - // SimpleMapStore::with_persistence_per_key with the other + // SketchStore::with_persistence_per_key with the other // --persistence-* flags as the config. Forces LockStrategy::PerKey // regardless of --lock-strategy; the Global variant is // intentionally left in-memory-only. - /// Enable the disk-backed persistence layer for SimpleMapStore + /// Enable the disk-backed persistence layer for SketchStore #[arg(long)] persistence_enabled: bool, @@ -364,12 +364,12 @@ async fn main() -> Result<()> { let hot_reload_config = data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config.clone()); - // Setup store (equivalent to Python's SimpleMapStore()) + // Setup store (equivalent to Python's SketchStore()) // Get cleanup policy from inference config let cleanup_policy = inference_config.cleanup_policy; info!("Using cleanup policy: {:?}", cleanup_policy); let store = if args.persistence_enabled { - use data_plane::stores::sketch_db::simple_map_store::persistence::SimpleMapStorePersistenceConfig; + use data_plane::stores::sketch_db::sketch_store::persistence::SketchStorePersistenceConfig; let disk_path = args .persistence_dir .clone() @@ -392,7 +392,7 @@ async fn main() -> Result<()> { let ten_pct = (memory_limit_bytes / 10) as u64; ten_pct.min(512 * 1024 * 1024) }); - let persistence_cfg = SimpleMapStorePersistenceConfig { + let persistence_cfg = SketchStorePersistenceConfig { memory_limit_bytes, memory_low_watermark_bytes: memory_limit_bytes * 8 / 10, hard_cap_bytes: memory_limit_bytes * 125 / 100, @@ -415,15 +415,15 @@ async fn main() -> Result<()> { info!("--persistence-enabled forces LockStrategy::PerKey (ignoring --lock-strategy)"); } Arc::new( - SimpleMapStore::with_persistence_per_key( + SketchStore::with_persistence_per_key( streaming_config.clone(), cleanup_policy, persistence_cfg, ) - .expect("SimpleMapStore::with_persistence_per_key failed"), + .expect("SketchStore::with_persistence_per_key failed"), ) } else { - Arc::new(SimpleMapStore::new_with_strategy( + Arc::new(SketchStore::new_with_strategy( streaming_config.clone(), cleanup_policy, args.lock_strategy, @@ -881,7 +881,7 @@ async fn main() -> Result<()> { // scans the schema registry for `Expired` schemas, cancels any // in-flight backfills targeting them, and drops the agg_id's // data from the store. Complements the age-based data retention - // in SimpleMapStore — see `SchemaEvictionService` module doc for + // in SketchStore — see `SchemaEvictionService` module doc for // the ordering rationale. let schema_eviction_handle = if let (true, Some(ingest_state)) = ( args.enable_schema_eviction, @@ -971,7 +971,7 @@ async fn main() -> Result<()> { /// Periodic memory diagnostics logger — runs every 30 seconds. async fn spawn_memory_diagnostics( - store: Arc, + store: Arc, worker_diagnostics: Option>, ) { use std::sync::atomic::Ordering; diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 3c0c23a1..6af9141a 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -2453,7 +2453,7 @@ aggregations: fn test_sketch_ingest_persists_and_query_returns_non_empty() { use crate::stores::schema::{CleanupPolicy, StreamingConfig}; use crate::precompute_engine::output_sink::StoreOutputSink; - use crate::stores::sketch_db::simple_map_store::per_key::SimpleMapStorePerKey; + use crate::stores::sketch_db::sketch_store::per_key::SketchStorePerKey; use crate::stores::Store; // Streaming config: agg_id=1, 30s tumbling, DDSketch, @@ -2473,7 +2473,7 @@ aggregations: // A real per_key store, so the test exercises the actual // insert + query path the production backend uses. - let store = Arc::new(SimpleMapStorePerKey::new( + let store = Arc::new(SketchStorePerKey::new( streaming_config.clone(), CleanupPolicy::CircularBuffer, )); diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 7e92fc76..c7ffd281 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -4929,7 +4929,7 @@ mod hot_reload_phase2_tests { AggregationType, CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, QueryLanguage, StreamingConfig, WindowType, }; - use crate::stores::sketch_db::simple_map_store::SimpleMapStore; + use crate::stores::sketch_db::sketch_store::SketchStore; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; fn dummy_agg(id: u64, metric: &str) -> crate::stores::schema::AggregationConfig { @@ -4962,7 +4962,7 @@ mod hot_reload_phase2_tests { fn build_engine(handle: HotReloadStreamingConfig) -> ASAPQueryEngine { let streaming_config = Arc::new(StreamingConfig::default()); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config, CleanupPolicy::NoCleanup, )); @@ -5033,7 +5033,7 @@ mod hot_reload_phase2_tests { let external_handle = HotReloadStreamingConfig::new(cfg_with_agg(101, "metric_a")); let streaming_config = external_handle.snapshot(); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( Arc::clone(&streaming_config), CleanupPolicy::NoCleanup, )); @@ -5091,7 +5091,7 @@ mod e2e_feedback_loop_tests { StreamingConfig, WindowType, }; use crate::drivers::query::controller_client::ControllerClient; - use crate::stores::sketch_db::simple_map_store::SimpleMapStore; + use crate::stores::sketch_db::sketch_store::SketchStore; use async_trait::async_trait; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; use promql_utilities::query_logics::enums::Statistic; @@ -5225,7 +5225,7 @@ mod e2e_feedback_loop_tests { })); // 3. Build ASAPQueryEngine with the handle and mock controller. - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( Arc::new(StreamingConfig::default()), CleanupPolicy::NoCleanup, )); @@ -5345,7 +5345,7 @@ mod e2e_feedback_loop_tests { streaming_config_with(&req.metric, 42) })); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( Arc::new(StreamingConfig::default()), CleanupPolicy::NoCleanup, )); @@ -5499,7 +5499,7 @@ mod aux_pushdown_tests { CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, PromQLSchema, QueryLanguage, SchemaConfig, StreamingConfig, }; - use crate::stores::sketch_db::simple_map_store::SimpleMapStore; + use crate::stores::sketch_db::sketch_store::SketchStore; let ic = InferenceConfig { schema: SchemaConfig::PromQL(PromQLSchema { @@ -5510,7 +5510,7 @@ mod aux_pushdown_tests { }; let sc = Arc::new(StreamingConfig::new(HashMap::new())); let hr = HotReloadStreamingConfig::from_arc(sc.clone()); - let store = Arc::new(SimpleMapStore::new(sc, CleanupPolicy::NoCleanup)); + let store = Arc::new(SketchStore::new(sc, CleanupPolicy::NoCleanup)); ASAPQueryEngine::new_with_hot_reload(store, ic, hr, 60, QueryLanguage::promql) } @@ -5759,7 +5759,7 @@ mod sketch_alias_resolver_tests { AggregationConfig, CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, PromQLSchema, QueryLanguage, SchemaConfig, StreamingConfig, WindowType, }; - use crate::stores::sketch_db::simple_map_store::SimpleMapStore; + use crate::stores::sketch_db::sketch_store::SketchStore; use std::sync::Arc; fn agg_for(id: u64, metric: &str, agg_type: AggregationType) -> AggregationConfig { @@ -5794,7 +5794,7 @@ mod sketch_alias_resolver_tests { configs.insert((i + 1) as u64, agg_for((i + 1) as u64, m, *t)); } let streaming_config = StreamingConfig::new(configs); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( Arc::new(streaming_config.clone()), CleanupPolicy::NoCleanup, )); @@ -6149,7 +6149,7 @@ mod warm_tier_classify_tests { use crate::stores::schema::{CleanupPolicy, HotReloadStreamingConfig, InferenceConfig}; use crate::query_engines::EngineError; use crate::query_engines::routing::query_engine_routing::QueryEngine as _; - use crate::stores::sketch_db::simple_map_store::SimpleMapStore; + use crate::stores::sketch_db::sketch_store::SketchStore; use crate::stores::sketch_db::sketch_index::{ AccuracyBound, Capability, SketchConfig, SketchIndex, SketchInstanceMetadata, SketchKindHandle, SketchSampleState, @@ -6158,7 +6158,7 @@ mod warm_tier_classify_tests { fn build_engine_with_index(idx: Arc) -> ASAPQueryEngine { let streaming_config = Arc::new(crate::stores::schema::StreamingConfig::default()); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); diff --git a/data_plane/src/query_engines/asap_query_engine/mod.rs b/data_plane/src/query_engines/asap_query_engine/mod.rs index 900ad0fe..cda5779a 100644 --- a/data_plane/src/query_engines/asap_query_engine/mod.rs +++ b/data_plane/src/query_engines/asap_query_engine/mod.rs @@ -2,7 +2,7 @@ //! //! `ASAPQueryEngine` is the long-standing PromQL/SQL/Elasticsearch-DSL //! query path that answers from the in-memory sketch DB -//! ([`crate::stores::sketch_db::SimpleMapStore`]) and its +//! ([`crate::stores::sketch_db::SketchStore`]) and its //! per-`agg_id` precomputed accumulators. It returns ε/δ-bounded //! approximate answers for sketch-resident queries and `None` on //! a capability miss (router falls through, which after Step-1 of diff --git a/data_plane/src/stores/mod.rs b/data_plane/src/stores/mod.rs index 1f157116..57c8eb81 100644 --- a/data_plane/src/stores/mod.rs +++ b/data_plane/src/stores/mod.rs @@ -8,13 +8,13 @@ //! * `sketch_db` — the in-memory + persisted sketch DB. Logical //! layer (schema registry, schema timeline, backfill types / //! workers / HTTP endpoints) AND the physical storage backend -//! (`sketch_db::simple_map_store`) are co-located under this +//! (`sketch_db::sketch_store`) are co-located under this //! path. //! * `gorilla_object_store` — S3/MinIO-backed Gorilla TSDB block //! store used by the archive tier. //! -//! `SimpleMapStore` is re-exported at the top level -//! (`crate::stores::SimpleMapStore`) for call-site stability. +//! `SketchStore` is re-exported at the top level +//! (`crate::stores::SketchStore`) for call-site stability. pub mod gorilla_object_store; pub mod schema; @@ -30,5 +30,5 @@ pub use sketch_db::sketch_index::{ AccuracyBound, Capability, SidLookup, SketchConfig, SketchEncoding, SketchIndex, SketchInstanceMetadata, SketchKindHandle, SketchSampleState, SketchTimeSeries, }; -pub use sketch_db::{AggSchema, AggStatus, SchemaRegistry, SimpleMapStore}; +pub use sketch_db::{AggSchema, AggStatus, SchemaRegistry, SketchStore}; pub use traits::*; diff --git a/data_plane/src/stores/schema/traits.rs b/data_plane/src/stores/schema/traits.rs index 854a8ccb..c342ffb9 100644 --- a/data_plane/src/stores/schema/traits.rs +++ b/data_plane/src/stores/schema/traits.rs @@ -52,7 +52,7 @@ pub trait AggregateCore: SerializableToSink + Send + Sync { /// Approximate in-memory byte footprint of this accumulator. /// - /// Used by the `SimpleMapStore` persistence layer to drive its + /// Used by the `SketchStore` persistence layer to drive its /// memory-pressure trigger. Not required to be exact — the flusher /// only needs rough proportionality. The default is a conservative /// 4 KiB constant; concrete types should override it with a diff --git a/data_plane/src/stores/sketch_db/backfill.rs b/data_plane/src/stores/sketch_db/backfill.rs index b8671436..43364cf0 100644 --- a/data_plane/src/stores/sketch_db/backfill.rs +++ b/data_plane/src/stores/sketch_db/backfill.rs @@ -293,7 +293,7 @@ pub enum CreateError { requested_end_ms: u64, created_at_ms: u64, }, - /// The requested `start_ms` is older than the `SimpleMapStore` + /// The requested `start_ms` is older than the `SketchStore` /// data-retention horizon — any windows the backfill writes /// at that range would immediately be evicted by the /// retention sweep. Method B from the design discussion: fail @@ -531,7 +531,7 @@ impl BackfillRegistry { /// provided): `time_range.0 >= now - data_retention_ms`. /// Method B from the design discussion — fail fast instead /// of letting the backfill produce windows that the - /// SimpleMapStore retention sweep would immediately evict. + /// SketchStore retention sweep would immediately evict. /// Pass `None` to skip the check (tests, or deployments /// where retention is disabled). /// diff --git a/data_plane/src/stores/sketch_db/backfill_processor.rs b/data_plane/src/stores/sketch_db/backfill_processor.rs index 176753da..f5c68ddc 100644 --- a/data_plane/src/stores/sketch_db/backfill_processor.rs +++ b/data_plane/src/stores/sketch_db/backfill_processor.rs @@ -264,7 +264,7 @@ mod tests { use crate::stores::sketch_db::backfill::BackfillSource; use crate::stores::sketch_db::backfill_worker::BackfillWorker; use crate::stores::sketch_db::raw_sample_reader::{LabelFilter, MockRawSampleReader}; - use crate::stores::sketch_db::simple_map_store::SimpleMapStore; + use crate::stores::sketch_db::sketch_store::SketchStore; use asap_types::enums::{AggregationType, WindowType}; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; use std::sync::Arc; @@ -308,7 +308,7 @@ mod tests { let streaming = streaming_config_with(cfg.clone()); let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); - let store: Arc = Arc::new(SimpleMapStore::new( + let store: Arc = Arc::new(SketchStore::new( streaming.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -358,7 +358,7 @@ mod tests { let streaming = streaming_config_with(cfg); let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); - let store: Arc = Arc::new(SimpleMapStore::new( + let store: Arc = Arc::new(SketchStore::new( streaming.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -385,7 +385,7 @@ mod tests { let streaming = streaming_config_with(cfg); let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); - let store: Arc = Arc::new(SimpleMapStore::new( + let store: Arc = Arc::new(SketchStore::new( streaming.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -411,7 +411,7 @@ mod tests { let streaming = streaming_config_with(cfg); let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); - let store: Arc = Arc::new(SimpleMapStore::new( + let store: Arc = Arc::new(SketchStore::new( streaming.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); diff --git a/data_plane/src/stores/sketch_db/backfill_service.rs b/data_plane/src/stores/sketch_db/backfill_service.rs index cf2d315e..935f2d02 100644 --- a/data_plane/src/stores/sketch_db/backfill_service.rs +++ b/data_plane/src/stores/sketch_db/backfill_service.rs @@ -295,7 +295,7 @@ mod tests { use super::*; use crate::stores::schema::StreamingConfig; use crate::stores::sketch_db::raw_sample_reader::{MockRawSampleReader, RawSample}; - use crate::stores::sketch_db::simple_map_store::SimpleMapStore; + use crate::stores::sketch_db::sketch_store::SketchStore; use asap_types::aggregation_config::AggregationConfig; use asap_types::enums::{AggregationType, WindowType}; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; @@ -358,7 +358,7 @@ mod tests { let streaming = streaming_with(cfg); let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); - let store: Arc = Arc::new(SimpleMapStore::new( + let store: Arc = Arc::new(SketchStore::new( streaming.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -411,7 +411,7 @@ mod tests { let streaming = streaming_with(cfg); let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); - let store: Arc = Arc::new(SimpleMapStore::new( + let store: Arc = Arc::new(SketchStore::new( streaming.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -451,7 +451,7 @@ mod tests { let streaming = streaming_with(cfg); let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); - let store: Arc = Arc::new(SimpleMapStore::new( + let store: Arc = Arc::new(SketchStore::new( streaming.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); @@ -520,7 +520,7 @@ mod tests { let streaming = streaming_with(cfg); let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); - let store: Arc = Arc::new(SimpleMapStore::new( + let store: Arc = Arc::new(SketchStore::new( streaming.clone(), crate::stores::schema::CleanupPolicy::NoCleanup, )); diff --git a/data_plane/src/stores/sketch_db/epoch_columnar.rs b/data_plane/src/stores/sketch_db/epoch_columnar.rs index c4905ef8..a1b78671 100644 --- a/data_plane/src/stores/sketch_db/epoch_columnar.rs +++ b/data_plane/src/stores/sketch_db/epoch_columnar.rs @@ -1,6 +1,6 @@ //! Epoch-partitioned columnar storage — generic payload type. //! -//! Lifted from `simple_map_store::common` (legacy SimpleMapStore index) +//! Lifted from `sketch_store::common` (legacy SketchStore index) //! with the payload column type made generic so the new SketchIndex //! (Phase 5) can reuse the legacy's six storage optimizations //! (`INDEX_DESIGN.md`) without dragging in `Arc` @@ -28,7 +28,7 @@ //! group-by VALUES vector to a compact ID, since the SketchIndex's //! sid already captures the metric identity at the level above. //! -//! See INDEX_DESIGN.md in `simple_map_store/` for the full complexity +//! See INDEX_DESIGN.md in `sketch_store/` for the full complexity //! analysis (Insert O(1), range query O(M) mutable / O(log N + k) //! sealed, etc.) — those bounds carry over verbatim because the //! algorithmic structure is unchanged. diff --git a/data_plane/src/stores/sketch_db/mod.rs b/data_plane/src/stores/sketch_db/mod.rs index 057b0cb5..ae35d6ad 100644 --- a/data_plane/src/stores/sketch_db/mod.rs +++ b/data_plane/src/stores/sketch_db/mod.rs @@ -2,7 +2,7 @@ //! //! See [`docs/design-sketch-db.md`](../../../../../docs/design-sketch-db.md) //! for the full architecture. This module houses the components that live -//! "above" the existing `SimpleMapStore` and turn it into a sketch-aware +//! "above" the existing `SketchStore` and turn it into a sketch-aware //! storage engine over time: //! //! * `schema` — per-`agg_id` `AggSchema` with `Active` / `Retired` / @@ -40,7 +40,7 @@ pub mod prometheus_reader; pub mod raw_sample_reader; pub mod schema; pub mod schema_eviction; -pub mod simple_map_store; +pub mod sketch_store; pub mod sketch_index; pub use accuracy::{AccuracyEnvelope, AccuracyKind, AccuracyProfile, PerSegmentAccuracy}; @@ -62,4 +62,4 @@ pub use schema::{AggSchema, AggStatus, SchemaRegistry, TimelineCoverage, Timelin pub use schema_eviction::{ warn_if_retention_inverted, SchemaEvictionConfig, SchemaEvictionHandle, SchemaEvictionService, }; -pub use simple_map_store::SimpleMapStore; +pub use sketch_store::SketchStore; diff --git a/data_plane/src/stores/sketch_db/schema.rs b/data_plane/src/stores/sketch_db/schema.rs index 49d57b84..213252bb 100644 --- a/data_plane/src/stores/sketch_db/schema.rs +++ b/data_plane/src/stores/sketch_db/schema.rs @@ -5,7 +5,7 @@ //! //! ## Why this exists //! -//! Today's `SimpleMapStore` is keyed by `aggregation_id` but does not know +//! Today's `SketchStore` is keyed by `aggregation_id` but does not know //! anything about the schema (sketch type, parameters, grouping labels, //! window) attached to that id beyond what's in `StreamingConfig`. The //! sketch DB design needs: @@ -181,7 +181,7 @@ impl AggSchema { /// Default retention for a retired schema before eviction. /// 24 hours — covers dashboards / ad-hoc queries that may still /// reference the old agg_id mid-reconfigure. Shorter than the -/// typical SimpleMapStore data retention (7d+) so schema eviction +/// typical SketchStore data retention (7d+) so schema eviction /// runs first, freeing space cleanly without fighting per-record /// retention. Override via `SchemaRegistry::set_retention_for_testing` /// or the CLI flag plumbed through `SchemaEvictionService`. @@ -654,7 +654,7 @@ impl SchemaRegistry { /// Override the retirement retention. Plumbed through from /// `SchemaEvictionService` at startup so deployments can pick - /// a retention that's ≤ their SimpleMapStore + /// a retention that's ≤ their SketchStore /// `persistence_delete_older_than` (see module-level doc on /// retention ordering). /// diff --git a/data_plane/src/stores/sketch_db/schema_eviction.rs b/data_plane/src/stores/sketch_db/schema_eviction.rs index b688c3fc..bde1ac0a 100644 --- a/data_plane/src/stores/sketch_db/schema_eviction.rs +++ b/data_plane/src/stores/sketch_db/schema_eviction.rs @@ -4,14 +4,14 @@ //! //! Implements the §6.2 "scheduled for deletion by the time-TTL //! sweep" semantics the lifecycle enum promises. Sits alongside -//! the SimpleMapStore's age-based `persistence_delete_older_than` +//! the SketchStore's age-based `persistence_delete_older_than` //! retention — the two are independent: //! //! * **Schema retention** (this module): lifecycle-driven. When a //! schema is removed from `StreamingConfig` it transitions //! `Active → Retired → Expired`; when `expires_at_ms` passes we //! drop its `agg_id`. -//! * **Data retention** (SimpleMapStore): age-driven. Records +//! * **Data retention** (SketchStore): age-driven. Records //! older than `persistence_delete_older_than` get swept up //! regardless of schema. //! @@ -269,7 +269,7 @@ mod tests { use super::*; use crate::stores::schema::{AggregationType, CleanupPolicy, LockStrategy, StreamingConfig}; use crate::precompute_engine::operators::SumAccumulator; - use crate::stores::sketch_db::{backfill::BackfillSource, simple_map_store::SimpleMapStore}; + use crate::stores::sketch_db::{backfill::BackfillSource, sketch_store::SketchStore}; use asap_types::aggregation_config::AggregationConfig; use asap_types::enums::WindowType; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; @@ -306,7 +306,7 @@ mod tests { Arc::new(StreamingConfig::new(map)) } - fn write_one(store: &SimpleMapStore, agg_id: u64, ts: u64) { + fn write_one(store: &SketchStore, agg_id: u64, ts: u64) { let acc = SumAccumulator::with_sum(1.0); let output = crate::stores::schema::PrecomputedOutput::new(ts, ts + 1000, None, agg_id); store @@ -314,7 +314,7 @@ mod tests { .unwrap(); } - fn total_buckets(store: &SimpleMapStore, metric: &str, agg_id: u64) -> usize { + fn total_buckets(store: &SketchStore, metric: &str, agg_id: u64) -> usize { let map = store .query_precomputed_output(metric, agg_id, 0, u64::MAX / 2) .unwrap(); @@ -327,7 +327,7 @@ mod tests { async fn fixture_with_expired_1() -> ( Arc, Arc, - Arc, + Arc, ) { let initial = make_streaming_config(&[1, 2]); let mut registry = SchemaRegistry::from_streaming_config(&initial); @@ -342,7 +342,7 @@ mod tests { assert_eq!(schemas.get(1).unwrap().status(), AggStatus::Expired); let backfill = Arc::new(BackfillRegistry::new()); - let store = Arc::new(SimpleMapStore::new_with_strategy( + let store = Arc::new(SketchStore::new_with_strategy( initial, CleanupPolicy::NoCleanup, LockStrategy::Global, @@ -390,7 +390,7 @@ mod tests { let initial = make_streaming_config(&[1]); let schemas = Arc::new(SchemaRegistry::from_streaming_config(&initial)); let backfill = Arc::new(BackfillRegistry::new()); - let store = Arc::new(SimpleMapStore::new_with_strategy( + let store = Arc::new(SketchStore::new_with_strategy( initial, CleanupPolicy::NoCleanup, LockStrategy::Global, diff --git a/data_plane/src/stores/sketch_db/simple_map_store/legacy/global.rs b/data_plane/src/stores/sketch_db/simple_map_store/legacy/global.rs deleted file mode 100644 index 86badb53..00000000 --- a/data_plane/src/stores/sketch_db/simple_map_store/legacy/global.rs +++ /dev/null @@ -1,550 +0,0 @@ -use crate::stores::schema::{ - AggregateCore, AggregationType, CleanupPolicy, KeyByLabelValues, PrecomputedOutput, - StreamingConfig, -}; -use crate::stores::{Store, StoreResult, TimestampedBucketsMap}; -use std::collections::HashMap; -use std::sync::Arc; -use std::sync::Mutex; -use std::time::Instant; -use tracing::{debug, error, info}; - -type TimestampRange = (u64, u64); // (start_timestamp, end_timestamp) -type StoreKey = u64; // aggregation_id -type StoreValue = Vec<(Option, Box)>; - -/// In-memory storage implementation using single mutex (like Python version) -pub struct LegacySimpleMapStoreGlobal { - // Single global mutex protecting all data structures - lock: Mutex, - - // Store the streaming configuration - streaming_config: Arc, - - // Policy for cleaning up old aggregates - cleanup_policy: CleanupPolicy, -} - -struct StoreData { - // Main storage: aggregation_id -> (start_time, end_time) -> [(key, precompute)] - store: HashMap>, - - // Track metrics that have been created - metrics: std::collections::HashSet, - - // Count items inserted per metric for logging - items_inserted: HashMap, - - // Track earliest timestamp per aggregation ID - earliest_timestamp_per_aggregation_id: HashMap, - - // Track how many times each aggregate window has been read - read_counts: HashMap>, -} - -impl LegacySimpleMapStoreGlobal { - pub fn new(streaming_config: Arc, cleanup_policy: CleanupPolicy) -> Self { - Self { - lock: Mutex::new(StoreData { - store: HashMap::new(), - metrics: std::collections::HashSet::new(), - items_inserted: HashMap::new(), - earliest_timestamp_per_aggregation_id: HashMap::new(), - read_counts: HashMap::new(), - }), - streaming_config, - cleanup_policy, - } - } - - fn create_table(&self, data: &mut StoreData, metric: &str) { - // In the in-memory implementation, "creating a table" just means - // marking the metric as known - data.metrics.insert(metric.to_string()); - } - - fn cleanup_old_aggregates_fixed_count( - &self, - data: &mut StoreData, - metric: &str, - aggregation_id: u64, - num_aggregates_to_retain: Option, - ) { - // Return early if no retention limit configured - let configured_limit = match num_aggregates_to_retain { - Some(limit) => limit as usize, - None => return, - }; - - let retention_limit = configured_limit * 4; - let store_key = aggregation_id; - - // Get the time map for this store key - if let Some(time_map) = data.store.get_mut(&store_key) { - if time_map.len() <= retention_limit { - return; // Nothing to clean up - } - - // Collect all timestamp ranges and sort by start timestamp (oldest first) - let mut timestamp_windows: Vec = time_map.keys().copied().collect(); - timestamp_windows.sort_by_key(|&(start, _end)| start); - - // Calculate which ones to remove (oldest first) - let num_to_remove = timestamp_windows.len() - retention_limit; - let windows_to_remove: Vec = - timestamp_windows.into_iter().take(num_to_remove).collect(); - - // Remove old windows - for window in windows_to_remove { - if time_map.remove(&window).is_some() { - debug!( - "Removed old aggregate for {} aggregation_id {} window {}-{} (retention limit: {}, configured: {})", - metric, - aggregation_id, - window.0, - window.1, - retention_limit, - configured_limit - ); - } - } - } - } - - fn cleanup_old_aggregates_read_based( - &self, - data: &mut StoreData, - metric: &str, - aggregation_id: u64, - read_count_threshold: Option, - ) { - // Return early if no threshold configured - let threshold = match read_count_threshold { - Some(t) => t, - None => return, - }; - - let store_key = aggregation_id; - - // Get both the time map and read count map - let time_map = match data.store.get_mut(&store_key) { - Some(map) => map, - None => return, - }; - - let read_count_map = data.read_counts.entry(store_key).or_default(); - - // Collect windows where read_count >= threshold - let mut windows_to_remove: Vec = Vec::new(); - - for (timestamp_range, _) in time_map.iter() { - let read_count = read_count_map.get(timestamp_range).copied().unwrap_or(0); - - if read_count >= threshold { - windows_to_remove.push(*timestamp_range); - } - } - - // Remove windows that exceeded threshold - for window in &windows_to_remove { - if time_map.remove(window).is_some() { - let read_count = read_count_map.get(window).copied().unwrap_or(0); - read_count_map.remove(window); - - debug!( - "Removed aggregate for {} aggregation_id {} window {}-{} (read_count: {} >= threshold: {})", - metric, - aggregation_id, - window.0, - window.1, - read_count, - threshold - ); - } - } - } - - fn cleanup_old_aggregates( - &self, - data: &mut StoreData, - metric: &str, - aggregation_id: u64, - num_aggregates_to_retain: Option, - read_count_threshold: Option, - ) { - match self.cleanup_policy { - CleanupPolicy::CircularBuffer => { - self.cleanup_old_aggregates_fixed_count( - data, - metric, - aggregation_id, - num_aggregates_to_retain, - ); - } - CleanupPolicy::ReadBased => { - self.cleanup_old_aggregates_read_based( - data, - metric, - aggregation_id, - read_count_threshold, - ); - } - CleanupPolicy::NoCleanup => { - // Do nothing - no cleanup - } - } - } -} - -#[async_trait::async_trait] -impl Store for LegacySimpleMapStoreGlobal { - fn insert_precomputed_output( - &self, - output: PrecomputedOutput, - precompute: Box, - ) -> StoreResult<()> { - self.insert_precomputed_output_batch(vec![(output, precompute)]) - } - - fn insert_precomputed_output_batch( - &self, - outputs: Vec<(PrecomputedOutput, Box)>, - ) -> StoreResult<()> { - let batch_insert_start_time = Instant::now(); - let batch_size = outputs.len(); - - // Measure lock acquisition time - #[cfg(feature = "lock_profiling")] - let lock_wait_start = Instant::now(); - - // Single lock for entire batch (like Python version) - let mut data = self.lock.lock().unwrap(); - - #[cfg(feature = "lock_profiling")] - { - let lock_wait_duration = lock_wait_start.elapsed(); - info!( - "🔒 Insert lock wait time: {:.2}ms (batch_size: {})", - lock_wait_duration.as_secs_f64() * 1000.0, - batch_size - ); - } - - #[cfg(feature = "lock_profiling")] - let lock_hold_start = Instant::now(); - - for (output, precompute) in outputs { - let aggregation_config = self - .streaming_config - .get_aggregation_config(output.aggregation_id); - - if aggregation_config.is_none() { - error!( - "Aggregation config not found for aggregation_id {}. Skipping insert.", - output.aggregation_id - ); - continue; - } - let aggregation_config = aggregation_config.unwrap(); - - let metric = aggregation_config.metric.clone(); - let aggregation_id = output.aggregation_id; - - // Create table if it doesn't exist - if !data.metrics.contains(&metric) { - self.create_table(&mut data, &metric); - } - - // Update earliest timestamp tracking - if let Some(current_earliest) = data - .earliest_timestamp_per_aggregation_id - .get_mut(&aggregation_id) - { - if output.start_timestamp < *current_earliest { - *current_earliest = output.start_timestamp; - } - } else { - data.earliest_timestamp_per_aggregation_id - .insert(aggregation_id, output.start_timestamp); - } - - let store_key = aggregation_id; - let timestamp_range = (output.start_timestamp, output.end_timestamp); - - // Get or create the time-based map for this aggregation - let time_map = data.store.entry(store_key).or_default(); - - // Get or create the value vector for this timestamp range - let store_value = time_map.entry(timestamp_range).or_default(); - - // Add the new entry with the real precompute data - store_value.push((output.key, precompute)); - - // Apply retention policy if configured (but exclude DeltaSetAggregator) - if aggregation_config.aggregation_type != AggregationType::DeltaSetAggregator { - self.cleanup_old_aggregates( - &mut data, - &metric, - aggregation_id, - aggregation_config.num_aggregates_to_retain, - aggregation_config.read_count_threshold, - ); - } - - // Update insertion count - let current_count = data.items_inserted.entry(metric.clone()).or_insert(0); - *current_count += 1; - - if (*current_count).is_multiple_of(1000) { - debug!("Inserted {} items into {}", current_count, metric); - } - } - - #[cfg(feature = "lock_profiling")] - { - let lock_hold_duration = lock_hold_start.elapsed(); - info!( - "🔓 Insert lock hold time: {:.2}ms (batch_size: {})", - lock_hold_duration.as_secs_f64() * 1000.0, - batch_size - ); - } - - // Lock will be dropped here when `data` goes out of scope - - let batch_insert_duration = batch_insert_start_time.elapsed(); - debug!( - "Batch insert of {} items took: {:.2}ms", - batch_size, - batch_insert_duration.as_secs_f64() * 1000.0 - ); - Ok(()) - } - - fn query_precomputed_output( - &self, - metric: &str, - aggregation_id: u64, - start: u64, - end: u64, - ) -> Result> { - let query_start_time = Instant::now(); - let store_key = aggregation_id; - - // Measure lock acquisition time - #[cfg(feature = "lock_profiling")] - let lock_wait_start = Instant::now(); - - // Single lock for entire query - now mutable to track read counts - let mut data = self.lock.lock().unwrap(); - - #[cfg(feature = "lock_profiling")] - { - let lock_wait_duration = lock_wait_start.elapsed(); - info!( - "🔒 Query lock wait time: {:.2}ms (metric: {}, agg_id: {})", - lock_wait_duration.as_secs_f64() * 1000.0, - metric, - aggregation_id - ); - } - - #[cfg(feature = "lock_profiling")] - let lock_hold_start = Instant::now(); - - let time_map = match data.store.get(&store_key) { - Some(map) => map, - None => { - info!("Metric {} not found in store", metric); - return Ok(HashMap::new()); - } - }; - - let mut results: TimestampedBucketsMap = HashMap::new(); - let mut total_entries = 0; - - // Find all timestamp ranges that overlap with our query range - let range_scan_start_time = Instant::now(); - - // First, collect all matching timestamp ranges - let mut matching_ranges: Vec = time_map - .keys() - .filter(|(range_start, range_end)| start <= *range_start && end >= *range_end) - .copied() - .collect(); - - // Sort by start timestamp to ensure chronological order - // This is important for range queries that use sliding windows - matching_ranges.sort_by_key(|(range_start, _)| *range_start); - - // Now iterate in sorted order, including timestamp with each bucket - for timestamp_range in &matching_ranges { - if let Some(store_values) = time_map.get(timestamp_range) { - for (key_opt, precompute) in store_values.iter() { - results - .entry(key_opt.clone()) - .or_default() - .push((*timestamp_range, precompute.clone_boxed_core().into())); - - total_entries += 1; - } - } - } - - // Update read counts for accessed ranges (after we're done with time_map to avoid borrow conflicts) - let read_count_map = data.read_counts.entry(store_key).or_default(); - for timestamp_range in &matching_ranges { - *read_count_map.entry(*timestamp_range).or_insert(0) += 1; - } - - let range_scan_duration = range_scan_start_time.elapsed(); - debug!( - "Range scanning took: {:.2}ms", - range_scan_duration.as_secs_f64() * 1000.0 - ); - - let query_duration = query_start_time.elapsed(); - debug!( - "Total query took: {:.2}ms", - query_duration.as_secs_f64() * 1000.0 - ); - - debug!( - "Found {} entries for query on {} (aggregation_id: {}, start: {}, end: {})", - total_entries, metric, aggregation_id, start, end - ); - debug!("Found {} unique keys", results.len()); - - #[cfg(feature = "lock_profiling")] - { - let lock_hold_duration = lock_hold_start.elapsed(); - info!( - "🔓 Query lock hold time: {:.2}ms (metric: {}, agg_id: {}, entries: {})", - lock_hold_duration.as_secs_f64() * 1000.0, - metric, - aggregation_id, - total_entries - ); - } - - // Lock will be dropped here when `data` goes out of scope - - Ok(results) - } - - fn query_precomputed_output_exact( - &self, - metric: &str, - aggregation_id: u64, - exact_start: u64, - exact_end: u64, - ) -> Result> { - let query_start_time = Instant::now(); - let store_key = aggregation_id; - - // Measure lock acquisition time - #[cfg(feature = "lock_profiling")] - let lock_wait_start = Instant::now(); - - let mut data = self.lock.lock().unwrap(); - - #[cfg(feature = "lock_profiling")] - { - let lock_wait_duration = lock_wait_start.elapsed(); - info!( - "🔒 Exact query lock wait time: {:.2}ms (metric: {}, agg_id: {})", - lock_wait_duration.as_secs_f64() * 1000.0, - metric, - aggregation_id - ); - } - - #[cfg(feature = "lock_profiling")] - let lock_hold_start = Instant::now(); - - let time_map = match data.store.get(&store_key) { - Some(map) => map, - None => { - debug!("Metric {} not found in store for exact query", metric); - return Ok(HashMap::new()); - } - }; - - let mut results: TimestampedBucketsMap = HashMap::new(); - - // Look for exact timestamp match (strict - no tolerance) - let timestamp_range = (exact_start, exact_end); - let mut found_match = false; - - // First, collect the results (immutable borrow of time_map) - if let Some(store_values) = time_map.get(×tamp_range) { - found_match = true; - - // Collect results with timestamp - let mut total_entries = 0; - for (key_opt, precompute) in store_values.iter() { - results - .entry(key_opt.clone()) - .or_default() - .push((timestamp_range, precompute.clone_boxed_core().into())); - total_entries += 1; - } - - debug!( - "Exact match FOUND for [{}, {}]: {} entries across {} keys", - exact_start, - exact_end, - total_entries, - results.len() - ); - } else { - debug!( - "Exact match NOT FOUND for metric: {}, agg_id: {}, range: [{}, {}]", - metric, aggregation_id, exact_start, exact_end - ); - } - - // Now update read count (mutable borrow of data.read_counts) - // This happens after we're done with time_map - if found_match { - let read_count_map = data.read_counts.entry(store_key).or_default(); - *read_count_map.entry(timestamp_range).or_insert(0) += 1; - } - - #[cfg(feature = "lock_profiling")] - { - let lock_hold_duration = lock_hold_start.elapsed(); - info!( - "🔓 Exact query lock hold time: {:.2}ms (metric: {}, agg_id: {}, found: {})", - lock_hold_duration.as_secs_f64() * 1000.0, - metric, - aggregation_id, - !results.is_empty() - ); - } - - let query_duration = query_start_time.elapsed(); - debug!( - "Exact timestamp query took: {:.2}ms (found: {})", - query_duration.as_secs_f64() * 1000.0, - !results.is_empty() - ); - - // Lock will be dropped here when `data` goes out of scope - - Ok(results) - } - - fn get_earliest_timestamp_per_aggregation_id( - &self, - ) -> Result, Box> { - let data = self.lock.lock().unwrap(); - Ok(data.earliest_timestamp_per_aggregation_id.clone()) - } - - fn close(&self) -> StoreResult<()> { - // For in-memory store, no cleanup needed - info!("LegacySimpleMapStoreGlobal closed"); - Ok(()) - } -} diff --git a/data_plane/src/stores/sketch_db/simple_map_store/legacy/mod.rs b/data_plane/src/stores/sketch_db/simple_map_store/legacy/mod.rs deleted file mode 100644 index 24a12f4b..00000000 --- a/data_plane/src/stores/sketch_db/simple_map_store/legacy/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod global; -mod per_key; - -pub use global::LegacySimpleMapStoreGlobal; -pub use per_key::LegacySimpleMapStorePerKey; diff --git a/data_plane/src/stores/sketch_db/simple_map_store/legacy/per_key.rs b/data_plane/src/stores/sketch_db/simple_map_store/legacy/per_key.rs deleted file mode 100644 index c82144e7..00000000 --- a/data_plane/src/stores/sketch_db/simple_map_store/legacy/per_key.rs +++ /dev/null @@ -1,639 +0,0 @@ -use crate::stores::schema::{ - AggregateCore, AggregationType, CleanupPolicy, KeyByLabelValues, PrecomputedOutput, - StreamingConfig, -}; -use crate::stores::{Store, StoreResult, TimestampedBucketsMap}; -use dashmap::DashMap; -use std::collections::HashMap; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, RwLock}; -use std::time::Instant; -use tracing::{debug, error, info}; - -type TimestampRange = (u64, u64); // (start_timestamp, end_timestamp) -type StoreKey = u64; // aggregation_id -type StoreValue = Vec<(Option, Box)>; - -/// Per-aggregation_id data protected by RwLock -struct StoreKeyData { - // Main storage: (start_time, end_time) -> [(key, precompute)] - time_map: HashMap, - - // Track how many times each timestamp range has been read - read_counts: HashMap, -} - -impl StoreKeyData { - fn new() -> Self { - Self { - time_map: HashMap::new(), - read_counts: HashMap::new(), - } - } -} - -/// In-memory storage implementation using per-key locks for concurrency -pub struct LegacySimpleMapStorePerKey { - // Lock-free concurrent outer map - per aggregation_id - store: DashMap>>, - - // Separate concurrent maps for global state - earliest_timestamps: DashMap, - metrics: DashMap, // HashSet equivalent - items_inserted: DashMap, - - // Store the streaming configuration - streaming_config: Arc, - - // Policy for cleaning up old aggregates - cleanup_policy: CleanupPolicy, -} - -impl LegacySimpleMapStorePerKey { - pub fn new(streaming_config: Arc, cleanup_policy: CleanupPolicy) -> Self { - Self { - store: DashMap::new(), - earliest_timestamps: DashMap::new(), - metrics: DashMap::new(), - items_inserted: DashMap::new(), - streaming_config, - cleanup_policy, - } - } - - fn cleanup_old_aggregates_fixed_count( - &self, - data: &mut StoreKeyData, - metric: &str, - aggregation_id: u64, - num_aggregates_to_retain: Option, - ) { - // Return early if no retention limit configured - let configured_limit = match num_aggregates_to_retain { - Some(limit) => limit as usize, - None => return, - }; - - let retention_limit = configured_limit * 4; - - if data.time_map.len() <= retention_limit { - return; // Nothing to clean up - } - - // Collect all timestamp ranges and sort by start timestamp (oldest first) - let mut timestamp_windows: Vec = data.time_map.keys().copied().collect(); - timestamp_windows.sort_by_key(|&(start, _end)| start); - - // Calculate which ones to remove (oldest first) - let num_to_remove = timestamp_windows.len() - retention_limit; - let windows_to_remove: Vec = - timestamp_windows.into_iter().take(num_to_remove).collect(); - - // Remove old windows from both time_map and read_counts - for window in windows_to_remove { - if data.time_map.remove(&window).is_some() { - data.read_counts.remove(&window); // Also remove from read_counts - debug!( - "Removed old aggregate for {} aggregation_id {} window {}-{} (retention limit: {}, configured: {})", - metric, - aggregation_id, - window.0, - window.1, - retention_limit, - configured_limit - ); - } - } - } - - fn cleanup_old_aggregates_read_based( - &self, - data: &mut StoreKeyData, - metric: &str, - aggregation_id: u64, - read_count_threshold: Option, - ) { - // Return early if no threshold configured - let threshold = match read_count_threshold { - Some(t) => t, - None => return, - }; - - // Collect windows where read_count >= threshold - let mut windows_to_remove: Vec = Vec::new(); - - for (timestamp_range, _) in data.time_map.iter() { - let read_count = data.read_counts.get(timestamp_range).copied().unwrap_or(0); - - if read_count >= threshold { - windows_to_remove.push(*timestamp_range); - } - } - - // Remove windows that exceeded threshold - for window in &windows_to_remove { - //if let Some(_) = data.time_map.remove(window) { - if data.time_map.remove(window).is_some() { - let read_count = data.read_counts.get(window).copied().unwrap_or(0); - data.read_counts.remove(window); - - debug!( - "Removed aggregate for {} aggregation_id {} window {}-{} (read_count: {} >= threshold: {})", - metric, - aggregation_id, - window.0, - window.1, - read_count, - threshold - ); - } - } - } - - fn cleanup_old_aggregates( - &self, - data: &mut StoreKeyData, - metric: &str, - aggregation_id: u64, - num_aggregates_to_retain: Option, - read_count_threshold: Option, - ) { - match self.cleanup_policy { - CleanupPolicy::CircularBuffer => { - self.cleanup_old_aggregates_fixed_count( - data, - metric, - aggregation_id, - num_aggregates_to_retain, - ); - } - CleanupPolicy::ReadBased => { - self.cleanup_old_aggregates_read_based( - data, - metric, - aggregation_id, - read_count_threshold, - ); - } - CleanupPolicy::NoCleanup => { - // Do nothing - no cleanup - } - } - } - - fn insert_for_store_key( - &self, - store_key: &StoreKey, - metric: &str, - items: Vec<(PrecomputedOutput, Box)>, - ) -> StoreResult<()> { - let aggregation_id = *store_key; - - // Measure lock acquisition time - #[cfg(feature = "lock_profiling")] - let lock_wait_start = Instant::now(); - - // Get or create the store data for this key - let store_data_lock = self - .store - .entry(*store_key) - .or_insert_with(|| Arc::new(RwLock::new(StoreKeyData::new()))); - - #[cfg(feature = "lock_profiling")] - { - let lock_wait_duration = lock_wait_start.elapsed(); - info!( - "🔒 Insert DashMap get time: {:.2}ms (metric: {}, agg_id: {}, items: {})", - lock_wait_duration.as_secs_f64() * 1000.0, - metric, - *store_key, - items.len() - ); - } - - #[cfg(feature = "lock_profiling")] - let rwlock_wait_start = Instant::now(); - - // Acquire write lock for this aggregation_id only - let mut data = store_data_lock.write().map_err(|e| { - format!( - "Failed to acquire write lock for aggregation_id {}: {}", - store_key, e - ) - })?; - - #[cfg(feature = "lock_profiling")] - { - let rwlock_wait_duration = rwlock_wait_start.elapsed(); - info!( - "🔒 Insert RwLock wait time: {:.2}ms (metric: {}, agg_id: {}, items: {})", - rwlock_wait_duration.as_secs_f64() * 1000.0, - metric, - *store_key, - items.len() - ); - } - - #[cfg(feature = "lock_profiling")] - let lock_hold_start = Instant::now(); - - for (output, precompute) in items { - // Create metric if needed (lock-free DashMap insert) - self.metrics.entry(metric.to_string()).or_insert(()); - - // Update earliest timestamp (lock-free atomic operation) - self.earliest_timestamps - .entry(aggregation_id) - .and_modify(|earliest| { - let current = earliest.load(Ordering::Relaxed); - if output.start_timestamp < current { - earliest.store(output.start_timestamp, Ordering::Relaxed); - } - }) - .or_insert_with(|| AtomicU64::new(output.start_timestamp)); - - // Insert into time map - let timestamp_range = (output.start_timestamp, output.end_timestamp); - data.time_map - .entry(timestamp_range) - .or_default() - .push((output.key, precompute)); - - // Update insertion count (lock-free atomic increment) - self.items_inserted - .entry(metric.to_string()) - .and_modify(|count| { - let new_count = count.fetch_add(1, Ordering::Relaxed) + 1; - if new_count.is_multiple_of(1000) { - debug!("Inserted {} items into {}", new_count, metric); - } - }) - .or_insert_with(|| AtomicU64::new(1)); - } - - // Apply retention policy if configured (but exclude DeltaSetAggregator) - let aggregation_config = self - .streaming_config - .get_aggregation_config(aggregation_id) - .ok_or_else(|| format!("Aggregation config not found for {}", aggregation_id))?; - - if aggregation_config.aggregation_type != AggregationType::DeltaSetAggregator { - self.cleanup_old_aggregates( - &mut data, - metric, - aggregation_id, - aggregation_config.num_aggregates_to_retain, - aggregation_config.read_count_threshold, - ); - } - - #[cfg(feature = "lock_profiling")] - { - let lock_hold_duration = lock_hold_start.elapsed(); - info!( - "🔓 Insert lock hold time: {:.2}ms (metric: {}, agg_id: {})", - lock_hold_duration.as_secs_f64() * 1000.0, - metric, - *store_key - ); - } - - Ok(()) - } -} - -#[async_trait::async_trait] -impl Store for LegacySimpleMapStorePerKey { - fn insert_precomputed_output( - &self, - output: PrecomputedOutput, - precompute: Box, - ) -> StoreResult<()> { - self.insert_precomputed_output_batch(vec![(output, precompute)]) - } - - fn insert_precomputed_output_batch( - &self, - outputs: Vec<(PrecomputedOutput, Box)>, - ) -> StoreResult<()> { - let batch_insert_start_time = Instant::now(); - let batch_size = outputs.len(); - - // Group by aggregation_id - #[allow(clippy::type_complexity)] - let mut grouped: HashMap< - StoreKey, - (String, Vec<(PrecomputedOutput, Box)>), - > = HashMap::new(); - - for (output, precompute) in outputs { - let aggregation_config = self - .streaming_config - .get_aggregation_config(output.aggregation_id); - - if aggregation_config.is_none() { - error!( - "Aggregation config not found for aggregation_id {}. Skipping insert.", - output.aggregation_id - ); - continue; - } - let aggregation_config = aggregation_config.unwrap(); - - let metric = aggregation_config.metric.clone(); - let store_key = output.aggregation_id; - - grouped - .entry(store_key) - .or_insert_with(|| (metric.clone(), Vec::new())) - .1 - .push((output, precompute)); - } - - // Sort keys to avoid deadlock when acquiring multiple locks - let mut keys: Vec<_> = grouped.keys().cloned().collect(); - keys.sort(); - - // Process each group - for store_key in keys { - let (metric, items) = grouped.remove(&store_key).unwrap(); - self.insert_for_store_key(&store_key, &metric, items)?; - } - - let batch_insert_duration = batch_insert_start_time.elapsed(); - debug!( - "Batch insert of {} items took: {:.2}ms", - batch_size, - batch_insert_duration.as_secs_f64() * 1000.0 - ); - Ok(()) - } - - fn query_precomputed_output( - &self, - metric: &str, - aggregation_id: u64, - start: u64, - end: u64, - ) -> Result> { - let query_start_time = Instant::now(); - let store_key = aggregation_id; - - // Measure lock acquisition time - #[cfg(feature = "lock_profiling")] - let lock_wait_start = Instant::now(); - - // Get the store data for this aggregation_id - let store_data_lock = match self.store.get(&store_key) { - Some(lock) => lock, - None => { - info!("Metric {} not found in store", metric); - return Ok(HashMap::new()); - } - }; - - #[cfg(feature = "lock_profiling")] - { - let lock_wait_duration = lock_wait_start.elapsed(); - info!( - "🔒 Query DashMap get time: {:.2}ms (metric: {}, agg_id: {})", - lock_wait_duration.as_secs_f64() * 1000.0, - metric, - aggregation_id - ); - } - - #[cfg(feature = "lock_profiling")] - let rwlock_wait_start = Instant::now(); - - // Acquire write lock (needed to update read_counts) - let mut data = store_data_lock.write().map_err(|e| { - format!( - "Failed to acquire write lock for query aggregation_id {}: {}", - store_key, e - ) - })?; - - #[cfg(feature = "lock_profiling")] - { - let rwlock_wait_duration = rwlock_wait_start.elapsed(); - info!( - "🔒 Query RwLock wait time: {:.2}ms (metric: {}, agg_id: {})", - rwlock_wait_duration.as_secs_f64() * 1000.0, - metric, - aggregation_id - ); - } - - #[cfg(feature = "lock_profiling")] - let lock_hold_start = Instant::now(); - - let mut results: TimestampedBucketsMap = HashMap::new(); - let mut total_entries = 0; - - // Find all timestamp ranges that overlap with our query range - let range_scan_start_time = Instant::now(); - - // First, collect all matching timestamp ranges - let mut matching_ranges: Vec = data - .time_map - .keys() - .filter(|(range_start, range_end)| start <= *range_start && end >= *range_end) - .copied() - .collect(); - - // Sort by start timestamp to ensure chronological order - // This is important for range queries that use sliding windows - matching_ranges.sort_by_key(|(range_start, _)| *range_start); - - // Now iterate in sorted order, including timestamp with each bucket - for timestamp_range in &matching_ranges { - if let Some(store_values) = data.time_map.get(timestamp_range) { - for (key_opt, precompute) in store_values.iter() { - results - .entry(key_opt.clone()) - .or_default() - .push((*timestamp_range, precompute.clone_boxed_core().into())); - - total_entries += 1; - } - } - } - - // Update read counts for accessed ranges - for timestamp_range in &matching_ranges { - *data.read_counts.entry(*timestamp_range).or_insert(0) += 1; - } - - let range_scan_duration = range_scan_start_time.elapsed(); - debug!( - "Range scanning took: {:.2}ms", - range_scan_duration.as_secs_f64() * 1000.0 - ); - - let query_duration = query_start_time.elapsed(); - debug!( - "Total query took: {:.2}ms", - query_duration.as_secs_f64() * 1000.0 - ); - - debug!( - "Found {} entries for query on {} (aggregation_id: {}, start: {}, end: {})", - total_entries, metric, aggregation_id, start, end - ); - debug!("Found {} unique keys", results.len()); - - #[cfg(feature = "lock_profiling")] - { - let lock_hold_duration = lock_hold_start.elapsed(); - info!( - "🔓 Query lock hold time: {:.2}ms (metric: {}, agg_id: {}, entries: {})", - lock_hold_duration.as_secs_f64() * 1000.0, - metric, - aggregation_id, - total_entries - ); - } - - Ok(results) - } - - fn query_precomputed_output_exact( - &self, - metric: &str, - aggregation_id: u64, - exact_start: u64, - exact_end: u64, - ) -> Result> { - let query_start_time = Instant::now(); - let store_key = aggregation_id; - - // Measure lock acquisition time - #[cfg(feature = "lock_profiling")] - let lock_wait_start = Instant::now(); - - // Get the store data for this aggregation_id - let store_data_lock = match self.store.get(&store_key) { - Some(lock) => lock, - None => { - debug!("Metric {} not found in store for exact query", metric); - return Ok(HashMap::new()); - } - }; - - #[cfg(feature = "lock_profiling")] - { - let lock_wait_duration = lock_wait_start.elapsed(); - info!( - "🔒 Exact query DashMap get time: {:.2}ms (metric: {}, agg_id: {})", - lock_wait_duration.as_secs_f64() * 1000.0, - metric, - aggregation_id - ); - } - - #[cfg(feature = "lock_profiling")] - let rwlock_wait_start = Instant::now(); - - // Acquire write lock (needed to update read_counts) - let mut data = store_data_lock.write().map_err(|e| { - format!( - "Failed to acquire write lock for exact query aggregation_id {}: {}", - store_key, e - ) - })?; - - #[cfg(feature = "lock_profiling")] - { - let rwlock_wait_duration = rwlock_wait_start.elapsed(); - info!( - "🔒 Exact query RwLock wait time: {:.2}ms (metric: {}, agg_id: {})", - rwlock_wait_duration.as_secs_f64() * 1000.0, - metric, - aggregation_id - ); - } - - #[cfg(feature = "lock_profiling")] - let lock_hold_start = Instant::now(); - - let mut results: TimestampedBucketsMap = HashMap::new(); - - // Look for exact timestamp match (strict - no tolerance) - let timestamp_range = (exact_start, exact_end); - let mut found_match = false; - - // First, collect the results (immutable borrow of time_map) - if let Some(store_values) = data.time_map.get(×tamp_range) { - found_match = true; - - // Collect results with timestamp - let mut total_entries = 0; - for (key_opt, precompute) in store_values.iter() { - results - .entry(key_opt.clone()) - .or_default() - .push((timestamp_range, precompute.clone_boxed_core().into())); - total_entries += 1; - } - - debug!( - "Exact match FOUND for [{}, {}]: {} entries across {} keys", - exact_start, - exact_end, - total_entries, - results.len() - ); - } else { - debug!( - "Exact match NOT FOUND for metric: {}, agg_id: {}, range: [{}, {}]", - metric, aggregation_id, exact_start, exact_end - ); - } - - // Now update read count (mutable borrow of data.read_counts) - if found_match { - *data.read_counts.entry(timestamp_range).or_insert(0) += 1; - } - - #[cfg(feature = "lock_profiling")] - { - let lock_hold_duration = lock_hold_start.elapsed(); - info!( - "🔓 Exact query lock hold time: {:.2}ms (metric: {}, agg_id: {}, found: {})", - lock_hold_duration.as_secs_f64() * 1000.0, - metric, - aggregation_id, - !results.is_empty() - ); - } - - let query_duration = query_start_time.elapsed(); - debug!( - "Exact timestamp query took: {:.2}ms (found: {})", - query_duration.as_secs_f64() * 1000.0, - !results.is_empty() - ); - - Ok(results) - } - - fn get_earliest_timestamp_per_aggregation_id( - &self, - ) -> Result, Box> { - // No lock needed - DashMap with AtomicU64 - let result = self - .earliest_timestamps - .iter() - .map(|entry| (*entry.key(), entry.value().load(Ordering::Relaxed))) - .collect(); - - Ok(result) - } - - fn close(&self) -> StoreResult<()> { - // For in-memory store, no cleanup needed - info!("LegacySimpleMapStorePerKey closed"); - Ok(()) - } -} diff --git a/data_plane/src/stores/sketch_db/sketch_index.rs b/data_plane/src/stores/sketch_db/sketch_index.rs index 6bc8518b..b50fb5d4 100644 --- a/data_plane/src/stores/sketch_db/sketch_index.rs +++ b/data_plane/src/stores/sketch_db/sketch_index.rs @@ -8,7 +8,7 @@ //! per-window sketch state. Intern table per sid maps the group-by //! VALUES vector to a compact `LabelValuesId = u32`; columnar //! `MutableEpoch` + sealed-epoch ring delivers the legacy -//! SimpleMapStore's six storage optimizations end-to-end. +//! SketchStore's six storage optimizations end-to-end. //! //! Ghost sids (registered but never carrying state) are valid — they //! exist when an agent registers a pre-merge identity that the gateway diff --git a/data_plane/src/stores/sketch_db/simple_map_store/INDEX_DESIGN.md b/data_plane/src/stores/sketch_db/sketch_store/INDEX_DESIGN.md similarity index 100% rename from data_plane/src/stores/sketch_db/simple_map_store/INDEX_DESIGN.md rename to data_plane/src/stores/sketch_db/sketch_store/INDEX_DESIGN.md diff --git a/data_plane/src/stores/sketch_db/simple_map_store/common.rs b/data_plane/src/stores/sketch_db/sketch_store/common.rs similarity index 100% rename from data_plane/src/stores/sketch_db/simple_map_store/common.rs rename to data_plane/src/stores/sketch_db/sketch_store/common.rs diff --git a/data_plane/src/stores/sketch_db/simple_map_store/global.rs b/data_plane/src/stores/sketch_db/sketch_store/global.rs similarity index 99% rename from data_plane/src/stores/sketch_db/simple_map_store/global.rs rename to data_plane/src/stores/sketch_db/sketch_store/global.rs index 9edc1332..5e8b5ab4 100644 --- a/data_plane/src/stores/sketch_db/simple_map_store/global.rs +++ b/data_plane/src/stores/sketch_db/sketch_store/global.rs @@ -1,7 +1,7 @@ use crate::stores::schema::{ AggregateCore, AggregationType, CleanupPolicy, PrecomputedOutput, StreamingConfig, }; -use crate::stores::sketch_db::simple_map_store::common::{ +use crate::stores::sketch_db::sketch_store::common::{ EpochID, InternTable, MetricBucketMap, MutableEpoch, SealedEpoch, TimestampRange, }; use crate::stores::{Store, StoreResult, TimestampedBucketsMap}; @@ -131,7 +131,7 @@ struct StoreData { } /// In-memory storage implementation using single mutex (like Python version) -pub struct SimpleMapStoreGlobal { +pub struct SketchStoreGlobal { // Single global mutex protecting all data structures lock: Mutex, @@ -142,7 +142,7 @@ pub struct SimpleMapStoreGlobal { cleanup_policy: CleanupPolicy, } -impl SimpleMapStoreGlobal { +impl SketchStoreGlobal { pub fn new(streaming_config: Arc, cleanup_policy: CleanupPolicy) -> Self { Self { lock: Mutex::new(StoreData { @@ -224,7 +224,7 @@ struct BatchConfig { } #[async_trait::async_trait] -impl Store for SimpleMapStoreGlobal { +impl Store for SketchStoreGlobal { fn insert_precomputed_output( &self, output: PrecomputedOutput, @@ -692,7 +692,7 @@ impl Store for SimpleMapStoreGlobal { fn close(&self) -> StoreResult<()> { // For in-memory store, no cleanup needed - info!("SimpleMapStoreGlobal closed"); + info!("SketchStoreGlobal closed"); Ok(()) } @@ -721,7 +721,7 @@ impl Store for SimpleMapStoreGlobal { info!( agg_id, evicted_windows = evicted, - "SimpleMapStoreGlobal::drop_agg_id" + "SketchStoreGlobal::drop_agg_id" ); Ok(evicted) } diff --git a/data_plane/src/stores/sketch_db/simple_map_store/mod.rs b/data_plane/src/stores/sketch_db/sketch_store/mod.rs similarity index 81% rename from data_plane/src/stores/sketch_db/simple_map_store/mod.rs rename to data_plane/src/stores/sketch_db/sketch_store/mod.rs index afddf6cd..c41ffa15 100644 --- a/data_plane/src/stores/sketch_db/simple_map_store/mod.rs +++ b/data_plane/src/stores/sketch_db/sketch_store/mod.rs @@ -1,6 +1,5 @@ mod common; pub mod global; -pub mod legacy; pub mod per_key; pub mod persistence; @@ -8,8 +7,8 @@ use crate::stores::schema::{ AggregateCore, CleanupPolicy, LockStrategy, PrecomputedOutput, StreamingConfig, }; use crate::stores::{Store, StoreResult, TimestampedBucketsMap}; -use global::SimpleMapStoreGlobal; -use per_key::SimpleMapStorePerKey; +use global::SketchStoreGlobal; +use per_key::SketchStorePerKey; use std::collections::HashMap; use std::sync::Arc; @@ -31,12 +30,12 @@ pub struct StoreDiagnostics { } /// Enum wrapper that dispatches to either global or per-key lock implementation -pub enum SimpleMapStore { - Global(SimpleMapStoreGlobal), - PerKey(SimpleMapStorePerKey), +pub enum SketchStore { + Global(SketchStoreGlobal), + PerKey(SketchStorePerKey), } -impl SimpleMapStore { +impl SketchStore { /// Constructor with default strategy (backward compatibility for tests) pub fn new(streaming_config: Arc, cleanup_policy: CleanupPolicy) -> Self { Self::new_with_strategy(streaming_config, cleanup_policy, LockStrategy::PerKey) @@ -45,8 +44,8 @@ impl SimpleMapStore { /// Collect diagnostic info for memory investigation. pub fn diagnostic_info(&self) -> StoreDiagnostics { match self { - SimpleMapStore::Global(store) => store.diagnostic_info(), - SimpleMapStore::PerKey(store) => store.diagnostic_info(), + SketchStore::Global(store) => store.diagnostic_info(), + SketchStore::PerKey(store) => store.diagnostic_info(), } } @@ -58,10 +57,10 @@ impl SimpleMapStore { ) -> Self { match lock_strategy { LockStrategy::Global => { - SimpleMapStore::Global(SimpleMapStoreGlobal::new(streaming_config, cleanup_policy)) + SketchStore::Global(SketchStoreGlobal::new(streaming_config, cleanup_policy)) } LockStrategy::PerKey => { - SimpleMapStore::PerKey(SimpleMapStorePerKey::new(streaming_config, cleanup_policy)) + SketchStore::PerKey(SketchStorePerKey::new(streaming_config, cleanup_policy)) } } } @@ -77,10 +76,10 @@ impl SimpleMapStore { pub fn with_persistence_per_key( streaming_config: Arc, cleanup_policy: CleanupPolicy, - persistence_cfg: persistence::SimpleMapStorePersistenceConfig, + persistence_cfg: persistence::SketchStorePersistenceConfig, ) -> persistence::PersistResult { - Ok(SimpleMapStore::PerKey( - SimpleMapStorePerKey::with_persistence( + Ok(SketchStore::PerKey( + SketchStorePerKey::with_persistence( streaming_config, cleanup_policy, persistence_cfg, @@ -90,15 +89,15 @@ impl SimpleMapStore { } #[async_trait::async_trait] -impl Store for SimpleMapStore { +impl Store for SketchStore { fn insert_precomputed_output( &self, output: PrecomputedOutput, precompute: Box, ) -> StoreResult<()> { match self { - SimpleMapStore::Global(store) => store.insert_precomputed_output(output, precompute), - SimpleMapStore::PerKey(store) => store.insert_precomputed_output(output, precompute), + SketchStore::Global(store) => store.insert_precomputed_output(output, precompute), + SketchStore::PerKey(store) => store.insert_precomputed_output(output, precompute), } } @@ -107,8 +106,8 @@ impl Store for SimpleMapStore { outputs: Vec<(PrecomputedOutput, Box)>, ) -> StoreResult<()> { match self { - SimpleMapStore::Global(store) => store.insert_precomputed_output_batch(outputs), - SimpleMapStore::PerKey(store) => store.insert_precomputed_output_batch(outputs), + SketchStore::Global(store) => store.insert_precomputed_output_batch(outputs), + SketchStore::PerKey(store) => store.insert_precomputed_output_batch(outputs), } } @@ -120,10 +119,10 @@ impl Store for SimpleMapStore { end: u64, ) -> Result> { match self { - SimpleMapStore::Global(store) => { + SketchStore::Global(store) => { store.query_precomputed_output(metric, aggregation_id, start, end) } - SimpleMapStore::PerKey(store) => { + SketchStore::PerKey(store) => { store.query_precomputed_output(metric, aggregation_id, start, end) } } @@ -137,10 +136,10 @@ impl Store for SimpleMapStore { exact_end: u64, ) -> Result> { match self { - SimpleMapStore::Global(store) => { + SketchStore::Global(store) => { store.query_precomputed_output_exact(metric, aggregation_id, exact_start, exact_end) } - SimpleMapStore::PerKey(store) => { + SketchStore::PerKey(store) => { store.query_precomputed_output_exact(metric, aggregation_id, exact_start, exact_end) } } @@ -150,22 +149,22 @@ impl Store for SimpleMapStore { &self, ) -> Result, Box> { match self { - SimpleMapStore::Global(store) => store.get_earliest_timestamp_per_aggregation_id(), - SimpleMapStore::PerKey(store) => store.get_earliest_timestamp_per_aggregation_id(), + SketchStore::Global(store) => store.get_earliest_timestamp_per_aggregation_id(), + SketchStore::PerKey(store) => store.get_earliest_timestamp_per_aggregation_id(), } } fn close(&self) -> StoreResult<()> { match self { - SimpleMapStore::Global(store) => store.close(), - SimpleMapStore::PerKey(store) => store.close(), + SketchStore::Global(store) => store.close(), + SketchStore::PerKey(store) => store.close(), } } fn drop_agg_id(&self, agg_id: u64) -> StoreResult { match self { - SimpleMapStore::Global(store) => store.drop_agg_id(agg_id), - SimpleMapStore::PerKey(store) => store.drop_agg_id(agg_id), + SketchStore::Global(store) => store.drop_agg_id(agg_id), + SketchStore::PerKey(store) => store.drop_agg_id(agg_id), } } } @@ -206,7 +205,7 @@ mod drop_agg_id_tests { Arc::new(StreamingConfig::new(map)) } - fn write_one(store: &SimpleMapStore, agg_id: u64, value: f64, ts: u64) { + fn write_one(store: &SketchStore, agg_id: u64, value: f64, ts: u64) { let acc = SumAccumulator::with_sum(value); let output = PrecomputedOutput::new(ts, ts + 1000, None, agg_id); store @@ -214,15 +213,15 @@ mod drop_agg_id_tests { .expect("insert ok"); } - fn total_buckets(store: &SimpleMapStore, metric: &str, agg_id: u64) -> usize { + fn total_buckets(store: &SketchStore, metric: &str, agg_id: u64) -> usize { let map = store .query_precomputed_output(metric, agg_id, 0, u64::MAX / 2) .expect("query ok"); map.values().map(|v| v.len()).sum() } - fn make_store(strategy: LockStrategy) -> SimpleMapStore { - SimpleMapStore::new_with_strategy( + fn make_store(strategy: LockStrategy) -> SketchStore { + SketchStore::new_with_strategy( two_agg_streaming_config(), CleanupPolicy::NoCleanup, strategy, diff --git a/data_plane/src/stores/sketch_db/simple_map_store/per_key.rs b/data_plane/src/stores/sketch_db/sketch_store/per_key.rs similarity index 97% rename from data_plane/src/stores/sketch_db/simple_map_store/per_key.rs rename to data_plane/src/stores/sketch_db/sketch_store/per_key.rs index 27d6fdd9..11d667eb 100644 --- a/data_plane/src/stores/sketch_db/simple_map_store/per_key.rs +++ b/data_plane/src/stores/sketch_db/sketch_store/per_key.rs @@ -2,7 +2,7 @@ use crate::stores::schema::{ AggregateCore, AggregationType, CleanupPolicy, KeyByLabelValues, PrecomputedOutput, StreamingConfig, }; -use crate::stores::sketch_db::simple_map_store::common::{ +use crate::stores::sketch_db::sketch_store::common::{ EpochID, InternTable, MetricBucketMap, MetricID, MutableEpoch, SealedEpoch, TimestampRange, }; use crate::stores::{Store, StoreResult, TimestampedBucketsMap}; @@ -21,7 +21,7 @@ use super::persistence::{ manifest::Manifest, recovery, source::{EpochSnapshot, EpochSnapshotEntry, EpochSource, SealedEpochRef}, - PersistError, PersistResult, SimpleMapStorePersistenceConfig, + PersistError, PersistResult, SketchStorePersistenceConfig, }; type StoreKey = u64; // aggregation_id @@ -201,7 +201,7 @@ impl StoreKeyData { } } -/// Shared state that both the outer `SimpleMapStorePerKey` and the +/// Shared state that both the outer `SketchStorePerKey` and the /// background flusher hold via `Arc`. Contains the DashMap of per-agg /// state plus the counters the flusher needs. pub struct PerKeyInner { @@ -243,7 +243,7 @@ pub struct PerKeyInner { } /// Persistence-related state owned by the outer store. Dropping this -/// (via `SimpleMapStorePerKey::Drop`) shuts the flusher down cleanly. +/// (via `SketchStorePerKey::Drop`) shuts the flusher down cleanly. struct PersistenceState { manifest: Arc, cache: PartCache, @@ -256,14 +256,14 @@ struct PersistenceState { } /// In-memory storage implementation using per-key locks for concurrency -pub struct SimpleMapStorePerKey { +pub struct SketchStorePerKey { inner: Arc, /// `None` when the store is in-memory-only (existing `new()` path). /// `Some` when constructed via `with_persistence`. persistence: Option, } -impl SimpleMapStorePerKey { +impl SketchStorePerKey { /// Backwards-compatible constructor. No persistence — behaves /// exactly like pre-persistence code. pub fn new(streaming_config: Arc, cleanup_policy: CleanupPolicy) -> Self { @@ -297,12 +297,12 @@ impl SimpleMapStorePerKey { pub fn with_persistence( streaming_config: Arc, cleanup_policy: CleanupPolicy, - persistence_cfg: SimpleMapStorePersistenceConfig, + persistence_cfg: SketchStorePersistenceConfig, ) -> PersistResult { // Run recovery first so the manifest reflects on-disk state. let (_loaded_manifest, report) = recovery::recover(&persistence_cfg.disk_path)?; info!( - "SimpleMapStorePerKey persistence recovery: live={}, corrupt_removed={}, orphans_removed={}", + "SketchStorePerKey persistence recovery: live={}, corrupt_removed={}, orphans_removed={}", report.live_parts, report.corrupt_parts_removed, report.orphan_parts_removed ); @@ -670,7 +670,7 @@ impl SimpleMapStorePerKey { // TODO: replace with non-datafusion path. The previous body // depended on `accumulator_serde::deserialize_accumulator` from // the removed `engines::physical` module to materialise sketches - // from persisted parts. Since SimpleMapStore is deprecated and + // from persisted parts. Since SketchStore is deprecated and // the warm-tier query path is being rebuilt on top of the // SketchIndex, this returns Ok(()) so that callers see "no disk // parts" rather than panicking; the live in-memory path still @@ -679,7 +679,7 @@ impl SimpleMapStorePerKey { } } -impl Drop for SimpleMapStorePerKey { +impl Drop for SketchStorePerKey { fn drop(&mut self) { // Dropping the PersistenceState (and therefore the FlusherHandle) // stops the flusher thread before the underlying Arc @@ -692,7 +692,7 @@ impl Drop for SimpleMapStorePerKey { } #[async_trait::async_trait] -impl Store for SimpleMapStorePerKey { +impl Store for SketchStorePerKey { fn insert_precomputed_output( &self, output: PrecomputedOutput, @@ -926,7 +926,7 @@ impl Store for SimpleMapStorePerKey { } fn close(&self) -> StoreResult<()> { - info!("SimpleMapStorePerKey closed"); + info!("SketchStorePerKey closed"); Ok(()) } @@ -959,7 +959,7 @@ impl Store for SimpleMapStorePerKey { info!( agg_id, evicted_windows = evicted, - "SimpleMapStorePerKey::drop_agg_id" + "SketchStorePerKey::drop_agg_id" ); Ok(evicted) } @@ -999,7 +999,7 @@ impl EpochSource for PerKeyInner { // TODO: replace with non-datafusion path. The previous body // serialised sketches via // `accumulator_serde::serialize_accumulator_arroyo` from the - // removed `engines::physical` module. SimpleMapStore is + // removed `engines::physical` module. SketchStore is // deprecated; persistence is being refactored on top of the // SketchIndex. // diff --git a/data_plane/src/stores/sketch_db/simple_map_store/persistence/cache.rs b/data_plane/src/stores/sketch_db/sketch_store/persistence/cache.rs similarity index 100% rename from data_plane/src/stores/sketch_db/simple_map_store/persistence/cache.rs rename to data_plane/src/stores/sketch_db/sketch_store/persistence/cache.rs diff --git a/data_plane/src/stores/sketch_db/simple_map_store/persistence/config.rs b/data_plane/src/stores/sketch_db/sketch_store/persistence/config.rs similarity index 95% rename from data_plane/src/stores/sketch_db/simple_map_store/persistence/config.rs rename to data_plane/src/stores/sketch_db/sketch_store/persistence/config.rs index fac29b2f..bb4d456a 100644 --- a/data_plane/src/stores/sketch_db/simple_map_store/persistence/config.rs +++ b/data_plane/src/stores/sketch_db/sketch_store/persistence/config.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use std::time::Duration; -/// Configuration for the SimpleMapStore persistence layer. +/// Configuration for the SketchStore persistence layer. /// /// The two bounding knobs, in priority order: /// @@ -22,7 +22,7 @@ use std::time::Duration; /// `now - delete_older_than_ms` is deleted from disk on the next tick. /// Bounds manifest size and long-running directory growth. #[derive(Debug, Clone)] -pub struct SimpleMapStorePersistenceConfig { +pub struct SketchStorePersistenceConfig { // ---- Primary: memory budget ---- pub memory_limit_bytes: usize, pub memory_low_watermark_bytes: usize, @@ -50,7 +50,7 @@ pub struct SimpleMapStorePersistenceConfig { pub part_cache_bytes: u64, } -impl SimpleMapStorePersistenceConfig { +impl SketchStorePersistenceConfig { /// Build a config with sensible defaults relative to a memory budget. pub fn with_memory_limit(memory_limit_bytes: usize, disk_path: PathBuf) -> Self { let low_water = memory_limit_bytes * 8 / 10; // 80% of high water diff --git a/data_plane/src/stores/sketch_db/simple_map_store/persistence/flusher.rs b/data_plane/src/stores/sketch_db/sketch_store/persistence/flusher.rs similarity index 98% rename from data_plane/src/stores/sketch_db/simple_map_store/persistence/flusher.rs rename to data_plane/src/stores/sketch_db/sketch_store/persistence/flusher.rs index f789690a..041d7552 100644 --- a/data_plane/src/stores/sketch_db/simple_map_store/persistence/flusher.rs +++ b/data_plane/src/stores/sketch_db/sketch_store/persistence/flusher.rs @@ -23,7 +23,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tracing::{debug, error, info, warn}; -use super::config::SimpleMapStorePersistenceConfig; +use super::config::SketchStorePersistenceConfig; use super::manifest::{Manifest, PartEntry}; use super::part::{part_dir_path, PartWriter}; use super::source::{EpochSnapshot, EpochSource, SealedEpochRef}; @@ -37,7 +37,7 @@ pub struct FlusherHandle { } pub(crate) struct FlusherShared { - pub cfg: SimpleMapStorePersistenceConfig, + pub cfg: SketchStorePersistenceConfig, pub manifest: Arc, pub next_part_id: AtomicU64, pub shutdown: AtomicBool, @@ -58,7 +58,7 @@ impl FlusherHandle { /// Start a flusher thread. Takes an `EpochSource` (typically the /// store itself, wrapped in `Arc`). pub fn start( - cfg: SimpleMapStorePersistenceConfig, + cfg: SketchStorePersistenceConfig, manifest: Arc, source: Arc, ) -> PersistResult @@ -425,7 +425,7 @@ fn now_ms() -> u64 { mod tests { use super::*; use crate::stores::schema::KeyByLabelValues; - use crate::stores::sketch_db::simple_map_store::persistence::source::{ + use crate::stores::sketch_db::sketch_store::persistence::source::{ EpochSnapshot, EpochSnapshotEntry, }; use std::sync::Mutex as StdMutex; @@ -511,8 +511,8 @@ mod tests { } } - fn test_cfg(disk_path: PathBuf, mem_limit: usize) -> SimpleMapStorePersistenceConfig { - SimpleMapStorePersistenceConfig { + fn test_cfg(disk_path: PathBuf, mem_limit: usize) -> SketchStorePersistenceConfig { + SketchStorePersistenceConfig { memory_limit_bytes: mem_limit, memory_low_watermark_bytes: mem_limit / 2, hard_cap_bytes: mem_limit * 2, diff --git a/data_plane/src/stores/sketch_db/simple_map_store/persistence/manifest.rs b/data_plane/src/stores/sketch_db/sketch_store/persistence/manifest.rs similarity index 100% rename from data_plane/src/stores/sketch_db/simple_map_store/persistence/manifest.rs rename to data_plane/src/stores/sketch_db/sketch_store/persistence/manifest.rs diff --git a/data_plane/src/stores/sketch_db/simple_map_store/persistence/mod.rs b/data_plane/src/stores/sketch_db/sketch_store/persistence/mod.rs similarity index 87% rename from data_plane/src/stores/sketch_db/simple_map_store/persistence/mod.rs rename to data_plane/src/stores/sketch_db/sketch_store/persistence/mod.rs index a7ed40dd..a0381142 100644 --- a/data_plane/src/stores/sketch_db/simple_map_store/persistence/mod.rs +++ b/data_plane/src/stores/sketch_db/sketch_store/persistence/mod.rs @@ -1,10 +1,10 @@ -//! Persistence layer for `SimpleMapStorePerKey`. +//! Persistence layer for `SketchStorePerKey`. //! //! See `docs/design-simple-map-store-persistence.md` for the design rationale. //! //! ## Structure //! -//! * [`config`] — [`SimpleMapStorePersistenceConfig`] +//! * [`config`] — [`SketchStorePersistenceConfig`] //! * [`part`] — on-disk part format (`meta.bin` + `data.bin` + `index.bin`), //! writer and reader. //! * [`manifest`] — append-only log + periodic binary snapshot of live parts. @@ -15,7 +15,7 @@ //! * [`recovery`] — startup: load snapshot, replay log, verify CRCs, sweep //! orphan part dirs. //! -//! The submodule is intentionally decoupled from `SimpleMapStorePerKey` +//! The submodule is intentionally decoupled from `SketchStorePerKey` //! via the [`EpochSource`] trait — the flusher knows nothing about the //! store's internal types and can be unit-tested against a fake source. @@ -28,7 +28,7 @@ pub mod cache; pub mod flusher; pub mod recovery; -pub use config::SimpleMapStorePersistenceConfig; +pub use config::SketchStorePersistenceConfig; pub use manifest::{Manifest, PartEntry}; pub use part::{PartId, PartReader, PartWriter, SnapshotEntry}; pub use source::{EpochSource, SealedEpochRef}; diff --git a/data_plane/src/stores/sketch_db/simple_map_store/persistence/part.rs b/data_plane/src/stores/sketch_db/sketch_store/persistence/part.rs similarity index 99% rename from data_plane/src/stores/sketch_db/simple_map_store/persistence/part.rs rename to data_plane/src/stores/sketch_db/sketch_store/persistence/part.rs index acf5a457..1a96c611 100644 --- a/data_plane/src/stores/sketch_db/simple_map_store/persistence/part.rs +++ b/data_plane/src/stores/sketch_db/sketch_store/persistence/part.rs @@ -594,7 +594,7 @@ fn map_file(path: &Path) -> PersistResult { mod tests { use super::*; use crate::stores::schema::KeyByLabelValues; - use crate::stores::sketch_db::simple_map_store::persistence::source::EpochSnapshotEntry; + use crate::stores::sketch_db::sketch_store::persistence::source::EpochSnapshotEntry; use tempfile::TempDir; fn make_snapshot() -> EpochSnapshot { diff --git a/data_plane/src/stores/sketch_db/simple_map_store/persistence/recovery.rs b/data_plane/src/stores/sketch_db/sketch_store/persistence/recovery.rs similarity index 96% rename from data_plane/src/stores/sketch_db/simple_map_store/persistence/recovery.rs rename to data_plane/src/stores/sketch_db/sketch_store/persistence/recovery.rs index 79a8b2df..40bcf6c7 100644 --- a/data_plane/src/stores/sketch_db/simple_map_store/persistence/recovery.rs +++ b/data_plane/src/stores/sketch_db/sketch_store/persistence/recovery.rs @@ -118,10 +118,10 @@ pub fn recover(disk_path: &Path) -> PersistResult<(Manifest, RecoveryReport)> { mod tests { use super::*; use crate::stores::schema::KeyByLabelValues; - use crate::stores::sketch_db::simple_map_store::persistence::part::{ + use crate::stores::sketch_db::sketch_store::persistence::part::{ part_dir_path, PartWriter, }; - use crate::stores::sketch_db::simple_map_store::persistence::source::{ + use crate::stores::sketch_db::sketch_store::persistence::source::{ EpochSnapshot, EpochSnapshotEntry, }; use tempfile::TempDir; @@ -183,7 +183,7 @@ mod tests { let report_write = PartWriter::write_part(&part_dir, 42, &[dummy_snapshot()]).unwrap(); manifest .append_add( - crate::stores::sketch_db::simple_map_store::persistence::manifest::PartEntry { + crate::stores::sketch_db::sketch_store::persistence::manifest::PartEntry { part_id: 42, min_ts: report_write.min_ts, max_ts: report_write.max_ts, diff --git a/data_plane/src/stores/sketch_db/simple_map_store/persistence/source.rs b/data_plane/src/stores/sketch_db/sketch_store/persistence/source.rs similarity index 96% rename from data_plane/src/stores/sketch_db/simple_map_store/persistence/source.rs rename to data_plane/src/stores/sketch_db/sketch_store/persistence/source.rs index ed095004..809f7ffb 100644 --- a/data_plane/src/stores/sketch_db/simple_map_store/persistence/source.rs +++ b/data_plane/src/stores/sketch_db/sketch_store/persistence/source.rs @@ -1,5 +1,5 @@ //! The trait the flusher uses to enumerate, snapshot, and evict sealed -//! epochs. Decouples `flusher.rs` from `SimpleMapStorePerKey` so the +//! epochs. Decouples `flusher.rs` from `SketchStorePerKey` so the //! flusher can be unit-tested against a fake source. use crate::stores::schema::KeyByLabelValues; @@ -68,7 +68,7 @@ pub struct EpochSnapshotEntry { } /// Trait the flusher uses to discover, snapshot, and evict sealed -/// epochs. Implemented by `SimpleMapStorePerKey`; a test fake lives in +/// epochs. Implemented by `SketchStorePerKey`; a test fake lives in /// `flusher.rs`'s unit tests. /// /// Implementors guarantee that: diff --git a/data_plane/src/tests/capability_matching_tests.rs b/data_plane/src/tests/capability_matching_tests.rs index 5472ddcb..60543b4d 100644 --- a/data_plane/src/tests/capability_matching_tests.rs +++ b/data_plane/src/tests/capability_matching_tests.rs @@ -14,7 +14,7 @@ use crate::precompute_engine::operators::count_min_sketch_accumulator::CountMinS use crate::precompute_engine::operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; use crate::precompute_engine::operators::delta_set_aggregator_accumulator::DeltaSetAggregatorAccumulator; use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; -use crate::stores::sketch_db::simple_map_store::SimpleMapStore; +use crate::stores::sketch_db::sketch_store::SketchStore; use crate::stores::traits::Store; use promql_utilities::data_model::KeyByLabelNames; use std::collections::HashMap; @@ -70,7 +70,7 @@ fn engine_no_query_configs( aggregation_configs: agg_map, storage_backend: Default::default(), }); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); @@ -132,7 +132,7 @@ fn engine_with_query_config( aggregation_configs: agg_map, storage_backend: Default::default(), }); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); diff --git a/data_plane/src/tests/capability_miss_http_e2e_tests.rs b/data_plane/src/tests/capability_miss_http_e2e_tests.rs index d4e7ada9..952ac56d 100644 --- a/data_plane/src/tests/capability_miss_http_e2e_tests.rs +++ b/data_plane/src/tests/capability_miss_http_e2e_tests.rs @@ -44,7 +44,7 @@ use crate::drivers::query::adapters::AdapterConfig; use crate::drivers::query::controller_client::{ControllerClient, HttpControllerClient}; use crate::drivers::query::servers::http::{HttpServer, HttpServerConfig}; use crate::query_engines::ASAPQueryEngine; -use crate::stores::sketch_db::simple_map_store::SimpleMapStore; +use crate::stores::sketch_db::sketch_store::SketchStore; use axum::{extract::State, routing::post, Router}; use reqwest::Client; use serde_json::Value; @@ -152,7 +152,7 @@ async fn start_mock_controller(state: MockControllerState) -> u16 { async fn start_backend(controller_url: String, hot_reload: HotReloadStreamingConfig) -> u16 { let streaming_config = hot_reload.snapshot(); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); diff --git a/data_plane/src/tests/persist_format_versioning_tests.rs b/data_plane/src/tests/persist_format_versioning_tests.rs index 829add33..2d29e3a1 100644 --- a/data_plane/src/tests/persist_format_versioning_tests.rs +++ b/data_plane/src/tests/persist_format_versioning_tests.rs @@ -7,8 +7,8 @@ //! `stores::sketch_db::schema::PERSIST_FORMAT_VERSION` (currently 1) //! * `BackfillRegistry` snapshot — JSON, //! `stores::sketch_db::backfill::PERSIST_FORMAT_VERSION` (currently 1) -//! * `SimpleMapStore` part `meta.bin` — binary, -//! `stores::sketch_db::simple_map_store::persistence::part::PART_FORMAT_VERSION` (currently 1) +//! * `SketchStore` part `meta.bin` — binary, +//! `stores::sketch_db::sketch_store::persistence::part::PART_FORMAT_VERSION` (currently 1) //! //! Each has a load path that tests `version == CURRENT`. The per- //! module unit tests already cover the happy-path roundtrip and a @@ -29,7 +29,7 @@ use crate::stores::sketch_db::backfill::{ BackfillJob, BackfillRegistry, BackfillSource, BackfillStatus, }; use crate::stores::sketch_db::schema::{AggStatus, SchemaRegistry}; -use crate::stores::sketch_db::simple_map_store::persistence::part::{ +use crate::stores::sketch_db::sketch_store::persistence::part::{ MAGIC_META, META_HEADER_SIZE, PART_FORMAT_VERSION, }; @@ -332,11 +332,11 @@ mod backfill { } } -// ─── SimpleMapStore part meta.bin format ──────────────────────────────── +// ─── SketchStore part meta.bin format ──────────────────────────────── mod part_meta { use super::*; - use crate::stores::sketch_db::simple_map_store::persistence::part::PartReader; + use crate::stores::sketch_db::sketch_store::persistence::part::PartReader; /// Build a valid 64-byte meta.bin header for part_id=1. fn valid_header() -> Vec { @@ -446,7 +446,7 @@ mod v2_forward_compat { use super::*; use crate::stores::sketch_db::backfill::PERSIST_FORMAT_VERSION as BACKFILL_V; use crate::stores::sketch_db::schema::PERSIST_FORMAT_VERSION as SCHEMA_V; - use crate::stores::sketch_db::simple_map_store::persistence::part::PartReader; + use crate::stores::sketch_db::sketch_store::persistence::part::PartReader; /// SchemaRegistry: snapshot tagged v_current+1 must trigger safe /// fallback, and the rewrite must be at v_current with the new @@ -567,7 +567,7 @@ mod v2_forward_compat { ); } - /// SimpleMapStore part meta.bin: header tagged PART_FORMAT_VERSION+1 + /// SketchStore part meta.bin: header tagged PART_FORMAT_VERSION+1 /// must surface a `PersistError::Format`. This is the /// "v2-on-disk-loaded-by-v1-code" path; for parts there is no /// fallback (each part is opaque), so a clean error is the contract. diff --git a/data_plane/src/tests/persistence_integration_tests.rs b/data_plane/src/tests/persistence_integration_tests.rs index 91b16f68..4fb0d345 100644 --- a/data_plane/src/tests/persistence_integration_tests.rs +++ b/data_plane/src/tests/persistence_integration_tests.rs @@ -1,9 +1,9 @@ -//! Minimal end-to-end test for `SimpleMapStorePerKey::with_persistence`. +//! Minimal end-to-end test for `SketchStorePerKey::with_persistence`. //! //! The deeper flush-and-readback tests previously exercised the //! datafusion-backed `accumulator_serde` path that PR #123 removed. //! Rather than rebuild that SerDe (the long-term plan is to migrate -//! persistence onto SketchIndex, not back onto SimpleMapStore), those +//! persistence onto SketchIndex, not back onto SketchStore), those //! tests were retired in this commit. The single remaining test //! verifies the construct/drop lifecycle of the flusher thread — it //! does not touch the disk path. @@ -15,8 +15,8 @@ use tempfile::TempDir; use std::time::Duration; use crate::stores::schema::{AggregationType, CleanupPolicy, StreamingConfig, WindowType}; -use crate::stores::sketch_db::simple_map_store::per_key::SimpleMapStorePerKey; -use crate::stores::sketch_db::simple_map_store::persistence::SimpleMapStorePersistenceConfig; +use crate::stores::sketch_db::sketch_store::per_key::SketchStorePerKey; +use crate::stores::sketch_db::sketch_store::persistence::SketchStorePersistenceConfig; use crate::AggregationConfig; fn make_streaming_config(agg_id: u64) -> Arc { @@ -44,8 +44,8 @@ fn make_streaming_config(agg_id: u64) -> Arc { Arc::new(StreamingConfig::new(map)) } -fn persistence_cfg(dir: &TempDir, hot_window_ms: Option) -> SimpleMapStorePersistenceConfig { - SimpleMapStorePersistenceConfig { +fn persistence_cfg(dir: &TempDir, hot_window_ms: Option) -> SketchStorePersistenceConfig { + SketchStorePersistenceConfig { memory_limit_bytes: 100 * 1024 * 1024, memory_low_watermark_bytes: 50 * 1024 * 1024, hard_cap_bytes: 200 * 1024 * 1024, @@ -62,7 +62,7 @@ fn construct_and_drop_shuts_flusher_cleanly() { let dir = TempDir::new().unwrap(); let cfg = make_streaming_config(1); let persistence = persistence_cfg(&dir, None); - let store = SimpleMapStorePerKey::with_persistence(cfg, CleanupPolicy::NoCleanup, persistence) + let store = SketchStorePerKey::with_persistence(cfg, CleanupPolicy::NoCleanup, persistence) .expect("with_persistence"); // Dropping the store should not deadlock or panic. drop(store); diff --git a/data_plane/src/tests/persistence_perf_tests.rs b/data_plane/src/tests/persistence_perf_tests.rs index 31905729..dba9d351 100644 --- a/data_plane/src/tests/persistence_perf_tests.rs +++ b/data_plane/src/tests/persistence_perf_tests.rs @@ -1,4 +1,4 @@ -//! Performance harness for the `SimpleMapStore` persistence layer. +//! Performance harness for the `SketchStore` persistence layer. //! //! All tests are `#[ignore]` so they don't slow down the normal //! `cargo test` run. Exercise them with: @@ -39,8 +39,8 @@ use crate::stores::schema::{ AggregationType, CleanupPolicy, PrecomputedOutput, StreamingConfig, WindowType, }; use crate::precompute_engine::operators::SumAccumulator; -use crate::stores::sketch_db::simple_map_store::per_key::SimpleMapStorePerKey; -use crate::stores::sketch_db::simple_map_store::persistence::SimpleMapStorePersistenceConfig; +use crate::stores::sketch_db::sketch_store::per_key::SketchStorePerKey; +use crate::stores::sketch_db::sketch_store::persistence::SketchStorePersistenceConfig; use crate::stores::Store; use crate::{AggregateCore, AggregationConfig}; @@ -78,8 +78,8 @@ fn persistence_cfg( memory_limit_bytes: usize, hot_window_ms: Option, flush_interval: Duration, -) -> SimpleMapStorePersistenceConfig { - SimpleMapStorePersistenceConfig { +) -> SketchStorePersistenceConfig { + SketchStorePersistenceConfig { memory_limit_bytes, memory_low_watermark_bytes: memory_limit_bytes * 8 / 10, hard_cap_bytes: memory_limit_bytes * 125 / 100, @@ -158,7 +158,7 @@ fn insert_throughput_in_memory_vs_persistent() { // -- baseline: in-memory, NoCleanup -- { - let store = SimpleMapStorePerKey::new(streaming_config(1, None), CleanupPolicy::NoCleanup); + let store = SketchStorePerKey::new(streaming_config(1, None), CleanupPolicy::NoCleanup); let items = gen_items(1, N); let d = insert_all(&store, items, BATCH); println!( @@ -182,7 +182,7 @@ fn insert_throughput_in_memory_vs_persistent() { None, Duration::from_secs(3600), ); - let store = SimpleMapStorePerKey::with_persistence( + let store = SketchStorePerKey::with_persistence( streaming_config(1, Some(1024)), CleanupPolicy::NoCleanup, cfg, @@ -207,7 +207,7 @@ fn insert_throughput_in_memory_vs_persistent() { Some(0), // flush everything ASAP Duration::from_millis(25), ); - let store = SimpleMapStorePerKey::with_persistence( + let store = SketchStorePerKey::with_persistence( streaming_config(1, Some(512)), CleanupPolicy::NoCleanup, cfg, @@ -241,7 +241,7 @@ fn query_latency_memory_only_vs_disk_through() { // -- in-memory baseline -- { - let store = SimpleMapStorePerKey::new(streaming_config(1, None), CleanupPolicy::NoCleanup); + let store = SketchStorePerKey::new(streaming_config(1, None), CleanupPolicy::NoCleanup); let items = gen_items(1, POPULATE); insert_all(&store, items, 1_000); @@ -253,7 +253,7 @@ fn query_latency_memory_only_vs_disk_through() { { let tmp = TempDir::new().unwrap(); let cfg = persistence_cfg(&tmp, 4 * 1024 * 1024, Some(0), Duration::from_millis(10)); - let store = SimpleMapStorePerKey::with_persistence( + let store = SketchStorePerKey::with_persistence( streaming_config(1, Some(256)), CleanupPolicy::NoCleanup, cfg, @@ -336,7 +336,7 @@ fn flush_throughput_sustained() { Some(0), // flush as fast as sealed epochs arrive Duration::from_millis(10), ); - let store = SimpleMapStorePerKey::with_persistence( + let store = SketchStorePerKey::with_persistence( streaming_config(1, Some(512)), CleanupPolicy::NoCleanup, cfg, @@ -450,7 +450,7 @@ fn memory_bound_adherence_under_overload() { let tmp = TempDir::new().unwrap(); let cfg = persistence_cfg(&tmp, LIMIT_BYTES, Some(0), Duration::from_millis(10)); - let store = SimpleMapStorePerKey::with_persistence( + let store = SketchStorePerKey::with_persistence( streaming_config(1, Some(128)), CleanupPolicy::NoCleanup, cfg, diff --git a/data_plane/src/tests/prometheus_forwarding_tests.rs b/data_plane/src/tests/prometheus_forwarding_tests.rs index bca7af2f..a98fcf25 100644 --- a/data_plane/src/tests/prometheus_forwarding_tests.rs +++ b/data_plane/src/tests/prometheus_forwarding_tests.rs @@ -3,7 +3,7 @@ use crate::stores::schema::{CleanupPolicy, InferenceConfig, QueryLanguage, Strea use crate::drivers::query::adapters::AdapterConfig; use crate::drivers::query::servers::http::{HttpServer, HttpServerConfig}; use crate::query_engines::ASAPQueryEngine; -use crate::stores::sketch_db::simple_map_store::SimpleMapStore; +use crate::stores::sketch_db::sketch_store::SketchStore; use reqwest::Client; use serde_json::Value; use std::sync::Arc; @@ -75,7 +75,7 @@ async fn setup_test_server(prometheus_port: u16) -> (HttpServer, u16) { let inference_config = InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::NoCleanup); let streaming_config = Arc::new(StreamingConfig::default()); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); @@ -168,7 +168,7 @@ async fn test_forwarding_disabled() { let inference_config = InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::NoCleanup); let streaming_config = Arc::new(StreamingConfig::default()); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); @@ -224,7 +224,7 @@ async fn test_prometheus_server_unreachable() { let inference_config = InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::NoCleanup); let streaming_config = Arc::new(StreamingConfig::default()); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); diff --git a/data_plane/src/tests/schema_timeline_dispatch_tests.rs b/data_plane/src/tests/schema_timeline_dispatch_tests.rs index 9eaf3a7e..caebadd3 100644 --- a/data_plane/src/tests/schema_timeline_dispatch_tests.rs +++ b/data_plane/src/tests/schema_timeline_dispatch_tests.rs @@ -3,7 +3,7 @@ //! Exercises the full path from a PromQL query → schema registry //! lookup → per-segment store query → `combine_statistic` → //! Prometheus `warnings`, on a real `ASAPQueryEngine` + -//! `SimpleMapStore` + `SchemaRegistry` with two agg_ids for the +//! `SketchStore` + `SchemaRegistry` with two agg_ids for the //! same metric and a reconfigure boundary inside the query range. //! //! Contract validated: queries that span a reconfigure boundary @@ -32,7 +32,7 @@ use crate::stores::schema::{ }; use crate::query_engines::{QueryResult, ASAPQueryEngine}; use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; -use crate::stores::sketch_db::simple_map_store::SimpleMapStore; +use crate::stores::sketch_db::sketch_store::SketchStore; use crate::stores::sketch_db::{AggSchema, SchemaRegistry}; use crate::stores::Store; @@ -135,7 +135,7 @@ fn build_engine( /// existing test-utility pattern in `engine_factories` — the engine /// treats those as single-point buckets aligned to the tumbling /// window, so a query whose range contains `ts` picks up the data. -fn seed_sum_at(store: &SimpleMapStore, agg_id: u64, ts: u64, host: &str, sum: f64) { +fn seed_sum_at(store: &SketchStore, agg_id: u64, ts: u64, host: &str, sum: f64) { let key = Some(KeyByLabelValues { labels: vec![host.to_string()], }); @@ -165,7 +165,7 @@ fn sum_query_across_reconfigure_boundary_returns_combined_full_result() { // agg_2: Active from the boundary onwards. schemas.insert_raw_for_testing(fixed_schema(2, BOUNDARY_MS, None, None)); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); @@ -222,7 +222,7 @@ fn sum_query_with_purged_segment_returns_partial_with_warnings() { schemas.insert_raw_for_testing(fixed_schema(1, 0, Some(BOUNDARY_MS), Some(1_000))); schemas.insert_raw_for_testing(fixed_schema(2, BOUNDARY_MS, None, None)); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); @@ -275,7 +275,7 @@ fn single_schema_query_falls_through_to_default_path() { let schemas = Arc::new(SchemaRegistry::empty()); schemas.insert_raw_for_testing(fixed_schema(7, 0, None, None)); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); diff --git a/data_plane/src/tests/store_correctness_tests.rs b/data_plane/src/tests/store_correctness_tests.rs index 100f9b70..c4247abf 100644 --- a/data_plane/src/tests/store_correctness_tests.rs +++ b/data_plane/src/tests/store_correctness_tests.rs @@ -38,7 +38,7 @@ use crate::precompute_engine::operators::{ SumAccumulator, }; use crate::stores::{Store, TimestampedBucketsMap}; -use crate::{AggregateCore, AggregationConfig, PrecomputedOutput, SimpleMapStore}; +use crate::{AggregateCore, AggregationConfig, PrecomputedOutput, SketchStore}; use promql_utilities::data_model::KeyByLabelNames; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -88,13 +88,13 @@ fn make_store( strategy: LockStrategy, policy: CleanupPolicy, ids: &[(u64, AggregationType, Option, Option)], -) -> SimpleMapStore { +) -> SketchStore { let config = make_streaming_config(ids); - SimpleMapStore::new_with_strategy(config, policy, strategy) + SketchStore::new_with_strategy(config, policy, strategy) } /// Convenience: single agg_id=1, type Sum, no cleanup. -fn make_store_simple(strategy: LockStrategy) -> SimpleMapStore { +fn make_store_simple(strategy: LockStrategy) -> SketchStore { make_store( strategy, CleanupPolicy::NoCleanup, @@ -985,7 +985,7 @@ fn test_concurrent_reads_return_complete_results(strategy: LockStrategy) { // ── test entry points ───────────────────────────────────────────────────────── -/// Contract suite against `SimpleMapStore` with [`LockStrategy::PerKey`]. +/// Contract suite against `SketchStore` with [`LockStrategy::PerKey`]. /// /// This is the reference implementation — all other stores must match its /// observable behaviour. @@ -994,7 +994,7 @@ fn contract_per_key() { run_contract_suite(LockStrategy::PerKey); } -/// Contract suite against `SimpleMapStore` with [`LockStrategy::Global`]. +/// Contract suite against `SketchStore` with [`LockStrategy::Global`]. #[test] fn contract_global() { run_contract_suite(LockStrategy::Global); diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index b83d5226..249e3342 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -1,6 +1,6 @@ //! Engine factory helpers for integration tests //! -//! Provides reusable construction helpers for ASAPQueryEngine + SimpleMapStore +//! Provides reusable construction helpers for ASAPQueryEngine + SketchStore //! populated with various accumulator types. Unlike TestConfigBuilder which //! hardcodes "SumAccumulator", these helpers build AggregationConfig with //! the correct aggregation_type string. @@ -12,7 +12,7 @@ use crate::stores::schema::{ }; use crate::query_engines::query_result::InstantVectorElement; use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; -use crate::stores::sketch_db::simple_map_store::SimpleMapStore; +use crate::stores::sketch_db::sketch_store::SketchStore; use crate::stores::Store; use crate::AggregateCore; use promql_utilities::data_model::KeyByLabelNames; @@ -98,7 +98,7 @@ pub fn create_engine_single_pop_with_aggregated( storage_backend: Default::default(), }); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); @@ -219,7 +219,7 @@ pub fn create_engine_dual_input( storage_backend: Default::default(), }); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); @@ -335,7 +335,7 @@ pub fn create_engine_two_metrics( storage_backend: Default::default(), }); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); @@ -440,7 +440,7 @@ pub fn create_engine_three_metrics( storage_backend: Default::default(), }); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); @@ -521,7 +521,7 @@ pub fn create_engine_multi_timestamp( storage_backend: Default::default(), }); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); @@ -602,7 +602,7 @@ pub fn create_engine_multi_timestamp_with_window( storage_backend: Default::default(), }); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); diff --git a/data_plane/tests/inference_yaml_pattern_coverage.rs b/data_plane/tests/inference_yaml_pattern_coverage.rs index b8625a31..f8562ba1 100644 --- a/data_plane/tests/inference_yaml_pattern_coverage.rs +++ b/data_plane/tests/inference_yaml_pattern_coverage.rs @@ -34,7 +34,7 @@ use data_plane::engines::ASAPQueryEngine; use data_plane::precompute_engine::operators::{ DDSketchAccumulator, DatasketchesKLLAccumulator, IncreaseAccumulator, SumAccumulator, }; -use data_plane::stores::SimpleMapStore; +use data_plane::stores::SketchStore; use data_plane::stores::Store; use data_plane::utils::file_io::read_inference_config; use data_plane::AggregateCore; @@ -157,7 +157,7 @@ fn build_engine( aggregation_configs, storage_backend: Default::default(), }); - let store = Arc::new(SimpleMapStore::new( + let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); From 347246a34c34c9b2dcd4a90fbf128c939912f496 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 12 May 2026 10:43:35 -0600 Subject: [PATCH 3/7] =?UTF-8?q?refactor(data=5Fplane):=20delete=20promethe?= =?UTF-8?q?us=5Fquery=5Fengine,=20reshape=20sketch=5Fdb,=20rename=20stores?= =?UTF-8?q?/schema=20=E2=86=92=20stores/types?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three independent cleanups on top of the prior reorg: 1) Delete `query_engines/prometheus_query_engine/` (the Phase-ε.2 HTTP-forwarder to Prometheus's `/api/v1/query`). No production call sites consume it; the http-handler's own `--forward-unsupported-queries` fallback is unaffected. The `StorageBackend::PrometheusRemote` routing identifier stays in `crates/asap_types` so controller-emitted configs targeting it surface a clean 503 NoEngineRegistered (already the current behaviour when the env var was unset). 2) Group `sketch_db/` files by concern into 4 subdirs: sketch_db/ accuracy.rs single-file concern, stays flat metrics.rs Prometheus exposition, stays flat schema/ per-`agg_id` lifecycle mod.rs (was schema.rs) eviction.rs (was schema_eviction.rs) index/ two-level sid index mod.rs (was sketch_index.rs) epoch_columnar.rs (storage layer used by SketchIndex) backfill/ job lifecycle + worker pool + readers mod.rs (was backfill.rs) processor.rs service.rs window_builder.rs worker.rs prometheus_reader.rs (BackfillSource::Prometheus impl) raw_sample_reader.rs (RawSampleReader trait + mock) store/ physical Store impl mod.rs (SketchStore enum, was sketch_store/) common.rs global.rs per_key.rs persistence/ tightly cohesive: cache + flusher + manifest + part format + recovery + source 3) Rename `stores/schema/` → `stores/types/` to remove the name clash with the new `sketch_db/schema/`. The two are unrelated concerns: `stores/types/` holds wire-format / config / data-model types used cross-module (drivers, precompute_engine, query_engines, AND sketch_db); `sketch_db/schema/` holds per-`agg_id` lifecycle (AggSchema, AggStatus, SchemaRegistry). Also delete the 6 tiny `pub use asap_types::X` shim files (aggregation_config.rs, aggregation_reference.rs, inference_config.rs, promql_schema.rs, query_config.rs, streaming_config.rs) — the equivalent re-exports are now consolidated in `stores/types/mod.rs`. Test counts: - data_plane lib: 792 passed / 2 pre-existing failures / 4 ignored (down from 804 due to deletion of 12 unit tests inside the removed prometheus_query_engine/forward.rs; documented failures unchanged) - controller lib: 710/710; bins: 27/27 Co-Authored-By: Claude Opus 4.7 (1M context) --- data_plane/src/drivers/ingest/kafka.rs | 12 +- data_plane/src/drivers/ingest/otel.rs | 32 +- .../src/drivers/query/adapters/config.rs | 2 +- .../src/drivers/query/adapters/factory.rs | 2 +- .../drivers/query/adapters/prometheus_http.rs | 2 +- data_plane/src/drivers/query/servers/http.rs | 108 +-- data_plane/src/lib.rs | 2 +- data_plane/src/main.rs | 18 +- .../precompute_engine/accumulator_factory.rs | 2 +- data_plane/src/precompute_engine/engine.rs | 4 +- .../src/precompute_engine/ingest_handler.rs | 14 +- .../operators/count_min_sketch_accumulator.rs | 6 +- .../count_min_sketch_with_heap_accumulator.rs | 4 +- .../operators/count_sketch_accumulator.rs | 2 +- .../operators/datasketches_kll_accumulator.rs | 6 +- .../operators/dd_sketch_accumulator.rs | 2 +- .../delta_set_aggregator_accumulator.rs | 4 +- .../operators/hll_sketch_accumulator.rs | 2 +- .../operators/hydra_kll_accumulator.rs | 4 +- .../operators/increase_accumulator.rs | 4 +- .../operators/min_max_accumulator.rs | 4 +- .../multiple_increase_accumulator.rs | 8 +- .../operators/multiple_min_max_accumulator.rs | 4 +- .../operators/multiple_sum_accumulator.rs | 4 +- .../operators/set_aggregator_accumulator.rs | 4 +- .../operators/sketch_envelope_accumulator.rs | 6 +- .../operators/sum_accumulator.rs | 4 +- .../src/precompute_engine/output_sink.rs | 2 +- .../src/precompute_engine/series_router.rs | 2 +- data_plane/src/precompute_engine/worker.rs | 14 +- .../query_engines/asap_query_engine/engine.rs | 88 +- data_plane/src/query_engines/mod.rs | 8 - .../prometheus_query_engine/forward.rs | 896 ------------------ .../prometheus_query_engine/mod.rs | 27 - data_plane/src/query_engines/query_result.rs | 2 +- .../routing/backend_storage_routing.rs | 2 +- .../thanos_query_engine/forward.rs | 2 +- .../query_engines/warm_tier/delta_apply.rs | 2 +- data_plane/src/query_engines/warm_tier/mod.rs | 4 +- .../query_engines/warm_tier/sketch_reducer.rs | 2 +- .../src/query_engines/warm_tier/tests.rs | 2 +- data_plane/src/query_engines/window_merger.rs | 4 +- .../src/stores/gorilla_object_store/mod.rs | 2 +- data_plane/src/stores/mod.rs | 15 +- .../src/stores/schema/aggregation_config.rs | 1 - .../stores/schema/aggregation_reference.rs | 1 - .../src/stores/schema/inference_config.rs | 1 - data_plane/src/stores/schema/mod.rs | 35 - data_plane/src/stores/schema/promql_schema.rs | 1 - data_plane/src/stores/schema/query_config.rs | 1 - .../src/stores/schema/streaming_config.rs | 1 - .../{backfill.rs => backfill/mod.rs} | 21 + .../processor.rs} | 32 +- .../{ => backfill}/prometheus_reader.rs | 0 .../{ => backfill}/raw_sample_reader.rs | 0 .../service.rs} | 22 +- .../window_builder.rs} | 4 +- .../worker.rs} | 2 +- .../sketch_db/{ => index}/epoch_columnar.rs | 0 .../{sketch_index.rs => index/mod.rs} | 6 +- data_plane/src/stores/sketch_db/mod.rs | 37 +- .../eviction.rs} | 10 +- .../sketch_db/{schema.rs => schema/mod.rs} | 8 +- .../{sketch_store => store}/INDEX_DESIGN.md | 0 .../{sketch_store => store}/common.rs | 2 +- .../{sketch_store => store}/global.rs | 4 +- .../sketch_db/{sketch_store => store}/mod.rs | 4 +- .../{sketch_store => store}/per_key.rs | 4 +- .../persistence/cache.rs | 0 .../persistence/config.rs | 0 .../persistence/flusher.rs | 4 +- .../persistence/manifest.rs | 0 .../persistence/mod.rs | 0 .../persistence/part.rs | 8 +- .../persistence/recovery.rs | 8 +- .../persistence/source.rs | 2 +- data_plane/src/stores/traits.rs | 2 +- .../src/stores/{schema => types}/enums.rs | 0 .../{schema => types}/hot_reload_config.rs | 4 +- .../{schema => types}/key_by_label_values.rs | 0 .../stores/{schema => types}/measurement.rs | 0 data_plane/src/stores/types/mod.rs | 39 + .../{schema => types}/precomputed_output.rs | 16 +- .../src/stores/{schema => types}/traits.rs | 2 +- .../src/tests/capability_matching_tests.rs | 4 +- .../tests/capability_miss_http_e2e_tests.rs | 6 +- .../tests/persist_format_versioning_tests.rs | 12 +- .../tests/persistence_integration_tests.rs | 6 +- .../src/tests/persistence_perf_tests.rs | 6 +- .../src/tests/prometheus_forwarding_tests.rs | 10 +- .../tests/schema_timeline_dispatch_tests.rs | 4 +- .../src/tests/store_correctness_tests.rs | 2 +- .../src/tests/test_utilities/comparison.rs | 2 +- .../tests/test_utilities/engine_factories.rs | 4 +- data_plane/src/tests/trait_design_tests.rs | 2 +- data_plane/src/utils/file_io.rs | 4 +- data_plane/src/utils/http.rs | 2 +- data_plane/src/utils/precompute_dumper.rs | 2 +- .../tests/e2e_modified_otlp_sketch_path.rs | 14 +- .../edge_runtime_consumes_precompute_rs.rs | 2 +- .../tests/inference_yaml_pattern_coverage.rs | 2 +- 101 files changed, 394 insertions(+), 1312 deletions(-) delete mode 100644 data_plane/src/query_engines/prometheus_query_engine/forward.rs delete mode 100644 data_plane/src/query_engines/prometheus_query_engine/mod.rs delete mode 100644 data_plane/src/stores/schema/aggregation_config.rs delete mode 100644 data_plane/src/stores/schema/aggregation_reference.rs delete mode 100644 data_plane/src/stores/schema/inference_config.rs delete mode 100644 data_plane/src/stores/schema/mod.rs delete mode 100644 data_plane/src/stores/schema/promql_schema.rs delete mode 100644 data_plane/src/stores/schema/query_config.rs delete mode 100644 data_plane/src/stores/schema/streaming_config.rs rename data_plane/src/stores/sketch_db/{backfill.rs => backfill/mod.rs} (98%) rename data_plane/src/stores/sketch_db/{backfill_processor.rs => backfill/processor.rs} (96%) rename data_plane/src/stores/sketch_db/{ => backfill}/prometheus_reader.rs (100%) rename data_plane/src/stores/sketch_db/{ => backfill}/raw_sample_reader.rs (100%) rename data_plane/src/stores/sketch_db/{backfill_service.rs => backfill/service.rs} (96%) rename data_plane/src/stores/sketch_db/{backfill_window_builder.rs => backfill/window_builder.rs} (98%) rename data_plane/src/stores/sketch_db/{backfill_worker.rs => backfill/worker.rs} (99%) rename data_plane/src/stores/sketch_db/{ => index}/epoch_columnar.rs (100%) rename data_plane/src/stores/sketch_db/{sketch_index.rs => index/mod.rs} (98%) rename data_plane/src/stores/sketch_db/{schema_eviction.rs => schema/eviction.rs} (97%) rename data_plane/src/stores/sketch_db/{schema.rs => schema/mod.rs} (99%) rename data_plane/src/stores/sketch_db/{sketch_store => store}/INDEX_DESIGN.md (100%) rename data_plane/src/stores/sketch_db/{sketch_store => store}/common.rs (99%) rename data_plane/src/stores/sketch_db/{sketch_store => store}/global.rs (99%) rename data_plane/src/stores/sketch_db/{sketch_store => store}/mod.rs (99%) rename data_plane/src/stores/sketch_db/{sketch_store => store}/per_key.rs (99%) rename data_plane/src/stores/sketch_db/{sketch_store => store}/persistence/cache.rs (100%) rename data_plane/src/stores/sketch_db/{sketch_store => store}/persistence/config.rs (100%) rename data_plane/src/stores/sketch_db/{sketch_store => store}/persistence/flusher.rs (99%) rename data_plane/src/stores/sketch_db/{sketch_store => store}/persistence/manifest.rs (100%) rename data_plane/src/stores/sketch_db/{sketch_store => store}/persistence/mod.rs (100%) rename data_plane/src/stores/sketch_db/{sketch_store => store}/persistence/part.rs (98%) rename data_plane/src/stores/sketch_db/{sketch_store => store}/persistence/recovery.rs (96%) rename data_plane/src/stores/sketch_db/{sketch_store => store}/persistence/source.rs (98%) rename data_plane/src/stores/{schema => types}/enums.rs (100%) rename data_plane/src/stores/{schema => types}/hot_reload_config.rs (98%) rename data_plane/src/stores/{schema => types}/key_by_label_values.rs (100%) rename data_plane/src/stores/{schema => types}/measurement.rs (100%) create mode 100644 data_plane/src/stores/types/mod.rs rename data_plane/src/stores/{schema => types}/precomputed_output.rs (97%) rename data_plane/src/stores/{schema => types}/traits.rs (99%) diff --git a/data_plane/src/drivers/ingest/kafka.rs b/data_plane/src/drivers/ingest/kafka.rs index ce962a9b..259695d9 100644 --- a/data_plane/src/drivers/ingest/kafka.rs +++ b/data_plane/src/drivers/ingest/kafka.rs @@ -5,10 +5,10 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use tracing::{debug, error, info, warn}; -use crate::stores::schema::enums::{InputFormat, StreamingEngine}; -use crate::stores::schema::traits::SerializableToSink; -use crate::stores::schema::PrecomputedOutput; -use crate::stores::schema::StreamingConfig; +use crate::stores::types::enums::{InputFormat, StreamingEngine}; +use crate::stores::types::traits::SerializableToSink; +use crate::stores::types::PrecomputedOutput; +use crate::stores::types::StreamingConfig; use crate::stores::Store; use crate::utils::PrecomputeDumper; @@ -182,7 +182,7 @@ impl KafkaConsumer { async fn process_batch( &self, - batch: &mut Vec<(PrecomputedOutput, Box)>, + batch: &mut Vec<(PrecomputedOutput, Box)>, ) -> Result<(), Box> { if batch.is_empty() { return Ok(()); @@ -230,7 +230,7 @@ impl KafkaConsumer { &self, message: &rdkafka::message::BorrowedMessage<'_>, ) -> Result< - Option<(PrecomputedOutput, Box)>, + Option<(PrecomputedOutput, Box)>, Box, > { let message_start_time = Instant::now(); diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 9322c661..303a9c22 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -24,7 +24,7 @@ use std::collections::HashMap; use std::io::Read; -use crate::stores::schema::AggregateCore; +use crate::stores::types::AggregateCore; use crate::precompute_engine::series_router::WorkerMessage; use crate::precompute_engine::IngestState; use crate::precompute_engine::operators::sketch_envelope_accumulator::SketchEnvelopeAccumulator; @@ -721,7 +721,7 @@ async fn route_modified_otlp_sketches_to_precompute( // then route each tuple through the same dispatcher. let dps: Vec = match &metric.data { Some(Data::Ddsketch(d)) => { - let cfg = crate::stores::sketch_db::sketch_index::SketchConfig::DDSketch { + let cfg = crate::stores::sketch_db::index::SketchConfig::DDSketch { relative_accuracy: d.relative_accuracy, }; d.data_points @@ -739,7 +739,7 @@ async fn route_modified_otlp_sketches_to_precompute( .collect() } Some(Data::Kllsketch(k)) => { - let cfg = crate::stores::sketch_db::sketch_index::SketchConfig::Kll { k: k.k }; + let cfg = crate::stores::sketch_db::index::SketchConfig::Kll { k: k.k }; k.data_points .iter() .map(|dp| ModifiedOtlpSketchDp { @@ -755,7 +755,7 @@ async fn route_modified_otlp_sketches_to_precompute( .collect() } Some(Data::Countsketch(c)) => { - let cfg = crate::stores::sketch_db::sketch_index::SketchConfig::CountSketch { + let cfg = crate::stores::sketch_db::index::SketchConfig::CountSketch { rows: c.rows, cols: c.cols, }; @@ -774,7 +774,7 @@ async fn route_modified_otlp_sketches_to_precompute( .collect() } Some(Data::Countminsketch(c)) => { - let cfg = crate::stores::sketch_db::sketch_index::SketchConfig::CountMin { + let cfg = crate::stores::sketch_db::index::SketchConfig::CountMin { rows: c.rows, cols: c.cols, }; @@ -793,7 +793,7 @@ async fn route_modified_otlp_sketches_to_precompute( .collect() } Some(Data::Hllsketch(h)) => { - let cfg = crate::stores::sketch_db::sketch_index::SketchConfig::Hll { + let cfg = crate::stores::sketch_db::index::SketchConfig::Hll { precision: h.precision, }; h.data_points @@ -879,7 +879,7 @@ async fn route_modified_otlp_sketches_to_precompute( // rollup, `attributes` is the group-by VALUES vector, // and its key set IS the group-by KEY set. { - use crate::stores::sketch_db::sketch_index::{ + use crate::stores::sketch_db::index::{ AccuracyBound, Capability, SketchEncoding, SketchInstanceMetadata, SketchKindHandle, SketchSampleState, }; @@ -942,7 +942,7 @@ async fn route_modified_otlp_sketches_to_precompute( .iter() .map(|(k, v)| (k.clone(), v.clone())) .collect(); - let window: crate::stores::sketch_db::epoch_columnar::TimestampRange = ( + let window: crate::stores::sketch_db::index::epoch_columnar::TimestampRange = ( dp.start_time_unix_nano / 1_000_000, dp.time_unix_nano / 1_000_000, ); @@ -1114,8 +1114,8 @@ async fn route_modified_otlp_sketches_to_precompute( /// directly for `topk` / `topk_over_time` queries. fn sketch_kind_handle_for( dp: &ModifiedOtlpSketchDp, -) -> crate::stores::sketch_db::sketch_index::SketchKindHandle { - use crate::stores::sketch_db::sketch_index::SketchKindHandle; +) -> crate::stores::sketch_db::index::SketchKindHandle { + use crate::stores::sketch_db::index::SketchKindHandle; match dp.kind { SketchKind::DdSketch => SketchKindHandle::DDSketch, SketchKind::Kll => SketchKindHandle::Kll, @@ -1146,8 +1146,8 @@ fn sketch_kind_handle_for( /// SketchIndex's `SketchEncoding` enum. Returns `None` for the unset /// (0) encoding so callers can default to `ProtoFull` (the dominant /// case for full-state frames). -fn encoding_to_handle(encoding: i32) -> Option { - use crate::stores::sketch_db::sketch_index::SketchEncoding; +fn encoding_to_handle(encoding: i32) -> Option { + use crate::stores::sketch_db::index::SketchEncoding; match encoding { ENCODING_PROTO => Some(SketchEncoding::ProtoFull), ENCODING_PROTO_DELTA => Some(SketchEncoding::ProtoDelta), @@ -1189,7 +1189,7 @@ struct ModifiedOtlpSketchDp { /// Phase 5 — sketch-instance configuration lifted off the parent /// container. Drives `SketchInstanceMetadata.sketch_config` and the /// derived `AccuracyBound`. - container_config: crate::stores::sketch_db::sketch_index::SketchConfig, + container_config: crate::stores::sketch_db::index::SketchConfig, } /// Decode the typed `sketch` bytes from a modified-OTLP @@ -1763,7 +1763,7 @@ fn attributes_to_map( #[cfg(test)] mod dispatcher_tests { use super::*; - use crate::stores::schema::AggregateCore; + use crate::stores::types::AggregateCore; use crate::precompute_engine::operators::{DDSketchAccumulator, HllSketchAccumulator}; use asap_sketchlib::sketches::ddsketch::DdSketch; use asap_sketchlib::sketches::hll::HllVariant; @@ -1888,11 +1888,11 @@ mod dispatcher_tests { #[cfg(test)] mod sid_resolution_tests { use super::*; - use crate::stores::schema::{HotReloadStreamingConfig, StreamingConfig}; + use crate::stores::types::{HotReloadStreamingConfig, StreamingConfig}; use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::series_router::SeriesRouter; use crate::stores::sketch_db::SchemaRegistry; - use crate::stores::sketch_db::sketch_index::SketchIndex; + use crate::stores::sketch_db::index::SketchIndex; use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; use asap_otel_proto::tonic::common::v1::{any_value::Value as AnyVal, AnyValue, KeyValue}; use asap_otel_proto::tonic::metrics::v1::{ diff --git a/data_plane/src/drivers/query/adapters/config.rs b/data_plane/src/drivers/query/adapters/config.rs index 2562c9cd..a4693d72 100644 --- a/data_plane/src/drivers/query/adapters/config.rs +++ b/data_plane/src/drivers/query/adapters/config.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::enums::{QueryLanguage, QueryProtocol}; +use crate::stores::types::enums::{QueryLanguage, QueryProtocol}; use crate::drivers::query::fallback::FallbackClient; use std::sync::Arc; diff --git a/data_plane/src/drivers/query/adapters/factory.rs b/data_plane/src/drivers/query/adapters/factory.rs index e28aed42..9e4464bd 100644 --- a/data_plane/src/drivers/query/adapters/factory.rs +++ b/data_plane/src/drivers/query/adapters/factory.rs @@ -1,7 +1,7 @@ use super::config::AdapterConfig; use super::prometheus_http::PrometheusHttpAdapter; use super::traits::HttpProtocolAdapter; -use crate::stores::schema::enums::QueryProtocol; +use crate::stores::types::enums::QueryProtocol; use std::sync::Arc; /// Factory function to create appropriate HTTP adapter based on protocol diff --git a/data_plane/src/drivers/query/adapters/prometheus_http.rs b/data_plane/src/drivers/query/adapters/prometheus_http.rs index 0cd98d0b..fabebcab 100644 --- a/data_plane/src/drivers/query/adapters/prometheus_http.rs +++ b/data_plane/src/drivers/query/adapters/prometheus_http.rs @@ -411,7 +411,7 @@ impl HttpProtocolAdapter for PrometheusHttpAdapter { #[cfg(test)] mod tests { use super::*; - use crate::stores::schema::enums::{QueryLanguage, QueryProtocol}; + use crate::stores::types::enums::{QueryLanguage, QueryProtocol}; fn create_test_adapter() -> PrometheusHttpAdapter { let config = AdapterConfig::new(QueryProtocol::PrometheusHttp, QueryLanguage::promql, None); diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 8a151c62..aa66d0b0 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -163,7 +163,7 @@ pub struct HttpServer { store: Arc, /// Hot-reloadable `StreamingConfig` source. `None` when hot-reload /// is not wired up by the caller (unit tests, legacy binaries). - hot_reload_config: Option, + hot_reload_config: Option, /// Per-metric storage-backend routing table consulted by the HTTP /// instant-query handler at request time. When `Some(..)` and the /// query parses, the handler extracts the metric name from the @@ -228,7 +228,7 @@ struct AppState { store: Arc, adapter: Arc, fallback: Option>, - hot_reload_config: Option, + hot_reload_config: Option, /// See [`HttpServer::backend_storage_routing`]. backend_storage_routing: Option, /// Per-`agg_id` schema registry (sketch DB §6). Phase 2b wires @@ -308,7 +308,7 @@ impl HttpServer { /// endpoints return `503 Service Unavailable`. pub fn with_hot_reload_config( mut self, - handle: crate::stores::schema::HotReloadStreamingConfig, + handle: crate::stores::types::HotReloadStreamingConfig, ) -> Self { self.hot_reload_config = Some(handle); self @@ -333,7 +333,7 @@ impl HttpServer { /// `StreamingConfig::storage_backend()`. pub fn with_backend_storage_routing( mut self, - routing: Arc, + routing: Arc, ) -> Self { self.backend_storage_routing = Some( crate::query_engines::routing::HotReloadBackendStorageRouting::from_arc(routing), @@ -740,7 +740,7 @@ async fn process_query_request( /// /// v7: when the routing table has multi-target rows for the metric, /// the parsed AST is also classified via -/// [`crate::stores::schema::classify_query_shape`] and the lookup picks +/// [`crate::stores::types::classify_query_shape`] and the lookup picks /// the target whose `applies_to_query_shape` matches. v6.1 /// single-target metrics keep their original semantics — every shape /// resolves to the one configured backend. @@ -757,7 +757,7 @@ fn resolve_metric_storage(state: &AppState, query: &str, tenant: &str) -> Storag match promql_parser::parser::parse(query) { Ok(expr) => { if let Some(metric_name) = first_metric_name(&expr) { - let shape = crate::stores::schema::classify_query_shape(&expr); + let shape = crate::stores::types::classify_query_shape(&expr); let backend = routing.lookup_with_shape(&metric_name, shape); debug!( "resolve_metric_storage: routing-table hit for tenant={} metric={} shape={:?} → {:?}", @@ -858,7 +858,7 @@ async fn try_answer_freshness_probe( ); let element = - InstantVectorElement::new(crate::stores::schema::KeyByLabelValues::new(), sample.value); + InstantVectorElement::new(crate::stores::types::KeyByLabelValues::new(), sample.value); // The instant-vector timestamp is unix milliseconds — match the // adapter's expectations downstream (the Prometheus adapter // divides by 1000 to render the wire `value: [, ...]`). @@ -1806,9 +1806,9 @@ async fn handle_range_query_post(State(state): State, body: Bytes) -> #[cfg(test)] mod tests { use super::*; - use crate::stores::schema::{HotReloadStreamingConfig, InferenceConfig, StreamingConfig}; + use crate::stores::types::{HotReloadStreamingConfig, InferenceConfig, StreamingConfig}; use crate::query_engines::ASAPQueryEngine; - use crate::stores::sketch_db::sketch_store::SketchStore; + use crate::stores::sketch_db::store::SketchStore; use reqwest::Client; use std::sync::Arc; @@ -1831,13 +1831,13 @@ mod tests { }; let inference_config = InferenceConfig::new( - crate::stores::schema::QueryLanguage::promql, - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::QueryLanguage::promql, + crate::stores::types::CleanupPolicy::NoCleanup, ); let streaming_config = Arc::new(StreamingConfig::default()); let store = Arc::new(SketchStore::new( streaming_config.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), @@ -1845,7 +1845,7 @@ mod tests { inference_config, streaming_config.clone(), 15000, - crate::stores::schema::QueryLanguage::promql, + crate::stores::types::QueryLanguage::promql, )); let mut server = HttpServer::new(config, query_engine, store); @@ -2088,20 +2088,20 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::stores::schema::QueryLanguage::promql, - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::QueryLanguage::promql, + crate::stores::types::CleanupPolicy::NoCleanup, ); let streaming_config = Arc::new(StreamingConfig::default()); let store = Arc::new(SketchStore::new( streaming_config.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_config.clone(), 15000, - crate::stores::schema::QueryLanguage::promql, + crate::stores::types::QueryLanguage::promql, )); let server = HttpServer::new(config, query_engine, store) .with_hot_reload_config(hot_reload) @@ -2571,20 +2571,20 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::stores::schema::QueryLanguage::promql, - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::QueryLanguage::promql, + crate::stores::types::CleanupPolicy::NoCleanup, ); let streaming_config = Arc::new(StreamingConfig::default()); let store = Arc::new(SketchStore::new( streaming_config.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_config.clone(), 15000, - crate::stores::schema::QueryLanguage::promql, + crate::stores::types::QueryLanguage::promql, )); let schemas = { use asap_types::aggregation_config::AggregationConfig; @@ -2965,8 +2965,8 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::stores::schema::QueryLanguage::promql, - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::QueryLanguage::promql, + crate::stores::types::CleanupPolicy::NoCleanup, ); // Pin `storage_backend` on the streaming config so the http // dispatcher reads it back through the hot-reload handle. @@ -2976,14 +2976,14 @@ aggregations: let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); let store = Arc::new(SketchStore::new( streaming_arc.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_arc, 15000, - crate::stores::schema::QueryLanguage::promql, + crate::stores::types::QueryLanguage::promql, )); let mut server = HttpServer::new(config, query_engine, store).with_hot_reload_config(hot_reload); @@ -3007,7 +3007,7 @@ aggregations: /// `setup_test_server_with_router` helper above which mocks the /// resolution by pinning `streaming_cfg.storage_backend` directly. async fn setup_test_server_with_routing_table( - routing: crate::stores::schema::BackendStorageRouting, + routing: crate::stores::types::BackendStorageRouting, extra_engines: Vec>, ) -> u16 { let adapter_config = @@ -3018,8 +3018,8 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::stores::schema::QueryLanguage::promql, - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::QueryLanguage::promql, + crate::stores::types::CleanupPolicy::NoCleanup, ); // Streaming-config stays on the default `SketchStore` axis // — exactly what the production deploy looks like (the YAML @@ -3030,14 +3030,14 @@ aggregations: let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); let store = Arc::new(SketchStore::new( streaming_arc.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_arc, 15000, - crate::stores::schema::QueryLanguage::promql, + crate::stores::types::QueryLanguage::promql, )); let mut server = HttpServer::new(config, query_engine, store) .with_hot_reload_config(hot_reload) @@ -3334,7 +3334,7 @@ aggregations: "http_requests_total".to_string(), StorageBackend::GorillaObjectStore, ); - let routing = crate::stores::schema::BackendStorageRouting::new_from_single_targets( + let routing = crate::stores::types::BackendStorageRouting::new_from_single_targets( StorageBackend::SketchStore, metrics, ); @@ -3377,7 +3377,7 @@ aggregations: "http_requests_total".to_string(), StorageBackend::GorillaObjectStore, ); - let routing = crate::stores::schema::BackendStorageRouting::new_from_single_targets( + let routing = crate::stores::types::BackendStorageRouting::new_from_single_targets( StorageBackend::SketchStore, metrics, ); @@ -3407,7 +3407,7 @@ aggregations: // top-level `default: thanos_query` — every metric must // route through the router. Pins the §8 "all-metrics-archive" // deploy mode. - let routing = crate::stores::schema::BackendStorageRouting::new_from_single_targets( + let routing = crate::stores::types::BackendStorageRouting::new_from_single_targets( StorageBackend::GorillaObjectStore, std::collections::HashMap::new(), ); @@ -3445,7 +3445,7 @@ aggregations: #[tokio::test] async fn http_v7_dual_routing_count_lands_on_archive() { - use crate::stores::schema::{BackendStorageRouting, QueryShape, RoutingTarget}; + use crate::stores::types::{BackendStorageRouting, QueryShape, RoutingTarget}; let mut metrics = std::collections::HashMap::new(); metrics.insert( "http_requests_total".to_string(), @@ -3491,7 +3491,7 @@ aggregations: #[tokio::test] async fn http_v7_dual_routing_quantile_stays_on_warm_tier() { - use crate::stores::schema::{BackendStorageRouting, QueryShape, RoutingTarget}; + use crate::stores::types::{BackendStorageRouting, QueryShape, RoutingTarget}; let mut metrics = std::collections::HashMap::new(); metrics.insert( "http_requests_total".to_string(), @@ -3561,7 +3561,7 @@ aggregations: /// shape-classifier and dispatches to the explicitly named engine. #[tokio::test] async fn http_engine_override_header_routes_to_named_engine() { - use crate::stores::schema::{BackendStorageRouting, QueryShape, RoutingTarget}; + use crate::stores::types::{BackendStorageRouting, QueryShape, RoutingTarget}; let (gorilla, gorilla_calls) = MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); @@ -3619,7 +3619,7 @@ aggregations: /// backwards-compatible. #[tokio::test] async fn http_engine_override_missing_uses_default_routing() { - use crate::stores::schema::{BackendStorageRouting, QueryShape, RoutingTarget}; + use crate::stores::types::{BackendStorageRouting, QueryShape, RoutingTarget}; let (gorilla, gorilla_calls) = MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); @@ -3767,7 +3767,7 @@ aggregations: /// the test can introspect the swap result. async fn setup_test_server_for_storage_routing( ) -> (u16, crate::query_engines::routing::HotReloadBackendStorageRouting) { - use crate::stores::schema::{HotReloadStreamingConfig, StreamingConfig}; + use crate::stores::types::{HotReloadStreamingConfig, StreamingConfig}; use crate::query_engines::routing::HotReloadBackendStorageRouting; let adapter_config = @@ -3778,22 +3778,22 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::stores::schema::QueryLanguage::promql, - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::QueryLanguage::promql, + crate::stores::types::CleanupPolicy::NoCleanup, ); let streaming_cfg = StreamingConfig::default(); let streaming_arc = Arc::new(streaming_cfg); let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); let store = Arc::new(SketchStore::new( streaming_arc.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_arc, 15000, - crate::stores::schema::QueryLanguage::promql, + crate::stores::types::QueryLanguage::promql, )); let routing_handle = HotReloadBackendStorageRouting::empty(); let server = HttpServer::new(config, query_engine, store) @@ -3904,7 +3904,7 @@ aggregations: let (port, handle) = setup_test_server_for_storage_routing().await; // Pre-load the table. let new = - crate::stores::schema::BackendStorageRouting::from_json_payload(&fixture_routing_json()) + crate::stores::types::BackendStorageRouting::from_json_payload(&fixture_routing_json()) .expect("parse"); handle.swap(new); @@ -4074,13 +4074,13 @@ aggregations: // the very next read. let snap = handle.snapshot(); assert_eq!( - snap.lookup_with_shape("http_requests_total", crate::stores::schema::QueryShape::Count,), + snap.lookup_with_shape("http_requests_total", crate::stores::types::QueryShape::Count,), StorageBackend::GorillaObjectStore, ); assert_eq!( snap.lookup_with_shape( "http_requests_total", - crate::stores::schema::QueryShape::Quantile, + crate::stores::types::QueryShape::Quantile, ), StorageBackend::SketchStore, ); @@ -4249,8 +4249,8 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::stores::schema::QueryLanguage::promql, - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::QueryLanguage::promql, + crate::stores::types::CleanupPolicy::NoCleanup, ); let streaming_cfg = StreamingConfig::with_storage_backend(Default::default(), metric_storage_backend); @@ -4258,14 +4258,14 @@ aggregations: let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); let store = Arc::new(SketchStore::new( streaming_arc.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_arc, 15000, - crate::stores::schema::QueryLanguage::promql, + crate::stores::types::QueryLanguage::promql, )); let mut server = HttpServer::new(config, query_engine, store).with_hot_reload_config(hot_reload); @@ -4308,20 +4308,20 @@ aggregations: adapter_config, }; let inference_config = InferenceConfig::new( - crate::stores::schema::QueryLanguage::promql, - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::QueryLanguage::promql, + crate::stores::types::CleanupPolicy::NoCleanup, ); let streaming_arc = Arc::new(StreamingConfig::default()); let store = Arc::new(SketchStore::new( streaming_arc.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let query_engine = Arc::new(ASAPQueryEngine::new( store.clone(), inference_config, streaming_arc, 15000, - crate::stores::schema::QueryLanguage::promql, + crate::stores::types::QueryLanguage::promql, )); let cache = Arc::new(crate::query_engines::routing::FreshnessProbeCache::new()); let server = HttpServer::new(config, query_engine, store).with_probe_cache(cache.clone()); @@ -4930,7 +4930,7 @@ async fn handle_post_storage_routing( return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); } }; - let new_table = match crate::stores::schema::BackendStorageRouting::from_json_payload(&json_value) { + let new_table = match crate::stores::types::BackendStorageRouting::from_json_payload(&json_value) { Ok(t) => t, Err(e) => { let body = serde_json::json!({ diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index 8d01ecbf..4c19257f 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -8,7 +8,7 @@ pub mod tests; pub mod utils; // Re-export commonly used types to avoid glob import conflicts -pub use stores::schema::{ +pub use stores::types::{ AccumulatorFactory, AggregateCore, AggregationConfig, InferenceConfig, KeyByLabelValues, Measurement, MergeableAccumulator, MultipleSubpopulationAggregate, MultipleSubpopulationAggregateFactory, PrecomputedOutput, PromQLSchema, QueryConfig, diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 67ad3fa7..4db11783 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -11,16 +11,16 @@ // this same backend process — there is no longer a separate // `asap-controller` container in `mvp-multinode/run_demo.sh`. use clap::Parser; -use data_plane::stores::schema::QueryLanguage; +use data_plane::stores::types::QueryLanguage; use std::fs; use std::sync::Arc; use tokio::signal; use tracing::{error, info, warn}; -use data_plane::stores::schema::enums::{ +use data_plane::stores::types::enums::{ CleanupPolicy, InputFormat, LockStrategy, StreamingEngine, }; -use data_plane::stores::schema::InferenceConfig; +use data_plane::stores::types::InferenceConfig; use data_plane::drivers::AdapterConfig; use data_plane::precompute_engine::config::LateDataPolicy; use data_plane::precompute_engine::PrecomputeWorkerDiagnostics; @@ -362,14 +362,14 @@ async fn main() -> Result<()> { // control-plane GET/POST endpoint. Phase 2 will extend the swap // to query execution and ingest routing. let hot_reload_config = - data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config.clone()); + data_plane::stores::types::HotReloadStreamingConfig::from_arc(streaming_config.clone()); // Setup store (equivalent to Python's SketchStore()) // Get cleanup policy from inference config let cleanup_policy = inference_config.cleanup_policy; info!("Using cleanup policy: {:?}", cleanup_policy); let store = if args.persistence_enabled { - use data_plane::stores::sketch_db::sketch_store::persistence::SketchStorePersistenceConfig; + use data_plane::stores::sketch_db::store::persistence::SketchStorePersistenceConfig; let disk_path = args .persistence_dir .clone() @@ -443,7 +443,7 @@ async fn main() -> Result<()> { let series_resolver = Arc::new(data_plane::drivers::ingest::series_resolver::SeriesIdResolver::new()); let sketch_index = - Arc::new(data_plane::stores::sketch_db::sketch_index::SketchIndex::new()); + Arc::new(data_plane::stores::sketch_db::index::SketchIndex::new()); // Setup query engine. ASAPQueryEngine shares the same // HotReloadStreamingConfig handle as the HTTP server, so a POST @@ -697,7 +697,7 @@ async fn main() -> Result<()> { // dev / standalone — the YAML supplies the bootstrap, controller // pushes overwrite it. let bootstrap_routing = if let Some(routing_path) = args.backend_storage_routing.as_deref() { - match data_plane::stores::schema::BackendStorageRouting::from_yaml_file(routing_path) { + match data_plane::stores::types::BackendStorageRouting::from_yaml_file(routing_path) { Ok(routing) => { info!( "Loaded backend-storage-routing from {:?}: default={:?}, entries={}", @@ -712,14 +712,14 @@ async fn main() -> Result<()> { "Failed to load backend-storage-routing from {:?}: {} — installing an empty routing table; the controller's first POST /api/v1/storage_routing push will fill it", routing_path, e, ); - data_plane::stores::schema::BackendStorageRouting::empty() + data_plane::stores::types::BackendStorageRouting::empty() } } } else { info!( "--backend-storage-routing not set — installing an empty routing table; the controller's first POST /api/v1/storage_routing push will fill it", ); - data_plane::stores::schema::BackendStorageRouting::empty() + data_plane::stores::types::BackendStorageRouting::empty() }; server = server.with_backend_storage_routing(Arc::new(bootstrap_routing)); diff --git a/data_plane/src/precompute_engine/accumulator_factory.rs b/data_plane/src/precompute_engine/accumulator_factory.rs index da7ec56f..a6205876 100644 --- a/data_plane/src/precompute_engine/accumulator_factory.rs +++ b/data_plane/src/precompute_engine/accumulator_factory.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{AggregateCore, AggregationType, KeyByLabelValues, Measurement}; +use crate::stores::types::{AggregateCore, AggregationType, KeyByLabelValues, Measurement}; use crate::precompute_engine::operators::{ CountMinSketchAccumulator, DDSketchAccumulator, DatasketchesKLLAccumulator, HydraKllSketchAccumulator, IncreaseAccumulator, MinMaxAccumulator, MultipleIncreaseAccumulator, diff --git a/data_plane/src/precompute_engine/engine.rs b/data_plane/src/precompute_engine/engine.rs index 5a971cae..935f4afe 100644 --- a/data_plane/src/precompute_engine/engine.rs +++ b/data_plane/src/precompute_engine/engine.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::HotReloadStreamingConfig; +use crate::stores::types::HotReloadStreamingConfig; use crate::precompute_engine::config::PrecomputeEngineConfig; use crate::precompute_engine::ingest_handler::IngestState; use crate::precompute_engine::output_sink::OutputSink; @@ -39,7 +39,7 @@ impl PrecomputeEngine { hot_reload_config: HotReloadStreamingConfig, output_sink: Arc, series_resolver: Arc, - sketch_index: Arc, + sketch_index: Arc, ) -> Self { let worker_group_counts = (0..config.num_workers) .map(|_| Arc::new(AtomicUsize::new(0))) diff --git a/data_plane/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs index 87cce418..721b3f32 100644 --- a/data_plane/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::HotReloadStreamingConfig; +use crate::stores::types::HotReloadStreamingConfig; use crate::precompute_engine::series_router::SeriesRouter; use crate::precompute_engine::worker::parse_labels_from_series_key; use crate::stores::sketch_db::SchemaRegistry; @@ -52,7 +52,7 @@ pub struct IngestState { /// follow-up will add TTL-based eviction keyed by last-seen /// timestamp so long-running deployments don't leak memory /// on retired series. - pub sketch_snapshots: dashmap::DashMap>, + pub sketch_snapshots: dashmap::DashMap>, /// Phase 4 — centralized series_id resolver. Shared across the OTLP /// receive path (sid resolution + `unknown_series_ids` population) and /// the `ResolveSeriesIDs` RPC (eager batch resolution from the agent's @@ -64,7 +64,7 @@ pub struct IngestState { /// every modified-OTLP first-class sketch DataPoint; queried by /// the `ASAPQueryEngine` query path (warm-tier hit / ghost / unknown /// classification drives the Phase 6 archive failover). - pub sketch_index: Arc, + pub sketch_index: Arc, } impl IngestState { @@ -76,7 +76,7 @@ impl IngestState { /// Returns the shared `Arc` — no cloning of /// individual AggregationConfig objects, just an atomic refcount /// increment (~5ns). - pub fn config_snapshot(&self) -> Arc { + pub fn config_snapshot(&self) -> Arc { self.hot_reload_config.snapshot() } } @@ -123,7 +123,7 @@ fn extract_group_key(series_key: &str, config: &AggregationConfig) -> String { #[cfg(test)] mod tests { use super::*; - use crate::stores::schema::StreamingConfig; + use crate::stores::types::StreamingConfig; use crate::precompute_engine::series_router::SeriesRouter; use crate::stores::sketch_db::SchemaRegistry; use asap_types::aggregation_config::AggregationConfig; @@ -168,7 +168,7 @@ mod tests { let mut map = std::collections::HashMap::new(); map.insert(agg_id, make_config(agg_id, metric)); let streaming = StreamingConfig::new(map); - let hot_reload = crate::stores::schema::HotReloadStreamingConfig::new(streaming.clone()); + let hot_reload = crate::stores::types::HotReloadStreamingConfig::new(streaming.clone()); let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); @@ -183,7 +183,7 @@ mod tests { series_resolver: Arc::new( crate::drivers::ingest::series_resolver::SeriesIdResolver::new(), ), - sketch_index: Arc::new(crate::stores::sketch_db::sketch_index::SketchIndex::new()), + sketch_index: Arc::new(crate::stores::sketch_db::index::SketchIndex::new()), }); let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} }); diff --git a/data_plane/src/precompute_engine/operators/count_min_sketch_accumulator.rs b/data_plane/src/precompute_engine/operators/count_min_sketch_accumulator.rs index 22bf9c09..21fb3872 100644 --- a/data_plane/src/precompute_engine/operators/count_min_sketch_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/count_min_sketch_accumulator.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, }; @@ -265,7 +265,7 @@ impl CountMinSketchAccumulator { /// Merge multiple accumulators efficiently without cloning all of them. pub fn merge_multiple( - accumulators: &[Box], + accumulators: &[Box], ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); @@ -382,7 +382,7 @@ impl AggregateCore for CountMinSketchAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::stores::schema::MultipleSubpopulationAggregate; + use crate::stores::types::MultipleSubpopulationAggregate; use promql_utilities::query_logics::enums::Statistic; // Key-provided path: route to MultipleSubpopulationAggregate::query diff --git a/data_plane/src/precompute_engine/operators/count_min_sketch_with_heap_accumulator.rs b/data_plane/src/precompute_engine/operators/count_min_sketch_with_heap_accumulator.rs index 47d0329b..361c11fb 100644 --- a/data_plane/src/precompute_engine/operators/count_min_sketch_with_heap_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/count_min_sketch_with_heap_accumulator.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, }; @@ -187,7 +187,7 @@ impl AggregateCore for CountMinSketchWithHeapAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::stores::schema::MultipleSubpopulationAggregate; + use crate::stores::types::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for CountMinSketchWithHeapAccumulator")?; diff --git a/data_plane/src/precompute_engine/operators/count_sketch_accumulator.rs b/data_plane/src/precompute_engine/operators/count_sketch_accumulator.rs index dae59a30..f46d0844 100644 --- a/data_plane/src/precompute_engine/operators/count_sketch_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/count_sketch_accumulator.rs @@ -19,7 +19,7 @@ //! The wire format carries the matrix losslessly, so the merge + store //! round-trip works end-to-end without that richer query surface. -use crate::stores::schema::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; +use crate::stores::types::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; use asap_sketchlib::sketches::countsketch::{CountSketch, CountSketchDelta}; use serde_json::Value; use std::collections::HashMap; diff --git a/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs b/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs index 458be54f..a744d8a1 100644 --- a/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, AggregationType, AuxStats, MergeableAccumulator, SerializableToSink, SingleSubpopulationAggregate, }; @@ -155,7 +155,7 @@ impl DatasketchesKLLAccumulator { /// Merge multiple accumulators efficiently without cloning all of them. pub fn merge_multiple( - accumulators: &[Box], + accumulators: &[Box], ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); @@ -315,7 +315,7 @@ impl AggregateCore for DatasketchesKLLAccumulator { _key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::stores::schema::SingleSubpopulationAggregate; + use crate::stores::types::SingleSubpopulationAggregate; self.query(statistic, Some(query_kwargs)) } } diff --git a/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs b/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs index 904fd9d4..35dfe04a 100644 --- a/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/dd_sketch_accumulator.rs @@ -11,7 +11,7 @@ //! offset, and aggregates losslessly, so the merge + store round-trip //! works end-to-end without that richer query surface. -use crate::stores::schema::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; +use crate::stores::types::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; use asap_sketchlib::sketches::ddsketch::{DdSketch, DdSketchDelta}; use serde_json::Value; use std::collections::HashMap; diff --git a/data_plane/src/precompute_engine/operators/delta_set_aggregator_accumulator.rs b/data_plane/src/precompute_engine/operators/delta_set_aggregator_accumulator.rs index aa44fb15..d868ddfd 100644 --- a/data_plane/src/precompute_engine/operators/delta_set_aggregator_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/delta_set_aggregator_accumulator.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, }; @@ -268,7 +268,7 @@ impl AggregateCore for DeltaSetAggregatorAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::stores::schema::MultipleSubpopulationAggregate; + use crate::stores::types::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for DeltaSetAggregatorAccumulator")?; diff --git a/data_plane/src/precompute_engine/operators/hll_sketch_accumulator.rs b/data_plane/src/precompute_engine/operators/hll_sketch_accumulator.rs index 7777c5fc..2684f290 100644 --- a/data_plane/src/precompute_engine/operators/hll_sketch_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/hll_sketch_accumulator.rs @@ -11,7 +11,7 @@ //! registers + variant + HIP accumulators losslessly, so the merge + //! store round-trip works end-to-end without that richer query surface. -use crate::stores::schema::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; +use crate::stores::types::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; use asap_sketchlib::sketches::hll::{HllSketch, HllSketchDelta, HllVariant}; use serde_json::Value; use std::collections::HashMap; diff --git a/data_plane/src/precompute_engine/operators/hydra_kll_accumulator.rs b/data_plane/src/precompute_engine/operators/hydra_kll_accumulator.rs index 02bc2f8a..474966a0 100644 --- a/data_plane/src/precompute_engine/operators/hydra_kll_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/hydra_kll_accumulator.rs @@ -1,5 +1,5 @@ use crate::{ - stores::schema::{ + stores::types::{ AggregateCore, AggregationType, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, }, @@ -136,7 +136,7 @@ impl AggregateCore for HydraKllSketchAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::stores::schema::MultipleSubpopulationAggregate; + use crate::stores::types::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for HydraKllSketchAccumulator")?; diff --git a/data_plane/src/precompute_engine/operators/increase_accumulator.rs b/data_plane/src/precompute_engine/operators/increase_accumulator.rs index 78669cec..69ca75d2 100644 --- a/data_plane/src/precompute_engine/operators/increase_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/increase_accumulator.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, AggregationType, Measurement, MergeableAccumulator, SerializableToSink, SingleSubpopulationAggregate, SingleSubpopulationAggregateFactory, }; @@ -264,7 +264,7 @@ impl AggregateCore for IncreaseAccumulator { _key: &Option, _query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::stores::schema::SingleSubpopulationAggregate; + use crate::stores::types::SingleSubpopulationAggregate; self.query(statistic, None) } } diff --git a/data_plane/src/precompute_engine/operators/min_max_accumulator.rs b/data_plane/src/precompute_engine/operators/min_max_accumulator.rs index 4b6ff999..df990b2f 100644 --- a/data_plane/src/precompute_engine/operators/min_max_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/min_max_accumulator.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, AggregationType, AuxStats, MergeableAccumulator, SerializableToSink, SingleSubpopulationAggregate, SingleSubpopulationAggregateFactory, }; @@ -225,7 +225,7 @@ impl AggregateCore for MinMaxAccumulator { _key: &Option, _query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::stores::schema::SingleSubpopulationAggregate; + use crate::stores::types::SingleSubpopulationAggregate; self.query(statistic, None) } } diff --git a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs b/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs index 56ded8ff..49cd6a70 100644 --- a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, SingleSubpopulationAggregate, }; @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::HashMap; -use crate::stores::schema::Measurement; +use crate::stores::types::Measurement; use promql_utilities::query_logics::enums::Statistic; /// Accumulator that maintains separate increase accumulators for multiple keys @@ -303,7 +303,7 @@ impl AggregateCore for MultipleIncreaseAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::stores::schema::MultipleSubpopulationAggregate; + use crate::stores::types::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for MultipleIncreaseAccumulator")?; @@ -367,7 +367,7 @@ impl MergeableAccumulator for MultipleIncreaseAccum #[cfg(test)] mod tests { use super::*; - use crate::stores::schema::Measurement; + use crate::stores::types::Measurement; fn create_test_increase_accumulator(start_val: f64, end_val: f64) -> IncreaseAccumulator { IncreaseAccumulator::new( diff --git a/data_plane/src/precompute_engine/operators/multiple_min_max_accumulator.rs b/data_plane/src/precompute_engine/operators/multiple_min_max_accumulator.rs index a4c90845..2926a49c 100644 --- a/data_plane/src/precompute_engine/operators/multiple_min_max_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/multiple_min_max_accumulator.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, }; @@ -259,7 +259,7 @@ impl AggregateCore for MultipleMinMaxAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::stores::schema::MultipleSubpopulationAggregate; + use crate::stores::types::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for MultipleMinMaxAccumulator")?; diff --git a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs b/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs index d4e3de5c..8c6d0cd4 100644 --- a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, MultipleSubpopulationAggregateFactory, SerializableToSink, }; @@ -251,7 +251,7 @@ impl AggregateCore for MultipleSumAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::stores::schema::MultipleSubpopulationAggregate; + use crate::stores::types::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for MultipleSumAccumulator")?; diff --git a/data_plane/src/precompute_engine/operators/set_aggregator_accumulator.rs b/data_plane/src/precompute_engine/operators/set_aggregator_accumulator.rs index 9a045e62..41cb0edb 100644 --- a/data_plane/src/precompute_engine/operators/set_aggregator_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/set_aggregator_accumulator.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, AggregationType, KeyByLabelValues, MergeableAccumulator, MultipleSubpopulationAggregate, SerializableToSink, }; @@ -201,7 +201,7 @@ impl AggregateCore for SetAggregatorAccumulator { key: &Option, query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::stores::schema::MultipleSubpopulationAggregate; + use crate::stores::types::MultipleSubpopulationAggregate; let key_val = key .as_ref() .ok_or("Key required for SetAggregatorAccumulator")?; diff --git a/data_plane/src/precompute_engine/operators/sketch_envelope_accumulator.rs b/data_plane/src/precompute_engine/operators/sketch_envelope_accumulator.rs index c1527a69..a1cf78af 100644 --- a/data_plane/src/precompute_engine/operators/sketch_envelope_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/sketch_envelope_accumulator.rs @@ -5,7 +5,7 @@ //! (via `SketchEnvelope::decode`) only when merge or query operations need //! the inner sketch type. -use crate::stores::schema::{AggregateCore, KeyByLabelValues, SerializableToSink}; +use crate::stores::types::{AggregateCore, KeyByLabelValues, SerializableToSink}; use asap_sketchlib::proto::sketchlib::{sketch_envelope, SketchEnvelope}; use prost::Message; use serde_json::Value; @@ -139,7 +139,7 @@ impl AggregateCore for SketchEnvelopeAccumulator { } } -impl crate::stores::schema::MultipleSubpopulationAggregate for SketchEnvelopeAccumulator { +impl crate::stores::types::MultipleSubpopulationAggregate for SketchEnvelopeAccumulator { fn query( &self, _statistic: Statistic, @@ -152,7 +152,7 @@ impl crate::stores::schema::MultipleSubpopulationAggregate for SketchEnvelopeAcc ) } - fn clone_boxed(&self) -> Box { + fn clone_boxed(&self) -> Box { Box::new(self.clone()) } } diff --git a/data_plane/src/precompute_engine/operators/sum_accumulator.rs b/data_plane/src/precompute_engine/operators/sum_accumulator.rs index ce65d278..baa9aeb2 100644 --- a/data_plane/src/precompute_engine/operators/sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/sum_accumulator.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, AggregationType, AuxStats, MergeableAccumulator, SerializableToSink, SingleSubpopulationAggregate, SingleSubpopulationAggregateFactory, }; @@ -146,7 +146,7 @@ impl AggregateCore for SumAccumulator { _key: &Option, _query_kwargs: &std::collections::HashMap, ) -> Result> { - use crate::stores::schema::SingleSubpopulationAggregate; + use crate::stores::types::SingleSubpopulationAggregate; self.query(statistic, None) } } diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index 2137feef..6841ec8b 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{AggregateCore, PrecomputedOutput}; +use crate::stores::types::{AggregateCore, PrecomputedOutput}; use crate::stores::Store; use std::sync::{Arc, Mutex}; use tracing::debug_span; diff --git a/data_plane/src/precompute_engine/series_router.rs b/data_plane/src/precompute_engine/series_router.rs index 6f536837..243f1c09 100644 --- a/data_plane/src/precompute_engine/series_router.rs +++ b/data_plane/src/precompute_engine/series_router.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::AggregateCore; +use crate::stores::types::AggregateCore; use futures::future::try_join_all; use std::collections::HashMap; use std::fmt; diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 6af9141a..890eacd9 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, HotReloadStreamingConfig, KeyByLabelValues, PrecomputedOutput, }; use crate::precompute_engine::accumulator_factory::{ @@ -1070,7 +1070,7 @@ mod tests { // Helpers // ----------------------------------------------------------------------- - use crate::stores::schema::StreamingConfig; + use crate::stores::types::StreamingConfig; use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::output_sink::CapturingOutputSink; use crate::precompute_engine::operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; @@ -1176,8 +1176,8 @@ mod tests { /// callsite. fn make_hot_reload( configs: HashMap, - ) -> crate::stores::schema::HotReloadStreamingConfig { - crate::stores::schema::HotReloadStreamingConfig::new(crate::stores::schema::StreamingConfig::new( + ) -> crate::stores::types::HotReloadStreamingConfig { + crate::stores::types::HotReloadStreamingConfig::new(crate::stores::types::StreamingConfig::new( configs, )) } @@ -2451,9 +2451,9 @@ aggregations: /// queried metric / agg_id. #[test] fn test_sketch_ingest_persists_and_query_returns_non_empty() { - use crate::stores::schema::{CleanupPolicy, StreamingConfig}; + use crate::stores::types::{CleanupPolicy, StreamingConfig}; use crate::precompute_engine::output_sink::StoreOutputSink; - use crate::stores::sketch_db::sketch_store::per_key::SketchStorePerKey; + use crate::stores::sketch_db::store::per_key::SketchStorePerKey; use crate::stores::Store; // Streaming config: agg_id=1, 30s tumbling, DDSketch, @@ -2487,7 +2487,7 @@ aggregations: 0, rx, sink, - crate::stores::schema::HotReloadStreamingConfig::new(StreamingConfig::new(configs_map)), + crate::stores::types::HotReloadStreamingConfig::new(StreamingConfig::new(configs_map)), WorkerRuntimeConfig { max_buffer_per_series: 10_000, allowed_lateness_ms: 0, diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index c7ffd281..e79a5116 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregationIdInfo, InferenceConfig, KeyByLabelValues, QueryConfig, QueryLanguage, SchemaConfig, StreamingConfig, }; @@ -262,7 +262,7 @@ pub struct ASAPQueryEngine { /// underlying `ArcSwap`, so when `main.rs` hands the same handle /// to both `ASAPQueryEngine` and `HttpServer::with_hot_reload_config`, /// a POST is immediately visible to the next query. - streaming_config_source: crate::stores::schema::HotReloadStreamingConfig, + streaming_config_source: crate::stores::types::HotReloadStreamingConfig, prometheus_scrape_interval: u64, controller_patterns: HashMap>, query_language: QueryLanguage, @@ -292,7 +292,7 @@ pub struct ASAPQueryEngine { /// EngineRouter's archive failover (Phase 6). When `None`, the /// engine behaves as it did before Phase 5 wire-in (every query /// goes through `handle_query`'s legacy path). - sketch_index: Option>, + sketch_index: Option>, /// Phase-5 hybrid-stitch hook — set by `with_archive_engine` from /// `main.rs`'s engine builder. When the warm-tier reducer reports a /// `WarmTierResult.coverage` narrower than the requested @@ -322,7 +322,7 @@ impl ASAPQueryEngine { prometheus_scrape_interval: u64, query_language: QueryLanguage, ) -> Self { - let hot_reload = crate::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config); + let hot_reload = crate::stores::types::HotReloadStreamingConfig::from_arc(streaming_config); Self::new_with_hot_reload( store, inference_config, @@ -339,7 +339,7 @@ impl ASAPQueryEngine { pub fn new_with_hot_reload( store: Arc, inference_config: InferenceConfig, - streaming_config_source: crate::stores::schema::HotReloadStreamingConfig, + streaming_config_source: crate::stores::types::HotReloadStreamingConfig, prometheus_scrape_interval: u64, query_language: QueryLanguage, ) -> Self { @@ -508,7 +508,7 @@ impl ASAPQueryEngine { /// query through `handle_query`). pub fn with_sketch_index( mut self, - index: Arc, + index: Arc, ) -> Self { self.sketch_index = Some(index); self @@ -2761,7 +2761,7 @@ impl ASAPQueryEngine { precomputed_outputs_map: &TimestampedBucketsMap, do_merge: bool, aggregation_type: AggregationType, - ) -> HashMap, Box> { + ) -> HashMap, Box> { #[cfg(feature = "extra_debugging")] let start_time = Instant::now(); #[cfg(feature = "extra_debugging")] @@ -2827,8 +2827,8 @@ impl ASAPQueryEngine { /// This follows the Python merge_accumulators approach fn merge_accumulators( &self, - accumulators: &[Box], - ) -> Box { + accumulators: &[Box], + ) -> Box { if accumulators.is_empty() { panic!("No accumulators to merge"); } @@ -3578,7 +3578,7 @@ fn warm_tier_result_to_query_result( result: crate::query_engines::warm_tier::WarmTierResult, _now_ms: u64, ) -> crate::query_engines::query_result::QueryResult { - use crate::stores::schema::KeyByLabelValues; + use crate::stores::types::KeyByLabelValues; use crate::query_engines::query_result::{QueryResult, RangeVectorElement}; let mut elements: Vec = Vec::with_capacity(result.series.len()); @@ -3727,14 +3727,14 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu // `Capability` enum (defined in the controller and // re-exported by `sketch_index`), so no `From` // conversion is needed — just clone. - let required: crate::stores::sketch_db::sketch_index::Capability = + let required: crate::stores::sketch_db::index::Capability = candidate.required_capability.clone(); let mut hit_sids: Vec = Vec::with_capacity(sids.len()); for sid in &sids { match idx.classify(*sid) { - crate::stores::sketch_db::sketch_index::SidLookup::Hit => {} - crate::stores::sketch_db::sketch_index::SidLookup::Ghost - | crate::stores::sketch_db::sketch_index::SidLookup::Unknown => { + crate::stores::sketch_db::index::SidLookup::Hit => {} + crate::stores::sketch_db::index::SidLookup::Ghost + | crate::stores::sketch_db::index::SidLookup::Unknown => { return Err(crate::query_engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!( @@ -3905,7 +3905,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu #[cfg(test)] mod range_query_tests { - use crate::stores::schema::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; + use crate::stores::types::{AggregateCore, AggregationType, KeyByLabelValues, SerializableToSink}; use crate::query_engines::window_merger::NaiveMerger; use serde_json::Value; use std::any::Any; @@ -4680,7 +4680,7 @@ mod range_query_tests { #[cfg(test)] mod sketch_query_tests { - // use crate::stores::schema::{CleanupPolicy, InferenceConfig, QueryLanguage, StreamingConfig}; + // use crate::stores::types::{CleanupPolicy, InferenceConfig, QueryLanguage, StreamingConfig}; // use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; // use crate::stores::promsketch_store::PromSketchStore; // use crate::stores::{Store, TimestampedBucketsMap}; @@ -4711,16 +4711,16 @@ mod sketch_query_tests { // } // fn insert_precomputed_output( // &self, - // _: crate::stores::schema::PrecomputedOutput, - // _: Box, + // _: crate::stores::types::PrecomputedOutput, + // _: Box, // ) -> Result<(), Box> { // panic!("NoOpStore should not be called for sketch queries"); // } // fn insert_precomputed_output_batch( // &self, // _: Vec<( - // crate::stores::schema::PrecomputedOutput, - // Box, + // crate::stores::types::PrecomputedOutput, + // Box, // )>, // ) -> Result<(), Box> { // panic!("NoOpStore should not be called for sketch queries"); @@ -4925,15 +4925,15 @@ mod sketch_query_tests { #[cfg(test)] mod hot_reload_phase2_tests { use super::*; - use crate::stores::schema::{ + use crate::stores::types::{ AggregationType, CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, QueryLanguage, StreamingConfig, WindowType, }; - use crate::stores::sketch_db::sketch_store::SketchStore; + use crate::stores::sketch_db::store::SketchStore; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; - fn dummy_agg(id: u64, metric: &str) -> crate::stores::schema::AggregationConfig { - crate::stores::schema::AggregationConfig::new( + fn dummy_agg(id: u64, metric: &str) -> crate::stores::types::AggregationConfig { + crate::stores::types::AggregationConfig::new( id, AggregationType::Sum, String::new(), @@ -5086,12 +5086,12 @@ mod hot_reload_phase2_tests { #[cfg(test)] mod e2e_feedback_loop_tests { use super::*; - use crate::stores::schema::{ + use crate::stores::types::{ AggregationType, CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, QueryLanguage, StreamingConfig, WindowType, }; use crate::drivers::query::controller_client::ControllerClient; - use crate::stores::sketch_db::sketch_store::SketchStore; + use crate::stores::sketch_db::store::SketchStore; use async_trait::async_trait; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; use promql_utilities::query_logics::enums::Statistic; @@ -5099,8 +5099,8 @@ mod e2e_feedback_loop_tests { use std::sync::Mutex; use std::time::{Duration, Instant}; - fn agg_for_metric(id: u64, metric: &str) -> crate::stores::schema::AggregationConfig { - crate::stores::schema::AggregationConfig::new( + fn agg_for_metric(id: u64, metric: &str) -> crate::stores::types::AggregationConfig { + crate::stores::types::AggregationConfig::new( id, AggregationType::Sum, String::new(), @@ -5439,7 +5439,7 @@ mod aux_pushdown_tests { query_calls: Arc, } - impl crate::stores::schema::SerializableToSink for SpyAccumulator { + impl crate::stores::types::SerializableToSink for SpyAccumulator { fn serialize_to_bytes(&self) -> Vec { Vec::new() } @@ -5486,20 +5486,20 @@ mod aux_pushdown_tests { self.query_calls.fetch_add(1, Ordering::Relaxed); Ok(-1.0) // sentinel: fast path should not return this } - fn aux_stats(&self) -> crate::stores::schema::AuxStats { - crate::stores::schema::AuxStats { + fn aux_stats(&self) -> crate::stores::types::AuxStats { + crate::stores::types::AuxStats { sum: Some(self.inner_sum), - ..crate::stores::schema::AuxStats::empty() + ..crate::stores::types::AuxStats::empty() } } } fn make_engine() -> ASAPQueryEngine { - use crate::stores::schema::{ + use crate::stores::types::{ CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, PromQLSchema, QueryLanguage, SchemaConfig, StreamingConfig, }; - use crate::stores::sketch_db::sketch_store::SketchStore; + use crate::stores::sketch_db::store::SketchStore; let ic = InferenceConfig { schema: SchemaConfig::PromQL(PromQLSchema { @@ -5755,11 +5755,11 @@ mod forced_agg_id_tests { #[cfg(test)] mod sketch_alias_resolver_tests { use super::*; - use crate::stores::schema::{ + use crate::stores::types::{ AggregationConfig, CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, PromQLSchema, QueryLanguage, SchemaConfig, StreamingConfig, WindowType, }; - use crate::stores::sketch_db::sketch_store::SketchStore; + use crate::stores::sketch_db::store::SketchStore; use std::sync::Arc; fn agg_for(id: u64, metric: &str, agg_type: AggregationType) -> AggregationConfig { @@ -6146,25 +6146,25 @@ mod cms_rate_capability_tests { #[cfg(test)] mod warm_tier_classify_tests { use super::*; - use crate::stores::schema::{CleanupPolicy, HotReloadStreamingConfig, InferenceConfig}; + use crate::stores::types::{CleanupPolicy, HotReloadStreamingConfig, InferenceConfig}; use crate::query_engines::EngineError; use crate::query_engines::routing::query_engine_routing::QueryEngine as _; - use crate::stores::sketch_db::sketch_store::SketchStore; - use crate::stores::sketch_db::sketch_index::{ + use crate::stores::sketch_db::store::SketchStore; + use crate::stores::sketch_db::index::{ AccuracyBound, Capability, SketchConfig, SketchIndex, SketchInstanceMetadata, SketchKindHandle, SketchSampleState, }; use std::collections::{BTreeMap, BTreeSet}; fn build_engine_with_index(idx: Arc) -> ASAPQueryEngine { - let streaming_config = Arc::new(crate::stores::schema::StreamingConfig::default()); + let streaming_config = Arc::new(crate::stores::types::StreamingConfig::default()); let store = Arc::new(SketchStore::new( streaming_config.clone(), CleanupPolicy::NoCleanup, )); let hot_reload = HotReloadStreamingConfig::from_arc(streaming_config); let inference_config = InferenceConfig::new( - crate::stores::schema::QueryLanguage::promql, + crate::stores::types::QueryLanguage::promql, CleanupPolicy::NoCleanup, ); ASAPQueryEngine::new_with_hot_reload( @@ -6172,7 +6172,7 @@ mod warm_tier_classify_tests { inference_config, hot_reload, 15000, - crate::stores::schema::QueryLanguage::promql, + crate::stores::types::QueryLanguage::promql, ) .with_sketch_index(idx) } @@ -6270,7 +6270,7 @@ mod warm_tier_classify_tests { (1_000, 1_010), SketchSampleState { bytes: vec![0], - encoding: crate::stores::sketch_db::sketch_index::SketchEncoding::ProtoFull, + encoding: crate::stores::sketch_db::index::SketchEncoding::ProtoFull, }, ); @@ -6297,7 +6297,7 @@ mod warm_tier_classify_tests { #[cfg(test)] mod hybrid_stitch_tests { use super::stitch_warm_and_archive; - use crate::stores::schema::KeyByLabelValues; + use crate::stores::types::KeyByLabelValues; use crate::query_engines::query_result::{QueryResult, RangeVectorElement, Sample}; fn matrix_with_samples(label: &str, samples: Vec<(u64, f64)>) -> QueryResult { diff --git a/data_plane/src/query_engines/mod.rs b/data_plane/src/query_engines/mod.rs index c1625212..663ddf75 100644 --- a/data_plane/src/query_engines/mod.rs +++ b/data_plane/src/query_engines/mod.rs @@ -15,16 +15,11 @@ //! engine. //! * [`thanos_query_engine::ThanosQueryEngine`] — archive-tier query //! engine. -//! * [`prometheus_query_engine::PrometheusForwardEngine`] — HTTP-forwarder -//! to a Prometheus `/api/v1/query` endpoint, registered under the -//! `prometheus_remote` engine id when -//! `ASAP_PROMETHEUS_QUERY_URL` is set (Phase ε.2). //! * [`EngineError`] — the trait-level error envelope every //! `crate::query_engines::routing::QueryEngine` impl returns. pub mod asap_query_engine; pub mod no_data_archive; -pub mod prometheus_query_engine; pub mod query_result; pub mod routing; pub mod thanos_query_engine; @@ -34,9 +29,6 @@ pub mod window_merger; pub use asap_query_engine::ASAPQueryEngine; pub use no_data_archive::{NoDataArchiveEngine, DATA_SOURCE_ID_NO_DATA_ARCHIVE}; -pub use prometheus_query_engine::{ - PrometheusForwardConfig, PrometheusForwardEngine, PrometheusForwardError, -}; pub use query_result::{InstantVector, QueryResult, RangeVector, RangeVectorElement, Sample}; pub use thanos_query_engine::{ thanos_engine_from_env, ThanosQueryConfig, ThanosQueryEngine, ThanosQueryError, diff --git a/data_plane/src/query_engines/prometheus_query_engine/forward.rs b/data_plane/src/query_engines/prometheus_query_engine/forward.rs deleted file mode 100644 index e77c5d7f..00000000 --- a/data_plane/src/query_engines/prometheus_query_engine/forward.rs +++ /dev/null @@ -1,896 +0,0 @@ -//! `PrometheusForwardEngine` — HTTP forwarder to a Prometheus -//! `/api/v1/query` endpoint for Phase ε.2 of the planner -//! consolidation. -//! -//! The controller's Mode 3 (`RawAtEdgePrometheusArchive`) routes a -//! metric's queries to Prometheus directly when the metric's data is -//! shipped raw to Prometheus's native OTLP receiver (no warm-tier -//! sketch, no Gorilla archive — Prometheus owns the storage). Phase -//! ε.1 (controller) emits `engine: prometheus_remote` in the -//! `BackendStorageRouting` JSON so the backend's dispatcher can pick -//! the new engine; Phase ε.2 (this file + main.rs wiring) registers -//! that engine on the `EngineRouter` so the dispatcher's -//! `engine_by_id` lookup hits the forwarder. -//! -//! Operating modes are selected by the -//! [`ASAP_PROMETHEUS_QUERY_URL_ENV`] env var, consulted at backend -//! startup: -//! -//! * **Phase ε.2 mode** (env set) — `PrometheusForwardEngine` is -//! registered in the [`crate::query_engines::routing::EngineRouter`] under id -//! `prometheus_remote`. Routing-table entries that target this -//! engine POST to `${ASAP_PROMETHEUS_QUERY_URL}/api/v1/query` and -//! the answer is wrapped in ASAP's standard -//! [`crate::query_engines::QueryResult`] shape. -//! * **Off** (env unset) — engine is not registered. Routing-table -//! entries that reference `prometheus_remote` surface a -//! `NoEngineRegistered` 503 from the HTTP handler — the correct -//! fail-loud behaviour for a misconfigured deploy. -//! -//! This is a near-mirror of [`crate::query_engines::thanos_query_engine::forward`] -//! (the Step-2.3 archive forwarder), pointed at Prometheus's standard -//! `/api/v1/query` endpoint instead of a `thanos-query` sidecar. The -//! two engines coexist: `thanos_query` answers archive-tier queries -//! over Prometheus TSDB blocks emitted by `gorillas3processor`; -//! `prometheus_remote` answers queries for metrics whose raw data is -//! shipped to Prometheus's native OTLP receiver (no ASAP archive at -//! all). - -use std::time::{Duration, Instant}; - -use async_trait::async_trait; -use serde::Deserialize; -use serde_json::Value; -use tracing::{debug, warn}; - -use crate::stores::schema::KeyByLabelValues; -use crate::query_engines::query_result::{InstantVectorElement, QueryResult, RangeVectorElement}; -use crate::query_engines::routing::query_engine_routing::{EngineCapabilities, QueryEngine}; -use crate::stores::sketch_db::accuracy::{AccuracyEnvelope, AccuracyProfile}; - -// --------------------------------------------------------------------------- -// Public constants. -// -// The env var name and the engine id are pinned strings so the -// binary, dashboards, and `BackendStorageRouting` configs can -// byte-compare without re-deriving them. -// --------------------------------------------------------------------------- - -/// Env var consulted at backend startup. When set, the binary -/// registers a [`PrometheusForwardEngine`] pointing at the URL and -/// the router dispatches `prometheus_remote` queries to it. When -/// unset, the engine is not registered. -pub const ASAP_PROMETHEUS_QUERY_URL_ENV: &str = "ASAP_PROMETHEUS_QUERY_URL"; - -/// Default upstream URL when `ASAP_PROMETHEUS_QUERY_URL` is set to -/// the empty string or contains only whitespace. Mirrors the demo -/// overlay's default service name + port (Prometheus's standard -/// HTTP API port is `9090`). -pub const DEFAULT_PROMETHEUS_QUERY_URL: &str = "http://prometheus:9090"; - -/// `data_source_id` the [`PrometheusForwardEngine`] registers under. -/// Pinned so dashboards / e2e scripts and the per-metric -/// `BackendStorageRouting` config can byte-compare without parsing. -pub const DATA_SOURCE_PROMETHEUS_REMOTE_ID: &str = "prometheus_remote"; - -/// Marker line every `PrometheusForwardEngine` answer carries on its -/// `infos` array. Pinned so dashboards and the upcoming Phase 3 -/// e2e demo can byte-compare without parsing. -pub const DATA_SOURCE_PROMETHEUS_REMOTE_INFO: &str = "data_source: prometheus_remote"; - -/// `data_source_quirk` line surfaced when the upstream Prometheus -/// instance is unreachable (network error / 5xx / timeout). Pinned -/// so the upcoming e2e demo can pin the fail-loud behaviour. -pub const QUIRK_PROMETHEUS_UNREACHABLE: &str = "data_source_quirk: prometheus_unreachable"; - -/// Default request timeout for the forwarded query. Generous enough -/// that Prometheus has room to answer big range queries, tight -/// enough that the backend doesn't pile up in-flight requests on a -/// wedged upstream. -pub const DEFAULT_PROMETHEUS_REQUEST_TIMEOUT: Duration = Duration::from_secs(60); - -// --------------------------------------------------------------------------- -// Config + engine. -// --------------------------------------------------------------------------- - -/// Tunable runtime knobs for [`PrometheusForwardEngine`]. Built -/// from env via [`PrometheusForwardConfig::from_env`]. -#[derive(Debug, Clone)] -pub struct PrometheusForwardConfig { - /// Base URL of the upstream Prometheus instance — e.g. - /// `http://prometheus:9090`. The engine appends `/api/v1/query` - /// (or `/api/v1/query_range`) when forwarding. Trailing slash is - /// tolerated; both forms are normalised. - pub base_url: String, - /// Wall-clock timeout per forwarded request. - pub request_timeout: Duration, -} - -impl Default for PrometheusForwardConfig { - fn default() -> Self { - Self { - base_url: DEFAULT_PROMETHEUS_QUERY_URL.to_string(), - request_timeout: DEFAULT_PROMETHEUS_REQUEST_TIMEOUT, - } - } -} - -impl PrometheusForwardConfig { - /// Build a config from the [`ASAP_PROMETHEUS_QUERY_URL_ENV`] env - /// var, returning `None` when the var is unset / empty / blank - /// (the binary should then skip registering the forwarder; a - /// routing-table entry referencing `prometheus_remote` will - /// surface a `NoEngineRegistered` 503). - /// - /// A whitespace-only value is treated as unset rather than as a - /// malformed URL: we don't want a stray - /// `ASAP_PROMETHEUS_QUERY_URL=` in a `.env` to silently flip - /// Phase ε.2 on with the default host name. - pub fn from_env() -> Option { - let raw = std::env::var(ASAP_PROMETHEUS_QUERY_URL_ENV).ok()?; - let trimmed = raw.trim(); - if trimmed.is_empty() { - return None; - } - Some(Self { - base_url: trimmed.trim_end_matches('/').to_string(), - request_timeout: DEFAULT_PROMETHEUS_REQUEST_TIMEOUT, - }) - } - - fn instant_endpoint(&self) -> String { - format!("{}/api/v1/query", self.base_url) - } -} - -/// Forwards PromQL queries to an upstream Prometheus instance over -/// HTTP and wraps the response in ASAP's standard -/// [`QueryResult`] shape. -/// -/// Implements the [`QueryEngine`] trait so the -/// [`crate::query_engines::routing::EngineRouter`] can hold it as `Arc`. Reports `data_source_id = "prometheus_remote"` -/// and (for the compatibility-list dispatch path) `storage_backend = -/// StorageBackend::PrometheusRemote` — Mode 3 gives the -/// Prometheus-native metrics their own slot in the routing matrix -/// (the data never lands in ASAP storage so no warm-tier / archive -/// failover is meaningful). -pub struct PrometheusForwardEngine { - config: PrometheusForwardConfig, - client: reqwest::Client, -} - -impl PrometheusForwardEngine { - /// Build with an explicit config. Used by tests + the binary's - /// startup wiring. - pub fn new(config: PrometheusForwardConfig) -> Result { - let client = reqwest::Client::builder() - .timeout(config.request_timeout) - .build() - .map_err(|e| PrometheusForwardError::ConfigInvalid(e.to_string()))?; - Ok(Self { config, client }) - } - - /// Build the production config from - /// [`ASAP_PROMETHEUS_QUERY_URL_ENV`] or return `None` when the - /// env var is unset / blank. The binary calls this; if it - /// returns `None`, the engine is not registered. - pub fn from_env() -> Option> { - PrometheusForwardConfig::from_env().map(Self::new) - } - - /// Read-only access to the configured base URL — useful for - /// diagnostics + the upcoming Phase 3 demo's startup banner. - pub fn base_url(&self) -> &str { - &self.config.base_url - } - - /// The infos a successful forwarded answer carries. Pinned so - /// tests + dashboards can byte-compare without re-implementing - /// the wire path. - pub fn success_infos(elapsed_ms: u128) -> Vec { - vec![ - DATA_SOURCE_PROMETHEUS_REMOTE_INFO.to_string(), - AccuracyProfile::exact().summary(), - format!("query_latency_ms: {elapsed_ms}"), - ] - } - - /// The infos a forwarded-but-failed answer carries. Includes the - /// quirk line so the upcoming e2e demo can pin fail-loud - /// behaviour. - pub fn unreachable_infos(reason: &str, elapsed_ms: u128) -> Vec { - vec![ - DATA_SOURCE_PROMETHEUS_REMOTE_INFO.to_string(), - QUIRK_PROMETHEUS_UNREACHABLE.to_string(), - format!("prometheus_unreachable_reason: {reason}"), - format!("query_latency_ms: {elapsed_ms}"), - ] - } - - /// Forward `query` to `${base_url}/api/v1/query` and parse the - /// Prometheus-format response back into a [`QueryResult`]. - /// - /// Errors are folded into [`PrometheusForwardError`] variants — - /// the [`QueryEngine`] impl decides how to surface each. - pub async fn query(&self, query: &str) -> Result { - let started = Instant::now(); - let url = self.config.instant_endpoint(); - debug!( - url = %url, - query = query, - "prometheus-forward: issuing instant query", - ); - - let resp = self - .client - .post(&url) - .form(&[("query", query)]) - .send() - .await - .map_err(|e| PrometheusForwardError::Unreachable(e.to_string()))?; - - let status = resp.status(); - if status.is_server_error() { - return Err(PrometheusForwardError::Unreachable(format!( - "upstream returned {status}", - ))); - } - if !status.is_success() { - // Treat 4xx as a "bad query" / capability miss — the - // upstream Prometheus didn't accept it (malformed - // PromQL, unknown metric, etc.). - let body = resp.text().await.unwrap_or_default(); - return Err(PrometheusForwardError::BadQuery { - status: status.as_u16(), - body, - }); - } - - let payload: PrometheusResponse = resp - .json() - .await - .map_err(|e| PrometheusForwardError::ParseError(e.to_string()))?; - - let elapsed_ms = started.elapsed().as_millis(); - let result = build_result_from_prometheus_payload(payload, elapsed_ms) - .map_err(PrometheusForwardError::ParseError)?; - Ok(result) - } -} - -#[async_trait] -impl QueryEngine for PrometheusForwardEngine { - async fn execute(&self, query: &str) -> Result { - match self.query(query).await { - Ok(result) => Ok(result), - Err(PrometheusForwardError::Unreachable(reason)) => { - // Surface fail-loud as a backend error. Mode 3 - // metrics have no failover slot, so the router will - // return AllFailed → the HTTP handler turns it into - // a 503 with the `prometheus_unreachable` quirk infos. - warn!( - engine = DATA_SOURCE_PROMETHEUS_REMOTE_ID, - error = %reason, - "prometheus-forward: upstream unreachable", - ); - Err(crate::query_engines::EngineError::backend( - DATA_SOURCE_PROMETHEUS_REMOTE_ID, - format!("prometheus_unreachable: {reason}"), - )) - } - Err(PrometheusForwardError::BadQuery { status, body }) => { - Err(crate::query_engines::EngineError::capability_miss( - DATA_SOURCE_PROMETHEUS_REMOTE_ID, - format!("prometheus rejected query (status {status}): {body}"), - )) - } - Err(PrometheusForwardError::ParseError(msg)) => { - Err(crate::query_engines::EngineError::backend( - DATA_SOURCE_PROMETHEUS_REMOTE_ID, - format!("prometheus response parse error: {msg}"), - )) - } - Err(PrometheusForwardError::ConfigInvalid(msg)) => { - Err(crate::query_engines::EngineError::backend( - DATA_SOURCE_PROMETHEUS_REMOTE_ID, - format!("prometheus client misconfigured: {msg}"), - )) - } - } - } - - fn capabilities(&self) -> EngineCapabilities { - EngineCapabilities { - data_source_id: DATA_SOURCE_PROMETHEUS_REMOTE_ID, - // Mode 3 owns its own routing slot; the metric's data is - // shipped to Prometheus, not to ASAP-managed storage. - storage_backend: asap_types::StorageBackend::PrometheusRemote, - // Forwarder doesn't materialise samples locally; - // upstream Prometheus owns the memory budget. We surface - // a generous ceiling so the cost-aware dispatcher - // (Phase-6) prefers the forwarder for large streams once - // it lands. - supports_streams_above_bytes: usize::MAX, - } - } -} - -// --------------------------------------------------------------------------- -// Wire helpers. -// --------------------------------------------------------------------------- - -/// Subset of the Prometheus HTTP API response shape we actually -/// consume. `serde` ignores unknown fields, so future Prometheus -/// extensions don't break parsing. -#[derive(Debug, Deserialize)] -struct PrometheusResponse { - status: String, - #[serde(default)] - data: Option, - #[serde(rename = "errorType", default)] - error_type: Option, - #[serde(default)] - error: Option, -} - -#[derive(Debug, Deserialize)] -struct PrometheusData { - #[serde(rename = "resultType", default)] - result_type: String, - #[serde(default)] - result: Vec, -} - -/// Build an ASAP [`QueryResult`] from a parsed Prometheus payload. -/// Pulled out as a pure function so the unit tests can pin the -/// wrapping behaviour without spinning up a TCP listener. -fn build_result_from_prometheus_payload( - payload: PrometheusResponse, - elapsed_ms: u128, -) -> Result { - if payload.status != "success" { - let detail = payload.error.unwrap_or_else(|| "unknown error".to_string()); - let kind = payload - .error_type - .unwrap_or_else(|| "execution".to_string()); - return Err(format!("prometheus error ({kind}): {detail}")); - } - let data = payload - .data - .ok_or_else(|| "prometheus response missing `data`".to_string())?; - - let mut result = match data.result_type.as_str() { - "vector" => parse_vector(&data.result)?, - "matrix" => parse_matrix(&data.result)?, - // Scalar / string result types are valid PromQL but the ASAP - // wire shape only models vector / matrix. Surface as a parse - // error so callers see the upstream type rather than an - // empty vector. - other => { - return Err(format!( - "prometheus response carried unsupported resultType={other:?}" - )); - } - }; - - // Pin an exact-accuracy envelope on every wrapped answer. Mode 3 - // reads from Prometheus's TSDB — the underlying samples are the - // raw OTLP-ingested points, so the answer is exact (ε = 0, - // δ = 0). - let envelope = AccuracyEnvelope::single(AccuracyProfile::exact()); - result = result.with_accuracy(envelope); - - let _ = elapsed_ms; // surfaced via tests directly via `success_infos`. - Ok(result) -} - -fn parse_vector(values: &[Value]) -> Result { - let mut elements = Vec::with_capacity(values.len()); - let mut latest_ts: u64 = 0; - for v in values { - let metric = v.get("metric").cloned().unwrap_or(Value::Null); - let value = v - .get("value") - .ok_or_else(|| "vector element missing `value`".to_string())?; - let pair = value - .as_array() - .ok_or_else(|| "vector element `value` is not an array".to_string())?; - if pair.len() != 2 { - return Err(format!( - "vector element `value` must be [ts, str_value], got {pair:?}" - )); - } - let ts_seconds = pair[0] - .as_f64() - .ok_or_else(|| format!("vector element ts is not a number: {:?}", pair[0]))?; - let ts_ms = (ts_seconds * 1000.0).round() as u64; - latest_ts = latest_ts.max(ts_ms); - let scalar = pair[1] - .as_str() - .ok_or_else(|| format!("vector element value is not a string: {:?}", pair[1]))?; - let parsed: f64 = scalar - .parse() - .map_err(|e| format!("vector element value parse error: {e} (raw={scalar:?})"))?; - let labels = labels_from_metric(&metric); - elements.push(InstantVectorElement::new(labels, parsed)); - } - Ok(QueryResult::vector(elements, latest_ts)) -} - -fn parse_matrix(values: &[Value]) -> Result { - let mut series = Vec::with_capacity(values.len()); - for v in values { - let metric = v.get("metric").cloned().unwrap_or(Value::Null); - let raw_samples = v - .get("values") - .and_then(Value::as_array) - .ok_or_else(|| "matrix element missing `values` array".to_string())?; - let labels = labels_from_metric(&metric); - let mut elem = RangeVectorElement::new(labels); - for sample in raw_samples { - let pair = sample - .as_array() - .ok_or_else(|| "matrix sample is not [ts, str_value]".to_string())?; - if pair.len() != 2 { - return Err(format!( - "matrix sample must be [ts, str_value], got {pair:?}" - )); - } - let ts_seconds = pair[0] - .as_f64() - .ok_or_else(|| format!("matrix sample ts is not a number: {:?}", pair[0]))?; - let ts_ms = (ts_seconds * 1000.0).round() as u64; - let scalar = pair[1] - .as_str() - .ok_or_else(|| format!("matrix sample value is not a string: {:?}", pair[1]))?; - let parsed: f64 = scalar - .parse() - .map_err(|e| format!("matrix sample value parse error: {e} (raw={scalar:?})"))?; - elem.add_sample(ts_ms, parsed); - } - series.push(elem); - } - Ok(QueryResult::matrix(series)) -} - -/// Best-effort label extraction. Prometheus returns the `metric` -/// field as a `{"__name__": "...", "label": "value"}` object; we -/// flatten the values into `KeyByLabelValues` (the same shape the -/// in-process engine pins on its results). Unknown / non-object -/// shapes fall through to an empty label set rather than failing -/// the parse — the wrapped `data_source: prometheus_remote` info is -/// the meaningful annotation. -fn labels_from_metric(metric: &Value) -> KeyByLabelValues { - if let Some(obj) = metric.as_object() { - let mut values: Vec = obj - .iter() - .filter(|(k, _)| k.as_str() != "__name__") - .filter_map(|(_, v)| v.as_str().map(|s| s.to_string())) - .collect(); - values.sort(); - KeyByLabelValues::new_with_labels(values) - } else { - KeyByLabelValues::new_with_labels(Vec::new()) - } -} - -// --------------------------------------------------------------------------- -// Errors. -// --------------------------------------------------------------------------- - -/// Failure modes of the HTTP-forwarder. The [`QueryEngine`] impl -/// folds these into the trait-level [`crate::query_engines::EngineError`] -/// envelope; the public `query` method returns the richer surface -/// for tests and direct callers. -#[derive(Debug, thiserror::Error)] -pub enum PrometheusForwardError { - /// Upstream returned a network error / timeout / 5xx — - /// dashboard-level "prometheus is down." - #[error("prometheus unreachable: {0}")] - Unreachable(String), - /// Upstream returned a 4xx — the query is malformed from - /// Prometheus's point of view, not a backend failure. - #[error("prometheus rejected query (HTTP {status}): {body}")] - BadQuery { - /// The 4xx status code Prometheus returned. - status: u16, - /// The (possibly empty) response body. - body: String, - }, - /// Upstream returned a 2xx but the body wasn't parseable as a - /// Prometheus-format response. - #[error("prometheus response parse error: {0}")] - ParseError(String), - /// reqwest client construction failed (TLS / DNS resolver init - /// etc.). Surfaces only at engine construction. - #[error("prometheus client config invalid: {0}")] - ConfigInvalid(String), -} - -// --------------------------------------------------------------------------- -// Helpers re-exported for the engine's test module + the binary's -// conditional registration. -// --------------------------------------------------------------------------- - -/// Convenience combinator the binary uses at startup: try -/// [`PrometheusForwardEngine::from_env`]; if it returns `None`, the -/// caller skips registration and the routing table will surface a -/// clear "engine not registered" error if it ever references -/// `prometheus_remote`. -/// -/// Returning `Result, ...>` instead of unwrapping in -/// `main.rs` keeps the construction failure (bad URL / bad TLS init) -/// inspectable so the binary can emit a helpful warning instead of -/// crashing on startup. -pub fn engine_from_env() -> Result, PrometheusForwardError> { - match PrometheusForwardEngine::from_env() { - Some(Ok(engine)) => Ok(Some(engine)), - Some(Err(e)) => Err(e), - None => Ok(None), - } -} - -// --------------------------------------------------------------------------- -// Test helpers -// --------------------------------------------------------------------------- - -#[doc(hidden)] -#[cfg(any(test, feature = "extra_debugging"))] -pub mod test_support { - //! Test-only helpers for spinning up an in-process mock - //! Prometheus instance. Used by the unit + integration tests - //! below. - - use std::net::SocketAddr; - use tokio::net::TcpListener; - use tokio::task::JoinHandle; - - use axum::{routing::post, Router}; - - /// Trivial in-process axum server that returns a canned - /// Prometheus-format JSON for every `POST /api/v1/query`. - /// - /// Returns `(base_url, join_handle)`. Drop the handle to stop - /// serving (the test runtime tears down anyway when the - /// `#[tokio::test]` future completes). - pub async fn spawn_mock_prometheus(canned_body: &'static str) -> (String, JoinHandle<()>) { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let local: SocketAddr = listener.local_addr().expect("local_addr"); - let base_url = format!("http://{local}"); - - let app: Router = Router::new() - .route("/api/v1/query", post(move || async move { canned_body })) - .route( - "/api/v1/query_range", - post(move || async move { canned_body }), - ); - - let handle = tokio::spawn(async move { - axum::serve(listener, app) - .await - .expect("mock_prometheus serve"); - }); - - // Best-effort: yield once so the listener is definitely - // bound before the test calls into the engine. - tokio::task::yield_now().await; - (base_url, handle) - } - - /// Same as [`spawn_mock_prometheus`] but the handler always - /// returns `503 Service Unavailable`. Used by the "unreachable - /// upstream" test. - pub async fn spawn_mock_prometheus_503() -> (String, JoinHandle<()>) { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let local: SocketAddr = listener.local_addr().expect("local_addr"); - let base_url = format!("http://{local}"); - - async fn always_503() -> axum::http::StatusCode { - axum::http::StatusCode::SERVICE_UNAVAILABLE - } - - let app: Router = Router::new() - .route("/api/v1/query", post(always_503)) - .route("/api/v1/query_range", post(always_503)); - - let handle = tokio::spawn(async move { - axum::serve(listener, app) - .await - .expect("mock_prometheus_503 serve"); - }); - tokio::task::yield_now().await; - (base_url, handle) - } - - /// Best-effort env-var override scope guard. Used by the - /// `from_env` tests to set / unset - /// `ASAP_PROMETHEUS_QUERY_URL` without leaking onto sibling - /// tests. Tests that touch this guard are serialised on a global - /// mutex so they don't race. - pub struct EnvGuard { - key: &'static str, - prev: Option, - } - - impl EnvGuard { - pub fn set(key: &'static str, value: &str) -> Self { - let prev = std::env::var(key).ok(); - std::env::set_var(key, value); - Self { key, prev } - } - - pub fn unset(key: &'static str) -> Self { - let prev = std::env::var(key).ok(); - std::env::remove_var(key); - Self { key, prev } - } - } - - impl Drop for EnvGuard { - fn drop(&mut self) { - match self.prev.take() { - Some(v) => std::env::set_var(self.key, v), - None => std::env::remove_var(self.key), - } - } - } - - /// Global mutex that serialises tests touching - /// `ASAP_PROMETHEUS_QUERY_URL` (and any other process-wide env - /// var). Use as `let _g = ENV_LOCK.lock().unwrap();` at the top - /// of every env-touching test. - pub static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); - - /// A minimal canned vector response — `up{job="prometheus"} 1` - /// at ts = 1.609 Mq. Pinned so multiple tests can share the - /// expected wrapped output. - pub const CANNED_VECTOR_BODY: &str = r#"{ - "status": "success", - "data": { - "resultType": "vector", - "result": [ - {"metric": {"__name__": "up", "job": "prometheus"}, "value": [1609459200.0, "1"]} - ] - } - }"#; -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::test_support::{ - spawn_mock_prometheus, spawn_mock_prometheus_503, CANNED_VECTOR_BODY, ENV_LOCK, - }; - use super::*; - use crate::query_engines::query_result::QueryResult; - use crate::query_engines::routing::query_engine_routing::{EngineRouter, QueryEngine as RouterQueryEngine}; - use std::sync::Arc; - - fn config_for(url: &str) -> PrometheusForwardConfig { - PrometheusForwardConfig { - base_url: url.trim_end_matches('/').to_string(), - request_timeout: Duration::from_secs(5), - } - } - - #[tokio::test] - async fn forwards_promql_and_wraps_response() { - let (url, _handle) = spawn_mock_prometheus(CANNED_VECTOR_BODY).await; - let engine = PrometheusForwardEngine::new(config_for(&url)).expect("engine"); - let result = engine.query("up").await.expect("ok response"); - - let infos = PrometheusForwardEngine::success_infos(0); - assert!( - infos - .iter() - .any(|s| s == DATA_SOURCE_PROMETHEUS_REMOTE_INFO), - "success_infos must carry the data_source marker; got {infos:?}", - ); - assert!( - infos.iter().any(|s| s.starts_with("accuracy: ε=0")), - "success_infos must carry the exact-accuracy marker; got {infos:?}", - ); - assert!( - infos.iter().any(|s| s.starts_with("query_latency_ms")), - "success_infos must surface a query_latency_ms info; got {infos:?}", - ); - - match result { - QueryResult::Vector(iv) => { - assert_eq!(iv.values.len(), 1); - assert_eq!(iv.values[0].value, 1.0); - let env = iv.accuracy.expect("envelope attached"); - assert!(env.summary().contains("kind=exact")); - } - other => panic!("expected Vector result, got {other:?}"), - } - } - - #[tokio::test] - async fn capabilities_report_prometheus_remote_id() { - // No upstream needed — we only inspect capabilities. - let engine = - PrometheusForwardEngine::new(config_for("http://127.0.0.1:1")).expect("engine"); - let caps = engine.capabilities(); - assert_eq!(caps.data_source_id, DATA_SOURCE_PROMETHEUS_REMOTE_ID); - assert_eq!( - caps.storage_backend, - asap_types::StorageBackend::PrometheusRemote, - "Phase ε.2 owns its own slot in the routing matrix", - ); - } - - #[tokio::test] - async fn unreachable_upstream_returns_503_quirk_via_engine_trait() { - let (url, _handle) = spawn_mock_prometheus_503().await; - let engine = PrometheusForwardEngine::new(config_for(&url)).expect("engine"); - - // The richer surface returns Unreachable. - let direct = engine.query("up").await; - match direct { - Err(PrometheusForwardError::Unreachable(_)) => {} - other => panic!("expected Unreachable error, got {other:?}"), - } - - // The trait surface folds it into a `Backend` error so the - // router's failover sequence (which for `PrometheusRemote` - // is just itself) returns AllFailed → the HTTP handler turns - // it into a 503 with the `prometheus_unreachable` quirk infos. - let trait_path = RouterQueryEngine::execute(&engine, "up").await; - match trait_path { - Err(crate::query_engines::EngineError::Backend { engine_id, message }) => { - assert_eq!(engine_id, DATA_SOURCE_PROMETHEUS_REMOTE_ID); - assert!( - message.contains("prometheus_unreachable"), - "Backend error must carry prometheus_unreachable marker; got {message:?}", - ); - } - other => panic!("expected Backend error, got {other:?}"), - } - - // The unreachable_infos helper exposes the wire shape - // dashboards / e2e demos pin against. - let infos = PrometheusForwardEngine::unreachable_infos("upstream returned 503", 0); - assert!(infos.iter().any(|s| s == QUIRK_PROMETHEUS_UNREACHABLE)); - assert!(infos - .iter() - .any(|s| s.contains("prometheus_unreachable_reason"))); - } - - #[tokio::test] - async fn config_from_env_returns_none_when_unset() { - let _g = ENV_LOCK.lock().expect("lock"); - let _scope = test_support::EnvGuard::unset(ASAP_PROMETHEUS_QUERY_URL_ENV); - assert!(PrometheusForwardConfig::from_env().is_none()); - } - - #[tokio::test] - async fn config_from_env_returns_none_when_blank() { - let _g = ENV_LOCK.lock().expect("lock"); - let _scope = test_support::EnvGuard::set(ASAP_PROMETHEUS_QUERY_URL_ENV, " "); - assert!(PrometheusForwardConfig::from_env().is_none()); - } - - #[tokio::test] - async fn config_from_env_strips_trailing_slash() { - let _g = ENV_LOCK.lock().expect("lock"); - let _scope = - test_support::EnvGuard::set(ASAP_PROMETHEUS_QUERY_URL_ENV, "http://prometheus:9090/"); - let cfg = PrometheusForwardConfig::from_env().expect("set"); - assert_eq!(cfg.base_url, "http://prometheus:9090"); - } - - #[tokio::test] - async fn engine_from_env_returns_some_when_set() { - let _g = ENV_LOCK.lock().expect("lock"); - let _scope = - test_support::EnvGuard::set(ASAP_PROMETHEUS_QUERY_URL_ENV, "http://127.0.0.1:1"); - let engine = engine_from_env().expect("ok"); - assert!(engine.is_some(), "env set → engine constructed"); - } - - #[tokio::test] - async fn engine_from_env_returns_none_when_unset() { - let _g = ENV_LOCK.lock().expect("lock"); - let _scope = test_support::EnvGuard::unset(ASAP_PROMETHEUS_QUERY_URL_ENV); - let engine = engine_from_env().expect("ok"); - assert!( - engine.is_none(), - "env unset → caller must skip registration", - ); - } - - /// Phase ε.2 router-startup contract: when the env var is set, - /// the binary should register the engine under `prometheus_remote`; - /// when unset, the engine should not be registered and the - /// router should not list `prometheus_remote` among its ids. - /// Mirrors the binary's wiring without re-running the full - /// `main.rs` startup sequence. - #[tokio::test] - async fn router_register_when_env_set_skip_when_unset() { - // Env set → engine registered. - { - let _g = ENV_LOCK.lock().expect("lock"); - let _scope = - test_support::EnvGuard::set(ASAP_PROMETHEUS_QUERY_URL_ENV, "http://127.0.0.1:1"); - let mut router = EngineRouter::new(); - if let Ok(Some(engine)) = engine_from_env() { - router.register(Arc::new(engine)); - } - assert!( - router - .engine_by_id(DATA_SOURCE_PROMETHEUS_REMOTE_ID) - .is_some(), - "env set must yield prometheus_remote engine in the router", - ); - } - - // Env unset → engine NOT registered. - { - let _g = ENV_LOCK.lock().expect("lock"); - let _scope = test_support::EnvGuard::unset(ASAP_PROMETHEUS_QUERY_URL_ENV); - let mut router = EngineRouter::new(); - if let Ok(Some(engine)) = engine_from_env() { - router.register(Arc::new(engine)); - } - assert!( - router - .engine_by_id(DATA_SOURCE_PROMETHEUS_REMOTE_ID) - .is_none(), - "env unset must leave prometheus_remote unregistered", - ); - } - } - - #[test] - fn build_result_from_prometheus_payload_rejects_non_success() { - let payload = PrometheusResponse { - status: "error".to_string(), - data: None, - error_type: Some("execution".to_string()), - error: Some("query timed out".to_string()), - }; - let err = build_result_from_prometheus_payload(payload, 0).unwrap_err(); - assert!(err.contains("query timed out")); - } - - #[test] - fn build_result_from_prometheus_payload_rejects_unsupported_result_type() { - // resultType = "scalar" is valid PromQL but unsupported in - // ASAP's wire shape — we want a clear parse error rather - // than an empty vector. - let payload = PrometheusResponse { - status: "success".to_string(), - data: Some(PrometheusData { - result_type: "scalar".to_string(), - result: vec![], - }), - error_type: None, - error: None, - }; - let err = build_result_from_prometheus_payload(payload, 0).unwrap_err(); - assert!(err.contains("unsupported resultType")); - } - - #[test] - fn parse_vector_extracts_value_and_timestamp() { - let raw: Value = serde_json::from_str( - r#"[{"metric":{"__name__":"x","job":"a"},"value":[1700000000.5,"3.14"]}]"#, - ) - .unwrap(); - let arr = raw.as_array().unwrap().clone(); - let result = parse_vector(&arr).expect("parse"); - match result { - QueryResult::Vector(iv) => { - assert_eq!(iv.values.len(), 1); - assert!((iv.values[0].value - 3.14).abs() < 1e-9); - assert_eq!(iv.timestamp, 1_700_000_000_500); - } - other => panic!("expected Vector, got {other:?}"), - } - } -} diff --git a/data_plane/src/query_engines/prometheus_query_engine/mod.rs b/data_plane/src/query_engines/prometheus_query_engine/mod.rs deleted file mode 100644 index d37af189..00000000 --- a/data_plane/src/query_engines/prometheus_query_engine/mod.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! Prometheus-remote query engine — HTTP-forwarder to a Prometheus -//! `/api/v1/query` endpoint. -//! -//! Phase ε.2 of the planner consolidation registers a -//! [`forward::PrometheusForwardEngine`] under the `prometheus_remote` -//! engine id so the controller's Mode 3 -//! (`RawAtEdgePrometheusArchive`) routing can target Prometheus -//! directly. The engine is conditional on the -//! [`forward::ASAP_PROMETHEUS_QUERY_URL_ENV`] env var; when unset, -//! the forwarder is not registered and a routing-table entry that -//! references `prometheus_remote` surfaces a `NoEngineRegistered` -//! 503 from the HTTP handler (the correct fail-loud behaviour for a -//! misconfigured deploy). -//! -//! Sibling of [`crate::query_engines::thanos_query_engine::forward`] (the -//! Step-2.3 archive forwarder); the two engines coexist in the -//! router under different ids and answer different routing-table -//! entries. - -pub mod forward; - -pub use forward::{ - engine_from_env as prometheus_engine_from_env, PrometheusForwardConfig, - PrometheusForwardEngine, PrometheusForwardError, ASAP_PROMETHEUS_QUERY_URL_ENV, - DATA_SOURCE_PROMETHEUS_REMOTE_ID, DATA_SOURCE_PROMETHEUS_REMOTE_INFO, - DEFAULT_PROMETHEUS_QUERY_URL, QUIRK_PROMETHEUS_UNREACHABLE, -}; diff --git a/data_plane/src/query_engines/query_result.rs b/data_plane/src/query_engines/query_result.rs index a1b6cc42..2f9f811f 100644 --- a/data_plane/src/query_engines/query_result.rs +++ b/data_plane/src/query_engines/query_result.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::KeyByLabelValues; +use crate::stores::types::KeyByLabelValues; use crate::stores::sketch_db::AccuracyEnvelope; use serde::{Deserialize, Serialize}; diff --git a/data_plane/src/query_engines/routing/backend_storage_routing.rs b/data_plane/src/query_engines/routing/backend_storage_routing.rs index db759f21..cc2d688c 100644 --- a/data_plane/src/query_engines/routing/backend_storage_routing.rs +++ b/data_plane/src/query_engines/routing/backend_storage_routing.rs @@ -892,7 +892,7 @@ pub fn routing_table_hash(table: &BackendStorageRouting) -> String { // --------------------------------------------------------------------------- /// Per-tenant atomic-swap wrapper around `BackendStorageRouting`, -/// mirroring [`crate::stores::schema::HotReloadStreamingConfig`]. Lets the +/// mirroring [`crate::stores::types::HotReloadStreamingConfig`]. Lets the /// `POST /api/v1/storage_routing` HTTP handler swap one tenant's table /// at runtime without restarting the backend or touching any other /// tenant's table. Cloneable; clones share the underlying `ArcSwap` so diff --git a/data_plane/src/query_engines/thanos_query_engine/forward.rs b/data_plane/src/query_engines/thanos_query_engine/forward.rs index 1ca9d757..826de332 100644 --- a/data_plane/src/query_engines/thanos_query_engine/forward.rs +++ b/data_plane/src/query_engines/thanos_query_engine/forward.rs @@ -39,7 +39,7 @@ use serde::Deserialize; use serde_json::Value; use tracing::{debug, warn}; -use crate::stores::schema::KeyByLabelValues; +use crate::stores::types::KeyByLabelValues; use crate::query_engines::query_result::{InstantVectorElement, QueryResult, RangeVectorElement}; use crate::query_engines::routing::query_engine_routing::{EngineCapabilities, QueryEngine}; use crate::stores::sketch_db::accuracy::{AccuracyEnvelope, AccuracyProfile}; diff --git a/data_plane/src/query_engines/warm_tier/delta_apply.rs b/data_plane/src/query_engines/warm_tier/delta_apply.rs index 16f4da89..4affe18b 100644 --- a/data_plane/src/query_engines/warm_tier/delta_apply.rs +++ b/data_plane/src/query_engines/warm_tier/delta_apply.rs @@ -34,7 +34,7 @@ use asap_sketchlib::sketches::ddsketch::DdSketch; use asap_sketchlib::sketches::hll::HllSketch; use asap_sketchlib::sketches::kll::KllSketch; -use crate::stores::sketch_db::sketch_index::{SketchEncoding, SketchSampleState}; +use crate::stores::sketch_db::index::{SketchEncoding, SketchSampleState}; /// Whether a sketch family supports delta-via-merge (DD/KLL) or /// delta-via-`apply_delta` (HLL). The reducer reads bytes through diff --git a/data_plane/src/query_engines/warm_tier/mod.rs b/data_plane/src/query_engines/warm_tier/mod.rs index 83b1f29a..2b7ff179 100644 --- a/data_plane/src/query_engines/warm_tier/mod.rs +++ b/data_plane/src/query_engines/warm_tier/mod.rs @@ -4,7 +4,7 @@ //! [`crate::query_engines::asap_query_engine::engine::ASAPQueryEngine`]'s //! `QueryEngine::execute` adapter: parse the PromQL, extract //! `(metric_name, label_keys)`, look up candidate sids via -//! [`crate::stores::sketch_db::sketch_index::SketchIndex::instances_matching`], +//! [`crate::stores::sketch_db::index::SketchIndex::instances_matching`], //! and classify each sid. On `Ghost`/`Unknown`, return //! `EngineError::CapabilityMiss(SketchStore, …)` so the //! `EngineRouter` fails over to the archive engine. @@ -14,7 +14,7 @@ //! that fall-through with **direct sketch evaluation** from //! [`SketchIndex::query_range`]'s output: deserialize each //! window's sketch state, dispatch on the per-instance -//! [`crate::stores::sketch_db::sketch_index::Capability`], and reduce to a +//! [`crate::stores::sketch_db::index::Capability`], and reduce to a //! per-window scalar via the canonical sketch query (DDSketch / //! KLL → quantile, HLL → cardinality estimate, CMS / CountSketch //! → frequency point query, CMS-with-heap → top-k items). diff --git a/data_plane/src/query_engines/warm_tier/sketch_reducer.rs b/data_plane/src/query_engines/warm_tier/sketch_reducer.rs index 7ea77520..39b57cd7 100644 --- a/data_plane/src/query_engines/warm_tier/sketch_reducer.rs +++ b/data_plane/src/query_engines/warm_tier/sketch_reducer.rs @@ -64,7 +64,7 @@ use crate::query_engines::warm_tier::decoders::{ use crate::query_engines::warm_tier::delta_apply::{ cumulative_evaluate, per_window_evaluate, DeltaSketchKind, }; -use crate::stores::sketch_db::sketch_index::{ +use crate::stores::sketch_db::index::{ Capability, SketchEncoding, SketchIndex, SketchInstanceMetadata, SketchKindHandle, SketchSampleState, }; diff --git a/data_plane/src/query_engines/warm_tier/tests.rs b/data_plane/src/query_engines/warm_tier/tests.rs index 06d65b32..b8a458a8 100644 --- a/data_plane/src/query_engines/warm_tier/tests.rs +++ b/data_plane/src/query_engines/warm_tier/tests.rs @@ -16,7 +16,7 @@ use asap_sketchlib::sketches::ddsketch::DdSketch; use asap_sketchlib::sketches::hll::{HllSketch, HllVariant}; use crate::query_engines::warm_tier::{SketchReducer, WarmTierError}; -use crate::stores::sketch_db::sketch_index::{ +use crate::stores::sketch_db::index::{ AccuracyBound, Capability, SketchConfig, SketchEncoding, SketchIndex, SketchInstanceMetadata, SketchKindHandle, SketchSampleState, }; diff --git a/data_plane/src/query_engines/window_merger.rs b/data_plane/src/query_engines/window_merger.rs index 3a2baa6a..1df020e9 100644 --- a/data_plane/src/query_engines/window_merger.rs +++ b/data_plane/src/query_engines/window_merger.rs @@ -8,7 +8,7 @@ //! - `IncrementalMerger`: Add/subtract for subtractable accumulators (future) //! - `SwagMerger`: Two-stack queue for non-subtractable accumulators (future) -use crate::stores::schema::{AggregateCore, AggregationType}; +use crate::stores::types::{AggregateCore, AggregationType}; /// Trait for merging buckets in a sliding window /// @@ -107,7 +107,7 @@ pub fn create_window_merger(_accumulator_type: AggregationType) -> Box)> = + let mut batch: Vec<(crate::stores::types::PrecomputedOutput, Box)> = Vec::with_capacity(by_group.len()); for (group_key, group_samples) in by_group { @@ -220,7 +220,7 @@ impl WindowProcessor for BackfillWindowProcessor { } else { Some(build_group_key_label_values(&group_key)) }; - let output = crate::stores::schema::PrecomputedOutput::new_backfilled( + let output = crate::stores::types::PrecomputedOutput::new_backfilled( window_range.0, window_range.1, key, @@ -260,11 +260,11 @@ impl WindowProcessor for BackfillWindowProcessor { #[cfg(test)] mod tests { use super::*; - use crate::stores::schema::StreamingConfig; + use crate::stores::types::StreamingConfig; use crate::stores::sketch_db::backfill::BackfillSource; - use crate::stores::sketch_db::backfill_worker::BackfillWorker; - use crate::stores::sketch_db::raw_sample_reader::{LabelFilter, MockRawSampleReader}; - use crate::stores::sketch_db::sketch_store::SketchStore; + use crate::stores::sketch_db::backfill::worker::BackfillWorker; + use crate::stores::sketch_db::backfill::raw_sample_reader::{LabelFilter, MockRawSampleReader}; + use crate::stores::sketch_db::store::SketchStore; use asap_types::enums::{AggregationType, WindowType}; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; use std::sync::Arc; @@ -310,7 +310,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SketchStore::new( streaming.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( @@ -360,7 +360,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SketchStore::new( streaming.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( @@ -387,7 +387,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SketchStore::new( streaming.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( @@ -413,7 +413,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SketchStore::new( streaming.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( @@ -461,7 +461,7 @@ mod tests { assert_eq!( registry.get(job_id).unwrap().status, - super::super::backfill::BackfillStatus::Complete + super::super::BackfillStatus::Complete ); // 4 windows × 1 write each (some windows have 1 group — the // writes are per-window batches, not per-group entries). diff --git a/data_plane/src/stores/sketch_db/prometheus_reader.rs b/data_plane/src/stores/sketch_db/backfill/prometheus_reader.rs similarity index 100% rename from data_plane/src/stores/sketch_db/prometheus_reader.rs rename to data_plane/src/stores/sketch_db/backfill/prometheus_reader.rs diff --git a/data_plane/src/stores/sketch_db/raw_sample_reader.rs b/data_plane/src/stores/sketch_db/backfill/raw_sample_reader.rs similarity index 100% rename from data_plane/src/stores/sketch_db/raw_sample_reader.rs rename to data_plane/src/stores/sketch_db/backfill/raw_sample_reader.rs diff --git a/data_plane/src/stores/sketch_db/backfill_service.rs b/data_plane/src/stores/sketch_db/backfill/service.rs similarity index 96% rename from data_plane/src/stores/sketch_db/backfill_service.rs rename to data_plane/src/stores/sketch_db/backfill/service.rs index 935f2d02..722dd8d8 100644 --- a/data_plane/src/stores/sketch_db/backfill_service.rs +++ b/data_plane/src/stores/sketch_db/backfill/service.rs @@ -50,11 +50,11 @@ use std::time::Duration; use tokio::sync::oneshot; use tracing::{debug, info, warn}; -use crate::stores::schema::HotReloadStreamingConfig; +use crate::stores::types::HotReloadStreamingConfig; use crate::stores::sketch_db::backfill::{BackfillRegistry, BackfillSource, BackfillStatus}; -use crate::stores::sketch_db::backfill_processor::BackfillWindowProcessor; -use crate::stores::sketch_db::backfill_worker::BackfillWorker; -use crate::stores::sketch_db::raw_sample_reader::{LabelFilter, RawSampleReader}; +use crate::stores::sketch_db::backfill::processor::BackfillWindowProcessor; +use crate::stores::sketch_db::backfill::worker::BackfillWorker; +use crate::stores::sketch_db::backfill::raw_sample_reader::{LabelFilter, RawSampleReader}; use crate::stores::sketch_db::schema::SchemaRegistry; use crate::stores::traits::Store; @@ -293,9 +293,9 @@ pub fn default_reader_factory() -> ReaderFactory { #[cfg(test)] mod tests { use super::*; - use crate::stores::schema::StreamingConfig; - use crate::stores::sketch_db::raw_sample_reader::{MockRawSampleReader, RawSample}; - use crate::stores::sketch_db::sketch_store::SketchStore; + use crate::stores::types::StreamingConfig; + use crate::stores::sketch_db::backfill::raw_sample_reader::{MockRawSampleReader, RawSample}; + use crate::stores::sketch_db::store::SketchStore; use asap_types::aggregation_config::AggregationConfig; use asap_types::enums::{AggregationType, WindowType}; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; @@ -360,7 +360,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SketchStore::new( streaming.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); @@ -413,7 +413,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SketchStore::new( streaming.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); @@ -453,7 +453,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SketchStore::new( streaming.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); @@ -522,7 +522,7 @@ mod tests { let schemas = Arc::new(SchemaRegistry::from_streaming_config(&streaming)); let store: Arc = Arc::new(SketchStore::new( streaming.clone(), - crate::stores::schema::CleanupPolicy::NoCleanup, + crate::stores::types::CleanupPolicy::NoCleanup, )); let registry = Arc::new(BackfillRegistry::new()); diff --git a/data_plane/src/stores/sketch_db/backfill_window_builder.rs b/data_plane/src/stores/sketch_db/backfill/window_builder.rs similarity index 98% rename from data_plane/src/stores/sketch_db/backfill_window_builder.rs rename to data_plane/src/stores/sketch_db/backfill/window_builder.rs index 4b22aea5..8690cd9f 100644 --- a/data_plane/src/stores/sketch_db/backfill_window_builder.rs +++ b/data_plane/src/stores/sketch_db/backfill/window_builder.rs @@ -47,12 +47,12 @@ //! MultipleSubpopulation (update_keyed) dispatch — mirrors //! `worker::apply_sample`. -use crate::stores::schema::{AggregateCore, KeyByLabelValues}; +use crate::stores::types::{AggregateCore, KeyByLabelValues}; use crate::precompute_engine::accumulator_factory::{ create_accumulator_updater, AccumulatorUpdater, }; use crate::precompute_engine::worker::parse_labels_from_series_key; -use crate::stores::sketch_db::raw_sample_reader::RawSample; +use crate::stores::sketch_db::backfill::raw_sample_reader::RawSample; use asap_types::aggregation_config::AggregationConfig; /// Extract the MultipleSubpopulation aggregated-label key from a diff --git a/data_plane/src/stores/sketch_db/backfill_worker.rs b/data_plane/src/stores/sketch_db/backfill/worker.rs similarity index 99% rename from data_plane/src/stores/sketch_db/backfill_worker.rs rename to data_plane/src/stores/sketch_db/backfill/worker.rs index 1d2fc8d3..d503d190 100644 --- a/data_plane/src/stores/sketch_db/backfill_worker.rs +++ b/data_plane/src/stores/sketch_db/backfill/worker.rs @@ -59,7 +59,7 @@ use std::sync::Arc; use async_trait::async_trait; -use super::backfill::{BackfillRegistry, BackfillStatus}; +use super::{BackfillRegistry, BackfillStatus}; use super::raw_sample_reader::{LabelFilter, RawSample, RawSampleReader}; /// Per-window callback invoked by [`BackfillWorker`] after reading diff --git a/data_plane/src/stores/sketch_db/epoch_columnar.rs b/data_plane/src/stores/sketch_db/index/epoch_columnar.rs similarity index 100% rename from data_plane/src/stores/sketch_db/epoch_columnar.rs rename to data_plane/src/stores/sketch_db/index/epoch_columnar.rs diff --git a/data_plane/src/stores/sketch_db/sketch_index.rs b/data_plane/src/stores/sketch_db/index/mod.rs similarity index 98% rename from data_plane/src/stores/sketch_db/sketch_index.rs rename to data_plane/src/stores/sketch_db/index/mod.rs index b50fb5d4..a01083bd 100644 --- a/data_plane/src/stores/sketch_db/sketch_index.rs +++ b/data_plane/src/stores/sketch_db/index/mod.rs @@ -24,7 +24,7 @@ use std::sync::{Arc, RwLock}; use dashmap::DashMap; -use super::epoch_columnar::{LabelValuesId, SidStoreData, TimestampRange}; +use self::epoch_columnar::{LabelValuesId, SidStoreData, TimestampRange}; // ── Capability re-exports ──────────────────────────────────────────────────── // @@ -474,3 +474,7 @@ mod tests { assert_eq!(series[0].samples.len(), 4); } } + +// 2026-05 reorg: generic epoch-partitioned columnar storage lives +// alongside the index that uses it. +pub mod epoch_columnar; diff --git a/data_plane/src/stores/sketch_db/mod.rs b/data_plane/src/stores/sketch_db/mod.rs index ae35d6ad..66f62f01 100644 --- a/data_plane/src/stores/sketch_db/mod.rs +++ b/data_plane/src/stores/sketch_db/mod.rs @@ -30,36 +30,21 @@ pub mod accuracy; pub mod backfill; -pub mod backfill_processor; -pub mod backfill_service; -pub mod backfill_window_builder; -pub mod backfill_worker; -pub mod epoch_columnar; +pub mod index; pub mod metrics; -pub mod prometheus_reader; -pub mod raw_sample_reader; pub mod schema; -pub mod schema_eviction; -pub mod sketch_store; -pub mod sketch_index; +pub mod store; pub use accuracy::{AccuracyEnvelope, AccuracyKind, AccuracyProfile, PerSegmentAccuracy}; pub use backfill::{ - BackfillJob, BackfillRegistry, BackfillSource, BackfillStatus, Coverage, CreateError, + build_backfilled_accumulator, default_reader_factory, noop_reader_factory, BackfillJob, + BackfillRegistry, BackfillService, BackfillServiceConfig, BackfillServiceHandle, + BackfillSource, BackfillStatus, BackfillWindowProcessor, BackfillWorker, BackfillWorkerError, + Coverage, CreateError, LabelFilter, MockRawSampleReader, PrometheusReader, RawSample, + RawSampleReader, RawSampleReaderError, ReaderFactory, WindowProcessor, }; -pub use backfill_processor::BackfillWindowProcessor; -pub use backfill_service::{ - default_reader_factory, noop_reader_factory, BackfillService, BackfillServiceConfig, - BackfillServiceHandle, ReaderFactory, +pub use schema::{ + warn_if_retention_inverted, AggSchema, AggStatus, SchemaEvictionConfig, SchemaEvictionHandle, + SchemaEvictionService, SchemaRegistry, TimelineCoverage, TimelineSegment, }; -pub use backfill_window_builder::build_backfilled_accumulator; -pub use backfill_worker::{BackfillWorker, BackfillWorkerError, WindowProcessor}; -pub use prometheus_reader::PrometheusReader; -pub use raw_sample_reader::{ - LabelFilter, MockRawSampleReader, RawSample, RawSampleReader, RawSampleReaderError, -}; -pub use schema::{AggSchema, AggStatus, SchemaRegistry, TimelineCoverage, TimelineSegment}; -pub use schema_eviction::{ - warn_if_retention_inverted, SchemaEvictionConfig, SchemaEvictionHandle, SchemaEvictionService, -}; -pub use sketch_store::SketchStore; +pub use store::SketchStore; diff --git a/data_plane/src/stores/sketch_db/schema_eviction.rs b/data_plane/src/stores/sketch_db/schema/eviction.rs similarity index 97% rename from data_plane/src/stores/sketch_db/schema_eviction.rs rename to data_plane/src/stores/sketch_db/schema/eviction.rs index bde1ac0a..9d855a0a 100644 --- a/data_plane/src/stores/sketch_db/schema_eviction.rs +++ b/data_plane/src/stores/sketch_db/schema/eviction.rs @@ -52,8 +52,8 @@ use tokio::sync::oneshot; use tokio::task::JoinHandle; use tracing::{info, warn}; -use super::backfill::{BackfillRegistry, BackfillStatus}; -use super::schema::{AggStatus, SchemaRegistry}; +use crate::stores::sketch_db::backfill::{BackfillRegistry, BackfillStatus}; +use super::{AggStatus, SchemaRegistry}; use crate::stores::traits::Store; /// Configuration for the eviction loop. Separate from @@ -267,9 +267,9 @@ pub fn warn_if_retention_inverted( #[cfg(test)] mod tests { use super::*; - use crate::stores::schema::{AggregationType, CleanupPolicy, LockStrategy, StreamingConfig}; + use crate::stores::types::{AggregationType, CleanupPolicy, LockStrategy, StreamingConfig}; use crate::precompute_engine::operators::SumAccumulator; - use crate::stores::sketch_db::{backfill::BackfillSource, sketch_store::SketchStore}; + use crate::stores::sketch_db::{backfill::BackfillSource, store::SketchStore}; use asap_types::aggregation_config::AggregationConfig; use asap_types::enums::WindowType; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; @@ -308,7 +308,7 @@ mod tests { fn write_one(store: &SketchStore, agg_id: u64, ts: u64) { let acc = SumAccumulator::with_sum(1.0); - let output = crate::stores::schema::PrecomputedOutput::new(ts, ts + 1000, None, agg_id); + let output = crate::stores::types::PrecomputedOutput::new(ts, ts + 1000, None, agg_id); store .insert_precomputed_output(output, Box::new(acc)) .unwrap(); diff --git a/data_plane/src/stores/sketch_db/schema.rs b/data_plane/src/stores/sketch_db/schema/mod.rs similarity index 99% rename from data_plane/src/stores/sketch_db/schema.rs rename to data_plane/src/stores/sketch_db/schema/mod.rs index 213252bb..3ae31515 100644 --- a/data_plane/src/stores/sketch_db/schema.rs +++ b/data_plane/src/stores/sketch_db/schema/mod.rs @@ -77,7 +77,7 @@ use asap_types::aggregation_config::AggregationConfig; use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; -use crate::stores::schema::StreamingConfig; +use crate::stores::types::StreamingConfig; /// Lifecycle state of an `aggregation_id`. Derived from the /// `AggSchema`'s timestamps and the current wall clock — never stored @@ -1276,3 +1276,9 @@ mod tests { assert!(r.force_expire(999).is_none()); } } + +// 2026-05 reorg: schema_eviction.rs moved alongside as a submodule. +pub mod eviction; +pub use eviction::{ + warn_if_retention_inverted, SchemaEvictionConfig, SchemaEvictionHandle, SchemaEvictionService, +}; diff --git a/data_plane/src/stores/sketch_db/sketch_store/INDEX_DESIGN.md b/data_plane/src/stores/sketch_db/store/INDEX_DESIGN.md similarity index 100% rename from data_plane/src/stores/sketch_db/sketch_store/INDEX_DESIGN.md rename to data_plane/src/stores/sketch_db/store/INDEX_DESIGN.md diff --git a/data_plane/src/stores/sketch_db/sketch_store/common.rs b/data_plane/src/stores/sketch_db/store/common.rs similarity index 99% rename from data_plane/src/stores/sketch_db/sketch_store/common.rs rename to data_plane/src/stores/sketch_db/store/common.rs index a6dadd37..8c91aede 100644 --- a/data_plane/src/stores/sketch_db/sketch_store/common.rs +++ b/data_plane/src/stores/sketch_db/store/common.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{AggregateCore, KeyByLabelValues}; +use crate::stores::types::{AggregateCore, KeyByLabelValues}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; diff --git a/data_plane/src/stores/sketch_db/sketch_store/global.rs b/data_plane/src/stores/sketch_db/store/global.rs similarity index 99% rename from data_plane/src/stores/sketch_db/sketch_store/global.rs rename to data_plane/src/stores/sketch_db/store/global.rs index 5e8b5ab4..4b5f0ad9 100644 --- a/data_plane/src/stores/sketch_db/sketch_store/global.rs +++ b/data_plane/src/stores/sketch_db/store/global.rs @@ -1,7 +1,7 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, AggregationType, CleanupPolicy, PrecomputedOutput, StreamingConfig, }; -use crate::stores::sketch_db::sketch_store::common::{ +use crate::stores::sketch_db::store::common::{ EpochID, InternTable, MetricBucketMap, MutableEpoch, SealedEpoch, TimestampRange, }; use crate::stores::{Store, StoreResult, TimestampedBucketsMap}; diff --git a/data_plane/src/stores/sketch_db/sketch_store/mod.rs b/data_plane/src/stores/sketch_db/store/mod.rs similarity index 99% rename from data_plane/src/stores/sketch_db/sketch_store/mod.rs rename to data_plane/src/stores/sketch_db/store/mod.rs index c41ffa15..5ce3a6ae 100644 --- a/data_plane/src/stores/sketch_db/sketch_store/mod.rs +++ b/data_plane/src/stores/sketch_db/store/mod.rs @@ -3,7 +3,7 @@ pub mod global; pub mod per_key; pub mod persistence; -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, CleanupPolicy, LockStrategy, PrecomputedOutput, StreamingConfig, }; use crate::stores::{Store, StoreResult, TimestampedBucketsMap}; @@ -172,7 +172,7 @@ impl Store for SketchStore { #[cfg(test)] mod drop_agg_id_tests { use super::*; - use crate::stores::schema::AggregationType; + use crate::stores::types::AggregationType; use crate::precompute_engine::operators::SumAccumulator; use asap_types::aggregation_config::AggregationConfig; use asap_types::enums::WindowType; diff --git a/data_plane/src/stores/sketch_db/sketch_store/per_key.rs b/data_plane/src/stores/sketch_db/store/per_key.rs similarity index 99% rename from data_plane/src/stores/sketch_db/sketch_store/per_key.rs rename to data_plane/src/stores/sketch_db/store/per_key.rs index 11d667eb..12b5df6a 100644 --- a/data_plane/src/stores/sketch_db/sketch_store/per_key.rs +++ b/data_plane/src/stores/sketch_db/store/per_key.rs @@ -1,8 +1,8 @@ -use crate::stores::schema::{ +use crate::stores::types::{ AggregateCore, AggregationType, CleanupPolicy, KeyByLabelValues, PrecomputedOutput, StreamingConfig, }; -use crate::stores::sketch_db::sketch_store::common::{ +use crate::stores::sketch_db::store::common::{ EpochID, InternTable, MetricBucketMap, MetricID, MutableEpoch, SealedEpoch, TimestampRange, }; use crate::stores::{Store, StoreResult, TimestampedBucketsMap}; diff --git a/data_plane/src/stores/sketch_db/sketch_store/persistence/cache.rs b/data_plane/src/stores/sketch_db/store/persistence/cache.rs similarity index 100% rename from data_plane/src/stores/sketch_db/sketch_store/persistence/cache.rs rename to data_plane/src/stores/sketch_db/store/persistence/cache.rs diff --git a/data_plane/src/stores/sketch_db/sketch_store/persistence/config.rs b/data_plane/src/stores/sketch_db/store/persistence/config.rs similarity index 100% rename from data_plane/src/stores/sketch_db/sketch_store/persistence/config.rs rename to data_plane/src/stores/sketch_db/store/persistence/config.rs diff --git a/data_plane/src/stores/sketch_db/sketch_store/persistence/flusher.rs b/data_plane/src/stores/sketch_db/store/persistence/flusher.rs similarity index 99% rename from data_plane/src/stores/sketch_db/sketch_store/persistence/flusher.rs rename to data_plane/src/stores/sketch_db/store/persistence/flusher.rs index 041d7552..cf877f4f 100644 --- a/data_plane/src/stores/sketch_db/sketch_store/persistence/flusher.rs +++ b/data_plane/src/stores/sketch_db/store/persistence/flusher.rs @@ -424,8 +424,8 @@ fn now_ms() -> u64 { #[cfg(test)] mod tests { use super::*; - use crate::stores::schema::KeyByLabelValues; - use crate::stores::sketch_db::sketch_store::persistence::source::{ + use crate::stores::types::KeyByLabelValues; + use crate::stores::sketch_db::store::persistence::source::{ EpochSnapshot, EpochSnapshotEntry, }; use std::sync::Mutex as StdMutex; diff --git a/data_plane/src/stores/sketch_db/sketch_store/persistence/manifest.rs b/data_plane/src/stores/sketch_db/store/persistence/manifest.rs similarity index 100% rename from data_plane/src/stores/sketch_db/sketch_store/persistence/manifest.rs rename to data_plane/src/stores/sketch_db/store/persistence/manifest.rs diff --git a/data_plane/src/stores/sketch_db/sketch_store/persistence/mod.rs b/data_plane/src/stores/sketch_db/store/persistence/mod.rs similarity index 100% rename from data_plane/src/stores/sketch_db/sketch_store/persistence/mod.rs rename to data_plane/src/stores/sketch_db/store/persistence/mod.rs diff --git a/data_plane/src/stores/sketch_db/sketch_store/persistence/part.rs b/data_plane/src/stores/sketch_db/store/persistence/part.rs similarity index 98% rename from data_plane/src/stores/sketch_db/sketch_store/persistence/part.rs rename to data_plane/src/stores/sketch_db/store/persistence/part.rs index 1a96c611..7a352715 100644 --- a/data_plane/src/stores/sketch_db/sketch_store/persistence/part.rs +++ b/data_plane/src/stores/sketch_db/store/persistence/part.rs @@ -99,7 +99,7 @@ pub struct SnapshotEntry { pub agg_id: u64, pub start_ts: u64, pub end_ts: u64, - pub label: Option, + pub label: Option, pub sketch_type_name: String, pub sketch_bytes: Vec, } @@ -559,7 +559,7 @@ impl PartReader { None } else { Some( - crate::stores::schema::KeyByLabelValues::deserialize_from_bytes(label_bytes) + crate::stores::types::KeyByLabelValues::deserialize_from_bytes(label_bytes) .map_err(|e| PersistError::Format(format!("label decode: {}", e)))?, ) }; @@ -593,8 +593,8 @@ fn map_file(path: &Path) -> PersistResult { #[cfg(test)] mod tests { use super::*; - use crate::stores::schema::KeyByLabelValues; - use crate::stores::sketch_db::sketch_store::persistence::source::EpochSnapshotEntry; + use crate::stores::types::KeyByLabelValues; + use crate::stores::sketch_db::store::persistence::source::EpochSnapshotEntry; use tempfile::TempDir; fn make_snapshot() -> EpochSnapshot { diff --git a/data_plane/src/stores/sketch_db/sketch_store/persistence/recovery.rs b/data_plane/src/stores/sketch_db/store/persistence/recovery.rs similarity index 96% rename from data_plane/src/stores/sketch_db/sketch_store/persistence/recovery.rs rename to data_plane/src/stores/sketch_db/store/persistence/recovery.rs index 40bcf6c7..afcf4938 100644 --- a/data_plane/src/stores/sketch_db/sketch_store/persistence/recovery.rs +++ b/data_plane/src/stores/sketch_db/store/persistence/recovery.rs @@ -117,11 +117,11 @@ pub fn recover(disk_path: &Path) -> PersistResult<(Manifest, RecoveryReport)> { #[cfg(test)] mod tests { use super::*; - use crate::stores::schema::KeyByLabelValues; - use crate::stores::sketch_db::sketch_store::persistence::part::{ + use crate::stores::types::KeyByLabelValues; + use crate::stores::sketch_db::store::persistence::part::{ part_dir_path, PartWriter, }; - use crate::stores::sketch_db::sketch_store::persistence::source::{ + use crate::stores::sketch_db::store::persistence::source::{ EpochSnapshot, EpochSnapshotEntry, }; use tempfile::TempDir; @@ -183,7 +183,7 @@ mod tests { let report_write = PartWriter::write_part(&part_dir, 42, &[dummy_snapshot()]).unwrap(); manifest .append_add( - crate::stores::sketch_db::sketch_store::persistence::manifest::PartEntry { + crate::stores::sketch_db::store::persistence::manifest::PartEntry { part_id: 42, min_ts: report_write.min_ts, max_ts: report_write.max_ts, diff --git a/data_plane/src/stores/sketch_db/sketch_store/persistence/source.rs b/data_plane/src/stores/sketch_db/store/persistence/source.rs similarity index 98% rename from data_plane/src/stores/sketch_db/sketch_store/persistence/source.rs rename to data_plane/src/stores/sketch_db/store/persistence/source.rs index 809f7ffb..7a4872b2 100644 --- a/data_plane/src/stores/sketch_db/sketch_store/persistence/source.rs +++ b/data_plane/src/stores/sketch_db/store/persistence/source.rs @@ -2,7 +2,7 @@ //! epochs. Decouples `flusher.rs` from `SketchStorePerKey` so the //! flusher can be unit-tested against a fake source. -use crate::stores::schema::KeyByLabelValues; +use crate::stores::types::KeyByLabelValues; use super::PersistResult; diff --git a/data_plane/src/stores/traits.rs b/data_plane/src/stores/traits.rs index 815afe7b..9e718af6 100644 --- a/data_plane/src/stores/traits.rs +++ b/data_plane/src/stores/traits.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{AggregateCore, KeyByLabelValues, PrecomputedOutput}; +use crate::stores::types::{AggregateCore, KeyByLabelValues, PrecomputedOutput}; use std::collections::HashMap; use std::sync::Arc; diff --git a/data_plane/src/stores/schema/enums.rs b/data_plane/src/stores/types/enums.rs similarity index 100% rename from data_plane/src/stores/schema/enums.rs rename to data_plane/src/stores/types/enums.rs diff --git a/data_plane/src/stores/schema/hot_reload_config.rs b/data_plane/src/stores/types/hot_reload_config.rs similarity index 98% rename from data_plane/src/stores/schema/hot_reload_config.rs rename to data_plane/src/stores/types/hot_reload_config.rs index c95cd7d4..3c2a7e05 100644 --- a/data_plane/src/stores/schema/hot_reload_config.rs +++ b/data_plane/src/stores/types/hot_reload_config.rs @@ -79,7 +79,7 @@ use std::sync::Arc; use arc_swap::ArcSwap; -use crate::stores::schema::StreamingConfig; +use crate::stores::types::StreamingConfig; /// Thin wrapper around `ArcSwap` with ergonomic /// snapshot + swap helpers. Cloneable; clones share the same @@ -137,7 +137,7 @@ impl std::fmt::Debug for HotReloadStreamingConfig { #[cfg(test)] mod tests { use super::*; - use crate::stores::schema::AggregationConfig; + use crate::stores::types::AggregationConfig; use asap_types::enums::{AggregationType, WindowType}; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; use std::collections::HashMap; diff --git a/data_plane/src/stores/schema/key_by_label_values.rs b/data_plane/src/stores/types/key_by_label_values.rs similarity index 100% rename from data_plane/src/stores/schema/key_by_label_values.rs rename to data_plane/src/stores/types/key_by_label_values.rs diff --git a/data_plane/src/stores/schema/measurement.rs b/data_plane/src/stores/types/measurement.rs similarity index 100% rename from data_plane/src/stores/schema/measurement.rs rename to data_plane/src/stores/types/measurement.rs diff --git a/data_plane/src/stores/types/mod.rs b/data_plane/src/stores/types/mod.rs new file mode 100644 index 00000000..20381216 --- /dev/null +++ b/data_plane/src/stores/types/mod.rs @@ -0,0 +1,39 @@ +//! Data types used by the storage layer (and consumed cross-module +//! by `drivers/`, `precompute_engine/`, `query_engines/`). +//! +//! Renamed from `stores/schema/` in the 2026-05 reorg to avoid the +//! name-clash with `stores/sketch_db/schema/` (per-`agg_id` schema +//! lifecycle, a different concern). Tiny `pub use asap_types::X` +//! shim files were dropped; import directly from `asap_types`. + +pub mod enums; +pub mod hot_reload_config; +pub mod key_by_label_values; +pub mod measurement; +pub mod precomputed_output; +pub mod traits; + +pub use enums::*; +pub use hot_reload_config::*; +pub use key_by_label_values::*; +pub use measurement::*; +pub use precomputed_output::*; +pub use traits::*; + +// Cross-module re-exports of asap_types data types so callers can +// write `crate::stores::types::StreamingConfig` instead of reaching +// across crates. (Previously these had per-type shim files like +// `aggregation_config.rs` containing only `pub use asap_types::...`.) +pub use asap_types::aggregation_config::*; +pub use asap_types::aggregation_reference::*; +pub use asap_types::inference_config::*; +pub use asap_types::promql_schema::*; +pub use asap_types::query_config::*; +pub use asap_types::streaming_config::*; + +// Re-export the query-side routing surface so existing call sites +// like `crate::stores::types::BackendStorageRouting` keep compiling. +pub use crate::query_engines::routing::{ + classify_query_shape, BackendStorageRouting, HotReloadBackendStorageRouting, QueryShape, + RoutingTarget, +}; diff --git a/data_plane/src/stores/schema/precomputed_output.rs b/data_plane/src/stores/types/precomputed_output.rs similarity index 97% rename from data_plane/src/stores/schema/precomputed_output.rs rename to data_plane/src/stores/types/precomputed_output.rs index 040336bb..f6242def 100644 --- a/data_plane/src/stores/schema/precomputed_output.rs +++ b/data_plane/src/stores/types/precomputed_output.rs @@ -4,8 +4,8 @@ use serde::{Deserialize, Serialize}; use std::io::Read as _; use tracing::error; -use crate::stores::schema::traits::SerializableToSink; -use crate::stores::schema::{AggregationType, KeyByLabelValues, StreamingConfig}; +use crate::stores::types::traits::SerializableToSink; +use crate::stores::types::{AggregationType, KeyByLabelValues, StreamingConfig}; /// §5.1 provenance tag on every precompute record: did this window /// come from live ingest or was it materialised by a backfill job? @@ -102,7 +102,7 @@ impl PrecomputedOutput { // /// Serialize PrecomputedOutput with precompute data to match Python JSON format // pub fn serialize_to_json_with_precompute( // &self, - // precompute: &dyn crate::stores::schema::AggregateCore, + // precompute: &dyn crate::stores::types::AggregateCore, // ) -> serde_json::Value { // serde_json::json!({ // // "config": self.config.serialize_to_json(), @@ -195,7 +195,7 @@ impl PrecomputedOutput { // streaming_config: &HashMap, streaming_config: &StreamingConfig, ) -> Result< - (Self, Box), + (Self, Box), Box, > { let aggregation_id = data @@ -295,7 +295,7 @@ impl PrecomputedOutput { // pub fn deserialize_from_json_with_precompute( // data: &serde_json::Value, // ) -> Result< - // (Self, Box), + // (Self, Box), // Box, // > { // debug!("Deserializing PrecomputedOutput with precompute from JSON: {data}"); @@ -324,7 +324,7 @@ impl PrecomputedOutput { // data: &[u8], // aggregation_type: &str, // ) -> Result< - // (Self, Box), + // (Self, Box), // Box, // > { // // First get the metadata and precompute bytes @@ -346,7 +346,7 @@ impl PrecomputedOutput { // fn create_precompute_from_json( // precompute_type: &str, // data: &serde_json::Value, - // ) -> Result, Box> + // ) -> Result, Box> // { // use crate::precompute_engine::operators::*; @@ -416,7 +416,7 @@ impl PrecomputedOutput { fn create_precompute_from_bytes( precompute_type: AggregationType, buffer: &[u8], - ) -> Result, Box> + ) -> Result, Box> { use crate::precompute_engine::operators::*; diff --git a/data_plane/src/stores/schema/traits.rs b/data_plane/src/stores/types/traits.rs similarity index 99% rename from data_plane/src/stores/schema/traits.rs rename to data_plane/src/stores/types/traits.rs index c342ffb9..f0f581df 100644 --- a/data_plane/src/stores/schema/traits.rs +++ b/data_plane/src/stores/types/traits.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::KeyByLabelValues; +use crate::stores::types::KeyByLabelValues; use serde_json::Value; use std::collections::HashMap; diff --git a/data_plane/src/tests/capability_matching_tests.rs b/data_plane/src/tests/capability_matching_tests.rs index 60543b4d..93d7a3f7 100644 --- a/data_plane/src/tests/capability_matching_tests.rs +++ b/data_plane/src/tests/capability_matching_tests.rs @@ -4,7 +4,7 @@ //! the engine falls back to searching StreamingConfig by capability, and that //! the existing query_config path still takes priority when an entry is present. -use crate::stores::schema::{ +use crate::stores::types::{ AggregationConfig, AggregationReference, AggregationType, CleanupPolicy, InferenceConfig, PrecomputedOutput, PromQLSchema, QueryConfig, QueryLanguage, SchemaConfig, StreamingConfig, WindowType, @@ -14,7 +14,7 @@ use crate::precompute_engine::operators::count_min_sketch_accumulator::CountMinS use crate::precompute_engine::operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; use crate::precompute_engine::operators::delta_set_aggregator_accumulator::DeltaSetAggregatorAccumulator; use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; -use crate::stores::sketch_db::sketch_store::SketchStore; +use crate::stores::sketch_db::store::SketchStore; use crate::stores::traits::Store; use promql_utilities::data_model::KeyByLabelNames; use std::collections::HashMap; diff --git a/data_plane/src/tests/capability_miss_http_e2e_tests.rs b/data_plane/src/tests/capability_miss_http_e2e_tests.rs index 952ac56d..ba27d5fc 100644 --- a/data_plane/src/tests/capability_miss_http_e2e_tests.rs +++ b/data_plane/src/tests/capability_miss_http_e2e_tests.rs @@ -37,14 +37,14 @@ //! (plan-arrival + idempotency on repeat query). #[cfg(test)] -use crate::stores::schema::{ +use crate::stores::types::{ CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, QueryLanguage, StreamingConfig, }; use crate::drivers::query::adapters::AdapterConfig; use crate::drivers::query::controller_client::{ControllerClient, HttpControllerClient}; use crate::drivers::query::servers::http::{HttpServer, HttpServerConfig}; use crate::query_engines::ASAPQueryEngine; -use crate::stores::sketch_db::sketch_store::SketchStore; +use crate::stores::sketch_db::store::SketchStore; use axum::{extract::State, routing::post, Router}; use reqwest::Client; use serde_json::Value; @@ -173,7 +173,7 @@ async fn start_backend(controller_url: String, hot_reload: HotReloadStreamingCon // No fallback — we want engine-miss to be visible to the // test and stay out of the hot-vs-cold routing question. let adapter_config = AdapterConfig::new( - crate::stores::schema::enums::QueryProtocol::PrometheusHttp, + crate::stores::types::enums::QueryProtocol::PrometheusHttp, QueryLanguage::promql, None, ); diff --git a/data_plane/src/tests/persist_format_versioning_tests.rs b/data_plane/src/tests/persist_format_versioning_tests.rs index 2d29e3a1..25ad580e 100644 --- a/data_plane/src/tests/persist_format_versioning_tests.rs +++ b/data_plane/src/tests/persist_format_versioning_tests.rs @@ -8,7 +8,7 @@ //! * `BackfillRegistry` snapshot — JSON, //! `stores::sketch_db::backfill::PERSIST_FORMAT_VERSION` (currently 1) //! * `SketchStore` part `meta.bin` — binary, -//! `stores::sketch_db::sketch_store::persistence::part::PART_FORMAT_VERSION` (currently 1) +//! `stores::sketch_db::store::persistence::part::PART_FORMAT_VERSION` (currently 1) //! //! Each has a load path that tests `version == CURRENT`. The per- //! module unit tests already cover the happy-path roundtrip and a @@ -29,7 +29,7 @@ use crate::stores::sketch_db::backfill::{ BackfillJob, BackfillRegistry, BackfillSource, BackfillStatus, }; use crate::stores::sketch_db::schema::{AggStatus, SchemaRegistry}; -use crate::stores::sketch_db::sketch_store::persistence::part::{ +use crate::stores::sketch_db::store::persistence::part::{ MAGIC_META, META_HEADER_SIZE, PART_FORMAT_VERSION, }; @@ -41,7 +41,7 @@ fn tmpdir() -> tempfile::TempDir { mod schema { use super::*; - use crate::stores::schema::StreamingConfig; + use crate::stores::types::StreamingConfig; use asap_types::aggregation_config::AggregationConfig; use asap_types::enums::{AggregationType, WindowType}; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; @@ -336,7 +336,7 @@ mod backfill { mod part_meta { use super::*; - use crate::stores::sketch_db::sketch_store::persistence::part::PartReader; + use crate::stores::sketch_db::store::persistence::part::PartReader; /// Build a valid 64-byte meta.bin header for part_id=1. fn valid_header() -> Vec { @@ -446,14 +446,14 @@ mod v2_forward_compat { use super::*; use crate::stores::sketch_db::backfill::PERSIST_FORMAT_VERSION as BACKFILL_V; use crate::stores::sketch_db::schema::PERSIST_FORMAT_VERSION as SCHEMA_V; - use crate::stores::sketch_db::sketch_store::persistence::part::PartReader; + use crate::stores::sketch_db::store::persistence::part::PartReader; /// SchemaRegistry: snapshot tagged v_current+1 must trigger safe /// fallback, and the rewrite must be at v_current with the new /// config's schemas — no leakage from the future-version blob. #[test] fn schema_v1_with_future_version_falls_back_and_rewrites_clean() { - use crate::stores::schema::StreamingConfig; + use crate::stores::types::StreamingConfig; use asap_types::aggregation_config::AggregationConfig; use asap_types::enums::{AggregationType, WindowType}; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; diff --git a/data_plane/src/tests/persistence_integration_tests.rs b/data_plane/src/tests/persistence_integration_tests.rs index 4fb0d345..b05fcca8 100644 --- a/data_plane/src/tests/persistence_integration_tests.rs +++ b/data_plane/src/tests/persistence_integration_tests.rs @@ -14,9 +14,9 @@ use promql_utilities::data_model::KeyByLabelNames; use tempfile::TempDir; use std::time::Duration; -use crate::stores::schema::{AggregationType, CleanupPolicy, StreamingConfig, WindowType}; -use crate::stores::sketch_db::sketch_store::per_key::SketchStorePerKey; -use crate::stores::sketch_db::sketch_store::persistence::SketchStorePersistenceConfig; +use crate::stores::types::{AggregationType, CleanupPolicy, StreamingConfig, WindowType}; +use crate::stores::sketch_db::store::per_key::SketchStorePerKey; +use crate::stores::sketch_db::store::persistence::SketchStorePersistenceConfig; use crate::AggregationConfig; fn make_streaming_config(agg_id: u64) -> Arc { diff --git a/data_plane/src/tests/persistence_perf_tests.rs b/data_plane/src/tests/persistence_perf_tests.rs index dba9d351..feadb886 100644 --- a/data_plane/src/tests/persistence_perf_tests.rs +++ b/data_plane/src/tests/persistence_perf_tests.rs @@ -35,12 +35,12 @@ use std::time::{Duration, Instant}; use promql_utilities::data_model::KeyByLabelNames; use tempfile::TempDir; -use crate::stores::schema::{ +use crate::stores::types::{ AggregationType, CleanupPolicy, PrecomputedOutput, StreamingConfig, WindowType, }; use crate::precompute_engine::operators::SumAccumulator; -use crate::stores::sketch_db::sketch_store::per_key::SketchStorePerKey; -use crate::stores::sketch_db::sketch_store::persistence::SketchStorePersistenceConfig; +use crate::stores::sketch_db::store::per_key::SketchStorePerKey; +use crate::stores::sketch_db::store::persistence::SketchStorePersistenceConfig; use crate::stores::Store; use crate::{AggregateCore, AggregationConfig}; diff --git a/data_plane/src/tests/prometheus_forwarding_tests.rs b/data_plane/src/tests/prometheus_forwarding_tests.rs index a98fcf25..21f060e9 100644 --- a/data_plane/src/tests/prometheus_forwarding_tests.rs +++ b/data_plane/src/tests/prometheus_forwarding_tests.rs @@ -1,9 +1,9 @@ #[cfg(test)] -use crate::stores::schema::{CleanupPolicy, InferenceConfig, QueryLanguage, StreamingConfig}; +use crate::stores::types::{CleanupPolicy, InferenceConfig, QueryLanguage, StreamingConfig}; use crate::drivers::query::adapters::AdapterConfig; use crate::drivers::query::servers::http::{HttpServer, HttpServerConfig}; use crate::query_engines::ASAPQueryEngine; -use crate::stores::sketch_db::sketch_store::SketchStore; +use crate::stores::sketch_db::store::SketchStore; use reqwest::Client; use serde_json::Value; use std::sync::Arc; @@ -85,7 +85,7 @@ async fn setup_test_server(prometheus_port: u16) -> (HttpServer, u16) { inference_config, streaming_config.clone(), 15000, // 15s scrape interval - crate::stores::schema::QueryLanguage::promql, + crate::stores::types::QueryLanguage::promql, )); let server = HttpServer::new(config, query_engine, store); @@ -179,7 +179,7 @@ async fn test_forwarding_disabled() { inference_config, streaming_config.clone(), 15000, // 15s scrape interval - crate::stores::schema::QueryLanguage::promql, + crate::stores::types::QueryLanguage::promql, )); let server = HttpServer::new(config, query_engine, store); @@ -235,7 +235,7 @@ async fn test_prometheus_server_unreachable() { inference_config, streaming_config.clone(), 15000, // 15s scrape interval - crate::stores::schema::QueryLanguage::promql, + crate::stores::types::QueryLanguage::promql, )); let server = HttpServer::new(config, query_engine, store); diff --git a/data_plane/src/tests/schema_timeline_dispatch_tests.rs b/data_plane/src/tests/schema_timeline_dispatch_tests.rs index caebadd3..b922fcae 100644 --- a/data_plane/src/tests/schema_timeline_dispatch_tests.rs +++ b/data_plane/src/tests/schema_timeline_dispatch_tests.rs @@ -26,13 +26,13 @@ use asap_types::promql_schema::PromQLSchema; use asap_types::query_config::QueryConfig; use promql_utilities::data_model::key_by_label_names::KeyByLabelNames; -use crate::stores::schema::{ +use crate::stores::types::{ CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, KeyByLabelValues, PrecomputedOutput, QueryLanguage, SchemaConfig, StreamingConfig, }; use crate::query_engines::{QueryResult, ASAPQueryEngine}; use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; -use crate::stores::sketch_db::sketch_store::SketchStore; +use crate::stores::sketch_db::store::SketchStore; use crate::stores::sketch_db::{AggSchema, SchemaRegistry}; use crate::stores::Store; diff --git a/data_plane/src/tests/store_correctness_tests.rs b/data_plane/src/tests/store_correctness_tests.rs index c4247abf..314d26d9 100644 --- a/data_plane/src/tests/store_correctness_tests.rs +++ b/data_plane/src/tests/store_correctness_tests.rs @@ -27,7 +27,7 @@ //! | `contract_per_key` | `LockStrategy::PerKey` (reference impl) | //! | `contract_global` | `LockStrategy::Global` | -use crate::stores::schema::{ +use crate::stores::types::{ AggregationType, CleanupPolicy, KeyByLabelValues, LockStrategy, Measurement, SerializableToSink, StreamingConfig, WindowType, }; diff --git a/data_plane/src/tests/test_utilities/comparison.rs b/data_plane/src/tests/test_utilities/comparison.rs index 1363d727..9021e281 100644 --- a/data_plane/src/tests/test_utilities/comparison.rs +++ b/data_plane/src/tests/test_utilities/comparison.rs @@ -2,7 +2,7 @@ //! //! Provides assertion helpers for deep equality checking of query execution contexts. -use crate::stores::schema::{AggregationIdInfo, AggregationType}; +use crate::stores::types::{AggregationIdInfo, AggregationType}; use crate::query_engines::asap_query_engine::engine::{ QueryExecutionContext, QueryMetadata, StoreQueryParams, StoreQueryPlan, }; diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 249e3342..0005787d 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -5,14 +5,14 @@ //! hardcodes "SumAccumulator", these helpers build AggregationConfig with //! the correct aggregation_type string. -use crate::stores::schema::{ +use crate::stores::types::{ AggregationConfig, AggregationReference, AggregationType, CleanupPolicy, InferenceConfig, KeyByLabelValues, PrecomputedOutput, PromQLSchema, QueryConfig, QueryLanguage, SchemaConfig, StreamingConfig, WindowType, }; use crate::query_engines::query_result::InstantVectorElement; use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; -use crate::stores::sketch_db::sketch_store::SketchStore; +use crate::stores::sketch_db::store::SketchStore; use crate::stores::Store; use crate::AggregateCore; use promql_utilities::data_model::KeyByLabelNames; diff --git a/data_plane/src/tests/trait_design_tests.rs b/data_plane/src/tests/trait_design_tests.rs index b07da6c2..2a84504c 100644 --- a/data_plane/src/tests/trait_design_tests.rs +++ b/data_plane/src/tests/trait_design_tests.rs @@ -1,5 +1,5 @@ #[cfg(test)] -use crate::stores::schema::{ +use crate::stores::types::{ KeyByLabelValues, MultipleSubpopulationAggregate, SingleSubpopulationAggregate, }; use crate::precompute_engine::operators::{MultipleSumAccumulator, SumAccumulator}; diff --git a/data_plane/src/utils/file_io.rs b/data_plane/src/utils/file_io.rs index 916bc961..97408e80 100644 --- a/data_plane/src/utils/file_io.rs +++ b/data_plane/src/utils/file_io.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{InferenceConfig, QueryLanguage, StreamingConfig}; +use crate::stores::types::{InferenceConfig, QueryLanguage, StreamingConfig}; // use crate::stores::promsketch_store::config::PromSketchConfig; use anyhow::{Context, Result}; @@ -37,7 +37,7 @@ pub fn read_streaming_config( #[cfg(test)] mod tests { use super::*; - use crate::stores::schema::QueryLanguage; + use crate::stores::types::QueryLanguage; use std::io::Write; use tempfile::NamedTempFile; diff --git a/data_plane/src/utils/http.rs b/data_plane/src/utils/http.rs index 02789b10..e8ece03a 100644 --- a/data_plane/src/utils/http.rs +++ b/data_plane/src/utils/http.rs @@ -229,7 +229,7 @@ pub fn convert_range_result_to_prometheus( #[cfg(test)] mod tests { use super::*; - use crate::stores::schema::KeyByLabelValues; + use crate::stores::types::KeyByLabelValues; use crate::query_engines::query_result::{InstantVectorElement, RangeVectorElement}; fn create_test_labels() -> KeyByLabelValues { diff --git a/data_plane/src/utils/precompute_dumper.rs b/data_plane/src/utils/precompute_dumper.rs index dfb29f31..d6cc341e 100644 --- a/data_plane/src/utils/precompute_dumper.rs +++ b/data_plane/src/utils/precompute_dumper.rs @@ -1,4 +1,4 @@ -use crate::stores::schema::{AggregateCore, PrecomputedOutput}; +use crate::stores::types::{AggregateCore, PrecomputedOutput}; use serde::Serialize; use std::fs::{create_dir_all, File}; use std::io::{BufWriter, Write}; diff --git a/data_plane/tests/e2e_modified_otlp_sketch_path.rs b/data_plane/tests/e2e_modified_otlp_sketch_path.rs index c2f9fc76..73e4f8e6 100644 --- a/data_plane/tests/e2e_modified_otlp_sketch_path.rs +++ b/data_plane/tests/e2e_modified_otlp_sketch_path.rs @@ -42,7 +42,7 @@ use prost::Message; use std::collections::HashMap; use std::sync::Arc; -use data_plane::stores::schema::StreamingConfig; +use data_plane::stores::types::StreamingConfig; use data_plane::drivers::ingest::{OtlpReceiver, OtlpReceiverConfig}; use data_plane::precompute_engine::config::{LateDataPolicy, PrecomputeEngineConfig}; use data_plane::precompute_engine::output_sink::CapturingOutputSink; @@ -223,7 +223,7 @@ async fn e2e_count_min_sketch_modified_otlp_path() { let sink = Arc::new(CapturingOutputSink::new()); let engine = PrecomputeEngine::new( engine_config(), - data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config), + data_plane::stores::types::HotReloadStreamingConfig::from_arc(streaming_config), sink.clone(), ); let ingest_state = engine.ingest_state(); @@ -452,7 +452,7 @@ async fn e2e_count_sketch_modified_otlp_path() { let sink = Arc::new(CapturingOutputSink::new()); let engine = PrecomputeEngine::new( engine_config(), - data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config), + data_plane::stores::types::HotReloadStreamingConfig::from_arc(streaming_config), sink.clone(), ); let ingest_state = engine.ingest_state(); @@ -650,7 +650,7 @@ async fn e2e_kll_sketch_modified_otlp_path() { let sink = Arc::new(CapturingOutputSink::new()); let engine = PrecomputeEngine::new( engine_config(), - data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config), + data_plane::stores::types::HotReloadStreamingConfig::from_arc(streaming_config), sink.clone(), ); let ingest_state = engine.ingest_state(); @@ -838,7 +838,7 @@ async fn e2e_dd_sketch_modified_otlp_path() { let sink = Arc::new(CapturingOutputSink::new()); let engine = PrecomputeEngine::new( engine_config(), - data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config), + data_plane::stores::types::HotReloadStreamingConfig::from_arc(streaming_config), sink.clone(), ); let ingest_state = engine.ingest_state(); @@ -1013,7 +1013,7 @@ async fn e2e_hll_sketch_modified_otlp_path() { let sink = Arc::new(CapturingOutputSink::new()); let engine = PrecomputeEngine::new( engine_config(), - data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config), + data_plane::stores::types::HotReloadStreamingConfig::from_arc(streaming_config), sink.clone(), ); let ingest_state = engine.ingest_state(); @@ -1162,7 +1162,7 @@ async fn e2e_count_min_sketch_msgpack_modified_otlp_path() { let sink = Arc::new(CapturingOutputSink::new()); let engine = PrecomputeEngine::new( engine_config(), - data_plane::stores::schema::HotReloadStreamingConfig::from_arc(streaming_config), + data_plane::stores::types::HotReloadStreamingConfig::from_arc(streaming_config), sink.clone(), ); let ingest_state = engine.ingest_state(); diff --git a/data_plane/tests/edge_runtime_consumes_precompute_rs.rs b/data_plane/tests/edge_runtime_consumes_precompute_rs.rs index ccc7620c..181ce1c4 100644 --- a/data_plane/tests/edge_runtime_consumes_precompute_rs.rs +++ b/data_plane/tests/edge_runtime_consumes_precompute_rs.rs @@ -23,7 +23,7 @@ use asap_precompute_rs::sketches::{ }; use asap_precompute_rs::Sketch; -use data_plane::stores::schema::AggregateCore; +use data_plane::stores::types::AggregateCore; use data_plane::precompute_engine::operators::edge_runtime_adapter::{ encode_ddsketch_envelope, reconstruct_via_runtime, snapshot_ddsketch_via_runtime, unwrap_envelope_state, ReconstructedSketch, SketchType, diff --git a/data_plane/tests/inference_yaml_pattern_coverage.rs b/data_plane/tests/inference_yaml_pattern_coverage.rs index f8562ba1..94f820a0 100644 --- a/data_plane/tests/inference_yaml_pattern_coverage.rs +++ b/data_plane/tests/inference_yaml_pattern_coverage.rs @@ -25,7 +25,7 @@ fn init_test_tracing() { let _ = tracing_subscriber::fmt::try_init(); } -use data_plane::stores::schema::{ +use data_plane::stores::types::{ AggregationConfig, AggregationReference, AggregationType, CleanupPolicy, InferenceConfig, KeyByLabelValues, PrecomputedOutput, PromQLSchema, QueryConfig, QueryLanguage, SchemaConfig, StreamingConfig, WindowType, From 0f1f119dcdc4c90e522aa1f396f45ded46082e76 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 12 May 2026 12:40:18 -0600 Subject: [PATCH 4/7] refactor(sketch_db): move warm_tier back inside asap_query_engine, consolidate MutableEpoch dedup, write phase 5 plan doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three follow-ups: 1) Move `query_engines/warm_tier/` back into `query_engines/asap_query_engine/warm_tier/`. The earlier reorg promoted it on the assumption it was shared infra; audit shows only `asap_query_engine::engine` consumes it. PR #137's original nesting was correct. 2) Consolidate the duplicate `MutableEpoch` / `SealedEpoch` / `InternTable` implementations. Before: ~290 LOC of identical storage logic in `sketch_db/store/common.rs` (non-generic, payload = `Arc`) and `sketch_db/index/epoch_columnar.rs` (generic over `P`). After: `store/common.rs` is a 50-LOC alias shim (`pub type MutableEpoch = epoch_columnar::MutableEpoch>`, etc.); the generic version is the single implementation. Required adapter methods added on the generic side, gated on `P: Clone` where the legacy semantics needed owning copies: - `MutableEpoch::range_query_into_grouped` — emits the `HashMap>` shape `SketchStore` callers consume. - `MutableEpoch::exact_query_owned` — `Option>` for cross-lock-boundary handoff. - `MutableEpoch::remove_windows` — ReadBased / CircularBuffer cleanup primitive. - `MutableEpoch::time_bounds` / `seal` — convenience wrappers. - Same triplet on `SealedEpoch` plus `distinct_window_count` and `unique_windows`. Fixed a latent UB in `SealedEpoch::from_mutable`: previously used `MaybeUninit::zeroed().assume_init()` + `mem::forget` to drain the payload column, which is undefined behavior for any `P` with non-trivial Drop. Replaced with safe `into_iter().zip(...)` — same O(M) cost, works for arbitrary `P` (including `Arc`). Made `SealedEpoch.entries` public to match the legacy access pattern (a handful of call sites in `store/{global,per_key}.rs` read it directly for diagnostics + persistence flush). 3) Wrote `data_plane/docs/phase5-unification-plan.md` covering the three in-flight Phase-5 migrations: - M1: `AggSchema` → `SketchInstanceMetadata` (lifecycle fold) - M2: `aggregation_id` → `sid` (data-path identifier) - M3: legacy `Arc` payload type retires File-by-file phases A–E, ordering, dependencies on the analyzer chain α-ε and Step Z legacy_expr, and a ~5-day estimate. Test counts unchanged: - data_plane lib: 792 passed / 2 pre-existing failures / 4 ignored - controller lib: 710/710; bins: 27/27 Co-Authored-By: Claude Opus 4.7 (1M context) --- data_plane/docs/phase5-unification-plan.md | 208 +++++++++ .../query_engines/asap_query_engine/engine.rs | 18 +- .../query_engines/asap_query_engine/mod.rs | 1 + .../warm_tier/decoders.rs | 2 +- .../warm_tier/delta_apply.rs | 2 +- .../{ => asap_query_engine}/warm_tier/mod.rs | 0 .../warm_tier/sketch_reducer.rs | 4 +- .../warm_tier/tests.rs | 2 +- data_plane/src/query_engines/mod.rs | 1 - .../stores/sketch_db/index/epoch_columnar.rs | 202 +++++++- .../src/stores/sketch_db/store/common.rs | 436 ++---------------- .../src/stores/sketch_db/store/global.rs | 27 +- .../src/stores/sketch_db/store/per_key.rs | 24 +- 13 files changed, 492 insertions(+), 435 deletions(-) create mode 100644 data_plane/docs/phase5-unification-plan.md rename data_plane/src/query_engines/{ => asap_query_engine}/warm_tier/decoders.rs (98%) rename data_plane/src/query_engines/{ => asap_query_engine}/warm_tier/delta_apply.rs (99%) rename data_plane/src/query_engines/{ => asap_query_engine}/warm_tier/mod.rs (100%) rename data_plane/src/query_engines/{ => asap_query_engine}/warm_tier/sketch_reducer.rs (99%) rename data_plane/src/query_engines/{ => asap_query_engine}/warm_tier/tests.rs (99%) diff --git a/data_plane/docs/phase5-unification-plan.md b/data_plane/docs/phase5-unification-plan.md new file mode 100644 index 00000000..2cb40b50 --- /dev/null +++ b/data_plane/docs/phase5-unification-plan.md @@ -0,0 +1,208 @@ +# Phase-5 unification plan: SchemaRegistry → SketchIndex, agg_id → sid, MutableEpoch dedup + +This is a planning doc, not an implementation. It explains how three +in-flight Phase-5 migrations close out together, and what the data +plane looks like after. + +**Status as of May 2026:** all three migrations have landed their +"new side" — `SketchIndex`, `SketchInstanceMetadata`, `sid`, +generic `MutableEpoch

` — but the "old side" is still the live +production path. Concretely: + +- 45 files reference `aggregation_id`, 48 reference `agg_id` + workspace-wide. Only 12 files mention `sid`. +- Ingest barrier `SchemaRegistry::is_writable(agg_id)` is the §6.3 + write-side gate, called by every OTLP ingest at + `data_plane/src/drivers/ingest/otel.rs:526,591,615,1049,1062`. +- Query path emits `aggregation_id_for_key` / `aggregation_id_for_value` + on the wire response (asap_query_engine/engine.rs:1019,1064). +- `data_plane/src/stores/sketch_db/store/{global,per_key}.rs` still + use the non-generic legacy `MutableEpoch` / `SealedEpoch` from + `store/common.rs`. The generic `MutableEpoch

` in + `index/epoch_columnar.rs` is used only by `SketchIndex`. +- There is a literal `// DEPRECATED: aggregation_id-keyed write — remove` + comment at `drivers/ingest/otel.rs:1054`, confirming the migration is + acknowledged but not finished. + +## The three overlapping migrations + +### M1 — Lifecycle metadata fold: `AggSchema` → `SketchInstanceMetadata` + +Today two structs describe overlapping per-aggregation metadata at +two granularities: + +| | `AggSchema` (sketch_db/schema/) | `SketchInstanceMetadata` (sketch_db/index/) | +|---|---|---| +| Primary key | `agg_id: u64` | `sid: u64` | +| Identity fields | metric_name, grouping_labels | metric_name, group-by KEY set, sketch_type, sketch_config, accuracy_bound | +| Lifecycle | `AggStatus { Active / Retired / Expired }`, retired_at_ms, expires_at_ms | none | +| Source of truth | `StreamingConfig` reconciliation | OTLP-ingest registration | + +After M1, `SketchInstanceMetadata` carries the lifecycle fields and +`SchemaRegistry` becomes `SketchInstanceRegistry` (or folds into +`SketchIndex::instances`). The `is_writable(sid)` gate uses sid; +the §6.3 invariant is preserved. + +### M2 — Identifier replacement: `agg_id` → `sid` + +`agg_id: u64` is a hash of `(metric, agg_type, grouping_labels)` +computed in `crates/asap_types/src/aggregation_config.rs` at +config-load time. It is stable across restarts because the inputs +are stable. + +`sid: u64` is assigned at OTLP-ingest registration time (collector +gateway), travels in the wire format, and is canonical from that +point forward. + +The migration order matters: +1. Both ids carry simultaneously through the pipeline (already + happens — wire format has both). +2. Switch the §6.3 barrier from `is_writable(agg_id)` to + `is_writable(sid)` — requires `SketchInstanceRegistry` keyed by + sid. (M1 prerequisite.) +3. Switch the store keys: `SketchStore::insert_precomputed_output_batch` + currently keys on `aggregation_id`; flip to sid. +4. Switch the query path: drop `aggregation_id_for_key / + aggregation_id_for_value` on the wire response in favor of sid. +5. Drop the `aggregation_id` field from `AggregationConfig` and + `PrecomputedOutput` (wire-format change — requires coordinated + collector release). + +### M3 — Storage primitive dedup: legacy `MutableEpoch` → generic `MutableEpoch

` + +The `index/epoch_columnar.rs::MutableEpoch

` is a generic version +of `store/common.rs::MutableEpoch` lifted from the legacy code and +parameterized on payload type `P`. Six storage optimizations preserved +(see `INDEX_DESIGN.md`). + +Today the legacy uses `P = Arc` (trait-object +dispatch). The new `SketchIndex` uses `P = SketchSampleState` (typed +bytes + encoding tag — no dyn dispatch, no Arc cloning). + +Two paths to the dedup: + +- **Eager M3:** rewrite `store/{global,per_key}.rs` to use + `MutableEpoch>` and delete the legacy copy. + ~3–5 hours of careful work on the hot ingest + query path, with + regression risk in subtle hot-path behavior. +- **Deferred M3:** add a deprecation banner to `store/common.rs`; let + the legacy version die naturally when M2 + M1 close out. After M1 + + M2, the path-of-record is `SketchIndex`-resident sketch state + (P = `SketchSampleState`), and the trait-object `AggregateCore` + payload type becomes vestigial. The `store/{global,per_key}.rs` + files either delete or become thin shims. + +## Sequencing + dependencies + +``` + +-------------------+ + | M1: AggSchema | + | → InstanceMeta | + +---------+---------+ + | + (lifecycle fields land on sid-keyed instance metadata) + | + v + +-------------------+ + | M2: agg_id → sid | + | (5 substeps) | + +---------+---------+ + | + (wire format + store keys + query path all on sid) + | + v + +-------------------+ + | M3: dedup | + | MutableEpoch | ← happens by itself + +-------------------+ + once `AggregateCore`-as-payload is gone +``` + +M1 blocks M2 (M2 needs sid-keyed lifecycle gate). M2 substeps 4 + 5 +require a coordinated ASAPCollector release because the wire format +changes. M3 happens automatically once M2 lands; or can be done +eagerly any time, at the cost of working on the live hot path twice. + +## Ordering against other in-flight chains + +From the May 12 controller_todo doc: + +- **Step Z legacy_expr retirement** (4 PRs, ~8000 LOC across ~340 + pattern-match sites) is orthogonal — operates on the controller- + side intent algebra, not on data-plane identifiers. Can run in + parallel. +- **Analyzer-unification α→ε** (5 PRs, 5–8 days) interacts only at + the `Capability` enum surface in `sketch_db/index/`, which both + the engine-side analyzer and the controller-side analyzer consume. + Coordinate the `Capability` shape once at α; downstream is + independent. + +## Concrete unification work, file-by-file + +### Phase A — M1 prep (does not change the wire format) + +1. Add lifecycle fields to `SketchInstanceMetadata`: + - `status: AggStatus` + - `retired_at_ms: Option` + - `expires_at_ms: Option` +2. Add `SketchIndex::is_writable(sid: u64) -> bool` mirroring + `SchemaRegistry::is_writable(agg_id)`. Internally consult the + status field on the matched `SketchInstanceMetadata`. +3. Add `SketchIndex::list_by_status(status: AggStatus) -> Vec` + so eviction can drive off it. +4. Tests: replicate every `SchemaRegistry` test against `SketchIndex`. + +### Phase B — M1 cutover + +1. Switch `SchemaEvictionService` to call `SketchIndex` methods. +2. Switch ingest barrier `is_writable(agg_id)` → `is_writable(sid)`. +3. Delete `SchemaRegistry`, `AggSchema`, `AggStatus` from + `sketch_db/schema/`. Folder remains for compat re-exports during + transition; can be deleted once all callers migrated. + +### Phase C — M2.1 (parallel-write) + +Ingest emits both `agg_id` and `sid` on every precompute (already +the case today). Store accepts either as a key; internally maps +agg_id → sid via a side table. Query path resolves either. + +### Phase D — M2.2 cutover + +Store keys flip to sid. Wire format drops `aggregation_id` fields. +Coordinated release with ASAPCollector. After this lands, +`aggregation_id` is dead. + +### Phase E — M3 freebie + +`Arc` payloads are no longer the path-of-record; +they only exist in the legacy `store/{global,per_key}.rs` code, which +either deletes (warm-tier sketch state now lives in +`SketchIndex.series.windows`) or becomes a thin adapter shim. The +`store/common.rs::MutableEpoch` duplication disappears with its only +caller. + +## Estimate + +- Phase A: 1 day (additive, low risk). +- Phase B: 1 day (cutover; SchemaRegistry deletion is touchy but + mechanical). +- Phase C: 1 day (parallel-write is already partially in place). +- Phase D: 1 day + ASAPCollector PR coordination + integration + testing window. +- Phase E: 0.5 day (mostly deletion). + +Total: ~5 working days end-to-end, plus coordination overhead. + +## Open questions for the reader + +1. Does the wire format have a stability commitment that constrains + Phase D? (Backwards-compat shim period? Versioned acceptance?) +2. After Phase E, does anything outside `SketchIndex` need to handle + the trait-object `AggregateCore` payload? E.g., the + `PrecomputeEngine` write path serializes via `SerializableToSink` + — does that path move to typed bytes too? +3. What's the agreed retention model for `sid` after a schema + transition? (Today: AggSchema lifecycle drops the agg_id when + Expired. Post-migration: SketchIndex drops the sid. Same + semantics, but verify nothing depends on the agg_id being + reusable post-eviction.) diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index e79a5116..76ca702c 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -3501,7 +3501,7 @@ impl ASAPQueryEngine { // to the next compatible backend. // --------------------------------------------------------------------------- -/// Adapt a [`crate::query_engines::warm_tier::WarmTierResult`] to the engine's +/// Adapt a [`crate::query_engines::asap_query_engine::warm_tier::WarmTierResult`] to the engine's /// existing `QueryResult` shape. The reducer hands back per-series /// time-stamped scalars; we materialize them as a /// `QueryResult::Matrix` whose [`crate::query_engines::query_result::RangeVectorElement`]s @@ -3575,7 +3575,7 @@ fn stitch_warm_and_archive( } fn warm_tier_result_to_query_result( - result: crate::query_engines::warm_tier::WarmTierResult, + result: crate::query_engines::asap_query_engine::warm_tier::WarmTierResult, _now_ms: u64, ) -> crate::query_engines::query_result::QueryResult { use crate::stores::types::KeyByLabelValues; @@ -3695,14 +3695,14 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu // for instant-vector candidates (range_seconds == 0). const DEFAULT_LOOKBACK_MS: u64 = 5 * 60 * 1000; - let reducer = crate::query_engines::warm_tier::SketchReducer::new(idx); + let reducer = crate::query_engines::asap_query_engine::warm_tier::SketchReducer::new(idx); // Multi-candidate aggregation is deferred (single-result // shapes today). On the first reducer error we surface // CapabilityMiss; on Ok we keep the result for the // hybrid-stitch path below. (When more than one // candidate is supported, a follow-up will fold // per-candidate WarmTierResults.) - let mut combined_result: Option = + let mut combined_result: Option = None; let mut combined_t0: u64 = u64::MAX; @@ -3783,7 +3783,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu ) { Ok(r) => r, Err( - crate::query_engines::warm_tier::WarmTierError::UnsupportedFunction( + crate::query_engines::asap_query_engine::warm_tier::WarmTierError::UnsupportedFunction( name, ), ) => { @@ -3795,7 +3795,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu ), )); } - Err(crate::query_engines::warm_tier::WarmTierError::UnsupportedCapability { + Err(crate::query_engines::asap_query_engine::warm_tier::WarmTierError::UnsupportedCapability { function, capability, }) => { @@ -3807,7 +3807,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu ), )); } - Err(crate::query_engines::warm_tier::WarmTierError::DeserializeFailure { + Err(crate::query_engines::asap_query_engine::warm_tier::WarmTierError::DeserializeFailure { sid, encoding, reason, @@ -3821,7 +3821,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu ), )); } - Err(crate::query_engines::warm_tier::WarmTierError::NoData { + Err(crate::query_engines::asap_query_engine::warm_tier::WarmTierError::NoData { metric_name: m, }) => { return Err(crate::query_engines::EngineError::capability_miss( @@ -3832,7 +3832,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu ), )); } - Err(crate::query_engines::warm_tier::WarmTierError::MissingHeap { + Err(crate::query_engines::asap_query_engine::warm_tier::WarmTierError::MissingHeap { sid, sketch_kind, }) => { diff --git a/data_plane/src/query_engines/asap_query_engine/mod.rs b/data_plane/src/query_engines/asap_query_engine/mod.rs index cda5779a..be2fa186 100644 --- a/data_plane/src/query_engines/asap_query_engine/mod.rs +++ b/data_plane/src/query_engines/asap_query_engine/mod.rs @@ -10,6 +10,7 @@ //! the JSONL leg has been deleted). pub mod engine; +pub mod warm_tier; #[cfg(test)] pub mod tests; diff --git a/data_plane/src/query_engines/warm_tier/decoders.rs b/data_plane/src/query_engines/asap_query_engine/warm_tier/decoders.rs similarity index 98% rename from data_plane/src/query_engines/warm_tier/decoders.rs rename to data_plane/src/query_engines/asap_query_engine/warm_tier/decoders.rs index 81e0fd08..08ce2c5e 100644 --- a/data_plane/src/query_engines/warm_tier/decoders.rs +++ b/data_plane/src/query_engines/asap_query_engine/warm_tier/decoders.rs @@ -1,7 +1,7 @@ //! Per-sketch-kind decoder helpers — out-of-line wrappers around //! `asap_sketchlib` deserialize / proto-decode paths. //! -//! Lifted from the inline closures in [`crate::query_engines::warm_tier::sketch_reducer`] +//! Lifted from the inline closures in [`crate::query_engines::asap_query_engine::warm_tier::sketch_reducer`] //! once the reducer started decoding CMS / CountSketch / CMS-with-heap //! payloads in addition to DDSketch / KLL / HLL. The CMS / CountSketch //! / CMS-with-heap decoders mirror diff --git a/data_plane/src/query_engines/warm_tier/delta_apply.rs b/data_plane/src/query_engines/asap_query_engine/warm_tier/delta_apply.rs similarity index 99% rename from data_plane/src/query_engines/warm_tier/delta_apply.rs rename to data_plane/src/query_engines/asap_query_engine/warm_tier/delta_apply.rs index 4affe18b..af6b58ca 100644 --- a/data_plane/src/query_engines/warm_tier/delta_apply.rs +++ b/data_plane/src/query_engines/asap_query_engine/warm_tier/delta_apply.rs @@ -9,7 +9,7 @@ //! format ships a sparse-but-mergeable sketch fragment. //! //! Two reducer modes, picked by the PromQL function name in -//! [`crate::query_engines::warm_tier::sketch_reducer`]: +//! [`crate::query_engines::asap_query_engine::warm_tier::sketch_reducer`]: //! //! * **per-window** (`quantile`, `histogram_quantile`, //! `cardinality_estimate`): emit one scalar per window. A `Full` diff --git a/data_plane/src/query_engines/warm_tier/mod.rs b/data_plane/src/query_engines/asap_query_engine/warm_tier/mod.rs similarity index 100% rename from data_plane/src/query_engines/warm_tier/mod.rs rename to data_plane/src/query_engines/asap_query_engine/warm_tier/mod.rs diff --git a/data_plane/src/query_engines/warm_tier/sketch_reducer.rs b/data_plane/src/query_engines/asap_query_engine/warm_tier/sketch_reducer.rs similarity index 99% rename from data_plane/src/query_engines/warm_tier/sketch_reducer.rs rename to data_plane/src/query_engines/asap_query_engine/warm_tier/sketch_reducer.rs index 39b57cd7..eb3d04cb 100644 --- a/data_plane/src/query_engines/warm_tier/sketch_reducer.rs +++ b/data_plane/src/query_engines/asap_query_engine/warm_tier/sketch_reducer.rs @@ -57,11 +57,11 @@ use asap_sketchlib::sketches::ddsketch::DdSketch; use asap_sketchlib::sketches::hll::HllSketch; use asap_sketchlib::sketches::kll::KllSketch; -use crate::query_engines::warm_tier::decoders::{ +use crate::query_engines::asap_query_engine::warm_tier::decoders::{ decode_cms_from_msgpack, decode_cms_from_proto, decode_cms_with_heap_from_msgpack, decode_cs_from_msgpack, decode_cs_from_proto, }; -use crate::query_engines::warm_tier::delta_apply::{ +use crate::query_engines::asap_query_engine::warm_tier::delta_apply::{ cumulative_evaluate, per_window_evaluate, DeltaSketchKind, }; use crate::stores::sketch_db::index::{ diff --git a/data_plane/src/query_engines/warm_tier/tests.rs b/data_plane/src/query_engines/asap_query_engine/warm_tier/tests.rs similarity index 99% rename from data_plane/src/query_engines/warm_tier/tests.rs rename to data_plane/src/query_engines/asap_query_engine/warm_tier/tests.rs index b8a458a8..4d6138f8 100644 --- a/data_plane/src/query_engines/warm_tier/tests.rs +++ b/data_plane/src/query_engines/asap_query_engine/warm_tier/tests.rs @@ -15,7 +15,7 @@ use std::collections::{BTreeMap, BTreeSet}; use asap_sketchlib::sketches::ddsketch::DdSketch; use asap_sketchlib::sketches::hll::{HllSketch, HllVariant}; -use crate::query_engines::warm_tier::{SketchReducer, WarmTierError}; +use crate::query_engines::asap_query_engine::warm_tier::{SketchReducer, WarmTierError}; use crate::stores::sketch_db::index::{ AccuracyBound, Capability, SketchConfig, SketchEncoding, SketchIndex, SketchInstanceMetadata, SketchKindHandle, SketchSampleState, diff --git a/data_plane/src/query_engines/mod.rs b/data_plane/src/query_engines/mod.rs index 663ddf75..d1d6ca44 100644 --- a/data_plane/src/query_engines/mod.rs +++ b/data_plane/src/query_engines/mod.rs @@ -24,7 +24,6 @@ pub mod query_result; pub mod routing; pub mod thanos_query_engine; pub mod timeline_dispatch; -pub mod warm_tier; pub mod window_merger; pub use asap_query_engine::ASAPQueryEngine; diff --git a/data_plane/src/stores/sketch_db/index/epoch_columnar.rs b/data_plane/src/stores/sketch_db/index/epoch_columnar.rs index a1b78671..40ceb13e 100644 --- a/data_plane/src/stores/sketch_db/index/epoch_columnar.rs +++ b/data_plane/src/stores/sketch_db/index/epoch_columnar.rs @@ -261,6 +261,93 @@ impl

MutableEpoch

{ pub fn distinct_windows(&self) -> usize { self.windows_set.len() } + + /// `(min_start, max_end)` across all windows, or `None` if empty. + /// Convenience for the epoch-skip check + /// `min_start > end || max_end < start`. + pub fn time_bounds(&self) -> Option<(u64, u64)> { + match (self.min_start, self.max_end) { + (Some(s), Some(e)) => Some((s, e)), + _ => None, + } + } + + /// Consume self and produce a `SealedEpoch

` — convenience for + /// epoch rotation. Equivalent to `SealedEpoch::from_mutable(self)`. + pub fn seal(self) -> SealedEpoch

{ + SealedEpoch::from_mutable(self) + } + + /// Remove all entries whose window is in `windows`. + /// Mirrors the legacy `SketchStore` ReadBased / CircularBuffer + /// cleanup contract. O(N) — rebuilds columns in one pass. + pub fn remove_windows(&mut self, windows: &[TimestampRange]) { + use std::collections::HashSet as StdHashSet; + let drop_set: StdHashSet = windows.iter().copied().collect(); + let old_windows = std::mem::take(&mut self.windows_col); + let old_ids = std::mem::take(&mut self.label_ids_col); + let old_payloads = std::mem::take(&mut self.payloads_col); + for ((w, id), p) in old_windows.into_iter().zip(old_ids).zip(old_payloads) { + if !drop_set.contains(&w) { + self.windows_col.push(w); + self.label_ids_col.push(id); + self.payloads_col.push(p); + } + } + for w in windows { + self.windows_set.remove(w); + } + self.window_to_ids = None; + self.last_window = None; + self.min_start = self.windows_col.iter().map(|w| w.0).min(); + self.max_end = self.windows_col.iter().map(|w| w.1).max(); + } +} + +impl MutableEpoch

{ + /// Range query into a caller-provided `HashMap>`, + /// matching the legacy `SketchStore`'s `MetricBucketMap` shape. + /// Also pushes each matched window into `matched_windows` for the + /// downstream `read_counts` accounting. + /// + /// Uses the same overlap-filter semantics as the flat + /// `range_query_into`: include any window whose `[w.0, w.1)` + /// intersects `[start, end)`. Tumbling panes that straddle the + /// query boundaries match; same fix that the legacy implementation + /// carried (see legacy module comment about + /// `quantile_over_time(...[1m])` against 30s panes). + pub fn range_query_into_grouped( + &self, + start: u64, + end: u64, + out: &mut HashMap>, + matched_windows: &mut Vec, + ) { + for (i, &w) in self.windows_col.iter().enumerate() { + if w.1 <= start || w.0 >= end { + continue; + } + out.entry(self.label_ids_col[i]) + .or_default() + .push((w, self.payloads_col[i].clone())); + matched_windows.push(w); + } + } + + /// Exact-window query returning OWNED payload clones — for callers + /// that need to hand the payload out across a lock boundary. + /// `None` when the window has no entries. + pub fn exact_query_owned( + &mut self, + target: TimestampRange, + ) -> Option> { + let r = self.exact_query(target); + if r.is_empty() { + None + } else { + Some(r.into_iter().map(|(id, p)| (id, p.clone())).collect()) + } + } } impl

Default for MutableEpoch

{ @@ -275,7 +362,7 @@ impl

Default for MutableEpoch

{ pub struct SealedEpoch

{ /// Sorted by `(TimestampRange, LabelValuesId)`. Binary search on /// `start_unix_ms` to seek; linear scan within the matched range. - entries: Vec<(TimestampRange, LabelValuesId, P)>, + pub entries: Vec<(TimestampRange, LabelValuesId, P)>, min_start: Option, max_end: Option, } @@ -283,25 +370,22 @@ pub struct SealedEpoch

{ impl

SealedEpoch

{ /// Consume a `MutableEpoch` and produce its sorted immutable form. /// O(M log M) — paid once at rotation, off the insert hot path. - pub fn from_mutable(mut m: MutableEpoch

) -> Self { + /// + /// Safe payload move: zip-consumes the three parallel columns + /// into owned tuples. Previously used `MaybeUninit::zeroed()` + + /// `mem::forget` which is UB for any `P` with non-trivial Drop + /// (e.g. `Arc<_>`); the new form has the same algorithmic cost + /// and works for arbitrary `P`. + pub fn from_mutable(m: MutableEpoch

) -> Self { let min_start = m.min_start; let max_end = m.max_end; - let len = m.windows_col.len(); - let mut entries: Vec<(TimestampRange, LabelValuesId, P)> = Vec::with_capacity(len); - // Drain via swap_remove from the back to move payloads without - // cloning. Equivalent to consuming the parallel arrays in order. - for i in 0..len { - entries.push(( - m.windows_col[i], - m.label_ids_col[i], - std::mem::replace(&mut m.payloads_col[i], unsafe { - std::mem::MaybeUninit::zeroed().assume_init() - }), - )); - } - // Forget the columns to avoid double-drop (the moved-out payloads - // were replaced with zeroed memory; their drop should not run). - std::mem::forget(m.payloads_col); + let mut entries: Vec<(TimestampRange, LabelValuesId, P)> = m + .windows_col + .into_iter() + .zip(m.label_ids_col) + .zip(m.payloads_col) + .map(|((w, lid), p)| (w, lid, p)) + .collect(); entries.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1))); Self { entries, @@ -364,6 +448,88 @@ impl

SealedEpoch

{ } out } + + /// `(min_start, max_end)` or `None` if empty. + pub fn time_bounds(&self) -> Option<(u64, u64)> { + match (self.min_start, self.max_end) { + (Some(s), Some(e)) => Some((s, e)), + _ => None, + } + } + + /// Count of distinct time windows in this sealed epoch — O(N) + /// scan (entries are sorted, so consecutive dupes are adjacent). + pub fn distinct_window_count(&self) -> usize { + let mut count = 0usize; + let mut last: Option = None; + for (w, _, _) in &self.entries { + if last != Some(*w) { + count += 1; + last = Some(*w); + } + } + count + } + + /// Sorted-deduplicated windows. Used by the legacy SketchStore to + /// purge `read_counts` when an epoch is dropped. + pub fn unique_windows(&self) -> Vec { + let mut windows: Vec = + self.entries.iter().map(|(w, _, _)| *w).collect(); + windows.dedup(); + windows + } + + /// Remove all entries whose window is in `windows`. O(N) scan; + /// preserves sortedness since `retain` keeps relative order. + pub fn remove_windows(&mut self, windows: &[TimestampRange]) { + use std::collections::HashSet as StdHashSet; + let drop_set: StdHashSet = windows.iter().copied().collect(); + self.entries.retain(|(w, _, _)| !drop_set.contains(w)); + self.min_start = self.entries.iter().map(|(w, _, _)| w.0).min(); + self.max_end = self.entries.iter().map(|(w, _, _)| w.1).max(); + } +} + +impl SealedEpoch

{ + /// Range query into a caller-provided + /// `HashMap>` — the + /// legacy `MetricBucketMap` shape used by `SketchStore`. + /// Binary-search start + linear scan. Same overlap semantics + /// as the mutable variant. + pub fn range_query_into_grouped( + &self, + start: u64, + end: u64, + out: &mut HashMap>, + matched_windows: &mut Vec, + ) { + // Entries are sorted by `(w.0, label_id)`. Bound the upper + // end with `w.0 < end`; entries past that point can't overlap. + let end_pos = self.entries.partition_point(|(w, _, _)| w.0 < end); + for (w, id, p) in &self.entries[..end_pos] { + if w.1 <= start { + continue; + } + out.entry(*id).or_default().push((*w, p.clone())); + matched_windows.push(*w); + } + } + + /// Exact-window query returning OWNED clones — used by callers + /// that need to release the lock before reading the payloads. + /// `None` when no entry matches. + pub fn exact_query_owned( + &self, + target: TimestampRange, + ) -> Option> { + let r = self.exact_query(target); + if r.is_empty() { + None + } else { + Some(r.into_iter().map(|(id, p)| (id, p.clone())).collect()) + } + } } /// Per-sid storage — drop-in replacement for the new SketchIndex's diff --git a/data_plane/src/stores/sketch_db/store/common.rs b/data_plane/src/stores/sketch_db/store/common.rs index 8c91aede..694807eb 100644 --- a/data_plane/src/stores/sketch_db/store/common.rs +++ b/data_plane/src/stores/sketch_db/store/common.rs @@ -1,395 +1,59 @@ +//! Legacy `SketchStore` columnar types — now thin type aliases over +//! the generic `index::epoch_columnar` implementation. +//! +//! Before the 2026-05 dedup: this file had its own non-generic +//! `MutableEpoch`, `SealedEpoch`, and `InternTable` (~290 LOC) which +//! duplicated `index/epoch_columnar.rs` with the payload type fixed +//! to `Arc`. The duplication carried the same six +//! storage optimizations and the same overlap semantics; the only +//! real difference was the payload type bound. +//! +//! The legacy implementation is gone. The aliases below preserve +//! every call-site identifier (`MutableEpoch`, `SealedEpoch`, +//! `MetricID`, `MetricBucketMap`, `InternTable`) so consumers in +//! `store/global.rs` and `store/per_key.rs` are unchanged at the +//! identifier level. +//! +//! The few API-shape mismatches between the legacy and generic +//! versions (legacy's grouped `range_query_into`, legacy's owned +//! `exact_query`, legacy's `remove_windows`) are now first-class +//! methods on the generic side, gated on `P: Clone` where the +//! legacy semantics needed owning copies — see +//! `index/epoch_columnar.rs` for those impls. +//! +//! See `docs/phase5-unification-plan.md` Phase E for the further +//! step that retires `Arc` payloads entirely in +//! favor of typed `SketchSampleState`. After Phase E, this file +//! itself goes away. + +use crate::stores::sketch_db::index::epoch_columnar; use crate::stores::types::{AggregateCore, KeyByLabelValues}; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::sync::Arc; -pub type MetricID = u32; -pub type EpochID = u64; -pub type TimestampRange = (u64, u64); -pub type MetricBucketMap = HashMap)>>; - -/// Assigns a compact MetricID (u32) to each unique label combination. -/// Label strings stored once; all internal maps use MetricID (O(1) key ops). -pub struct InternTable { - label_to_id: HashMap, MetricID>, - id_to_label: Vec>, -} - -impl InternTable { - pub fn new() -> Self { - Self { - label_to_id: HashMap::new(), - id_to_label: Vec::new(), - } - } - - /// Intern a label, assigning a new MetricID if first seen. - /// Uses HashMap::entry to avoid double-hashing. - pub fn intern(&mut self, label: Option) -> MetricID { - let next_id = self.id_to_label.len() as MetricID; - match self.label_to_id.entry(label) { - std::collections::hash_map::Entry::Occupied(e) => *e.get(), - std::collections::hash_map::Entry::Vacant(e) => { - self.id_to_label.push(e.key().clone()); - *e.insert(next_id) - } - } - } - - /// O(1) resolution by MetricID. - pub fn resolve(&self, id: MetricID) -> &Option { - &self.id_to_label[id as usize] - } - - /// Number of interned labels. - pub fn len(&self) -> usize { - self.id_to_label.len() - } -} - -/// Mutable (active) epoch: pure append-only insert, O(1) amortized. -/// -/// # Optimizations applied -/// -/// **Opt 5 — Columnar storage**: timestamps, MetricIDs, and aggregates are kept in three -/// separate parallel arrays instead of one array of tuples. The range-query hot loop only -/// scans `windows_col` (contiguous u64 pairs) and does not touch aggregate pointers unless a -/// window actually matches, cutting cache pressure significantly for sparse range queries. -/// -/// **Opt 1 + 2 — Lazy offset index**: `window_to_ids` is built on the *first* `exact_query` -/// after any write batch and stores u32 column offsets rather than Arc clones. Any `insert` -/// simply sets the field to `None` (one pointer-width write); there are no HashMap lookups, -/// no `HashSet::insert` calls for the index, and no atomic refcount bumps on the hot insert -/// path. The index is rebuilt in O(M) on demand from `windows_col` alone. -/// -/// **Opt 3 — Monotonic ingest fast path**: `last_window` tracks the most recently inserted -/// window. Consecutive inserts to the same window (multiple label combinations for one time -/// bucket — the common case in ordered TSDB ingestion) skip the `windows_set` HashSet probe -/// entirely. -/// -/// **Opt 6 — Pre-allocated column buffers**: `with_capacity(n)` reserves space upfront using -/// the previous epoch's entry count, avoiding Vec reallocation during the next epoch fill. -pub struct MutableEpoch { - // Columnar storage: three parallel arrays (Opt 5) - windows_col: Vec, - metric_ids_col: Vec, - aggregates_col: Vec>, - - // Distinct-window count for epoch rotation threshold - windows_set: HashSet, - - // Monotonic ingest fast path: skip HashSet probe for consecutive same-window inserts (Opt 3) - last_window: Option, - - // Lazy offset index: built on first exact_query, invalidated on any insert (Opt 1 + 2). - // Stores column indices (u32) instead of Arc clones — zero atomic ops during insert. - window_to_ids: Option>>, - - /// Epoch time bounds for O(1) skip check, updated incrementally on insert. - min_start: Option, - max_end: Option, -} - -impl MutableEpoch { - pub fn new() -> Self { - Self::with_capacity(0) - } - - /// Pre-allocate column buffers with a capacity hint (Opt 6). - /// Pass the previous epoch's `len()` to avoid reallocation during the next epoch fill. - pub fn with_capacity(cap: usize) -> Self { - Self { - windows_col: Vec::with_capacity(cap), - metric_ids_col: Vec::with_capacity(cap), - aggregates_col: Vec::with_capacity(cap), - windows_set: HashSet::new(), - last_window: None, - window_to_ids: None, - min_start: None, - max_end: None, - } - } - - pub fn window_count(&self) -> usize { - self.windows_set.len() - } - - /// Total raw entries across all windows and labels. - pub fn len(&self) -> usize { - self.windows_col.len() - } - - /// Returns `(min_start, max_end)` across all windows, or `None` if empty. - /// Used for the epoch-skip check: `min_start > end || max_end < start`. - pub fn time_bounds(&self) -> Option<(u64, u64)> { - match (self.min_start, self.max_end) { - (Some(s), Some(e)) => Some((s, e)), - _ => None, - } - } +/// Compact metric identifier — 4 bytes. Renamed in the generic to +/// `LabelValuesId`; the alias preserves legacy naming. +pub type MetricID = epoch_columnar::LabelValuesId; - /// O(1) amortized insert: three column pushes + conditional HashSet insert + bounds update. - /// - /// No secondary-index maintenance and no Arc clone for any index. The lazy `window_to_ids` - /// is invalidated by setting it to `None` — a single pointer-width write with no HashMap or - /// HashSet work. - pub fn insert( - &mut self, - metric_id: MetricID, - range: TimestampRange, - agg: Arc, - ) { - // Opt 3: skip HashSet probe when the incoming window equals the last inserted window. - // Multiple label combinations arriving for the same time bucket (the common ordered- - // ingest pattern) cost zero HashSet operations after the first. - if self.last_window != Some(range) { - self.windows_set.insert(range); - self.last_window = Some(range); - } +/// Monotonically increasing epoch counter. +pub type EpochID = epoch_columnar::EpochId; - // Opt 5: columnar append — no secondary index, no Arc clone - self.windows_col.push(range); - self.metric_ids_col.push(metric_id); - self.aggregates_col.push(agg); +/// `(start_unix_ms, end_unix_ms)`. +pub type TimestampRange = epoch_columnar::TimestampRange; - // Opt 1: invalidate lazy index at zero cost - self.window_to_ids = None; +/// Legacy intern table keyed by `Option` (the +/// `None` slot represents a missing group-by; that semantic predates +/// the SketchIndex's `BTreeMap` key shape). +pub type InternTable = epoch_columnar::InternTable>; - self.min_start = Some(self.min_start.map_or(range.0, |m| m.min(range.0))); - self.max_end = Some(self.max_end.map_or(range.1, |m| m.max(range.1))); - } +/// Active (mutable) epoch holding `Arc` payloads. +pub type MutableEpoch = epoch_columnar::MutableEpoch>; - /// Consume this epoch and produce an immutable SealedEpoch by sorting in-place. - /// Zips the three columns into tuples and sorts — moves Arcs without cloning. - /// O(M log M) paid once at rotation time, not at query time. - pub fn seal(self) -> SealedEpoch { - let min_start = self.min_start; - let max_end = self.max_end; - let mut entries: Vec<(TimestampRange, MetricID, Arc)> = self - .windows_col - .into_iter() - .zip(self.metric_ids_col) - .zip(self.aggregates_col) - .map(|((tr, mid), agg)| (tr, mid, agg)) - .collect(); - entries.sort_unstable_by_key(|(tr, metric_id, _)| (*tr, *metric_id)); - // Count distinct windows in the sorted entries (consecutive dupes are adjacent). - let distinct_window_count = entries.windows(2).filter(|w| w[0].0 != w[1].0).count() - + if entries.is_empty() { 0 } else { 1 }; - SealedEpoch { - entries, - min_start, - max_end, - distinct_window_count, - } - } +/// Sealed (immutable, sorted) epoch holding `Arc` +/// payloads. +pub type SealedEpoch = epoch_columnar::SealedEpoch>; - /// Opt 5: scans only `windows_col` for time-range filtering — cache-friendly because - /// only contiguous TimestampRange values are touched in the hot loop. Aggregate pointers - /// are chased only for entries that actually match the range. - /// O(M) where M ≤ epoch_capacity × labels_per_window. - pub fn range_query_into( - &self, - start: u64, - end: u64, - out: &mut MetricBucketMap, - matched_windows: &mut Vec, - ) { - // Overlap filter, not fully-contained: include any window whose - // [tr.0, tr.1) interval intersects [start, end). The previous - // form (`tr.0 < start || tr.0 > end || tr.1 > end → skip`) - // required `start ≤ tr.0 ≤ tr.1 ≤ end`, which excluded windows - // that crossed the query boundaries — typical for tumbling - // windows with a query range that doesn't align to the window - // grid (e.g. 60s query range over 30s panes with an unaligned - // query end timestamp returns 0 panes instead of the 2 it - // should). `quantile_over_time(...[1m])` against a sketch - // emitted into a 30s pane otherwise reports - // "No precomputed outputs found" even when the data is - // demonstrably in the store. - for (i, &tr) in self.windows_col.iter().enumerate() { - if tr.1 <= start || tr.0 >= end { - continue; - } - let metric_id = self.metric_ids_col[i]; - out.entry(metric_id) - .or_default() - .push((tr, Arc::clone(&self.aggregates_col[i]))); - matched_windows.push(tr); - } - } - - /// Opt 1 + 2: lazy exact match — O(m) after the index is built, O(M) to build once. - /// - /// The offset index (`HashMap>`) is constructed from `windows_col` - /// on the first call after any write batch, then cached. Building it scans `windows_col` - /// once with no Arc clones (only integer offsets are stored). The index remains valid - /// until the next `insert`, which sets `window_to_ids = None`. - /// - /// Takes `&mut self` because building the index mutates `window_to_ids`. - /// Callers must hold exclusive (write) access to the containing epoch. - pub fn exact_query( - &mut self, - range: TimestampRange, - ) -> Option)>> { - if self.window_to_ids.is_none() { - let mut idx: HashMap> = - HashMap::with_capacity(self.windows_set.len()); - for (i, &tr) in self.windows_col.iter().enumerate() { - idx.entry(tr).or_default().push(i as u32); - } - self.window_to_ids = Some(idx); - } - let offsets = self.window_to_ids.as_ref().unwrap().get(&range)?; - Some( - offsets - .iter() - .map(|&i| { - let i = i as usize; - (self.metric_ids_col[i], Arc::clone(&self.aggregates_col[i])) - }) - .collect(), - ) - } - - /// Remove specific windows (ReadBased cleanup). - /// Drains all three columns in lockstep — moves Arcs without cloning. - pub fn remove_windows(&mut self, windows: &[TimestampRange]) { - let window_set: HashSet = windows.iter().copied().collect(); - - let old_windows = std::mem::take(&mut self.windows_col); - let old_metrics = std::mem::take(&mut self.metric_ids_col); - let old_aggs = std::mem::take(&mut self.aggregates_col); - - for ((tr, mid), agg) in old_windows.into_iter().zip(old_metrics).zip(old_aggs) { - if !window_set.contains(&tr) { - self.windows_col.push(tr); - self.metric_ids_col.push(mid); - self.aggregates_col.push(agg); - } - } - - for window in windows { - self.windows_set.remove(window); - } - - // Invalidate lazy index and monotonic fast-path hint. - self.window_to_ids = None; - self.last_window = None; - - // Recompute bounds (cleanup is rare; linear scan is fine). - self.min_start = self.windows_col.iter().map(|tr| tr.0).min(); - self.max_end = self.windows_col.iter().map(|tr| tr.1).max(); - } -} - -/// Sealed (immutable) epoch: flat sorted `Vec` for cache-friendly range scans. -/// -/// Produced by `MutableEpoch::seal()`. Entries are sorted by `(TimestampRange, MetricID)`: -/// all entries for the same window are contiguous, which is cache-friendly for both -/// range queries (binary-search start + linear scan) and exact queries. -pub struct SealedEpoch { - /// Sorted by (TimestampRange, MetricID). - pub entries: Vec<(TimestampRange, MetricID, Arc)>, - /// Precomputed for O(1) epoch-skip check. - pub min_start: Option, - pub max_end: Option, - /// Number of distinct time windows in this epoch — O(1) read. - distinct_window_count: usize, -} - -impl SealedEpoch { - pub fn is_empty(&self) -> bool { - self.entries.is_empty() - } - - /// O(1) count of distinct time windows in this epoch. - pub fn distinct_window_count(&self) -> usize { - self.distinct_window_count - } - - /// Returns `(min_start, max_end)`, or `None` if empty. - pub fn time_bounds(&self) -> Option<(u64, u64)> { - match (self.min_start, self.max_end) { - (Some(s), Some(e)) => Some((s, e)), - _ => None, - } - } - - /// Binary-search start + linear scan — O(log N + actual_matches), cache-friendly. - /// - /// Overlap filter (not fully-contained): include any window whose - /// `[tr.0, tr.1)` interval intersects `[start, end)`. See the - /// matching change on the columnar variant above for the longer - /// rationale — short version: tumbling windows that cross the - /// query boundary should still match, otherwise unaligned query - /// ranges silently return no data. - pub fn range_query_into( - &self, - start: u64, - end: u64, - out: &mut MetricBucketMap, - matched_windows: &mut Vec, - ) { - // Entries are sorted by `tr.0`. Bound the upper end with - // `tr.0 < end`; entries past that point can't overlap. - let end_pos = self.entries.partition_point(|(tr, _, _)| tr.0 < end); - for (tr, metric_id, agg) in &self.entries[..end_pos] { - // Lower-end overlap check: skip entries that ended at or - // before the query start. - if tr.1 <= start { - continue; - } - out.entry(*metric_id) - .or_default() - .push((*tr, Arc::clone(agg))); - matched_windows.push(*tr); - } - } - - /// Binary-search exact window match — O(log N + m) where m = labels in that window. - pub fn exact_query( - &self, - range: TimestampRange, - ) -> Option)>> { - let start_pos = self.entries.partition_point(|(tr, _, _)| *tr < range); - let mut out = Vec::new(); - for (tr, metric_id, agg) in &self.entries[start_pos..] { - if *tr != range { - break; - } - out.push((*metric_id, Arc::clone(agg))); - } - if out.is_empty() { - None - } else { - Some(out) - } - } - - /// Remove specific windows (ReadBased / CircularBuffer cleanup). Rebuilds Vec in one pass. - /// Also updates `distinct_window_count`. - pub fn remove_windows(&mut self, windows: &[TimestampRange]) { - let window_set: HashSet = windows.iter().copied().collect(); - self.entries.retain(|(tr, _, _)| !window_set.contains(tr)); - self.min_start = self.entries.iter().map(|(tr, _, _)| tr.0).min(); - self.max_end = self.entries.iter().map(|(tr, _, _)| tr.1).max(); - // Recount distinct windows (entries remain sorted; dedup in one pass). - let mut count = 0usize; - let mut last: Option = None; - for (tr, _, _) in &self.entries { - if last != Some(*tr) { - count += 1; - last = Some(*tr); - } - } - self.distinct_window_count = count; - } - - /// Deduplicated windows (entries sorted, so consecutive dupes are adjacent). - /// Used to purge `read_counts` when this epoch is dropped. - pub fn unique_windows(&self) -> Vec { - let mut windows: Vec = self.entries.iter().map(|(tr, _, _)| *tr).collect(); - windows.dedup(); - windows - } -} +/// Range-query output shape used by `SketchStore` callers: per-metric +/// list of `(window, aggregate)` pairs. Matches the legacy +/// `MetricBucketMap`. +pub type MetricBucketMap = HashMap)>>; diff --git a/data_plane/src/stores/sketch_db/store/global.rs b/data_plane/src/stores/sketch_db/store/global.rs index 4b5f0ad9..03e9fd8b 100644 --- a/data_plane/src/stores/sketch_db/store/global.rs +++ b/data_plane/src/stores/sketch_db/store/global.rs @@ -339,7 +339,7 @@ impl Store for SketchStoreGlobal { // Insert into current (mutable) epoch. per_key .current_epoch - .insert(metric_id, timestamp_range, Arc::from(precompute)); + .insert(timestamp_range, metric_id, Arc::from(precompute)); // Apply retention policy if configured (but exclude DeltaSetAggregator). // per_key is last used above; NLL ends its borrow so data.read_counts can @@ -484,7 +484,7 @@ impl Store for SketchStoreGlobal { // Query current (mutable) epoch. if let Some((min_start, max_end)) = per_key.current_epoch.time_bounds() { if !(min_start > end || max_end < start) { - per_key.current_epoch.range_query_into( + per_key.current_epoch.range_query_into_grouped( start, end, &mut mid, @@ -501,7 +501,12 @@ impl Store for SketchStoreGlobal { if min_start > end || max_end < start { continue; } - epoch.range_query_into(start, end, &mut mid, &mut matched_windows); + epoch.range_query_into_grouped( + start, + end, + &mut mid, + &mut matched_windows, + ); } mid @@ -513,7 +518,11 @@ impl Store for SketchStoreGlobal { let mut r = HashMap::with_capacity(mid.len()); for (metric_id, buckets) in mid.drain() { total_entries += buckets.len(); - let label = per_key.intern.resolve(metric_id).clone(); + let label = per_key + .intern + .resolve(metric_id) + .cloned() + .unwrap_or(None); r.insert(label, buckets); } r @@ -614,13 +623,13 @@ impl Store for SketchStoreGlobal { // sealed_epochs scan below. per_key .current_epoch - .exact_query(timestamp_range) + .exact_query_owned(timestamp_range) .or_else(|| { per_key .sealed_epochs .values() .rev() - .find_map(|epoch| epoch.exact_query(timestamp_range)) + .find_map(|epoch| epoch.exact_query_owned(timestamp_range)) }) }; // &mut borrow of data.stores ends here @@ -631,7 +640,11 @@ impl Store for SketchStoreGlobal { if let Some(entries) = entries_opt { let per_key = data.stores.get(&store_key).unwrap(); for (metric_id, agg) in entries { - let label = per_key.intern.resolve(metric_id).clone(); + let label = per_key + .intern + .resolve(metric_id) + .cloned() + .unwrap_or(None); results .entry(label) .or_default() diff --git a/data_plane/src/stores/sketch_db/store/per_key.rs b/data_plane/src/stores/sketch_db/store/per_key.rs index 12b5df6a..7bb77aa3 100644 --- a/data_plane/src/stores/sketch_db/store/per_key.rs +++ b/data_plane/src/stores/sketch_db/store/per_key.rs @@ -606,7 +606,7 @@ impl SketchStorePerKey { let metric_id: MetricID = data.intern.intern(output.key); data.current_epoch - .insert(metric_id, timestamp_range, Arc::from(precompute)); + .insert(timestamp_range, metric_id, Arc::from(precompute)); // When persistence is on, always run rotation so sealed // epochs accumulate. Otherwise preserve the old @@ -787,8 +787,12 @@ impl Store for SketchStorePerKey { if let Some((min_start, max_end)) = data.current_epoch.time_bounds() { if !(min_start > end || max_end < start) { - data.current_epoch - .range_query_into(start, end, &mut mid, &mut matched_windows); + data.current_epoch.range_query_into_grouped( + start, + end, + &mut mid, + &mut matched_windows, + ); } } @@ -799,11 +803,11 @@ impl Store for SketchStorePerKey { if min_start > end || max_end < start { continue; } - epoch.range_query_into(start, end, &mut mid, &mut matched_windows); + epoch.range_query_into_grouped(start, end, &mut mid, &mut matched_windows); } for (metric_id, buckets) in mid { - let label = data.intern.resolve(metric_id).clone(); + let label = data.intern.resolve(metric_id).cloned().unwrap_or(None); results.entry(label).or_default().extend(buckets); } @@ -876,12 +880,14 @@ impl Store for SketchStorePerKey { let timestamp_range = (exact_start, exact_end); - let entries_opt: Option)>> = - data.current_epoch.exact_query(timestamp_range).or_else(|| { + let entries_opt: Option)>> = data + .current_epoch + .exact_query_owned(timestamp_range) + .or_else(|| { data.sealed_epochs .values() .rev() - .find_map(|epoch| epoch.exact_query(timestamp_range)) + .find_map(|epoch| epoch.exact_query_owned(timestamp_range)) }); let mut results: TimestampedBucketsMap = HashMap::new(); @@ -889,7 +895,7 @@ impl Store for SketchStorePerKey { if let Some(entries) = entries_opt { for (metric_id, agg) in entries { - let label = data.intern.resolve(metric_id).clone(); + let label = data.intern.resolve(metric_id).cloned().unwrap_or(None); results .entry(label) .or_default() From e44d0b1e327c9f6a8123c77171597fdf04cced43 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 12 May 2026 14:09:14 -0600 Subject: [PATCH 5/7] refactor(sketch_db): remove read-count-based eviction (outdated) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `CleanupPolicy::ReadBased` was the eviction policy where windows are dropped after their `read_count` reaches a threshold. The mechanism was outdated and not used in any production deployment — every production call site passes `CleanupPolicy::CircularBuffer` or `NoCleanup`. The remaining surface was test-only + the example YAML. Removed surface: **asap_types crate:** - `CleanupPolicy::ReadBased` variant + its Display / FromStr branches - `AggregationConfig::read_count_threshold` field (also dropped from `AggregationConfig::new` + `from_yaml_data` signatures and the JSON serialize / deserialize paths — wire-format field `readCountThreshold` no longer emitted) - `AggregationReference::read_count_threshold` field + `with_read_count_threshold` constructor - `read_based` parsing in `InferenceConfig::parse_cleanup_policy` **data_plane crate:** - `read_counts: HashMap<...>` from `SketchStoreGlobal`'s `StoreData` and from `SketchStorePerKey`'s `StoreKeyData` - `read_count_threshold` parameter on `cleanup_old_aggregates` and `BatchConfig` - `ReadBased` match-arm in both store backends - `cleanup_read_based` helper on `StoreKeyData` - Per-query read-count update logic in range/exact query paths - Read-count purge logic in epoch eviction - `read_counts_len` field on `AggregationDiagnostic`; matching `[MEMORY_DIAG]` log line in `main.rs` **Tests:** - Two contract tests: `test_cleanup_read_based_evicts_after_threshold_reads`, `test_cleanup_read_based_unread_window_is_retained` - 23 `AggregationConfig::new` call sites updated (the 15th positional `read_count_threshold` arg removed across data_plane + crates + tests + integration tests) **Docs / example:** - `data_plane/examples/promql/inference_config.yaml` switched from `read_based` to `circular_buffer`, `read_count_threshold: 1` lines stripped - `sketch_db/store/INDEX_DESIGN.md` updated to drop the read_counts field, the ReadBased complexity row, and the ReadBased policy section - Stale doc-comment references cleaned in store/global.rs, store/per_key.rs, index/epoch_columnar.rs Tests still match the post-PR baseline: - data_plane lib: 792 passed / 2 pre-existing failures / 4 ignored - controller lib: 710/710; bins: 27/27 Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/asap_types/src/aggregation_config.rs | 12 --- .../asap_types/src/aggregation_reference.rs | 15 --- crates/asap_types/src/capability_matching.rs | 1 - crates/asap_types/src/enums.rs | 4 - crates/asap_types/src/inference_config.rs | 15 +-- crates/asap_types/src/streaming_config.rs | 8 -- .../examples/promql/inference_config.yaml | 35 +----- data_plane/src/drivers/query/servers/http.rs | 1 - data_plane/src/main.rs | 3 +- .../precompute_engine/accumulator_factory.rs | 2 - .../src/precompute_engine/ingest_handler.rs | 1 - data_plane/src/precompute_engine/worker.rs | 2 - .../query_engines/asap_query_engine/engine.rs | 3 - data_plane/src/stores/sketch_db/accuracy.rs | 1 - .../stores/sketch_db/backfill/processor.rs | 1 - .../src/stores/sketch_db/backfill/service.rs | 1 - .../sketch_db/backfill/window_builder.rs | 1 - .../stores/sketch_db/index/epoch_columnar.rs | 6 +- .../src/stores/sketch_db/schema/eviction.rs | 1 - data_plane/src/stores/sketch_db/schema/mod.rs | 1 - .../stores/sketch_db/store/INDEX_DESIGN.md | 27 +---- .../src/stores/sketch_db/store/global.rs | 82 +++----------- data_plane/src/stores/sketch_db/store/mod.rs | 2 - .../src/stores/sketch_db/store/per_key.rs | 84 ++------------- .../src/stores/types/hot_reload_config.rs | 1 - .../accuracy_empirical_validation_tests.rs | 1 - .../src/tests/capability_matching_tests.rs | 1 - .../tests/persist_format_versioning_tests.rs | 2 - .../tests/persistence_integration_tests.rs | 1 - .../src/tests/persistence_perf_tests.rs | 1 - .../tests/schema_timeline_dispatch_tests.rs | 1 - .../src/tests/store_correctness_tests.rs | 101 +++--------------- .../tests/test_utilities/engine_factories.rs | 8 -- .../tests/e2e_modified_otlp_sketch_path.rs | 5 - .../tests/inference_yaml_pattern_coverage.rs | 1 - 35 files changed, 45 insertions(+), 387 deletions(-) diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index a25493e6..52c12d92 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -28,7 +28,6 @@ pub struct AggregationConfig { pub spatial_filter_normalized: String, pub metric: String, // PromQL mode: metric name; SQL mode: derived from table_name.value_column pub num_aggregates_to_retain: Option, - pub read_count_threshold: Option, // SQL-specific fields (optional, used when query_language=sql) pub table_name: Option, // SQL mode: table name @@ -65,7 +64,6 @@ impl AggregationConfig { spatial_filter: String, metric: String, num_aggregates_to_retain: Option, - read_count_threshold: Option, // SQL-specific fields table_name: Option, value_column: Option, @@ -89,7 +87,6 @@ impl AggregationConfig { spatial_filter_normalized, metric, num_aggregates_to_retain, - read_count_threshold, table_name, value_column, } @@ -163,7 +160,6 @@ impl AggregationConfig { let metric = data["metric"].as_str().ok_or("Missing metric")?.to_string(); let num_aggregates_to_retain = data.get("numAggregatesToRetain").and_then(|v| v.as_u64()); - let read_count_threshold = data.get("readCountThreshold").and_then(|v| v.as_u64()); // SQL-specific fields (optional) let table_name = data @@ -190,7 +186,6 @@ impl AggregationConfig { spatial_filter, metric, num_aggregates_to_retain, - read_count_threshold, table_name, value_column, )) @@ -207,7 +202,6 @@ impl AggregationConfig { pub fn from_yaml_data( aggregation_data: &serde_yaml::Value, num_aggregates_to_retain: Option, - read_count_threshold: Option, query_language: QueryLanguage, ) -> Result { let aggregation_id = aggregation_data["aggregationId"] @@ -313,7 +307,6 @@ impl AggregationConfig { spatial_filter, metric, num_aggregates_to_retain, - read_count_threshold, table_name, value_column, )) @@ -340,11 +333,6 @@ impl SerializableToSink for AggregationConfig { json["numAggregatesToRetain"] = serde_json::json!(num_aggregates); } - // Only include readCountThreshold if it's Some - if let Some(threshold) = self.read_count_threshold { - json["readCountThreshold"] = serde_json::json!(threshold); - } - // SQL-specific fields (only include if present) if let Some(ref table_name) = self.table_name { json["tableName"] = serde_json::json!(table_name); diff --git a/crates/asap_types/src/aggregation_reference.rs b/crates/asap_types/src/aggregation_reference.rs index ccd18c5e..b34b9aba 100644 --- a/crates/asap_types/src/aggregation_reference.rs +++ b/crates/asap_types/src/aggregation_reference.rs @@ -6,9 +6,6 @@ pub struct AggregationReference { /// For circular_buffer policy: keep this many most recent aggregates #[serde(skip_serializing_if = "Option::is_none")] pub num_aggregates_to_retain: Option, - /// For read_based policy: remove aggregate after this many reads - #[serde(skip_serializing_if = "Option::is_none")] - pub read_count_threshold: Option, } impl AggregationReference { @@ -16,18 +13,6 @@ impl AggregationReference { Self { aggregation_id, num_aggregates_to_retain, - read_count_threshold: None, - } - } - - pub fn with_read_count_threshold( - aggregation_id: u64, - read_count_threshold: Option, - ) -> Self { - Self { - aggregation_id, - num_aggregates_to_retain: None, - read_count_threshold, } } } diff --git a/crates/asap_types/src/capability_matching.rs b/crates/asap_types/src/capability_matching.rs index 986d1407..9d36d49b 100644 --- a/crates/asap_types/src/capability_matching.rs +++ b/crates/asap_types/src/capability_matching.rs @@ -574,7 +574,6 @@ mod tests { spatial_filter_normalized, metric: metric.to_string(), num_aggregates_to_retain: None, - read_count_threshold: None, table_name: None, value_column: None, } diff --git a/crates/asap_types/src/enums.rs b/crates/asap_types/src/enums.rs index dc82bd8b..6534a8e5 100644 --- a/crates/asap_types/src/enums.rs +++ b/crates/asap_types/src/enums.rs @@ -18,8 +18,6 @@ pub enum QueryLanguage { pub enum CleanupPolicy { /// Keep only the N most recent aggregates (circular buffer behavior) CircularBuffer, - /// Remove aggregates after they've been read N times - ReadBased, /// Never clean up aggregates NoCleanup, } @@ -28,7 +26,6 @@ impl fmt::Display for CleanupPolicy { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { CleanupPolicy::CircularBuffer => write!(f, "circular_buffer"), - CleanupPolicy::ReadBased => write!(f, "read_based"), CleanupPolicy::NoCleanup => write!(f, "no_cleanup"), } } @@ -40,7 +37,6 @@ impl FromStr for CleanupPolicy { fn from_str(s: &str) -> Result { match s { "circular_buffer" => Ok(CleanupPolicy::CircularBuffer), - "read_based" => Ok(CleanupPolicy::ReadBased), "no_cleanup" => Ok(CleanupPolicy::NoCleanup), _ => Err(format!("Unknown cleanup policy: '{s}'")), } diff --git a/crates/asap_types/src/inference_config.rs b/crates/asap_types/src/inference_config.rs index 3112abf1..f6447450 100644 --- a/crates/asap_types/src/inference_config.rs +++ b/crates/asap_types/src/inference_config.rs @@ -91,7 +91,7 @@ impl InferenceConfig { let cleanup_policy_data = data.get("cleanup_policy").ok_or_else(|| { anyhow::anyhow!( "Missing cleanup_policy section in inference_config.yaml. \ - Must specify cleanup_policy.name as one of: circular_buffer, read_based, no_cleanup" + Must specify cleanup_policy.name as one of: circular_buffer, no_cleanup" ) })?; @@ -101,13 +101,13 @@ impl InferenceConfig { .ok_or_else(|| { anyhow::anyhow!( "Missing cleanup_policy.name in inference_config.yaml. \ - Must be one of: circular_buffer, read_based, no_cleanup" + Must be one of: circular_buffer, no_cleanup" ) })?; name.parse::().map_err(|_| { anyhow::anyhow!( - "Invalid cleanup policy: '{}'. Valid options: circular_buffer, read_based, no_cleanup", + "Invalid cleanup policy: '{}'. Valid options: circular_buffer, no_cleanup", name ) }) @@ -146,15 +146,6 @@ impl InferenceConfig { .and_then(|v| v.as_u64()); AggregationReference::new(aggregation_id, num_aggregates_to_retain) } - CleanupPolicy::ReadBased => { - let read_count_threshold = agg_data - .get("read_count_threshold") - .and_then(|v| v.as_u64()); - AggregationReference::with_read_count_threshold( - aggregation_id, - read_count_threshold, - ) - } CleanupPolicy::NoCleanup => { AggregationReference::new(aggregation_id, None) } diff --git a/crates/asap_types/src/streaming_config.rs b/crates/asap_types/src/streaming_config.rs index 63467a9b..d3aea7b3 100644 --- a/crates/asap_types/src/streaming_config.rs +++ b/crates/asap_types/src/streaming_config.rs @@ -78,19 +78,13 @@ impl StreamingConfig { inference_config: Option<&InferenceConfig>, ) -> Result { let mut retention_map: HashMap = HashMap::new(); - let mut read_count_threshold_map: HashMap = HashMap::new(); if let Some(inference_config) = inference_config { for query_config in &inference_config.query_configs { for aggregation in &query_config.aggregations { let aggregation_id = aggregation.aggregation_id; if let Some(num_aggregates) = aggregation.num_aggregates_to_retain { - // OLD: Keep last value only (for backwards compatibility) retention_map.insert(aggregation_id, num_aggregates); - - // NEW: Sum up num_aggregates_to_retain across all queries - *read_count_threshold_map.entry(aggregation_id).or_insert(0) += - num_aggregates; } } } @@ -115,11 +109,9 @@ impl StreamingConfig { ) })?; let num_aggregates_to_retain = retention_map.get(&aggregation_id_u64); - let read_count_threshold = read_count_threshold_map.get(&aggregation_id_u64); let config = AggregationConfig::from_yaml_data( aggregation_data, num_aggregates_to_retain.copied(), - read_count_threshold.copied(), query_language, )?; aggregation_configs.insert(aggregation_id_u64, config); diff --git a/data_plane/examples/promql/inference_config.yaml b/data_plane/examples/promql/inference_config.yaml index 24fd878b..ecbbf9e4 100644 --- a/data_plane/examples/promql/inference_config.yaml +++ b/data_plane/examples/promql/inference_config.yaml @@ -5,7 +5,7 @@ metrics: - label_0 - label_1 cleanup_policy: - name: read_based + name: circular_buffer # Each (query, aggregation_id) pair binds a PromQL pattern to a precompute # plan. The exact-string match in `find_query_config` requires the request # query to canonicalize to one of the listed strings — wider-range / wider- @@ -17,96 +17,74 @@ queries: # ── Spatial quantile (existing canonical pattern + multi-quantile) ───── - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile by (label_0) (0.5, fake_metric) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile by (label_0) (0.9, fake_metric) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile by (label_0) (0.95, fake_metric) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile by (label_0) (0.99, fake_metric) # ── quantile_over_time: multi-quantile × wider ranges ────────────────── # Routes to `Statistic::Quantile`; supported by DDSketch / KLL accumulators. - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile_over_time(0.5, fake_metric[1m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile_over_time(0.9, fake_metric[1m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile_over_time(0.95, fake_metric[1m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile_over_time(0.99, fake_metric[1m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile_over_time(0.5, fake_metric[2m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile_over_time(0.9, fake_metric[2m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile_over_time(0.95, fake_metric[2m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile_over_time(0.99, fake_metric[2m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile_over_time(0.5, fake_metric[5m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile_over_time(0.9, fake_metric[5m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile_over_time(0.95, fake_metric[5m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: quantile_over_time(0.99, fake_metric[5m]) # ── sum_over_time / count_over_time: wider ranges ────────────────────── # Routes to `Statistic::Sum` / `Statistic::Count`; supported by DDSketch / # CountSketch / CountMinSketch accumulators (no-key total-volume fallback). - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: sum_over_time(fake_metric[1m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: sum_over_time(fake_metric[2m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: sum_over_time(fake_metric[5m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: count_over_time(fake_metric[1m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: count_over_time(fake_metric[2m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: count_over_time(fake_metric[5m]) # ── rate / increase: delta-capable sketches ──────────────────────────── # Routes to `Statistic::Rate` / `Statistic::Increase`; supported by @@ -115,23 +93,18 @@ queries: # streaming aggregation when serving rate/increase queries. - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: rate(fake_metric[1m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: rate(fake_metric[2m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: rate(fake_metric[5m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: increase(fake_metric[1m]) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: increase(fake_metric[5m]) # ── Cardinality / generic spatial aggregations ───────────────────────── # `count` over an HLL-backed aggregation routes to `Statistic::Count`, @@ -139,15 +112,12 @@ queries: # `sum` / `avg` route to `Statistic::Sum` / `(Sum, Count)`. - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: count(fake_metric) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: sum(fake_metric) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: avg(fake_metric) # ── Top-K (CountSketch heavy-hitter readout) ─────────────────────────── # `topk(N, …)` routes to `Statistic::Topk`; CountSketch's `query_statistic` @@ -156,13 +126,10 @@ queries: # upstream). - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: topk(5, fake_metric) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: topk(10, fake_metric) - aggregations: - aggregation_id: 1 - read_count_threshold: 1 query: topk(50, fake_metric) diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index aa66d0b0..55edda22 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2610,7 +2610,6 @@ aggregations: None, None, None, - None, ); map.insert(*agg_id, cfg); } diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 4db11783..af4ad4c2 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -990,10 +990,9 @@ async fn spawn_memory_diagnostics( ); for agg in &store_diag.per_aggregation { info!( - "[MEMORY_DIAG] agg_id={}: time_map_len={}, read_counts_len={}, aggregate_objects={}, sketch_bytes={:.2} KB", + "[MEMORY_DIAG] agg_id={}: time_map_len={}, aggregate_objects={}, sketch_bytes={:.2} KB", agg.aggregation_id, agg.time_map_len, - agg.read_counts_len, agg.num_aggregate_objects, agg.sketch_bytes as f64 / 1024.0, ); diff --git a/data_plane/src/precompute_engine/accumulator_factory.rs b/data_plane/src/precompute_engine/accumulator_factory.rs index a6205876..f4aca6e0 100644 --- a/data_plane/src/precompute_engine/accumulator_factory.rs +++ b/data_plane/src/precompute_engine/accumulator_factory.rs @@ -844,7 +844,6 @@ mod tests { None, None, None, - None, ) }; @@ -928,7 +927,6 @@ mod tests { None, None, None, - None, ); let updater = create_accumulator_updater(&config); let acc = updater.snapshot_accumulator(); diff --git a/data_plane/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs index 721b3f32..98b0ecdb 100644 --- a/data_plane/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -151,7 +151,6 @@ mod tests { None, None, None, - None, ) } diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 890eacd9..5a59290b 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -1137,7 +1137,6 @@ mod tests { None, None, None, - None, ) } @@ -2109,7 +2108,6 @@ aggregations: "http_requests_total".to_string(), "http_requests_total".to_string(), Some(60), - Some(0), None, None, ); diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 76ca702c..f69daa0e 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -4950,7 +4950,6 @@ mod hot_reload_phase2_tests { None, None, None, - None, ) } @@ -5117,7 +5116,6 @@ mod e2e_feedback_loop_tests { None, None, None, - None, ) } @@ -5780,7 +5778,6 @@ mod sketch_alias_resolver_tests { None, None, None, - None, ) } diff --git a/data_plane/src/stores/sketch_db/accuracy.rs b/data_plane/src/stores/sketch_db/accuracy.rs index bf68fe86..76207632 100644 --- a/data_plane/src/stores/sketch_db/accuracy.rs +++ b/data_plane/src/stores/sketch_db/accuracy.rs @@ -450,7 +450,6 @@ mod tests { None, None, None, - None, ) } diff --git a/data_plane/src/stores/sketch_db/backfill/processor.rs b/data_plane/src/stores/sketch_db/backfill/processor.rs index f1a205be..f3afefa9 100644 --- a/data_plane/src/stores/sketch_db/backfill/processor.rs +++ b/data_plane/src/stores/sketch_db/backfill/processor.rs @@ -292,7 +292,6 @@ mod tests { None, None, None, - None, ) } diff --git a/data_plane/src/stores/sketch_db/backfill/service.rs b/data_plane/src/stores/sketch_db/backfill/service.rs index 722dd8d8..736077be 100644 --- a/data_plane/src/stores/sketch_db/backfill/service.rs +++ b/data_plane/src/stores/sketch_db/backfill/service.rs @@ -319,7 +319,6 @@ mod tests { None, None, None, - None, ) } diff --git a/data_plane/src/stores/sketch_db/backfill/window_builder.rs b/data_plane/src/stores/sketch_db/backfill/window_builder.rs index 8690cd9f..3bb8bb0b 100644 --- a/data_plane/src/stores/sketch_db/backfill/window_builder.rs +++ b/data_plane/src/stores/sketch_db/backfill/window_builder.rs @@ -140,7 +140,6 @@ mod tests { None, None, None, - None, ) } diff --git a/data_plane/src/stores/sketch_db/index/epoch_columnar.rs b/data_plane/src/stores/sketch_db/index/epoch_columnar.rs index 40ceb13e..39b5c3d3 100644 --- a/data_plane/src/stores/sketch_db/index/epoch_columnar.rs +++ b/data_plane/src/stores/sketch_db/index/epoch_columnar.rs @@ -279,7 +279,7 @@ impl

MutableEpoch

{ } /// Remove all entries whose window is in `windows`. - /// Mirrors the legacy `SketchStore` ReadBased / CircularBuffer + /// Mirrors the legacy `SketchStore` CircularBuffer /// cleanup contract. O(N) — rebuilds columns in one pass. pub fn remove_windows(&mut self, windows: &[TimestampRange]) { use std::collections::HashSet as StdHashSet; @@ -308,7 +308,7 @@ impl MutableEpoch

{ /// Range query into a caller-provided `HashMap>`, /// matching the legacy `SketchStore`'s `MetricBucketMap` shape. /// Also pushes each matched window into `matched_windows` for the - /// downstream `read_counts` accounting. + /// downstream accounting (e.g. metrics). /// /// Uses the same overlap-filter semantics as the flat /// `range_query_into`: include any window whose `[w.0, w.1)` @@ -472,7 +472,7 @@ impl

SealedEpoch

{ } /// Sorted-deduplicated windows. Used by the legacy SketchStore to - /// purge `read_counts` when an epoch is dropped. + /// surface the windows that were dropped on epoch eviction. pub fn unique_windows(&self) -> Vec { let mut windows: Vec = self.entries.iter().map(|(w, _, _)| *w).collect(); diff --git a/data_plane/src/stores/sketch_db/schema/eviction.rs b/data_plane/src/stores/sketch_db/schema/eviction.rs index 9d855a0a..52160807 100644 --- a/data_plane/src/stores/sketch_db/schema/eviction.rs +++ b/data_plane/src/stores/sketch_db/schema/eviction.rs @@ -292,7 +292,6 @@ mod tests { spatial_filter_normalized: String::new(), metric: format!("metric_{id}"), num_aggregates_to_retain: None, - read_count_threshold: None, table_name: None, value_column: None, } diff --git a/data_plane/src/stores/sketch_db/schema/mod.rs b/data_plane/src/stores/sketch_db/schema/mod.rs index 3ae31515..2fda099a 100644 --- a/data_plane/src/stores/sketch_db/schema/mod.rs +++ b/data_plane/src/stores/sketch_db/schema/mod.rs @@ -795,7 +795,6 @@ mod tests { None, None, None, - None, ) } diff --git a/data_plane/src/stores/sketch_db/store/INDEX_DESIGN.md b/data_plane/src/stores/sketch_db/store/INDEX_DESIGN.md index 594e6a94..d1bc2d74 100644 --- a/data_plane/src/stores/sketch_db/store/INDEX_DESIGN.md +++ b/data_plane/src/stores/sketch_db/store/INDEX_DESIGN.md @@ -110,12 +110,9 @@ StoreKeyData { current_epoch_id: EpochID epoch_capacity: Option // None = unlimited max_epochs: usize // default 4 - read_counts: Mutex> } ``` -`read_counts` is behind an inner `Mutex` so queries can hold the outer `RwLock::read` and still update counts. - ### Global Store (`global.rs`) Same per-key epoch structure, but all aggregation_ids share a single `Mutex`: @@ -124,9 +121,8 @@ Same per-key epoch structure, but all aggregation_ids share a single `Mutex StoreData { - stores: HashMap - read_counts: HashMap> - metrics: HashSet + stores: HashMap + metrics: HashSet } PerKeyState { @@ -139,8 +135,6 @@ PerKeyState { } ``` -No inner `Mutex` for `read_counts` — the outer `Mutex` already serializes all access. - --- ## Complexity @@ -170,7 +164,6 @@ No inner `Mutex` for `read_counts` — the outer `Mutex` already serializes all | **Exact query** (first after write) | O(M) | Build `window_to_ids` from `windows_col` | | **Exact query** (cached) | O(m) | HashMap lookup + `Arc::clone` per offset | | **Exact query** (sealed epoch) | O(log N + m) | Binary search to window + linear scan | -| **ReadBased cleanup** | O(N + k · m) | Scan `read_counts` + targeted removal via `remove_windows` | | **get_earliest_timestamp** | O(A) | DashMap iteration with AtomicU64 loads | ### Space @@ -181,7 +174,6 @@ No inner `Mutex` for `read_counts` — the outer `Mutex` already serializes all | `MutableEpoch` columns | O(M) | | `SealedEpoch` entries | O(M) per sealed epoch | | `window_to_ids` (when built) | O(M) | -| `read_counts` | O(N) total | | **Total** | **O(A · E · M)** where E ≤ `max_epochs` | --- @@ -196,7 +188,6 @@ No inner `Mutex` for `read_counts` — the outer `Mutex` already serializes all - Skip if `min_start > end || max_end < start` — O(1) bounds check - `sealed_epoch.range_query_into(start, end)` — O(log N + k) binary search + scan 4. Resolve MetricIDs → labels via `InternTable` in one pass -5. Briefly acquire inner `Mutex` to update `read_counts` ### Exact Query `(exact_start, exact_end)` @@ -217,15 +208,7 @@ Epoch-based eviction — O(1) amortized per insert: 1. On first insert: set `epoch_capacity` from `num_aggregates_to_retain` 2. After each insert: call `maybe_rotate_epoch()` - If `current_epoch.window_count() >= epoch_capacity`: seal current epoch, open new one with `with_capacity(hint)` (Opt 6) - - If `1 + sealed_epochs.len() > max_epochs`: pop oldest sealed epoch in O(1), purge its windows from `read_counts` - -### ReadBased - -Read-count triggered eviction: - -1. Scan `read_counts` for windows with `count >= threshold` -2. For each such window, call `MutableEpoch::remove_windows` or `SealedEpoch::remove_windows` -3. Drop any epochs that become empty + - If `1 + sealed_epochs.len() > max_epochs`: pop oldest sealed epoch in O(1) ### NoCleanup @@ -238,8 +221,8 @@ No eviction — data accumulates indefinitely. | Operation | Lock | |-----------|------| | **Insert** | `RwLock::write` for the batch duration | -| **Range query** | `RwLock::read` → brief `Mutex::lock` on `read_counts` | +| **Range query** | `RwLock::read` for the scan | | **Exact query** | `RwLock::write` (lazy index build) → drop → `RwLock::read` for label resolution | -| **Cleanup** | Under existing write lock; `Mutex::get_mut()` bypasses inner lock | +| **Cleanup** | Under existing write lock | Multiple readers per `aggregation_id` run concurrently. Writers only block readers of the same `aggregation_id`. diff --git a/data_plane/src/stores/sketch_db/store/global.rs b/data_plane/src/stores/sketch_db/store/global.rs index 03e9fd8b..6a6750ba 100644 --- a/data_plane/src/stores/sketch_db/store/global.rs +++ b/data_plane/src/stores/sketch_db/store/global.rs @@ -58,7 +58,7 @@ impl PerKeyState { /// Seal the current epoch when full, then evict the minimum number of oldest windows /// to keep total distinct windows ≤ `epoch_capacity * max_epochs`. - /// Returns the evicted windows so the caller can clean up `read_counts`. + /// Returns the evicted windows. fn maybe_rotate_epoch(&mut self) -> Vec { let capacity = match self.epoch_capacity { Some(c) if c > 0 => c, @@ -124,10 +124,6 @@ struct StoreData { /// Track earliest timestamp per aggregation ID earliest_timestamp_per_aggregation_id: HashMap, - - /// Track how many times each aggregate window has been read (per store key) - /// No inner Mutex needed — outer Mutex serializes everything. - read_counts: HashMap>, } /// In-memory storage implementation using single mutex (like Python version) @@ -150,7 +146,6 @@ impl SketchStoreGlobal { metrics: HashSet::new(), items_inserted: HashMap::new(), earliest_timestamp_per_aggregation_id: HashMap::new(), - read_counts: HashMap::new(), }), streaming_config, cleanup_policy, @@ -173,11 +168,6 @@ impl SketchStoreGlobal { .values() .map(|e| e.distinct_window_count()) .sum::(); - let read_counts_len = data - .read_counts - .get(&agg_id) - .map(|rc| rc.len()) - .unwrap_or(0); total_time_map_entries += time_map_len; let num_aggregate_objects = per_key.current_epoch.len() @@ -190,7 +180,6 @@ impl SketchStoreGlobal { per_aggregation.push(AggregationDiagnostic { aggregation_id: agg_id, time_map_len, - read_counts_len, num_aggregate_objects, sketch_bytes: 0, // skip serialization for diagnostics }); @@ -220,7 +209,6 @@ struct BatchConfig { metric: String, is_delta: bool, num_aggregates_to_retain: Option, - read_count_threshold: Option, } #[async_trait::async_trait] @@ -270,7 +258,6 @@ impl Store for SketchStoreGlobal { is_delta: aggregation_config.aggregation_type == AggregationType::DeltaSetAggregator, num_aggregates_to_retain: aggregation_config.num_aggregates_to_retain, - read_count_threshold: aggregation_config.read_count_threshold, }, u64::MAX, Vec::new(), @@ -327,9 +314,6 @@ impl Store for SketchStoreGlobal { } // per_key borrow ends here for (output, precompute) in items { - // Get per_key fresh each iteration so the borrow of data.stores ends before - // the cleanup branches borrow data.read_counts (different field — NLL splits - // them, but only if the per_key borrow scope is confined to each iteration). let per_key = data.stores.get_mut(&store_key).unwrap(); // Intern the label key (Optimization 1) @@ -342,8 +326,6 @@ impl Store for SketchStoreGlobal { .insert(timestamp_range, metric_id, Arc::from(precompute)); // Apply retention policy if configured (but exclude DeltaSetAggregator). - // per_key is last used above; NLL ends its borrow so data.read_counts can - // be accessed in the cleanup branches below. if !cfg.is_delta { match self.cleanup_policy { CleanupPolicy::CircularBuffer => { @@ -352,45 +334,11 @@ impl Store for SketchStoreGlobal { .get_mut(&store_key) .unwrap() .maybe_rotate_epoch(); - if !dropped_windows.is_empty() { - if let Some(rc_map) = data.read_counts.get_mut(&store_key) { - for window in &dropped_windows { - rc_map.remove(window); - } - } - for window in &dropped_windows { - debug!( - "Removed old aggregate for {} aggregation_id {} window {}-{} (epoch rotation)", - cfg.metric, store_key, window.0, window.1 - ); - } - } - } - CleanupPolicy::ReadBased => { - if let Some(threshold) = cfg.read_count_threshold { - let rc_map = data.read_counts.entry(store_key).or_default(); - let windows_to_remove: Vec = rc_map - .iter() - .filter(|(_, &count)| count >= threshold) - .map(|(range, _)| *range) - .collect(); - - if !windows_to_remove.is_empty() { - for window in &windows_to_remove { - debug!( - "Removed aggregate for {} aggregation_id {} window {}-{} (read_count >= threshold: {})", - cfg.metric, store_key, window.0, window.1, threshold - ); - rc_map.remove(window); - } - - let per_key = data.stores.get_mut(&store_key).unwrap(); - per_key.current_epoch.remove_windows(&windows_to_remove); - per_key.sealed_epochs.retain(|_, epoch| { - epoch.remove_windows(&windows_to_remove); - !epoch.is_empty() - }); - } + for window in &dropped_windows { + debug!( + "Removed old aggregate for {} aggregation_id {} window {}-{} (epoch rotation)", + cfg.metric, store_key, window.0, window.1 + ); } } CleanupPolicy::NoCleanup => {} @@ -512,7 +460,7 @@ impl Store for SketchStoreGlobal { mid }; - // Resolve MetricIDs → labels in a single pass (scope ends before read_counts borrow) + // Resolve MetricIDs → labels in a single pass. let results: TimestampedBucketsMap = { let per_key = data.stores.get(&store_key).unwrap(); let mut r = HashMap::with_capacity(mid.len()); @@ -528,11 +476,9 @@ impl Store for SketchStoreGlobal { r }; - // Update read counts (outer Mutex already held — no inner Mutex needed) - let rc_map = data.read_counts.entry(store_key).or_default(); - for window in &matched_windows { - *rc_map.entry(*window).or_insert(0) += 1; - } + // matched_windows kept for potential telemetry — read-count-based + // eviction was removed; tracking is no-op now. + let _ = matched_windows; let range_scan_duration = range_scan_start_time.elapsed(); debug!( @@ -668,11 +614,8 @@ impl Store for SketchStoreGlobal { ); } - // Update read count (outer Mutex held — no inner Mutex needed) - if found_match { - let rc_map = data.read_counts.entry(store_key).or_default(); - *rc_map.entry(timestamp_range).or_insert(0) += 1; - } + // read-count tracking removed. + let _ = found_match; #[cfg(feature = "lock_profiling")] { @@ -726,7 +669,6 @@ impl Store for SketchStoreGlobal { }) .unwrap_or(0); data.stores.remove(&agg_id); - data.read_counts.remove(&agg_id); data.earliest_timestamp_per_aggregation_id.remove(&agg_id); // `metrics` and `items_inserted` are keyed by metric name, // not agg_id, so we don't touch them — other agg_ids for the diff --git a/data_plane/src/stores/sketch_db/store/mod.rs b/data_plane/src/stores/sketch_db/store/mod.rs index 5ce3a6ae..c80d4901 100644 --- a/data_plane/src/stores/sketch_db/store/mod.rs +++ b/data_plane/src/stores/sketch_db/store/mod.rs @@ -16,7 +16,6 @@ use std::sync::Arc; pub struct AggregationDiagnostic { pub aggregation_id: u64, pub time_map_len: usize, - pub read_counts_len: usize, pub num_aggregate_objects: usize, pub sketch_bytes: usize, } @@ -195,7 +194,6 @@ mod drop_agg_id_tests { spatial_filter_normalized: String::new(), metric: format!("metric_{id}"), num_aggregates_to_retain: None, - read_count_threshold: None, table_name: None, value_column: None, }; diff --git a/data_plane/src/stores/sketch_db/store/per_key.rs b/data_plane/src/stores/sketch_db/store/per_key.rs index 7bb77aa3..a3e0e6dc 100644 --- a/data_plane/src/stores/sketch_db/store/per_key.rs +++ b/data_plane/src/stores/sketch_db/store/per_key.rs @@ -72,10 +72,6 @@ struct StoreKeyData { /// Max total epochs (1 current + sealed) to retain before dropping the oldest. max_epochs: usize, - - /// Track how many times each timestamp range has been read. - /// Behind Mutex so range queries can use a read lock on the outer RwLock. - read_counts: Mutex>, } impl StoreKeyData { @@ -87,7 +83,6 @@ impl StoreKeyData { current_epoch_id: 0, epoch_capacity: None, max_epochs: 4, - read_counts: Mutex::new(HashMap::new()), } } @@ -151,12 +146,6 @@ impl StoreKeyData { let to_remove = oldest_windows[..n_evict].to_vec(); over -= n_evict; - { - let read_counts = self.read_counts.get_mut().unwrap(); - for w in &to_remove { - read_counts.remove(w); - } - } if n_evict == oldest_windows.len() { self.sealed_epochs.remove(&oldest_id); } else { @@ -167,38 +156,6 @@ impl StoreKeyData { } } } - - /// Apply ReadBased cleanup across current and sealed epochs. - fn cleanup_read_based(&mut self, metric: &str, aggregation_id: u64, threshold: u64) { - let read_counts = self.read_counts.get_mut().unwrap(); - - let windows_to_remove: Vec = read_counts - .iter() - .filter(|(_, &count)| count >= threshold) - .map(|(range, _)| *range) - .collect(); - - if windows_to_remove.is_empty() { - return; - } - - for window in &windows_to_remove { - debug!( - "Removed aggregate for {} aggregation_id {} window {}-{} (read_count >= threshold: {})", - metric, aggregation_id, window.0, window.1, threshold - ); - read_counts.remove(window); - } - - // Remove from current epoch. - self.current_epoch.remove_windows(&windows_to_remove); - - // Remove from sealed epochs; drop any that become empty. - self.sealed_epochs.retain(|_, epoch| { - epoch.remove_windows(&windows_to_remove); - !epoch.is_empty() - }); - } } /// Shared state that both the outer `SketchStorePerKey` and the @@ -289,7 +246,7 @@ impl SketchStorePerKey { /// memory / time pressure. /// /// Cleanup policy still applies, but when persistence is on, the - /// destructive eviction step of `CircularBuffer` / `ReadBased` is + /// destructive eviction step of `CircularBuffer` is /// bypassed in favor of the flusher. `NoCleanup` + persistence is /// the typical production configuration: the flusher bounds RAM, /// nothing is ever dropped from memory without first being on @@ -380,7 +337,6 @@ impl SketchStorePerKey { .values() .map(|e| e.distinct_window_count()) .sum::(); - let read_counts_len = data.read_counts.lock().map(|rc| rc.len()).unwrap_or(0); total_time_map_entries += time_map_len; let num_aggregate_objects = data.current_epoch.len() @@ -393,7 +349,6 @@ impl SketchStorePerKey { per_aggregation.push(AggregationDiagnostic { aggregation_id: agg_id, time_map_len, - read_counts_len, num_aggregate_objects, sketch_bytes: 0, // per-agg sketch byte sizing is a follow-up }); @@ -413,30 +368,19 @@ impl SketchStorePerKey { metric: &str, aggregation_id: u64, num_aggregates_to_retain: Option, - read_count_threshold: Option, ) { // When persistence is enabled, eviction is the flusher's job. // Skip destructive cleanup entirely — parts on disk are the // source of truth for cold data. if self.inner.persistence_enabled { - let _ = ( - num_aggregates_to_retain, - metric, - aggregation_id, - read_count_threshold, - ); + let _ = (num_aggregates_to_retain, metric, aggregation_id, data); return; } match self.inner.cleanup_policy { CleanupPolicy::CircularBuffer => { // Handled by maybe_rotate_epoch() during insert. - let _ = (num_aggregates_to_retain, metric, aggregation_id); - } - CleanupPolicy::ReadBased => { - if let Some(threshold) = read_count_threshold { - data.cleanup_read_based(metric, aggregation_id, threshold); - } + let _ = (num_aggregates_to_retain, metric, aggregation_id, data); } CleanupPolicy::NoCleanup => {} } @@ -639,7 +583,6 @@ impl SketchStorePerKey { metric, aggregation_id, aggregation_config.num_aggregates_to_retain, - aggregation_config.read_count_threshold, ); } @@ -811,12 +754,9 @@ impl Store for SketchStorePerKey { results.entry(label).or_default().extend(buckets); } - { - let mut read_counts = data.read_counts.lock().unwrap(); - for window in &matched_windows { - *read_counts.entry(*window).or_insert(0) += 1; - } - } + // read-count tracking removed; matched_windows kept available for + // telemetry if needed in the future. + let _ = matched_windows; } else if self.persistence.is_none() { // Nothing in memory and no disk layer → empty result, // matching the old behavior. @@ -903,10 +843,8 @@ impl Store for SketchStorePerKey { } } - if found_match { - let mut read_counts = data.read_counts.lock().unwrap(); - *read_counts.entry(timestamp_range).or_insert(0) += 1; - } + // read-count tracking removed. + let _ = found_match; let query_duration = query_start_time.elapsed(); debug!( @@ -1036,12 +974,6 @@ impl EpochSource for PerKeyInner { if let Some(epoch) = data.sealed_epochs.remove(&epoch_id) { let freed = epoch_approx_bytes(&epoch); self.mem_bytes_sealed.fetch_sub(freed, Ordering::Relaxed); - // Also purge the epoch's windows from read_counts so they - // don't leak. - let mut read_counts = data.read_counts.lock().unwrap(); - for w in epoch.unique_windows() { - read_counts.remove(&w); - } } } diff --git a/data_plane/src/stores/types/hot_reload_config.rs b/data_plane/src/stores/types/hot_reload_config.rs index 3c2a7e05..b5e5606f 100644 --- a/data_plane/src/stores/types/hot_reload_config.rs +++ b/data_plane/src/stores/types/hot_reload_config.rs @@ -161,7 +161,6 @@ mod tests { None, None, None, - None, ) } diff --git a/data_plane/src/tests/accuracy_empirical_validation_tests.rs b/data_plane/src/tests/accuracy_empirical_validation_tests.rs index b3296ec7..708c6a64 100644 --- a/data_plane/src/tests/accuracy_empirical_validation_tests.rs +++ b/data_plane/src/tests/accuracy_empirical_validation_tests.rs @@ -53,7 +53,6 @@ fn cfg(agg_type: AggregationType, params: HashMap) -> Aggregation None, None, None, - None, ) } diff --git a/data_plane/src/tests/capability_matching_tests.rs b/data_plane/src/tests/capability_matching_tests.rs index 93d7a3f7..5a842f6a 100644 --- a/data_plane/src/tests/capability_matching_tests.rs +++ b/data_plane/src/tests/capability_matching_tests.rs @@ -49,7 +49,6 @@ fn make_agg_config( spatial_filter_normalized: String::new(), metric: metric.to_string(), num_aggregates_to_retain: None, - read_count_threshold: None, table_name: None, value_column: None, } diff --git a/data_plane/src/tests/persist_format_versioning_tests.rs b/data_plane/src/tests/persist_format_versioning_tests.rs index 25ad580e..a38e8742 100644 --- a/data_plane/src/tests/persist_format_versioning_tests.rs +++ b/data_plane/src/tests/persist_format_versioning_tests.rs @@ -64,7 +64,6 @@ mod schema { None, None, None, - None, ) } @@ -476,7 +475,6 @@ mod v2_forward_compat { None, None, None, - None, ) } diff --git a/data_plane/src/tests/persistence_integration_tests.rs b/data_plane/src/tests/persistence_integration_tests.rs index b05fcca8..b23e57d5 100644 --- a/data_plane/src/tests/persistence_integration_tests.rs +++ b/data_plane/src/tests/persistence_integration_tests.rs @@ -37,7 +37,6 @@ fn make_streaming_config(agg_id: u64) -> Arc { Some(2), None, None, - None, ); let mut map = std::collections::HashMap::new(); map.insert(agg_id, cfg); diff --git a/data_plane/src/tests/persistence_perf_tests.rs b/data_plane/src/tests/persistence_perf_tests.rs index feadb886..dd5beade 100644 --- a/data_plane/src/tests/persistence_perf_tests.rs +++ b/data_plane/src/tests/persistence_perf_tests.rs @@ -66,7 +66,6 @@ fn streaming_config(agg_id: u64, retention: Option) -> Arc retention, None, None, - None, ); let mut map = HashMap::new(); map.insert(agg_id, cfg); diff --git a/data_plane/src/tests/schema_timeline_dispatch_tests.rs b/data_plane/src/tests/schema_timeline_dispatch_tests.rs index b922fcae..657d39b6 100644 --- a/data_plane/src/tests/schema_timeline_dispatch_tests.rs +++ b/data_plane/src/tests/schema_timeline_dispatch_tests.rs @@ -73,7 +73,6 @@ fn make_agg_config(id: u64) -> AggregationConfig { None, None, None, - None, ) } diff --git a/data_plane/src/tests/store_correctness_tests.rs b/data_plane/src/tests/store_correctness_tests.rs index 314d26d9..39ce6424 100644 --- a/data_plane/src/tests/store_correctness_tests.rs +++ b/data_plane/src/tests/store_correctness_tests.rs @@ -8,7 +8,7 @@ //! - Partial-range filtering //! - Aggregation-ID isolation //! - Earliest-timestamp tracking -//! - Cleanup policies (circular-buffer and read-based) +//! - Cleanup policies (circular-buffer + no-cleanup) //! - Concurrent insert and read safety //! - **Clone fidelity** for every supported accumulator type //! - **Keyed (label-grouped) entries** @@ -49,7 +49,6 @@ fn make_agg_config( agg_id: u64, aggregation_type: AggregationType, num_aggregates_to_retain: Option, - read_count_threshold: Option, ) -> AggregationConfig { AggregationConfig::new( agg_id, @@ -66,20 +65,17 @@ fn make_agg_config( "".to_string(), // spatial_filter "cpu_usage".to_string(), num_aggregates_to_retain, - read_count_threshold, None, // table_name None, // value_column ) } fn make_streaming_config( - ids: &[(u64, AggregationType, Option, Option)], + ids: &[(u64, AggregationType, Option)], ) -> Arc { let configs = ids .iter() - .map(|&(id, agg_type, retain, threshold)| { - (id, make_agg_config(id, agg_type, retain, threshold)) - }) + .map(|&(id, agg_type, retain)| (id, make_agg_config(id, agg_type, retain))) .collect(); Arc::new(StreamingConfig::new(configs)) } @@ -87,7 +83,7 @@ fn make_streaming_config( fn make_store( strategy: LockStrategy, policy: CleanupPolicy, - ids: &[(u64, AggregationType, Option, Option)], + ids: &[(u64, AggregationType, Option)], ) -> SketchStore { let config = make_streaming_config(ids); SketchStore::new_with_strategy(config, policy, strategy) @@ -98,7 +94,7 @@ fn make_store_simple(strategy: LockStrategy) -> SketchStore { make_store( strategy, CleanupPolicy::NoCleanup, - &[(1, AggregationType::Sum, None, None)], + &[(1, AggregationType::Sum, None)], ) } @@ -196,8 +192,6 @@ pub fn run_contract_suite(strategy: LockStrategy) { // Cleanup policies test_cleanup_circular_buffer_evicts_oldest_window(strategy); test_cleanup_circular_buffer_retains_newest_windows(strategy); - test_cleanup_read_based_evicts_after_threshold_reads(strategy); - test_cleanup_read_based_unread_window_is_retained(strategy); test_delta_set_aggregator_bypasses_cleanup(strategy); // Keyed (label-grouped) entries @@ -417,8 +411,8 @@ fn test_multiple_agg_ids_are_isolated(strategy: LockStrategy) { strategy, CleanupPolicy::NoCleanup, &[ - (1, AggregationType::Sum, None, None), - (2, AggregationType::Sum, None, None), + (1, AggregationType::Sum, None), + (2, AggregationType::Sum, None), ], ); let (o1, a1) = sum_entry(1, 1_000, 2_000, 10.0); @@ -481,8 +475,8 @@ fn test_earliest_timestamp_tracked_per_agg_id(strategy: LockStrategy) { strategy, CleanupPolicy::NoCleanup, &[ - (1, AggregationType::Sum, None, None), - (2, AggregationType::Sum, None, None), + (1, AggregationType::Sum, None), + (2, AggregationType::Sum, None), ], ); let (o1, a1) = sum_entry(1, 1_000, 2_000, 1.0); @@ -513,7 +507,7 @@ fn test_cleanup_circular_buffer_evicts_oldest_window(strategy: LockStrategy) { let store = make_store( strategy, CleanupPolicy::CircularBuffer, - &[(1, AggregationType::Sum, Some(2), None)], + &[(1, AggregationType::Sum, Some(2))], ); for i in 0u64..9 { let (out, acc) = sum_entry(1, i * 60_000, (i + 1) * 60_000, i as f64); @@ -533,7 +527,7 @@ fn test_cleanup_circular_buffer_retains_newest_windows(strategy: LockStrategy) { let store = make_store( strategy, CleanupPolicy::CircularBuffer, - &[(1, AggregationType::Sum, Some(2), None)], + &[(1, AggregationType::Sum, Some(2))], ); for i in 0u64..9 { let (out, acc) = sum_entry(1, i * 60_000, (i + 1) * 60_000, i as f64); @@ -552,77 +546,6 @@ fn test_cleanup_circular_buffer_retains_newest_windows(strategy: LockStrategy) { // ── cleanup: read-based ─────────────────────────────────────────────────────── -fn test_cleanup_read_based_evicts_after_threshold_reads(strategy: LockStrategy) { - // read_count_threshold = 2: evicted once read count reaches 2. - // Cleanup runs on every insert. - let store = make_store( - strategy, - CleanupPolicy::ReadBased, - &[(1, AggregationType::Sum, None, Some(2))], - ); - let (out, acc) = sum_entry(1, 1_000, 2_000, 1.0); - store.insert_precomputed_output(out, acc).unwrap(); - - // Read 1 — count becomes 1, window kept on next insert. - store - .query_precomputed_output("cpu_usage", 1, 0, u64::MAX) - .unwrap(); - let (o2, a2) = sum_entry(1, 3_000, 4_000, 2.0); - store.insert_precomputed_output(o2, a2).unwrap(); - - let still_there = store - .query_precomputed_output_exact("cpu_usage", 1, 1_000, 2_000) - .unwrap(); - assert_eq!( - total_bucket_count(&still_there), - 1, - "[{}] window must survive until read count reaches threshold", - label(strategy) - ); - - // Read 2 — count becomes 2, evicted on the next insert. - store - .query_precomputed_output("cpu_usage", 1, 0, 2_000) - .unwrap(); - let (o3, a3) = sum_entry(1, 5_000, 6_000, 3.0); - store.insert_precomputed_output(o3, a3).unwrap(); - - let evicted = store - .query_precomputed_output_exact("cpu_usage", 1, 1_000, 2_000) - .unwrap(); - assert!( - evicted.is_empty(), - "[{}] window must be evicted once read count reaches threshold", - label(strategy) - ); -} - -fn test_cleanup_read_based_unread_window_is_retained(strategy: LockStrategy) { - let store = make_store( - strategy, - CleanupPolicy::ReadBased, - &[(1, AggregationType::Sum, None, Some(1))], - ); - let (out, acc) = sum_entry(1, 1_000, 2_000, 1.0); - store.insert_precomputed_output(out, acc).unwrap(); - - // Insert more windows without reading window 0 — cleanup runs each time. - for i in 1u64..5 { - let (o, a) = sum_entry(1, i * 10_000, (i + 1) * 10_000, i as f64); - store.insert_precomputed_output(o, a).unwrap(); - } - - let result = store - .query_precomputed_output_exact("cpu_usage", 1, 1_000, 2_000) - .unwrap(); - assert_eq!( - total_bucket_count(&result), - 1, - "[{}] unread window must not be evicted by read-based cleanup", - label(strategy) - ); -} - // ── cleanup: DeltaSetAggregator exclusion ───────────────────────────────────── fn test_delta_set_aggregator_bypasses_cleanup(strategy: LockStrategy) { @@ -631,7 +554,7 @@ fn test_delta_set_aggregator_bypasses_cleanup(strategy: LockStrategy) { let store = make_store( strategy, CleanupPolicy::CircularBuffer, - &[(1, AggregationType::DeltaSetAggregator, Some(2), None)], + &[(1, AggregationType::DeltaSetAggregator, Some(2))], ); let n = 10u64; for i in 0..n { diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 0005787d..495c6dfb 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -87,7 +87,6 @@ pub fn create_engine_single_pop_with_aggregated( spatial_filter_normalized: String::new(), metric: metric.to_string(), num_aggregates_to_retain: None, - read_count_threshold: None, table_name: None, value_column: None, }; @@ -185,7 +184,6 @@ pub fn create_engine_dual_input( spatial_filter_normalized: String::new(), metric: metric.to_string(), num_aggregates_to_retain: None, - read_count_threshold: None, table_name: None, value_column: None, }; @@ -208,7 +206,6 @@ pub fn create_engine_dual_input( spatial_filter_normalized: String::new(), metric: metric.to_string(), num_aggregates_to_retain: None, - read_count_threshold: None, table_name: None, value_column: None, }; @@ -302,7 +299,6 @@ pub fn create_engine_two_metrics( spatial_filter_normalized: String::new(), metric: metric_a.to_string(), num_aggregates_to_retain: None, - read_count_threshold: None, table_name: None, value_column: None, }; @@ -324,7 +320,6 @@ pub fn create_engine_two_metrics( spatial_filter_normalized: String::new(), metric: metric_b.to_string(), num_aggregates_to_retain: None, - read_count_threshold: None, table_name: None, value_column: None, }; @@ -428,7 +423,6 @@ pub fn create_engine_three_metrics( spatial_filter_normalized: String::new(), metric: metric.to_string(), num_aggregates_to_retain: None, - read_count_threshold: None, table_name: None, value_column: None, }, @@ -510,7 +504,6 @@ pub fn create_engine_multi_timestamp( spatial_filter_normalized: String::new(), metric: metric.to_string(), num_aggregates_to_retain: None, - read_count_threshold: None, table_name: None, value_column: None, }; @@ -591,7 +584,6 @@ pub fn create_engine_multi_timestamp_with_window( spatial_filter_normalized: String::new(), metric: metric.to_string(), num_aggregates_to_retain: None, - read_count_threshold: None, table_name: None, value_column: None, }; diff --git a/data_plane/tests/e2e_modified_otlp_sketch_path.rs b/data_plane/tests/e2e_modified_otlp_sketch_path.rs index 73e4f8e6..a6f50fa6 100644 --- a/data_plane/tests/e2e_modified_otlp_sketch_path.rs +++ b/data_plane/tests/e2e_modified_otlp_sketch_path.rs @@ -85,7 +85,6 @@ fn make_count_min_agg_config( None, None, None, - None, ) } @@ -353,7 +352,6 @@ fn make_count_sketch_agg_config( None, None, None, - None, ) } @@ -566,7 +564,6 @@ fn make_kll_agg_config( None, None, None, - None, ) } @@ -747,7 +744,6 @@ fn make_dd_sketch_agg_config( None, None, None, - None, ) } @@ -931,7 +927,6 @@ fn make_hll_agg_config( None, None, None, - None, ) } diff --git a/data_plane/tests/inference_yaml_pattern_coverage.rs b/data_plane/tests/inference_yaml_pattern_coverage.rs index 94f820a0..3d92dbb8 100644 --- a/data_plane/tests/inference_yaml_pattern_coverage.rs +++ b/data_plane/tests/inference_yaml_pattern_coverage.rs @@ -148,7 +148,6 @@ fn build_engine( spatial_filter_normalized: String::new(), metric: metric.to_string(), num_aggregates_to_retain: None, - read_count_threshold: None, table_name: None, value_column: None, }, From 3e3329176cde5b28073fbc343415dca0b36dfe84 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 12 May 2026 14:29:32 -0600 Subject: [PATCH 6/7] refactor(data_plane): audit precomputed_output.rs + remove Arroyo streaming-engine path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **precomputed_output.rs cleanup (~470 LOC removed):** Deleted commented-out and otherwise-dead code: - 7 commented-out serialization / deserialization methods (serialize_to_json_with_precompute, deserialize_from_bytes, deserialize_from_json, deserialize_from_json_with_precompute, deserialize_from_bytes_with_precompute_and_type, create_precompute_from_json) - The entire commented-out test module (references long-removed signatures like `PrecomputedOutput::new(_, _, key, config)` from before AggregationConfig was lifted out of the struct) - `// pub config: AggregationConfig` field comments + stale TODO Deleted live but unused surface: - `deserialize_from_bytes_with_precompute` — `Not implemented` stub; the only call site was a commented-out branch in kafka.rs that no longer exists - `impl SerializableToSink for PrecomputedOutput` — the trait is consumed only for accumulators in the workspace; this impl was never invoked Kept (live): - `start_timestamp`, `end_timestamp`, `key`, `aggregation_id` - `new()` + `get_freshness_debug_string()` Kept and rewrote doc-comment for honesty: - `origin: Origin` + `Origin { Native, Backfilled { job_id } }` + `new_backfilled()`. Backfill is a real subsystem under `sketch_db/backfill/`; the field is the provenance hook for it. Read-side consumers (HTTP listing, coverage UI, audit logs) belong to backfill scope and don't exist yet — doc-comment now states that honestly rather than promising "Phase 5f-b consults this tag" in present tense. **Arroyo streaming-engine path deleted (workspace-wide):** Background: `StreamingEngine::Arroyo` was the alternate ingest mode where a Kafka consumer parsed Arroyo-format JSON (gzip-compressed sketch bytes + JSON metadata). It was never selected in any production deployment — `--streaming-engine=precompute` is the only selection, and grep confirmed zero `StreamingEngine::Arroyo` use sites outside the enum definition itself. Removed: - `data_plane/src/drivers/ingest/kafka.rs` — the Kafka consumer (its only purpose was Arroyo JSON parsing). ~400 LOC. - `KafkaConsumer*` re-exports in `drivers/ingest/mod.rs`, `drivers/mod.rs`, and `lib.rs` - `precomputed_output.rs::deserialize_from_json_arroyo` + `create_precompute_from_bytes` factory (only called from kafka.rs and from the three deleted worker.rs Arroyo tests below) - Three `test_arroyosketch_*` tests in `precompute_engine/worker.rs` - `StreamingEngine` enum (only variants were Arroyo, Precompute — with Arroyo gone the enum was a one-variant tautology) and `InputFormat` enum (only consumed by the Kafka consumer) - `--streaming-engine`, `--kafka-topic`, `--kafka-broker`, `--input-format`, `--decompress-json` CLI flags from `main.rs` - The `kafka_handle` setup + shutdown in `main.rs`; the `if enable_precompute { ... } else { ... }` branch collapsed to the precompute-always-enabled body - `rdkafka = "0.34"` dep from `data_plane/Cargo.toml` Preserved (these are not Arroyo-the-streaming-engine): - `*_arroyo` suffixed methods on accumulators (`SumAccumulator::deserialize_from_bytes_arroyo` etc.) — the "Arroyo" name there refers to the historical sketch byte format, which is the production sketch serialization used by warm_tier query path + the persistence read-back. Renaming those is a separate concern. **Tests:** - data_plane lib: 789 passed / 2 pre-existing failures / 4 ignored (down from 792 due to the 3 deleted `test_arroyosketch_*` tests) - controller lib: 710/710; bins: 27/27 Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 113 ---- data_plane/Cargo.toml | 1 - data_plane/src/drivers/ingest/kafka.rs | 340 ---------- data_plane/src/drivers/ingest/mod.rs | 2 - data_plane/src/drivers/mod.rs | 2 +- data_plane/src/lib.rs | 5 +- data_plane/src/main.rs | 101 +-- data_plane/src/precompute_engine/worker.rs | 279 -------- data_plane/src/stores/types/enums.rs | 12 - .../src/stores/types/precomputed_output.rs | 624 +----------------- 10 files changed, 33 insertions(+), 1446 deletions(-) delete mode 100644 data_plane/src/drivers/ingest/kafka.rs diff --git a/Cargo.lock b/Cargo.lock index 65e4e3ce..c51332a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1071,7 +1071,6 @@ dependencies = [ "promql-parser 0.5.1", "promql_utilities", "prost", - "rdkafka", "regex", "reqwest 0.11.27", "rmp-serde", @@ -2059,18 +2058,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "libz-sys" -version = "1.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc3a226e576f50782b3305c5ccf458698f92798987f551c6a02efe8276721e22" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -2368,28 +2355,6 @@ dependencies = [ "libm", ] -[[package]] -name = "num_enum" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" -dependencies = [ - "num_enum_derive", - "rustversion", -] - -[[package]] -name = "num_enum_derive" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "num_threads" version = "0.1.7" @@ -2610,15 +2575,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "proc-macro-crate" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" -dependencies = [ - "toml_edit", -] - [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3030,36 +2986,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "rdkafka" -version = "0.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053adfa02fab06e86c01d586cc68aa47ee0ff4489a59469081dc12cbcde578bf" -dependencies = [ - "futures-channel", - "futures-util", - "libc", - "log", - "rdkafka-sys", - "serde", - "serde_derive", - "serde_json", - "slab", - "tokio", -] - -[[package]] -name = "rdkafka-sys" -version = "4.10.0+2.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e234cf318915c1059d4921ef7f75616b5219b10b46e9f3a511a15eb4b56a3f77" -dependencies = [ - "libc", - "libz-sys", - "num_enum", - "pkg-config", -] - [[package]] name = "recursive" version = "0.1.1" @@ -4065,36 +3991,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml_datetime" -version = "1.1.1+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" -dependencies = [ - "serde_core", -] - -[[package]] -name = "toml_edit" -version = "0.25.11+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" -dependencies = [ - "indexmap 2.14.0", - "toml_datetime", - "toml_parser", - "winnow", -] - -[[package]] -name = "toml_parser" -version = "1.1.2+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" -dependencies = [ - "winnow", -] - [[package]] name = "tonic" version = "0.12.3" @@ -4980,15 +4876,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winnow" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" -dependencies = [ - "memchr", -] - [[package]] name = "winreg" version = "0.50.0" diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index d6e03734..ea7e7aea 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -32,7 +32,6 @@ arc-swap.workspace = true form_urlencoded = "1.2" rmp-serde = "1.1" axum = "0.7" -rdkafka = "0.34" rusqlite = { version = "0.31", features = ["bundled"] } bincode = "1.3" dashmap = "5.5" diff --git a/data_plane/src/drivers/ingest/kafka.rs b/data_plane/src/drivers/ingest/kafka.rs deleted file mode 100644 index 259695d9..00000000 --- a/data_plane/src/drivers/ingest/kafka.rs +++ /dev/null @@ -1,340 +0,0 @@ -use rdkafka::config::ClientConfig; -use rdkafka::consumer::{Consumer, StreamConsumer}; -use rdkafka::Message; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use tracing::{debug, error, info, warn}; - -use crate::stores::types::enums::{InputFormat, StreamingEngine}; -use crate::stores::types::traits::SerializableToSink; -use crate::stores::types::PrecomputedOutput; -use crate::stores::types::StreamingConfig; -use crate::stores::Store; -use crate::utils::PrecomputeDumper; - -#[derive(Debug, Clone)] -pub struct KafkaConsumerConfig { - pub broker: String, - pub topic: String, - pub group_id: String, - pub auto_offset_reset: String, - pub input_format: InputFormat, - pub decompress_json: bool, - pub batch_size: usize, - pub poll_timeout_ms: u64, - pub streaming_engine: StreamingEngine, - pub dump_precomputes: bool, - pub dump_output_dir: Option, -} - -pub struct KafkaConsumer { - config: KafkaConsumerConfig, - store: Arc, - consumer: StreamConsumer, - streaming_config: Arc, - previous_consume_time: Option, - precompute_dumper: Option, -} - -impl KafkaConsumer { - pub fn new( - config: KafkaConsumerConfig, - store: Arc, - streaming_config: Arc, - ) -> Result> { - let consumer: StreamConsumer = ClientConfig::new() - .set("bootstrap.servers", &config.broker) - .set("group.id", &config.group_id) - .set("auto.offset.reset", &config.auto_offset_reset) - .set("enable.partition.eof", "false") - .set("session.timeout.ms", "6000") - .set("enable.auto.commit", "true") - .create()?; - - // Subscribe to the topic - consumer.subscribe(&[&config.topic])?; - - // Initialize precompute dumper if enabled - let precompute_dumper = if config.dump_precomputes { - match &config.dump_output_dir { - Some(output_dir) => match PrecomputeDumper::new(output_dir) { - Ok(dumper) => { - info!("Precompute dumping enabled to: {}", dumper.get_file_path()); - Some(dumper) - } - Err(e) => { - error!("Failed to create precompute dumper: {}", e); - info!("Continuing without precompute dumping"); - None - } - }, - None => { - warn!("Precompute dumping requested but no output directory provided"); - None - } - } - } else { - None - }; - - Ok(Self { - config, - store, - consumer, - streaming_config, - previous_consume_time: None, - precompute_dumper, - }) - } - - pub async fn run(&mut self) -> Result<(), Box> { - info!( - "Starting Kafka consumer for topic: {} on broker: {}", - self.config.topic, self.config.broker - ); - - let mut batch = Vec::new(); - - loop { - // Collect messages into batches like Python implementation - let timeout_duration = Duration::from_millis(self.config.poll_timeout_ms); - - // StreamConsumer uses recv() for async message reception - match tokio::time::timeout(timeout_duration, self.consumer.recv()).await { - Ok(Ok(message)) => { - // Add timing debug similar to Python - let current_consume_time = Instant::now(); - if let Some(previous_time) = self.previous_consume_time { - let elapsed = current_consume_time.duration_since(previous_time); - debug!( - "Time since last consume: {:.2} seconds", - elapsed.as_secs_f64() - ); - } - self.previous_consume_time = Some(current_consume_time); - // Process single message and add to batch - match self.process_message(&message) { - Ok(Some((precomputed_output, precompute_accumulator))) => { - // Check if this is an empty DeltaSetAggregator and skip it - if let Some(delta_acc) = precompute_accumulator - .as_any() - .downcast_ref::() - { - if delta_acc.is_empty() { - debug!("Skipping empty DeltaSetAggregatorAccumulator"); - continue; - } - } - - // Dump precompute if enabled - if let Some(ref mut dumper) = self.precompute_dumper { - if let Err(e) = dumper.dump_precompute( - &precomputed_output, - precompute_accumulator.as_ref(), - ) { - error!("Failed to dump precompute: {}", e); - } - } - - // Store both the metadata and the real accumulator data - batch.push((precomputed_output, precompute_accumulator)); - } - Ok(None) => { - debug!("Message processed but no precomputed output produced"); - } - Err(e) => { - error!("Error processing message: {e}"); - continue; // Skip this message and continue - } - } - - // Process batch when we reach batch_size or periodically - if batch.len() >= self.config.batch_size { - self.process_batch(&mut batch).await?; - } - } - Ok(Err(kafka_err)) => { - if kafka_err.rdkafka_error_code() - == Some(rdkafka::types::RDKafkaErrorCode::PartitionEOF) - { - debug!("Reached end of partition"); - continue; - } else { - error!("Kafka error: {kafka_err}"); - return Err(Box::new(kafka_err)); - } - } - Err(_) => { - // Timeout occurred - process any accumulated batch - if !batch.is_empty() { - debug!( - "Poll timeout, processing accumulated batch of {} items", - batch.len() - ); - self.process_batch(&mut batch).await?; - } else { - debug!("Poll timeout, no messages to process"); - } - } - } - } - } - - async fn process_batch( - &self, - batch: &mut Vec<(PrecomputedOutput, Box)>, - ) -> Result<(), Box> { - if batch.is_empty() { - return Ok(()); - } - - let batch_start_time = Instant::now(); - debug!("Processing batch of {} messages", batch.len()); - - // Batch insert with real precompute data like Python implementation - let store_insert_start_time = Instant::now(); - match self.store.insert_precomputed_output_batch(batch.to_vec()) { - Ok(_) => { - let store_insert_duration = store_insert_start_time.elapsed(); - debug!( - "Store batch insert took: {:.2}ms", - store_insert_duration.as_secs_f64() * 1000.0 - ); - debug!("{}", batch[0].0.get_freshness_debug_string()); - for (item, _) in batch.iter() { - debug!( - "Received message: {} with aggregation_id: {}", - serde_json::to_string(&item.serialize_to_json()) - .unwrap_or_else(|_| "failed to serialize".to_string()), - item.aggregation_id - ); - } - } - Err(e) => { - error!("Error inserting precomputed output batch: {}", e); - return Err(e); - } - } - - batch.clear(); - let total_batch_duration = batch_start_time.elapsed(); - debug!( - "Total batch processing took: {:.2}ms", - total_batch_duration.as_secs_f64() * 1000.0 - ); - Ok(()) - } - - #[allow(clippy::type_complexity)] - fn process_message( - &self, - message: &rdkafka::message::BorrowedMessage<'_>, - ) -> Result< - Option<(PrecomputedOutput, Box)>, - Box, - > { - let message_start_time = Instant::now(); - let payload = match message.payload() { - Some(payload) => payload, - None => { - warn!("Received message with no payload"); - return Ok(None); - } - }; - - match self.config.input_format { - InputFormat::Byte => { - // For binary format, we need to first extract metadata to get aggregation_type - // Then use it to create the proper accumulator - // let (metadata, _precompute_bytes) = - // match PrecomputedOutput::deserialize_from_bytes_with_precompute(payload) { - // Ok(result) => result, - // Err(e) => { - // error!("Error deserializing binary message metadata: {}", e); - // return Err(format!("Binary deserialization error: {e}").into()); - // } - // }; - - // // Now deserialize with the correct accumulator type - // match PrecomputedOutput::deserialize_from_bytes_with_precompute_and_type( - // payload, - // &metadata.config.aggregation_type, - // ) { - // Ok((output, precompute)) => { - // debug!("Successfully deserialized binary message with precompute data"); - // Ok(Some((output, precompute))) - // } - // Err(e) => { - // error!("Error deserializing binary message with precompute: {}", e); - // Err(e) - // } - // } - error!("Binary input format with precompute not implemented"); - Err("Binary input format with precompute not implemented".into()) - } - InputFormat::Json => { - // Arroyo messages - gzip decompression is applied at precompute level, not message level - let json_str = match String::from_utf8(payload.to_vec()) { - Ok(s) => s, - Err(e) => { - error!("Error converting bytes to UTF-8: {}", e); - return Err(format!("UTF-8 conversion error: {e}").into()); - } - }; - - let json_dict: serde_json::Value = match serde_json::from_str(&json_str) { - Ok(dict) => dict, - Err(e) => { - error!("Error parsing Arroyo JSON: {}", e); - debug!("JSON content: {}", json_str); - return Err(format!("JSON parsing error: {e}").into()); - } - }; - - let deserialize_start_time = Instant::now(); - match PrecomputedOutput::deserialize_from_json_arroyo( - &json_dict, - &self.streaming_config, - ) { - Ok((output, precompute)) => { - let deserialize_duration = deserialize_start_time.elapsed(); - debug!( - "Arroyo deserialization took: {:.2}ms", - deserialize_duration.as_secs_f64() * 1000.0 - ); - debug!( - "Successfully deserialized Arroyo JSON message with precompute data" - ); - let total_message_duration = message_start_time.elapsed(); - debug!( - "Total Arroyo message processing took: {:.2}ms", - total_message_duration.as_secs_f64() * 1000.0 - ); - Ok(Some((output, precompute))) - } - Err(e) => { - error!( - "Error deserializing Arroyo PrecomputedOutput from JSON with precompute: {e}" - ); - debug!("JSON content: {}", json_str); - Err(e) - } - } - } - } - } - - pub async fn stop(&mut self) -> Result<(), Box> { - info!("Stopping Kafka consumer"); - - // Flush precompute dumper if it exists - if let Some(ref mut dumper) = self.precompute_dumper { - if let Err(e) = dumper.flush() { - error!("Failed to flush precompute dumper on stop: {}", e); - } - } - - // The consumer will be dropped automatically - Ok(()) - } -} diff --git a/data_plane/src/drivers/ingest/mod.rs b/data_plane/src/drivers/ingest/mod.rs index c7ae11f9..21ca4d43 100644 --- a/data_plane/src/drivers/ingest/mod.rs +++ b/data_plane/src/drivers/ingest/mod.rs @@ -1,7 +1,5 @@ -pub mod kafka; pub mod otel; pub mod series_resolver; -pub use kafka::{KafkaConsumer, KafkaConsumerConfig}; pub use otel::{OtlpReceiver, OtlpReceiverConfig}; pub use series_resolver::{canonical_attrs_fingerprint, SeriesIdResolver}; diff --git a/data_plane/src/drivers/mod.rs b/data_plane/src/drivers/mod.rs index b58936d4..aed76b7a 100644 --- a/data_plane/src/drivers/mod.rs +++ b/data_plane/src/drivers/mod.rs @@ -4,5 +4,5 @@ pub mod query; // Re-export commonly used types for convenience pub use controller_client::{spawn_capability_miss_notify, ControllerClient, HttpControllerClient}; -pub use ingest::{KafkaConsumer, KafkaConsumerConfig, OtlpReceiver, OtlpReceiverConfig}; +pub use ingest::{OtlpReceiver, OtlpReceiverConfig}; pub use query::{AdapterConfig, HttpServer, HttpServerConfig}; diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index 4c19257f..6af89d0d 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -23,10 +23,7 @@ pub use stores::{SketchStore, Store, StoreResult}; pub use query_engines::{ASAPQueryEngine, InstantVector, QueryResult}; -pub use drivers::{ - HttpServer, HttpServerConfig, KafkaConsumer, KafkaConsumerConfig, OtlpReceiver, - OtlpReceiverConfig, -}; +pub use drivers::{HttpServer, HttpServerConfig, OtlpReceiver, OtlpReceiverConfig}; pub use precompute_engine::config::{LateDataPolicy, PrecomputeEngineConfig}; pub use precompute_engine::output_sink::StoreOutputSink; diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index af4ad4c2..04e4e90d 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -17,31 +17,20 @@ use std::sync::Arc; use tokio::signal; use tracing::{error, info, warn}; -use data_plane::stores::types::enums::{ - CleanupPolicy, InputFormat, LockStrategy, StreamingEngine, -}; +use data_plane::stores::types::enums::{CleanupPolicy, LockStrategy}; use data_plane::stores::types::InferenceConfig; use data_plane::drivers::AdapterConfig; use data_plane::precompute_engine::config::LateDataPolicy; use data_plane::precompute_engine::PrecomputeWorkerDiagnostics; use data_plane::utils::file_io::{read_inference_config, read_streaming_config}; use data_plane::{ - HttpServer, HttpServerConfig, KafkaConsumer, KafkaConsumerConfig, OtlpReceiver, - OtlpReceiverConfig, PrecomputeEngine, PrecomputeEngineConfig, Result, ASAPQueryEngine, - SketchStore, StoreOutputSink, + HttpServer, HttpServerConfig, OtlpReceiver, OtlpReceiverConfig, PrecomputeEngine, + PrecomputeEngineConfig, Result, ASAPQueryEngine, SketchStore, StoreOutputSink, }; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] struct Args { - /// Kafka topic to consume from (required when streaming-engine=arroyo) - #[arg(long)] - kafka_topic: Option, - - /// Input format for Kafka messages (required when streaming-engine=arroyo) - #[arg(long, value_enum)] - input_format: Option, - /// Path to the inference config YAML — maps query patterns to /// aggregation IDs so the query engine can pick the right /// stored sketch for an incoming PromQL. Both spellings are @@ -57,12 +46,6 @@ struct Args { #[arg(long)] streaming_config: String, - /// Streaming engine to use. Default `precompute` matches the - /// formerly-deployed `precompute_engine` binary's behavior; - /// override to `arroyo` (Kafka consumer) only when needed. - #[arg(long, value_enum, default_value = "precompute")] - streaming_engine: StreamingEngine, - /// Prometheus scrape interval (seconds). Default 30 matches /// the e2e harness's 30s window. ASAPQueryEngine uses this as the /// instant-query lookback window — for tumbling-window @@ -108,10 +91,6 @@ struct Args { #[arg(long)] forward_unsupported_queries: bool, - /// Kafka broker address - #[arg(long, default_value = "localhost:9092")] - kafka_broker: String, - /// Database path (currently unused, kept for compatibility) #[arg(long, default_value = "sketchdb.db")] db_path: String, @@ -132,10 +111,6 @@ struct Args { #[arg(long)] do_profiling: bool, - /// Decompress JSON messages - #[arg(long)] - decompress_json: bool, - /// Enable dumping received precomputes to files for debugging #[arg(long)] dump_precomputes: bool, @@ -490,59 +465,7 @@ async fn main() -> Result<()> { engine }; - // Setup Kafka consumer (only when not using precompute engine as the streaming backend) - let kafka_handle = if args.streaming_engine == StreamingEngine::Precompute { - info!("Using precompute engine as streaming backend — skipping Kafka consumer"); - None - } else { - let kafka_topic = args.kafka_topic.clone().unwrap_or_else(|| { - error!("--kafka-topic is required when --streaming-engine is not precompute"); - std::process::exit(1); - }); - let input_format = args.input_format.unwrap_or_else(|| { - error!("--input-format is required when --streaming-engine is not precompute"); - std::process::exit(1); - }); - let kafka_config = KafkaConsumerConfig { - broker: args.kafka_broker.clone(), - topic: kafka_topic.clone(), - group_id: "query-engine-rust".to_string(), - auto_offset_reset: "beginning".to_string(), - input_format, - decompress_json: args.decompress_json, - batch_size: 1000, - poll_timeout_ms: 1000, - streaming_engine: args.streaming_engine.clone(), - dump_precomputes: args.dump_precomputes, - dump_output_dir: if args.dump_precomputes { - Some(args.output_dir.clone()) - } else { - None - }, - }; - - let store_for_kafka = store.clone(); - let kafka_consumer_result = - KafkaConsumer::new(kafka_config, store_for_kafka, streaming_config.clone()); - match kafka_consumer_result { - Ok(mut consumer) => { - info!("Starting Kafka consumer for topic: {}", kafka_topic); - Some(tokio::spawn(async move { - if let Err(e) = consumer.run().await { - error!("Kafka consumer error: {}", e); - } - })) - } - Err(e) => { - error!("Failed to create Kafka consumer: {}", e); - info!("Continuing without Kafka consumer"); - None - } - } - }; - - // Setup precompute engine. Automatically enabled when the configured - // streaming engine is Precompute. Backend ingest is OTLP-only — the + // Setup precompute engine. Backend ingest is OTLP-only — the // precompute engine no longer hosts an HTTP listener of its own; the // OTLP receiver below pushes envelopes / raw points into the worker // pool via the `IngestState` handle returned by `engine.ingest_state()`. @@ -550,8 +473,7 @@ async fn main() -> Result<()> { // NOTE: precompute is constructed BEFORE the OTLP receiver so the receiver // can obtain an `Arc` handle and push OTLP metrics / sketches // into the same worker pool (and not just write directly to the store). - let enable_precompute = args.streaming_engine == StreamingEngine::Precompute; - let (precompute_handle, precompute_ingest_state) = if enable_precompute { + let (precompute_handle, precompute_ingest_state) = { let precompute_config = PrecomputeEngineConfig { num_workers: args.precompute_num_workers, allowed_lateness_ms: args.precompute_allowed_lateness_ms, @@ -588,13 +510,6 @@ async fn main() -> Result<()> { } }); (Some(handle), Some(ingest_state)) - } else { - // Even without precompute, log store diagnostics - let diag_store = store.clone(); - tokio::spawn(async move { - spawn_memory_diagnostics(diag_store, None).await; - }); - (None, None) }; // Hand the precompute engine's `SchemaRegistry` to the query @@ -947,12 +862,6 @@ async fn main() -> Result<()> { handle.shutdown().await; } - if let Some(handle) = kafka_handle { - info!("Shutting down Kafka consumer..."); - handle.abort(); - let _ = handle.await; - } - if let Some(handle) = otel_handle { info!("Shutting down OTLP receiver..."); handle.abort(); diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 5a59290b..333056f2 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -1619,285 +1619,6 @@ mod tests { assert!(found_b, "expected key B inside accumulator"); } - // ----------------------------------------------------------------------- - // Test: Arroyo KLL equivalence — same output as Arroyo pipeline - // ----------------------------------------------------------------------- - #[test] - fn test_arroyosketch_multiple_sum_matches_handcrafted_precompute_output() { - let config = make_agg_config( - 11, - "cpu", - AggregationType::MultipleSum, - "sum", - 10, - 0, - vec!["host"], - ); - let mut agg_configs = HashMap::new(); - agg_configs.insert(11, config.clone()); - - let sink = Arc::new(CapturingOutputSink::new()); - let mut worker = make_worker( - agg_configs.clone(), - sink.clone(), - false, - 0, - LateDataPolicy::Drop, - ); - - worker - .process_group_samples( - 11, - "A", - group_samples("cpu{host=\"A\"}", vec![(1_000_i64, 1.0)]), - ) - .unwrap(); - worker - .process_group_samples( - 11, - "A", - group_samples("cpu{host=\"A\"}", vec![(5_000_i64, 2.0)]), - ) - .unwrap(); - worker - .process_group_samples( - 11, - "A", - group_samples("cpu{host=\"A\"}", vec![(9_000_i64, 3.0)]), - ) - .unwrap(); - worker - .process_group_samples( - 11, - "A", - group_samples("cpu{host=\"A\"}", vec![(10_000_i64, 0.0)]), - ) - .unwrap(); - - let captured = sink.drain(); - assert_eq!(captured.len(), 1, "expected one closed window output"); - - let (handcrafted_output, handcrafted_acc) = &captured[0]; - let handcrafted_acc = handcrafted_acc - .as_any() - .downcast_ref::() - .expect("hand-crafted engine should emit MultipleSumAccumulator"); - - // grouping=["host"] means the host value goes in the outer key ("A"), - // and aggregated=[] means the accumulator sub-key has no labels. - assert_eq!(handcrafted_output.aggregation_id, 11); - assert_eq!(handcrafted_output.start_timestamp, 0); - assert_eq!(handcrafted_output.end_timestamp, 10_000); - assert_eq!( - handcrafted_output.key, - Some(KeyByLabelValues::new_with_labels(vec!["A".to_string()])) - ); - - let mut expected_sums = HashMap::new(); - expected_sums.insert(KeyByLabelValues::new_with_labels(vec![]), 6.0); - assert_eq!(handcrafted_acc.sums, expected_sums); - } - - #[test] - fn test_arroyosketch_kll_matches_handcrafted_precompute_output() { - let mut config = make_agg_config( - 12, - "latency", - AggregationType::DatasketchesKLL, - "", - 10, - 0, - vec![], - ); - config - .parameters - .insert("K".to_string(), serde_json::Value::from(20_u64)); - - let mut agg_configs = HashMap::new(); - agg_configs.insert(12, config); - - let sink = Arc::new(CapturingOutputSink::new()); - let mut worker = make_worker( - agg_configs.clone(), - sink.clone(), - false, - 0, - LateDataPolicy::Drop, - ); - - let samples = vec![(1_000_i64, 10.0), (5_000_i64, 20.0), (9_000_i64, 30.0)]; - for &(ts, value) in &samples { - worker - .process_group_samples(12, "", group_samples("latency", vec![(ts, value)])) - .unwrap(); - } - worker - .process_group_samples(12, "", group_samples("latency", vec![(10_000, 0.0)])) - .unwrap(); - - let captured = sink.drain(); - assert_eq!(captured.len(), 1, "expected one closed window output"); - - let (handcrafted_output, handcrafted_acc) = &captured[0]; - let handcrafted_acc = handcrafted_acc - .as_any() - .downcast_ref::() - .expect("hand-crafted engine should emit DatasketchesKLLAccumulator"); - - let arroyo_precompute_bytes = KllSketch::aggregate_kll(20, &[10.0, 20.0, 30.0]) - .expect("Arroyo KLL aggregation should produce bytes"); - - let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); - encoder - .write_all(&arroyo_precompute_bytes) - .expect("gzip encoding should succeed"); - let arroyo_json = json!({ - "aggregation_id": 12, - "window": { - "start": "1970-01-01T00:00:00", - "end": "1970-01-01T00:00:10" - }, - "key": "", - "precompute": hex::encode(encoder.finish().expect("gzip finalize should succeed")) - }); - - let streaming_config = StreamingConfig::new(agg_configs); - let (arroyo_output, arroyo_acc) = - PrecomputedOutput::deserialize_from_json_arroyo(&arroyo_json, &streaming_config) - .expect("Arroyo KLL precompute should deserialize"); - let arroyo_acc = arroyo_acc - .as_any() - .downcast_ref::() - .expect("Arroyo payload should deserialize to DatasketchesKLLAccumulator"); - - assert_eq!( - handcrafted_output.aggregation_id, - arroyo_output.aggregation_id - ); - assert_eq!( - handcrafted_output.start_timestamp, - arroyo_output.start_timestamp - ); - assert_eq!( - handcrafted_output.end_timestamp, - arroyo_output.end_timestamp - ); - assert_eq!(handcrafted_acc.inner.k, arroyo_acc.inner.k); - assert_eq!(handcrafted_acc.inner.count(), arroyo_acc.inner.count()); - - for quantile in [0.0, 0.5, 1.0] { - assert_eq!( - handcrafted_acc.get_quantile(quantile), - arroyo_acc.get_quantile(quantile) - ); - } - } - - // ----------------------------------------------------------------------- - // Test: Arroyo MultipleSum equivalence - // ----------------------------------------------------------------------- - - #[test] - fn test_arroyosketch_multiple_sum_empty_grouping_matches_handcrafted_precompute_output() { - // Like planner output: grouping=[], aggregated=[host] - let config = make_agg_config_full( - 11, - "cpu", - AggregationType::MultipleSum, - "sum", - 10, - 0, - vec![], - vec!["host"], - ); - let mut agg_configs = HashMap::new(); - agg_configs.insert(11, config.clone()); - - let sink = Arc::new(CapturingOutputSink::new()); - let mut worker = make_worker( - agg_configs.clone(), - sink.clone(), - false, - 0, - LateDataPolicy::Drop, - ); - - // All samples go to group "" (empty group key since grouping=[]). - // The host label is the aggregated key inside the accumulator. - worker - .process_group_samples(11, "", group_samples("cpu{host=\"A\"}", vec![(1_000, 1.0)])) - .unwrap(); - worker - .process_group_samples(11, "", group_samples("cpu{host=\"A\"}", vec![(5_000, 2.0)])) - .unwrap(); - worker - .process_group_samples(11, "", group_samples("cpu{host=\"A\"}", vec![(9_000, 3.0)])) - .unwrap(); - worker - .process_group_samples( - 11, - "", - group_samples("cpu{host=\"A\"}", vec![(10_000, 0.0)]), - ) - .unwrap(); - - let captured = sink.drain(); - assert_eq!(captured.len(), 1, "expected one closed window output"); - - let (handcrafted_output, handcrafted_acc) = &captured[0]; - let handcrafted_acc = handcrafted_acc - .as_any() - .downcast_ref::() - .expect("hand-crafted engine should emit MultipleSumAccumulator"); - - // Arroyo: GROUP BY '' (empty key), UDF gets host="A" as aggregated key - let mut arroyo_sums = HashMap::new(); - arroyo_sums.insert("A".to_string(), 6.0); - let arroyo_precompute_bytes = - rmp_serde::to_vec(&arroyo_sums).expect("Arroyo MessagePack encoding should succeed"); - - let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); - encoder - .write_all(&arroyo_precompute_bytes) - .expect("gzip encoding should succeed"); - let arroyo_json = json!({ - "aggregation_id": 11, - "window": { - "start": "1970-01-01T00:00:00", - "end": "1970-01-01T00:00:10" - }, - "key": "", - "precompute": hex::encode(encoder.finish().expect("gzip finalize should succeed")) - }); - - let streaming_config = StreamingConfig::new(agg_configs); - let (arroyo_output, arroyo_acc) = - PrecomputedOutput::deserialize_from_json_arroyo(&arroyo_json, &streaming_config) - .expect("Arroyo precompute should deserialize"); - let arroyo_acc = arroyo_acc - .as_any() - .downcast_ref::() - .expect("Arroyo payload should deserialize to MultipleSumAccumulator"); - - assert_eq!( - handcrafted_output.aggregation_id, - arroyo_output.aggregation_id - ); - assert_eq!( - handcrafted_output.start_timestamp, - arroyo_output.start_timestamp - ); - assert_eq!( - handcrafted_output.end_timestamp, - arroyo_output.end_timestamp - ); - assert_eq!(handcrafted_output.key, arroyo_output.key); - assert_eq!(handcrafted_acc.sums, arroyo_acc.sums); - } - - // ----------------------------------------------------------------------- - // Test: late data drop - // ----------------------------------------------------------------------- #[test] fn test_late_data_drop() { diff --git a/data_plane/src/stores/types/enums.rs b/data_plane/src/stores/types/enums.rs index b3440bc3..224e2a55 100644 --- a/data_plane/src/stores/types/enums.rs +++ b/data_plane/src/stores/types/enums.rs @@ -1,15 +1,3 @@ -#[derive(clap::ValueEnum, Clone, Debug)] -pub enum InputFormat { - Json, - Byte, -} - -#[derive(clap::ValueEnum, Clone, Debug, PartialEq)] -pub enum StreamingEngine { - Arroyo, - Precompute, -} - pub use asap_types::enums::{CleanupPolicy, QueryLanguage, WindowType}; pub use promql_utilities::query_logics::enums::AggregationType; diff --git a/data_plane/src/stores/types/precomputed_output.rs b/data_plane/src/stores/types/precomputed_output.rs index f6242def..1c9d518c 100644 --- a/data_plane/src/stores/types/precomputed_output.rs +++ b/data_plane/src/stores/types/precomputed_output.rs @@ -1,26 +1,26 @@ -use chrono::DateTime; -use flate2::read::GzDecoder; use serde::{Deserialize, Serialize}; -use std::io::Read as _; -use tracing::error; -use crate::stores::types::traits::SerializableToSink; -use crate::stores::types::{AggregationType, KeyByLabelValues, StreamingConfig}; +use crate::stores::types::KeyByLabelValues; -/// §5.1 provenance tag on every precompute record: did this window -/// come from live ingest or was it materialised by a backfill job? +/// Provenance tag stamped on every `PrecomputedOutput`: did this +/// window come from live ingest or was it materialised by a backfill +/// job? /// -/// `Native` is the default (pre-existing records on disk deserialise -/// to `Native` via `#[serde(default)]`), so the tag is -/// forward-compatible with older on-disk formats. +/// `Native` is the default (records without the field deserialise as +/// `Native` via `#[serde(default)]`), so the tag is forward-compatible +/// with older on-disk formats. /// -/// Phase 5f-b / eviction / audit logs consult this tag to distinguish -/// live vs backfilled windows at read / cleanup time without having -/// to join against `BackfillRegistry::windows_written_by`. +/// Read-side consumers (HTTP listing of backfilled windows, coverage +/// UI, audit logs) belong to the backfill subsystem under +/// [`crate::stores::sketch_db::backfill`]; today this field is +/// written but no production read site branches on it yet. Recording +/// it eagerly means a window written before the read-side consumer +/// lands still carries its `job_id` — readers can recover history, +/// not just future. #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] pub enum Origin { - /// Emitted by the live ingest pipeline (PrecomputeEngine workers - /// closing a window). The common case. + /// Emitted by the live ingest pipeline (the + /// `PrecomputeEngine` workers closing a window). The common case. #[default] Native, /// Materialised by a backfill job. The tag carries the `job_id` @@ -35,26 +35,21 @@ pub struct PrecomputedOutput { pub end_timestamp: u64, pub key: Option, pub aggregation_id: u64, - /// §5.1 provenance tag. `#[serde(default)]` on read means old - /// records without the field deserialise as `Native`, preserving - /// behaviour for anything persisted before Phase 5.1. + /// Provenance tag. `#[serde(default)]` on read means records + /// without the field deserialise as `Native`, preserving + /// forward-compat with older on-disk payloads. #[serde(default)] pub origin: Origin, - // pub config: AggregationConfig, - // Note: precompute will be handled separately as it's a trait object } impl PrecomputedOutput { /// Construct a `Native` precompute — the default used by the - /// live ingest pipeline. Existing callers keep their signature - /// untouched; `origin` defaults to `Origin::Native`. + /// live ingest pipeline. pub fn new( start_timestamp: u64, end_timestamp: u64, key: Option, aggregation_id: u64, - // TODO: we should remove AggregationConfig from here. Configs should only be accessed from the StreamingConfig read in main.rs - // config: AggregationConfig, ) -> Self { Self { start_timestamp, @@ -62,14 +57,13 @@ impl PrecomputedOutput { key, aggregation_id, origin: Origin::Native, - // config, } } /// Construct a `Backfilled { job_id }` precompute. Called by - /// the Phase 5e `BackfillWindowProcessor` so each backfilled - /// window carries its provenance back to the originating - /// `BackfillJob`. + /// [`crate::stores::sketch_db::backfill::processor::BackfillWindowProcessor`] + /// so each backfilled window carries its provenance back to the + /// originating `BackfillJob`. pub fn new_backfilled( start_timestamp: u64, end_timestamp: u64, @@ -87,7 +81,6 @@ impl PrecomputedOutput { } pub fn get_freshness_debug_string(&self) -> String { - // Match Python implementation more closely let current_time = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() @@ -98,574 +91,9 @@ impl PrecomputedOutput { self.end_timestamp, current_time, freshness ) } - - // /// Serialize PrecomputedOutput with precompute data to match Python JSON format - // pub fn serialize_to_json_with_precompute( - // &self, - // precompute: &dyn crate::stores::types::AggregateCore, - // ) -> serde_json::Value { - // serde_json::json!({ - // // "config": self.config.serialize_to_json(), - // "start_timestamp": self.start_timestamp, - // "end_timestamp": self.end_timestamp, - // "key": self.key.as_ref().map(|k| k.serialize_to_json()), - // "precompute": precompute.serialize_to_json() - // }) - // } - - /// Deserialize from bytes using Python-compatible format - pub fn deserialize_from_bytes_with_precompute( - _data: &[u8], - ) -> Result<(Self, Vec), Box> { - error!("Not implemented: deserialize_from_bytes_with_precompute"); - Err(("Not implemented: deserialize_from_bytes_with_precompute").into()) - } - - // /// Simple deserialization from bytes (compatibility method for Kafka consumer) - // /// This doesn't include precompute data and is primarily for compatibility - // pub fn deserialize_from_bytes( - // data: &[u8], - // ) -> Result> { - // // Try to deserialize as JSON first (common case) - // if let Ok(json_str) = String::from_utf8(data.to_vec()) { - // if let Ok(json_value) = serde_json::from_str::(&json_str) { - // return Self::deserialize_from_json(&json_value); - // } - // } - - // // If JSON fails, try binary format - // let (output, _precompute_bytes) = Self::deserialize_from_bytes_with_precompute(data) - // .map_err(|e| -> Box { - // format!("Failed to deserialize from bytes: {e}").into() - // })?; - // Ok(output) - // } - - // /// Legacy deserialization method for backward compatibility - // pub fn deserialize_from_json( - // data: &serde_json::Value, - // ) -> Result> { - // // Extract required fields - // let config_data = data.get("config").ok_or("Missing 'config' field in JSON")?; - // // Use custom deserialization for the config - // let config = AggregationConfig::deserialize_from_json(config_data).map_err( - // |e| -> Box { - // format!("Failed to deserialize config: {e}").into() - // }, - // )?; - - // let start_timestamp = data - // .get("start_timestamp") - // .and_then(|v| v.as_u64()) - // .ok_or("Missing or invalid 'start_timestamp' field")?; - - // let end_timestamp = data - // .get("end_timestamp") - // .and_then(|v| v.as_u64()) - // .ok_or("Missing or invalid 'end_timestamp' field")?; - - // let key = if let Some(key_data) = data.get("key") { - // if key_data.is_null() { - // None - // } else { - // // Use the custom deserialize_from_json method which expects the direct HashMap format - // Some(KeyByLabelValues::deserialize_from_json(key_data).map_err( - // |e| -> Box { - // format!("Failed to deserialize key: {e}").into() - // }, - // )?) - // } - // } else { - // None - // }; - - // // For now, we create a PrecomputedOutput without precompute data - // // In a full implementation, we would deserialize the precompute field as well - // Ok(Self { - // start_timestamp, - // end_timestamp, - // key, - // config, - // }) - // } - - /// Deserialization for Arroyo streaming engine - pub fn deserialize_from_json_arroyo( - data: &serde_json::Value, - // streaming_config: &HashMap, - streaming_config: &StreamingConfig, - ) -> Result< - (Self, Box), - Box, - > { - let aggregation_id = data - .get("aggregation_id") - .and_then(|v| v.as_u64()) - .ok_or("Missing or invalid 'aggregation_id' field")?; - - // Parse window timestamps from Arroyo format - let window = data - .get("window") - .ok_or("Missing 'window' field in Arroyo data")?; - let start_str = window - .get("start") - .and_then(|v| v.as_str()) - .ok_or("Missing or invalid 'start' field in window")?; - let end_str = window - .get("end") - .and_then(|v| v.as_str()) - .ok_or("Missing or invalid 'end' field in window")?; - - // Parse timestamps with Z suffix - convert to milliseconds - let start_timestamp = (DateTime::parse_from_rfc3339(&format!("{start_str}Z")) - .map_err(|e| format!("Failed to parse start timestamp: {e}"))? - .timestamp() as u64) - * 1000; - let end_timestamp = (DateTime::parse_from_rfc3339(&format!("{end_str}Z")) - .map_err(|e| format!("Failed to parse end timestamp: {e}"))? - .timestamp() as u64) - * 1000; - - // Parse key from semicolon-separated format - always create KeyByLabelValues (even if empty) - let key_str = data.get("key").and_then(|v| v.as_str()).unwrap_or(""); - let labels: Vec = key_str.split(';').map(|s| s.to_string()).collect(); - // let key = Some(KeyByLabelValues::new_with_labels( - // labels - // .into_iter() - // .enumerate() - // .map(|(i, v)| (format!("label_{i}"), v)) - // .collect(), - // )); - let key = Some(KeyByLabelValues::new_with_labels(labels)); - - // Get aggregation type from streaming config lookup - let config = streaming_config - .get_aggregation_config(aggregation_id) - .ok_or_else(|| { - format!("Aggregation ID {aggregation_id} not found in streaming config") - })? - .clone(); - - let precomputed_output = Self { - start_timestamp, - end_timestamp, - key, - aggregation_id, - origin: Origin::Native, - }; - - // data["precompute"] has been compressed using the following logic - // fn gzip_compress(data: &[u8]) -> Option> { - // let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); - // encoder.write_all(&data).ok()?; - // encoder.finish().ok() - // } - - // Extract and decompress precompute data - // Equivalent python code: - // precompute_bytes = bytes.fromhex(data["precompute"]) - // precompute_bytes = gzip.decompress(precompute_bytes) - let precompute_hex = data - .get("precompute") - .and_then(|v| v.as_str()) - .ok_or("Missing or invalid 'precompute' field")?; - - // NOTE: Check if hex decoding is actually needed - might depend on Arroyo's JSON serialization - let compressed_bytes = hex::decode(precompute_hex) - .map_err(|e| format!("Failed to decode hex precompute data: {e}"))?; - - // Decompress gzip data - - let mut decoder = GzDecoder::new(&compressed_bytes[..]); - let mut precompute_bytes = Vec::new(); - decoder - .read_to_end(&mut precompute_bytes) - .map_err(|e| format!("Failed to decompress precompute data: {e}"))?; - - let precompute = Self::create_precompute_from_bytes( - config.aggregation_type, - Vec::as_slice(&precompute_bytes), - )?; - - Ok((precomputed_output, precompute)) - } - - // /// Deserialize from JSON and extract precompute data following Python implementation - // /// This is the public method that should be used by Kafka consumer - // pub fn deserialize_from_json_with_precompute( - // data: &serde_json::Value, - // ) -> Result< - // (Self, Box), - // Box, - // > { - // debug!("Deserializing PrecomputedOutput with precompute from JSON: {data}"); - // // First get the metadata - // let precomputed_output = Self::deserialize_from_json(data)?; - // debug!( - // "Deserialized PrecomputedOutput metadata: {:?}", - // precomputed_output - // ); - - // // Then deserialize the precompute data based on aggregation type - // let precompute_data = data - // .get("precompute") - // .ok_or("Missing 'precompute' field in JSON")?; - // let precompute = Self::create_precompute_from_json( - // &precomputed_output.config.aggregation_type, - // precompute_data, - // )?; - - // Ok((precomputed_output, precompute)) - // } - - // /// Deserialize from bytes and extract precompute data following Python implementation - // /// This is the public method that should be used by Kafka consumer - // pub fn deserialize_from_bytes_with_precompute_and_type( - // data: &[u8], - // aggregation_type: &str, - // ) -> Result< - // (Self, Box), - // Box, - // > { - // // First get the metadata and precompute bytes - // let (precomputed_output, precompute_bytes) = Self::deserialize_from_bytes_with_precompute( - // data, - // ) - // .map_err(|e| -> Box { - // format!("Failed to deserialize from bytes: {e}").into() - // })?; - - // // Then create the accumulator from the precompute bytes - // let precompute = - // Self::create_precompute_from_bytes(aggregation_type, &precompute_bytes, "flink")?; - - // Ok((precomputed_output, precompute)) - // } - - // /// Factory method to create precompute accumulator from JSON data - // fn create_precompute_from_json( - // precompute_type: &str, - // data: &serde_json::Value, - // ) -> Result, Box> - // { - // use crate::precompute_engine::operators::*; - - // match precompute_type { - // "Sum" | "sum" => { - // let accumulator = SumAccumulator::deserialize_from_json(data) - // .map_err(|e| format!("Failed to deserialize SumAccumulator: {e}"))?; - // Ok(Box::new(accumulator)) - // } - // "MinMax" => { - // let accumulator = MinMaxAccumulator::deserialize_from_json(data) - // .map_err(|e| format!("Failed to deserialize MinMaxAccumulator: {e}"))?; - // Ok(Box::new(accumulator)) - // } - // "Increase" => { - // let accumulator = IncreaseAccumulator::deserialize_from_json(data) - // .map_err(|e| format!("Failed to deserialize IncreaseAccumulator: {e}"))?; - // Ok(Box::new(accumulator)) - // } - // "MultipleSum" => { - // let accumulator = MultipleSumAccumulator::deserialize_from_json(data) - // .map_err(|e| format!("Failed to deserialize MultipleSumAccumulator: {e}"))?; - // Ok(Box::new(accumulator)) - // } - // "MultipleMinMax" => { - // // Extract sub_type from data - // let _sub_type = data - // .get("sub_type") - // .and_then(|v| v.as_str()) - // .unwrap_or("min") - // .to_string(); - // let accumulator = MultipleMinMaxAccumulator::deserialize_from_json(data) - // .map_err(|e| format!("Failed to deserialize MultipleMinMaxAccumulator: {e}"))?; - // Ok(Box::new(accumulator)) - // } - // "MultipleIncrease" => { - // let accumulator = MultipleIncreaseAccumulator::deserialize_from_json(data) - // .map_err(|e| { - // format!("Failed to deserialize MultipleIncreaseAccumulator: {e}") - // })?; - // Ok(Box::new(accumulator)) - // } - // "CountMinSketch" => { - // let accumulator = CountMinSketchAccumulator::deserialize_from_json(data) - // .map_err(|e| format!("Failed to deserialize CountMinSketchAccumulator: {e}"))?; - // Ok(Box::new(accumulator)) - // } - // "DatasketchesKLL" => { - // let accumulator = - // DatasketchesKLLAccumulator::deserialize_from_json(data).map_err(|e| { - // format!("Failed to deserialize DatasketchesKLLAccumulator: {e}") - // })?; - // Ok(Box::new(accumulator)) - // } - // "DeltaSetAggregator" => { - // let accumulator = DeltaSetAggregatorAccumulator::deserialize_from_json(data) - // .map_err(|e| { - // format!("Failed to deserialize DeltaSetAggregatorAccumulator: {e}") - // })?; - // Ok(Box::new(accumulator)) - // } - // _ => Err(format!("Unknown precompute type: {precompute_type}").into()), - // } - // } - - /// Factory method to create precompute accumulator from bytes - fn create_precompute_from_bytes( - precompute_type: AggregationType, - buffer: &[u8], - ) -> Result, Box> - { - use crate::precompute_engine::operators::*; - - match precompute_type { - AggregationType::Sum => { - let accumulator = SumAccumulator::deserialize_from_bytes_arroyo(buffer) - .map_err(|e| format!("Failed to deserialize SumAccumulator: {e}"))?; - Ok(Box::new(accumulator)) - } - AggregationType::MinMax => { - let accumulator = MinMaxAccumulator::deserialize_from_bytes(buffer) - .map_err(|e| format!("Failed to deserialize MinMaxAccumulator: {e}"))?; - Ok(Box::new(accumulator)) - } - AggregationType::Increase => { - let accumulator = IncreaseAccumulator::deserialize_from_bytes(buffer) - .map_err(|e| format!("Failed to deserialize IncreaseAccumulator: {e}"))?; - Ok(Box::new(accumulator)) - } - AggregationType::MultipleSum => { - let accumulator = MultipleSumAccumulator::deserialize_from_bytes_arroyo(buffer) - .map_err(|e| format!("Failed to deserialize MultipleSumAccumulator: {e}"))?; - Ok(Box::new(accumulator)) - } - AggregationType::MultipleMinMax => { - let accumulator = - MultipleMinMaxAccumulator::deserialize_from_bytes(buffer, "min".to_string()) - .map_err(|e| { - format!("Failed to deserialize MultipleMinMaxAccumulator: {e}") - })?; - Ok(Box::new(accumulator)) - } - AggregationType::MultipleIncrease => { - let accumulator = MultipleIncreaseAccumulator::deserialize_from_bytes_arroyo( - buffer, - ) - .map_err(|e| format!("Failed to deserialize MultipleIncreaseAccumulator: {e}"))?; - Ok(Box::new(accumulator)) - } - AggregationType::CountMinSketch => { - let accumulator = CountMinSketchAccumulator::deserialize_from_bytes_arroyo(buffer) - .map_err(|e| format!("Failed to deserialize CountMinSketchAccumulator: {e}"))?; - Ok(Box::new(accumulator)) - } - AggregationType::CountMinSketchWithHeap => { - let accumulator = - CountMinSketchWithHeapAccumulator::deserialize_from_bytes_arroyo(buffer) - .map_err(|e| { - format!("Failed to deserialize CountMinSketchWithHeapAccumulator: {e}") - })?; - Ok(Box::new(accumulator)) - } - AggregationType::DatasketchesKLL => { - let accumulator = DatasketchesKLLAccumulator::deserialize_from_bytes_arroyo(buffer) - .map_err(|e| { - format!("Failed to deserialize DatasketchesKLLAccumulator: {e}") - })?; - Ok(Box::new(accumulator)) - } - AggregationType::HydraKLL => { - let accumulator = HydraKllSketchAccumulator::deserialize_from_bytes_arroyo(buffer) - .map_err(|e| format!("Failed to deserialize HydraKllSketchAccumulator: {e}"))?; - Ok(Box::new(accumulator)) - } - AggregationType::DeltaSetAggregator => { - let accumulator = DeltaSetAggregatorAccumulator::deserialize_from_bytes_arroyo( - buffer, - ) - .map_err(|e| format!("Failed to deserialize DeltaSetAggregatorAccumulator: {e}"))?; - Ok(Box::new(accumulator)) - } - _ => Err(format!("Unknown precompute type: {precompute_type:?}").into()), - } - } -} - -impl SerializableToSink for PrecomputedOutput { - fn serialize_to_json(&self) -> serde_json::Value { - // Default implementation without precompute data for backward compatibility - serde_json::json!({ - // "config": self.config.serialize_to_json(), - "start_timestamp": self.start_timestamp, - "end_timestamp": self.end_timestamp, - "key": self.key.as_ref().map(|k| k.serialize_to_json()) - }) - } - - fn serialize_to_bytes(&self) -> Vec { - // Default implementation without precompute data for backward compatibility - serde_json::to_vec(self).unwrap_or_else(|_| Vec::new()) - } } -// #[cfg(test)] -// mod tests { -// use super::*; - -// #[test] -// fn test_aggregation_config_creation() { -// let labels = KeyByLabelNames::from_names(vec!["instance".to_string(), "job".to_string()]); -// let empty_labels = KeyByLabelNames::new(vec![]); -// let config = AggregationConfig::new( -// 1, -// "cpu_usage".to_string(), -// labels, -// empty_labels.clone(), -// empty_labels, -// "".to_string(), -// "sum".to_string(), -// 10, -// ); - -// assert_eq!(config.aggregation_id, 1); -// assert_eq!(config.metric, "cpu_usage"); -// assert_eq!(config.aggregation_type, "sum"); -// assert_eq!(config.window_size, 10); -// } - -// #[test] -// fn test_query_config_builder() { -// let labels = KeyByLabelNames::from_names(vec!["instance".to_string()]); -// let empty_labels = KeyByLabelNames::new(vec![]); -// let aggregation = AggregationConfig::new( -// 1, -// "cpu_usage".to_string(), -// labels, -// empty_labels.clone(), -// empty_labels, -// "".to_string(), -// "sum".to_string(), -// 10, -// ); - -// let query_config = QueryConfig::new("sum_over_time(cpu_usage[5m])".to_string()) -// .add_aggregation(aggregation); - -// assert_eq!(query_config.query, "sum_over_time(cpu_usage[5m])"); -// assert_eq!(query_config.aggregations.len(), 1); -// } - -// #[test] -// fn test_precomputed_output_json_serialization_with_precompute() { -// // Test Issue 9: PrecomputedOutput JSON serialization alignment with Python behavior -// use crate::precompute_engine::operators::SumAccumulator; -// use std::collections::BTreeMap; - -// let labels = KeyByLabelNames::from_names(vec!["instance".to_string()]); -// let empty_labels = KeyByLabelNames::new(vec![]); -// let config = AggregationConfig::new( -// 1, -// "cpu_usage".to_string(), -// labels, -// empty_labels.clone(), -// empty_labels, -// "".to_string(), -// "sum".to_string(), -// 10, -// ); - -// let mut key_labels = BTreeMap::new(); -// key_labels.insert("instance".to_string(), "server1".to_string()); -// let key = Some(KeyByLabelValues::new_with_labels(key_labels)); - -// let precomputed_output = PrecomputedOutput::new( -// 1000, // start_timestamp -// 2000, // end_timestamp -// key.clone(), -// config.clone(), -// ); - -// let accumulator = SumAccumulator::with_sum(42.5); - -// // Test JSON serialization with precompute data (matching Python format) -// let json_with_precompute = -// precomputed_output.serialize_to_json_with_precompute(&accumulator); - -// // Verify the JSON structure matches Python implementation -// assert!(json_with_precompute["config"].is_object()); -// assert_eq!(json_with_precompute["start_timestamp"], 1000); -// assert_eq!(json_with_precompute["end_timestamp"], 2000); -// assert!(json_with_precompute["key"].is_object()); -// assert!(json_with_precompute["precompute"].is_object()); - -// // Verify precompute data is included (this is the key difference from default serialization) -// assert_eq!(json_with_precompute["precompute"]["sum"], 42.5); - -// // Test default JSON serialization without precompute data -// let json_default = precomputed_output.serialize_to_json(); - -// // Verify default serialization does NOT include precompute data -// assert!( -// json_default["precompute"].is_null() -// || !json_default.as_object().unwrap().contains_key("precompute") -// ); -// assert_eq!(json_default["start_timestamp"], 1000); -// assert_eq!(json_default["end_timestamp"], 2000); -// } - -// #[test] -// fn test_precomputed_output_byte_serialization_with_precompute() { -// // Test Issue 9: PrecomputedOutput byte serialization alignment with Python behavior -// use crate::precompute_engine::operators::SumAccumulator; - -// let labels = KeyByLabelNames::from_names(vec!["instance".to_string()]); -// let empty_labels = KeyByLabelNames::new(vec![]); -// let config = AggregationConfig::new( -// 1, -// "cpu_usage".to_string(), -// labels, -// empty_labels.clone(), -// empty_labels, -// "".to_string(), -// "sum".to_string(), -// 10, -// ); - -// let precomputed_output = PrecomputedOutput::new( -// 1000, // start_timestamp -// 2000, // end_timestamp -// None, // key -// config, -// ); - -// let accumulator = SumAccumulator::with_sum(42.5); - -// // Test byte serialization with precompute data (matching Python format) -// let bytes_with_precompute = -// precomputed_output.serialize_to_bytes_with_precompute(&accumulator); - -// // Test round-trip: serialize then deserialize -// let (deserialized_output, precompute_bytes) = -// PrecomputedOutput::deserialize_from_bytes_with_precompute(&bytes_with_precompute) -// .unwrap(); - -// // Verify round-trip works correctly -// assert_eq!(deserialized_output.start_timestamp, 1000); -// assert_eq!(deserialized_output.end_timestamp, 2000); -// assert!(deserialized_output.key.is_none()); -// assert_eq!(deserialized_output.config.aggregation_id, 1); -// assert_eq!(deserialized_output.config.metric, "cpu_usage"); - -// // Verify precompute data can be deserialized back to SumAccumulator -// let deserialized_accumulator = -// SumAccumulator::deserialize_from_bytes(&precompute_bytes).unwrap(); -// assert_eq!(deserialized_accumulator.sum, 42.5); -// } -// } - -// ─── §5.1 Origin tag tests ───────────────────────────────────────────── +// ─── Origin tag tests ───────────────────────────────────────────── #[cfg(test)] mod origin_tests { @@ -706,8 +134,8 @@ mod origin_tests { /// Forward-compat guard: old on-disk records without the /// `origin` field must deserialise as `Native`, not fail with - /// "missing field". Matters because Phase 2c persisted schemas - /// and older precompute records predate §5.1. + /// "missing field". Matters because the persisted payload format + /// predates the addition of `origin`. #[test] fn old_serialized_record_without_origin_deserialises_as_native() { let old_json = r#"{ From 177175cd21e791100aa0b5856e777a484ffe3339 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 12 May 2026 15:22:47 -0600 Subject: [PATCH 7/7] refactor(precompute_operators): delete dead *_arroyo MessagePack legacy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ASAPCollector emits sketches in the canonical prost-encoded `SketchEnvelope` proto from `asap_sketchlib`. The data-plane consumes it via `SketchEnvelopeAccumulator::from_proto_bytes` (drivers/ingest/otel.rs) and `edge_runtime_adapter::reconstruct_via_runtime` (for DDSketch / KLL via the shared `asap-precompute-rs` runtime, which is what asap-precompute-rs itself uses internally). The `*_arroyo` accumulator methods (`SumAccumulator::deserialize_from_bytes_arroyo` and friends) decoded a different format — MessagePack via rmp_serde, inherited from the deleted Arroyo streaming engine path. Their only remaining callers were their own files' round-trip unit tests; the production read-back path through `accumulator_serde` was removed in PR #123 and `snapshot_sealed_epoch` is now a TODO stub returning `Ok(None)` pending the SketchIndex-backed refactor. Removed: - 13 `*_arroyo` method blocks across 9 accumulator files (deserialize_from_bytes_arroyo / serialize_to_bytes_arroyo on: sum, multiple_sum, count_min_sketch, count_min_sketch_with_heap, set_aggregator, delta_set_aggregator, hydra_kll, datasketches_kll, multiple_increase) - 5 round-trip unit tests that exercised those methods - `data_plane/src/utils/precompute_dumper.rs` — only consumer was the deleted KafkaConsumer; CLI flag `--dump-precomputes` (which fed `dump_output_dir` into the now-deleted KafkaConsumerConfig) also removed - `pub mod precompute_dumper;` + glob re-export from utils/mod.rs - `rmp-serde = "1.1"` dep from data_plane/Cargo.toml (no remaining consumers) Doc-comments updated to be honest about the persistence layer's current state (sketch_bytes are opaque; the legacy `accumulator_serde::deserialize_accumulator` callers are gone; `snapshot_sealed_epoch` returns `Ok(None)` until the SketchIndex refactor lands). Tests: data_plane lib 784 → 782 passed (-5 round-trip + -2 PrecomputeDumper tests = -7; new total 782). 2 pre-existing failures unchanged. controller lib 710/710; bins 27/27. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 1 - data_plane/Cargo.toml | 1 - data_plane/src/main.rs | 4 - .../operators/count_min_sketch_accumulator.rs | 29 --- .../count_min_sketch_with_heap_accumulator.rs | 45 ----- .../operators/datasketches_kll_accumulator.rs | 29 --- .../delta_set_aggregator_accumulator.rs | 39 ---- .../operators/hydra_kll_accumulator.rs | 8 - .../multiple_increase_accumulator.rs | 63 ------- .../operators/multiple_sum_accumulator.rs | 33 ---- .../operators/set_aggregator_accumulator.rs | 52 ----- .../operators/sum_accumulator.rs | 12 -- .../asap_query_engine/warm_tier/decoders.rs | 4 +- .../sketch_db/store/persistence/source.rs | 14 +- data_plane/src/utils/mod.rs | 2 - data_plane/src/utils/precompute_dumper.rs | 177 ------------------ 16 files changed, 11 insertions(+), 502 deletions(-) delete mode 100644 data_plane/src/utils/precompute_dumper.rs diff --git a/Cargo.lock b/Cargo.lock index c51332a1..57791cc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1073,7 +1073,6 @@ dependencies = [ "prost", "regex", "reqwest 0.11.27", - "rmp-serde", "rusqlite", "rust-s3", "serde", diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index ea7e7aea..428b11c5 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -30,7 +30,6 @@ arc-swap.workspace = true # Crate-specific (keep version pinned here) form_urlencoded = "1.2" -rmp-serde = "1.1" axum = "0.7" rusqlite = { version = "0.31", features = ["bundled"] } bincode = "1.3" diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 04e4e90d..c88cd1a0 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -111,10 +111,6 @@ struct Args { #[arg(long)] do_profiling: bool, - /// Enable dumping received precomputes to files for debugging - #[arg(long)] - dump_precomputes: bool, - /// Differentiate between query languages of input query. /// Default `promql` matches every production deploy. #[arg(long, value_enum, default_value = "promql")] diff --git a/data_plane/src/precompute_engine/operators/count_min_sketch_accumulator.rs b/data_plane/src/precompute_engine/operators/count_min_sketch_accumulator.rs index 21fb3872..4df399b6 100644 --- a/data_plane/src/precompute_engine/operators/count_min_sketch_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/count_min_sketch_accumulator.rs @@ -60,14 +60,6 @@ impl CountMinSketchAccumulator { }) } - pub fn deserialize_from_bytes_arroyo( - buffer: &[u8], - ) -> Result> { - Ok(Self { - inner: CountMinSketch::deserialize_msgpack(buffer) - .map_err(|e| -> Box { e.to_string().into() })?, - }) - } /// Decode from the modified OTLP wire format's /// `CountMinSketchDataPoint.sketch` bytes when @@ -556,27 +548,6 @@ mod tests { assert!(result.is_err()); } - #[test] - fn test_count_min_sketch_serialization() { - let cms = CountMinSketchAccumulator { - inner: CountMinSketch::from_legacy_matrix( - vec![vec![0.0, 42.0, 0.0], vec![0.0, 0.0, 100.0]], - 2, - 3, - ), - }; - - let bytes = cms.serialize_to_bytes(); - let deserialized = - CountMinSketchAccumulator::deserialize_from_bytes_arroyo(&bytes).unwrap(); - - assert_eq!(deserialized.inner.rows(), 2); - assert_eq!(deserialized.inner.cols(), 3); - let deser_sketch = deserialized.inner.sketch(); - assert_eq!(deser_sketch[0][1], 42.0); - assert_eq!(deser_sketch[1][2], 100.0); - } - #[test] fn test_count_min_sketch_as_aggregate_core() { let cms = CountMinSketchAccumulator::new(2, 3); diff --git a/data_plane/src/precompute_engine/operators/count_min_sketch_with_heap_accumulator.rs b/data_plane/src/precompute_engine/operators/count_min_sketch_with_heap_accumulator.rs index 361c11fb..87becdc6 100644 --- a/data_plane/src/precompute_engine/operators/count_min_sketch_with_heap_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/count_min_sketch_with_heap_accumulator.rs @@ -81,14 +81,6 @@ impl CountMinSketchWithHeapAccumulator { }) } - pub fn deserialize_from_bytes_arroyo( - buffer: &[u8], - ) -> Result> { - Ok(Self { - inner: CountMinSketchWithHeap::deserialize_msgpack(buffer) - .map_err(|e| -> Box { e.to_string().into() })?, - }) - } pub fn deserialize_from_bytes(_buffer: &[u8]) -> Result> { Err("deserialize_from_bytes for CountMinSketchWithHeapAccumulator not implemented".into()) @@ -317,43 +309,6 @@ mod tests { assert!(result.unwrap_err().to_string().contains("dimension")); } - #[test] - fn test_count_min_sketch_with_heap_serialization() { - // Use from_legacy_matrix for a controlled state that round-trips correctly with both backends. - let sketch = vec![vec![0.0, 42.0, 0.0], vec![0.0, 0.0, 100.0]]; - let topk_heap = vec![CmsHeapItem { - key: "test_key".to_string(), - value: 99.0, - }]; - let cms = CountMinSketchWithHeapAccumulator { - inner: CountMinSketchWithHeap::from_legacy_matrix(sketch, topk_heap, 2, 3, 5), - }; - - let bytes = cms.serialize_to_bytes(); - let deserialized = - CountMinSketchWithHeapAccumulator::deserialize_from_bytes_arroyo(&bytes).unwrap(); - - assert_eq!(deserialized.inner.rows(), 2); - assert_eq!(deserialized.inner.cols(), 3); - assert_eq!(deserialized.inner.heap_size, 5); - assert_eq!(deserialized.inner.sketch_matrix()[0][1], 42.0); - // [1][2] may be 100 (legacy, no hash collision) or 199 (100+99 when test_key hashes there) - assert!( - deserialized.inner.sketch_matrix()[1][2] >= 100.0, - "expected >= 100, got {}", - deserialized.inner.sketch_matrix()[1][2] - ); - assert_eq!(deserialized.inner.topk_heap_items().len(), 1); - assert_eq!(deserialized.inner.topk_heap_items()[0].key, "test_key"); - // With sketchlib backend, heap stores CMS estimate (min over buckets for key). - // "test_key" may hash to (0,1) and (1,2) giving min(42,100)=42, or other values. - assert!( - deserialized.inner.topk_heap_items()[0].value >= 42.0, - "expected >= 42, got {}", - deserialized.inner.topk_heap_items()[0].value - ); - } - #[test] fn test_count_min_sketch_with_heap_as_aggregate_core() { let cms = CountMinSketchWithHeapAccumulator::new(2, 3, 5); diff --git a/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs b/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs index a744d8a1..e803146e 100644 --- a/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/datasketches_kll_accumulator.rs @@ -34,18 +34,6 @@ impl DatasketchesKLLAccumulator { self.inner.quantile(quantile) } - pub fn deserialize_from_bytes_arroyo( - buffer: &[u8], - ) -> Result> { - debug!( - "Deserializing DatasketchesKLLAccumulator from MessagePack buffer of size {}", - buffer.len() - ); - Ok(Self { - inner: KllSketch::deserialize_msgpack(buffer) - .map_err(|e| -> Box { e.to_string().into() })?, - }) - } /// Decode from the modified OTLP wire format's /// `KLLSketchDataPoint.sketch` bytes when @@ -442,23 +430,6 @@ mod tests { assert_eq!(merged.get_quantile(1.0), 10.0); } - #[test] - fn test_datasketches_kll_serialization() { - let mut kll = DatasketchesKLLAccumulator::new(200); - for i in 1..=5 { - kll.update(i as f64); - } - - let bytes = kll.serialize_to_bytes(); - let deserialized = - DatasketchesKLLAccumulator::deserialize_from_bytes_arroyo(&bytes).unwrap(); - - assert_eq!(deserialized.inner.k, 200); - assert_eq!(deserialized.inner.count(), 5); - assert_eq!(deserialized.get_quantile(0.0), 1.0); - assert_eq!(deserialized.get_quantile(1.0), 5.0); - } - #[test] fn test_datasketches_kll_get_keys() { let kll = DatasketchesKLLAccumulator::new(200); diff --git a/data_plane/src/precompute_engine/operators/delta_set_aggregator_accumulator.rs b/data_plane/src/precompute_engine/operators/delta_set_aggregator_accumulator.rs index d868ddfd..b45ab524 100644 --- a/data_plane/src/precompute_engine/operators/delta_set_aggregator_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/delta_set_aggregator_accumulator.rs @@ -148,26 +148,6 @@ impl DeltaSetAggregatorAccumulator { Ok(Self { added, removed }) } - - pub fn deserialize_from_bytes_arroyo( - buffer: &[u8], - ) -> Result> { - // Delegate to sketch-core canonical DeltaResult msgpack format - let delta = deserialize_msgpack(buffer) - .map_err(|e| -> Box { e.to_string().into() })?; - - let mut added = HashSet::new(); - for item in &delta.added { - added.insert(KeyByLabelValues::from_semicolon_str(item)); - } - - let mut removed = HashSet::new(); - for item in &delta.removed { - removed.insert(KeyByLabelValues::from_semicolon_str(item)); - } - - Ok(Self { added, removed }) - } } impl Default for DeltaSetAggregatorAccumulator { @@ -378,25 +358,6 @@ mod tests { assert_eq!(merged.removed.len(), 1); } - #[test] - fn test_delta_set_aggregator_serialization() { - let mut acc = DeltaSetAggregatorAccumulator::new(); - let key1 = create_test_key("web"); - let key2 = create_test_key("api"); - acc.add_key(key1.clone()); - acc.remove_key(key2.clone()); - - // Test binary (msgpack) serialization roundtrip - let bytes = acc.serialize_to_bytes(); - let deserialized_bytes = - DeltaSetAggregatorAccumulator::deserialize_from_bytes_arroyo(&bytes).unwrap(); - - assert_eq!(deserialized_bytes.added.len(), 1); - assert_eq!(deserialized_bytes.removed.len(), 1); - assert!(deserialized_bytes.added.contains(&key1)); - assert!(deserialized_bytes.removed.contains(&key2)); - } - #[test] fn test_delta_set_aggregator_query() { let acc = DeltaSetAggregatorAccumulator::new(); diff --git a/data_plane/src/precompute_engine/operators/hydra_kll_accumulator.rs b/data_plane/src/precompute_engine/operators/hydra_kll_accumulator.rs index 474966a0..35701a53 100644 --- a/data_plane/src/precompute_engine/operators/hydra_kll_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/hydra_kll_accumulator.rs @@ -34,14 +34,6 @@ impl HydraKllSketchAccumulator { Err("deserialize_from_bytes for HydraKllSketchAccumulator not implemented".into()) } - pub fn deserialize_from_bytes_arroyo( - buffer: &[u8], - ) -> Result> { - Ok(Self { - inner: HydraKllSketch::deserialize_msgpack(buffer) - .map_err(|e| -> Box { e.to_string().into() })?, - }) - } pub fn query_key(&self, key: &KeyByLabelValues, quantile: f64) -> f64 { self.inner.quantile(&key.to_semicolon_str(), quantile) diff --git a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs b/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs index 49cd6a70..e7df968a 100644 --- a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs @@ -116,69 +116,6 @@ impl MultipleIncreaseAccumulator { Ok(accumulator) } - - pub fn deserialize_from_bytes_arroyo( - buffer: &[u8], - ) -> Result> { - let precompute: HashMap = - rmp_serde::from_slice(buffer).map_err(|e| { - format!("Failed to deserialize MultipleIncreaseAccumulator from MessagePack: {e}") - })?; - - let mut accumulator = Self::new(); - for (key_str, values) in precompute { - // Parse semicolon-separated key values - let key_values: Vec = key_str.split(';').map(|s| s.to_string()).collect(); - // let mut labels = std::collections::BTreeMap::new(); - // for (i, value) in key_values.into_iter().enumerate() { - // labels.insert(format!("label_{i}"), value); - // } - let key_obj = KeyByLabelValues::new_with_labels(key_values); - - let starting_measurement = Measurement::new(values.starting_measurement); - let starting_timestamp = values.starting_timestamp; - let last_seen_measurement = Measurement::new(values.last_seen_measurement); - let last_seen_timestamp = values.last_seen_timestamp; - - let increase_accumulator = IncreaseAccumulator::new( - starting_measurement, - starting_timestamp, - last_seen_measurement, - last_seen_timestamp, - ); - - accumulator.increases.insert(key_obj, increase_accumulator); - } - - Ok(accumulator) - } - - /// Serialize to Arroyo-compatible format (MessagePack HashMap) - /// Matches the Arroyo multipleincrease_ UDF format - pub fn serialize_to_bytes_arroyo(&self) -> Vec { - use serde::Serialize; - let mut per_key_storage: HashMap = HashMap::new(); - - for (key, increase_acc) in &self.increases { - // Keys are semicolon-separated label values - let key_str = key.labels.join(";"); - per_key_storage.insert( - key_str, - MeasurementData { - starting_measurement: increase_acc.starting_measurement.value, - starting_timestamp: increase_acc.starting_timestamp, - last_seen_measurement: increase_acc.last_seen_measurement.value, - last_seen_timestamp: increase_acc.last_seen_timestamp, - }, - ); - } - - let mut buf = Vec::new(); - per_key_storage - .serialize(&mut rmp_serde::Serializer::new(&mut buf)) - .expect("Failed to serialize MultipleIncreaseAccumulator to MessagePack"); - buf - } } impl Default for MultipleIncreaseAccumulator { diff --git a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs b/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs index 8c6d0cd4..83aa9e22 100644 --- a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs @@ -50,39 +50,6 @@ impl MultipleSumAccumulator { Ok(Self { sums }) } - pub fn deserialize_from_bytes_arroyo( - buffer: &[u8], - ) -> Result> { - let precompute: HashMap = rmp_serde::from_slice(buffer).map_err(|e| { - format!("Failed to deserialize MultipleSumAccumulator from MessagePack: {e}") - })?; - - let mut sums = HashMap::new(); - for (key_str, sum) in precompute { - let key_values: Vec = key_str.split(';').map(|s| s.to_string()).collect(); - let key = KeyByLabelValues::new_with_labels(key_values); - sums.insert(key, sum); - } - - Ok(Self { sums }) - } - - /// Serialize to Arroyo-compatible format (MessagePack HashMap) - pub fn serialize_to_bytes_arroyo(&self) -> Vec { - use serde::Serialize; - let per_key_storage: HashMap = self - .sums - .iter() - .map(|(key, &sum)| (key.labels.join(";"), sum)) - .collect(); - - let mut buf = Vec::new(); - per_key_storage - .serialize(&mut rmp_serde::Serializer::new(&mut buf)) - .expect("Failed to serialize MultipleSumAccumulator to MessagePack"); - buf - } - pub fn deserialize_from_bytes(buffer: &[u8]) -> Result> { let mut offset = 0; diff --git a/data_plane/src/precompute_engine/operators/set_aggregator_accumulator.rs b/data_plane/src/precompute_engine/operators/set_aggregator_accumulator.rs index 41cb0edb..75ef0620 100644 --- a/data_plane/src/precompute_engine/operators/set_aggregator_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/set_aggregator_accumulator.rs @@ -88,29 +88,6 @@ impl SetAggregatorAccumulator { Ok(Self { added }) } - - pub fn deserialize_from_bytes_arroyo( - buffer: &[u8], - ) -> Result> { - let sa = SetAggregator::deserialize_msgpack(buffer) - .map_err(|e| -> Box { e.to_string().into() })?; - let added = sa - .values - .into_iter() - .map(|s| KeyByLabelValues::from_semicolon_str(&s)) - .collect(); - Ok(Self { added }) - } - - /// Serialize to Arroyo-compatible format (MessagePack StringSet). - /// Delegates to sketch-core's canonical wire format. - pub fn serialize_to_bytes_arroyo(&self) -> Vec { - let mut sa = SetAggregator::new(); - for key in &self.added { - sa.update(&key.to_semicolon_str()); - } - sa.serialize_msgpack().unwrap_or_default() - } } impl Default for SetAggregatorAccumulator { @@ -342,33 +319,4 @@ mod tests { assert_eq!(keys.len(), 0); } - #[test] - fn test_arroyo_roundtrip() { - // Verify serialize_to_bytes_arroyo / deserialize_from_bytes_arroyo round-trip. - // Both now delegate to sketch-core's SetAggregator which uses the same - // StringSet { values: HashSet } format as Arroyo's setaggregator_ UDF. - let mut acc = SetAggregatorAccumulator::new(); - acc.add_key(KeyByLabelValues::new_with_labels(vec![ - "web".to_string(), - "prod".to_string(), - ])); - acc.add_key(KeyByLabelValues::new_with_labels(vec!["api".to_string()])); - - let bytes = acc.serialize_to_bytes_arroyo(); - let deserialized = SetAggregatorAccumulator::deserialize_from_bytes_arroyo(&bytes).expect( - "deserialize_from_bytes_arroyo failed — format mismatch with serialize_to_bytes_arroyo", - ); - - assert_eq!( - deserialized.added.len(), - acc.added.len(), - "roundtrip changed the number of keys" - ); - for key in &acc.added { - assert!( - deserialized.added.contains(key), - "key {key:?} missing after arroyo roundtrip" - ); - } - } } diff --git a/data_plane/src/precompute_engine/operators/sum_accumulator.rs b/data_plane/src/precompute_engine/operators/sum_accumulator.rs index baa9aeb2..7fef9d64 100644 --- a/data_plane/src/precompute_engine/operators/sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/sum_accumulator.rs @@ -42,19 +42,7 @@ impl SumAccumulator { Ok(Self::with_sum(sum)) } - pub fn deserialize_from_bytes_arroyo( - buffer: &[u8], - ) -> Result> { - // Arroyo uses MessagePack format - let sum: f64 = rmp_serde::from_slice(buffer) - .map_err(|e| format!("Failed to deserialize from MessagePack: {e}"))?; - Ok(Self::with_sum(sum)) - } - /// Serialize to Arroyo-compatible format (MessagePack f64) - pub fn serialize_to_bytes_arroyo(&self) -> Vec { - rmp_serde::to_vec(&self.sum).expect("Failed to serialize sum to MessagePack") - } } impl Default for SumAccumulator { diff --git a/data_plane/src/query_engines/asap_query_engine/warm_tier/decoders.rs b/data_plane/src/query_engines/asap_query_engine/warm_tier/decoders.rs index 08ce2c5e..a9fa8dd0 100644 --- a/data_plane/src/query_engines/asap_query_engine/warm_tier/decoders.rs +++ b/data_plane/src/query_engines/asap_query_engine/warm_tier/decoders.rs @@ -173,8 +173,8 @@ pub fn decode_cs_from_msgpack(buffer: &[u8]) -> Result { /// Decode a `CountMinSketchWithHeap` from msgpack bytes — the OTLP /// `CountMinSketch` wire bytes when the gateway/precompute layer /// marked the sid as CmsWithHeap (heap embedded in the -/// `CountMinSketchWithHeapSerialized` outer wrapper). Mirrors -/// `precompute_operators::count_min_sketch_with_heap_accumulator::deserialize_from_bytes_arroyo`. +/// `CountMinSketchWithHeapSerialized` outer wrapper). Delegates to +/// `asap_sketchlib::sketches::CountMinSketchWithHeap::deserialize_msgpack`. pub fn decode_cms_with_heap_from_msgpack(buffer: &[u8]) -> Result { CountMinSketchWithHeap::deserialize_msgpack(buffer) .map_err(|e| format!("deserialize CountMinSketchWithHeap msgpack: {e}")) diff --git a/data_plane/src/stores/sketch_db/store/persistence/source.rs b/data_plane/src/stores/sketch_db/store/persistence/source.rs index 7a4872b2..6cf06850 100644 --- a/data_plane/src/stores/sketch_db/store/persistence/source.rs +++ b/data_plane/src/stores/sketch_db/store/persistence/source.rs @@ -28,8 +28,11 @@ pub struct SealedEpochRef { /// /// The `entries` are ready to write to disk: labels are already resolved /// to `Option` (no intern-table lookup needed) and the -/// sketch bytes are already in the Arroyo/MessagePack format used by -/// `crate::query_engines::physical::accumulator_serde::deserialize_accumulator`. +/// sketch bytes are opaque to the persistence layer — the writer carries +/// whatever format the SketchStore put in. (The legacy Arroyo/MessagePack +/// path that consumed these bytes lives in deleted modules; the +/// production read-back path will land with the SketchIndex-backed +/// refactor — `snapshot_sealed_epoch` returns `Ok(None)` until then.) #[derive(Debug, Clone)] pub struct EpochSnapshot { pub agg_id: u64, @@ -60,10 +63,11 @@ pub struct EpochSnapshotEntry { pub end_ts: u64, /// Optional label set, already resolved from the per-agg intern table. pub label: Option, - /// `AggregateCore::type_name()` of the underlying sketch, used on - /// read-back to pick the right `deserialize_from_bytes_arroyo` impl. + /// `AggregateCore::type_name()` of the underlying sketch — recorded + /// so a future read-back path can dispatch to the right + /// deserializer once the SketchIndex-backed snapshot lands. pub sketch_type_name: String, - /// Serialized sketch payload (Arroyo / MessagePack format). + /// Serialized sketch payload (opaque to the persistence layer). pub sketch_bytes: Vec, } diff --git a/data_plane/src/utils/mod.rs b/data_plane/src/utils/mod.rs index 3e5e2745..5d620636 100644 --- a/data_plane/src/utils/mod.rs +++ b/data_plane/src/utils/mod.rs @@ -1,7 +1,5 @@ pub mod file_io; pub mod http; -pub mod precompute_dumper; pub use file_io::*; pub use http::*; -pub use precompute_dumper::*; diff --git a/data_plane/src/utils/precompute_dumper.rs b/data_plane/src/utils/precompute_dumper.rs deleted file mode 100644 index d6cc341e..00000000 --- a/data_plane/src/utils/precompute_dumper.rs +++ /dev/null @@ -1,177 +0,0 @@ -use crate::stores::types::{AggregateCore, PrecomputedOutput}; -use serde::Serialize; -use std::fs::{create_dir_all, File}; -use std::io::{BufWriter, Write}; -use std::path::Path; -use std::time::{SystemTime, UNIX_EPOCH}; -use tracing::{debug, error, info}; - -#[derive(Serialize)] -struct PrecomputeDump { - timestamp: u64, - metadata: PrecomputedOutput, - accumulator_type: String, - accumulator_data_bytes: Vec, -} - -pub struct PrecomputeDumper { - file: BufWriter, - dump_count: u64, - file_path: String, -} - -impl PrecomputeDumper { - pub fn new(output_dir: &str) -> Result> { - // Create precompute_dumps subdirectory - let dump_dir = Path::new(output_dir).join("precompute_dumps"); - create_dir_all(&dump_dir)?; - - // Generate filename with timestamp - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - let filename = format!("precomputes_{timestamp}.msgpack"); - let file_path = dump_dir.join(filename); - - let file = File::create(&file_path)?; - let buffered_writer = BufWriter::new(file); - - info!("Created precompute dump file: {:?}", file_path); - - Ok(Self { - file: buffered_writer, - dump_count: 0, - file_path: file_path.to_string_lossy().to_string(), - }) - } - - pub fn dump_precompute( - &mut self, - output: &PrecomputedOutput, - accumulator: &dyn AggregateCore, - ) -> Result<(), Box> { - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - - // Create the dump record - let dump = PrecomputeDump { - timestamp, - metadata: output.clone(), - accumulator_type: accumulator.type_name().to_string(), - accumulator_data_bytes: accumulator.serialize_to_bytes(), - }; - - // Serialize to MessagePack - let serialized_data = rmp_serde::to_vec(&dump) - .map_err(|e| format!("Failed to serialize precompute dump: {e}"))?; - - // Write length prefix (4 bytes, little-endian) - let length = serialized_data.len() as u32; - self.file.write_all(&length.to_le_bytes())?; - - // Write the serialized data - self.file.write_all(&serialized_data)?; - - self.dump_count += 1; - - debug!( - "Dumped precompute #{}: type={}, aggregation_id={}, size={} bytes", - self.dump_count, - dump.accumulator_type, - output.aggregation_id, - serialized_data.len() - ); - - // Flush every 100 records to ensure data is written - if self.dump_count.is_multiple_of(100) { - self.file.flush()?; - debug!( - "Flushed precompute dump file after {} records", - self.dump_count - ); - } - - Ok(()) - } - - pub fn flush(&mut self) -> Result<(), Box> { - self.file.flush()?; - debug!( - "Flushed precompute dump file with {} total records", - self.dump_count - ); - Ok(()) - } - - pub fn get_dump_count(&self) -> u64 { - self.dump_count - } - - pub fn get_file_path(&self) -> &str { - &self.file_path - } -} - -impl Drop for PrecomputeDumper { - fn drop(&mut self) { - if let Err(e) = self.flush() { - error!("Failed to flush precompute dump file on drop: {}", e); - } else { - info!( - "Closed precompute dump file {} with {} records", - self.file_path, self.dump_count - ); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::precompute_engine::operators::SumAccumulator; - use tempfile::TempDir; - - #[test] - fn test_precompute_dumper_creation() { - let temp_dir = TempDir::new().unwrap(); - let output_dir = temp_dir.path().to_str().unwrap(); - - let dumper = PrecomputeDumper::new(output_dir); - assert!(dumper.is_ok()); - - let dumper = dumper.unwrap(); - assert_eq!(dumper.get_dump_count(), 0); - assert!(dumper.get_file_path().contains("precomputes_")); - assert!(dumper.get_file_path().ends_with(".msgpack")); - } - - #[test] - fn test_precompute_dumping() { - let temp_dir = TempDir::new().unwrap(); - let output_dir = temp_dir.path().to_str().unwrap(); - - let mut dumper = PrecomputeDumper::new(output_dir).unwrap(); - - // Create test precompute data - let accumulator = SumAccumulator::with_sum(42.5); - let output = PrecomputedOutput { - start_timestamp: 1000, - end_timestamp: 2000, - key: None, - aggregation_id: 1, - origin: Default::default(), - }; - - // Dump the precompute - let result = dumper.dump_precompute(&output, &accumulator); - assert!(result.is_ok()); - assert_eq!(dumper.get_dump_count(), 1); - - // Test flushing - let flush_result = dumper.flush(); - assert!(flush_result.is_ok()); - } -}