From 4983454b3ff0a2504c9924821da33f5d842a909a Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 13:53:14 -0600 Subject: [PATCH] feat: restore Remote Write compatibility profile --- Cargo.lock | 7 + data_plane/Cargo.toml | 1 + .../examples/asapquery/streaming-config.yaml | 35 + data_plane/src/drivers/ingest/mod.rs | 4 + .../drivers/ingest/prometheus_remote_write.rs | 727 ++++++++++++++++++ data_plane/src/drivers/query/servers/http.rs | 232 +++++- data_plane/src/lib.rs | 3 + data_plane/src/main.rs | 162 +++- .../src/precompute_engine/series_router.rs | 66 ++ docs/README.md | 1 + docs/user_guide/asapquery-profile.md | 59 ++ 11 files changed, 1288 insertions(+), 9 deletions(-) create mode 100644 data_plane/examples/asapquery/streaming-config.yaml create mode 100644 data_plane/src/drivers/ingest/prometheus_remote_write.rs create mode 100644 docs/user_guide/asapquery-profile.md diff --git a/Cargo.lock b/Cargo.lock index e84f81018..cb41bc184 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1012,6 +1012,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "snap", "structopt", "tempfile", "thiserror 1.0.69", @@ -3250,6 +3251,12 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + [[package]] name = "socket2" version = "0.5.10" diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index e42164a84..89ec7817c 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -71,6 +71,7 @@ hex = "0.4" arrow = "53.4.1" futures = "0.3" prost = "0.13" +snap = "1.1" # Vendored modified opentelemetry-proto with first-class sketch metric # variants from DataCollector (DDSketch / KLLSketch / CountSketch / # CountMinSketch / HLLSketch). See docs/pipeline-query-catalog.md §5.4 in the diff --git a/data_plane/examples/asapquery/streaming-config.yaml b/data_plane/examples/asapquery/streaming-config.yaml new file mode 100644 index 000000000..c2174a06c --- /dev/null +++ b/data_plane/examples/asapquery/streaming-config.yaml @@ -0,0 +1,35 @@ +aggregations: + - aggregationType: Sum + aggregationSubType: '' + metric: compat_value + labels: + grouping: [job] + rollup: [] + aggregated: [] + parameters: {} + windowSize: 30 + windowType: tumbling + spatialFilter: '' + - aggregationType: Increase + aggregationSubType: '' + metric: compat_counter_total + labels: + grouping: [job] + rollup: [] + aggregated: [] + parameters: {} + windowSize: 30 + windowType: tumbling + spatialFilter: '' + - aggregationType: DDSketch + aggregationSubType: '' + metric: compat_latency_seconds + labels: + grouping: [job] + rollup: [] + aggregated: [] + parameters: + relative_accuracy: 0.01 + windowSize: 30 + windowType: tumbling + spatialFilter: '' diff --git a/data_plane/src/drivers/ingest/mod.rs b/data_plane/src/drivers/ingest/mod.rs index 21ca4d43f..39d0e8eff 100644 --- a/data_plane/src/drivers/ingest/mod.rs +++ b/data_plane/src/drivers/ingest/mod.rs @@ -1,5 +1,9 @@ pub mod otel; +pub mod prometheus_remote_write; pub mod series_resolver; pub use otel::{OtlpReceiver, OtlpReceiverConfig}; +pub use prometheus_remote_write::{ + PrometheusRemoteWriteConfig, PrometheusRemoteWriteReceiver, RemoteWriteStats, +}; pub use series_resolver::{canonical_attrs_fingerprint, SeriesIdResolver}; diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs new file mode 100644 index 000000000..55d1a3767 --- /dev/null +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -0,0 +1,727 @@ +//! Prometheus Remote Write v1 adapter for the backend-local precompute path. +//! +//! This module deliberately stops at wire validation, canonical series +//! identity, retry deduplication, and config-driven routing. Aggregation-family +//! selection remains owned by the installed precompute plan. + +use crate::precompute_engine::ingest_handler::IngestState; +use crate::precompute_engine::series_router::{TryRouteError, WorkerMessage}; +use prost::Message; +use std::collections::{HashMap, VecDeque}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +pub const STALE_NAN_BITS: u64 = 0x7ff0_0000_0000_0002; + +#[derive(Clone, PartialEq, Message)] +pub struct WriteRequest { + #[prost(message, repeated, tag = "1")] + pub timeseries: Vec, +} + +#[derive(Clone, PartialEq, Message)] +pub struct TimeSeries { + #[prost(message, repeated, tag = "1")] + pub labels: Vec