Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 50 additions & 3 deletions asap-query-engine/src/drivers/query/adapters/prometheus_http.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,17 @@ use std::collections::HashMap;
use std::sync::Arc;
use tracing::{debug, error};

/// Prometheus-compatible response structure
/// Prometheus-compatible response structure, with two ASAP
/// extensions:
///
/// * `infos` — Prometheus 3.0-style informational annotations
/// (Grafana 11+ renders these inline). Mirrors a one-liner
/// summary of `accuracy` so older / non-JSON-aware UIs still
/// see the ε/δ bound.
/// * `accuracy` — structured [`AccuracyEnvelope`] carrying ε, δ,
/// kind, and optional `per_segment` for schema-timeline-
/// crossing queries. Standard Prometheus clients ignore
/// unknown top-level fields, so this is a zero-risk extension.
#[derive(Debug, Serialize, Deserialize)]
pub struct PrometheusResponse {
pub status: String,
Expand All @@ -31,6 +41,19 @@ pub struct PrometheusResponse {
/// Purged segment.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub warnings: Vec<String>,
/// Prometheus 3.0 `infos: []`. Grafana 11+ renders each
/// string inline. Used to mirror a human-readable
/// `accuracy: ε=..., δ=..., kind=...` line when the
/// structured `accuracy` field is present.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub infos: Vec<String>,
/// ASAP extension: theoretical accuracy envelope for the
/// answer (§6.4 of docs/design-sketch-db.md). Unknown to
/// standard Prometheus clients (they ignore unknown fields),
/// consumed by Grafana panels / paper artifacts that want
/// the machine-readable (ε, δ) bound.
#[serde(skip_serializing_if = "Option::is_none")]
pub accuracy: Option<crate::stores::sketch_db::AccuracyEnvelope>,
}

impl PrometheusResponse {
Expand All @@ -41,6 +64,8 @@ impl PrometheusResponse {
error_type: None,
error: None,
warnings: Vec::new(),
infos: Vec::new(),
accuracy: None,
}
}

Expand All @@ -53,16 +78,30 @@ impl PrometheusResponse {
error_type: None,
error: None,
warnings,
infos: Vec::new(),
accuracy: None,
}
}

/// Attach an accuracy envelope: sets the structured `accuracy`
/// field and mirrors a human-readable one-liner to `infos`.
/// Chainable so both warning + accuracy paths can decorate
/// the same `success(...)` construction.
pub fn with_accuracy(mut self, envelope: crate::stores::sketch_db::AccuracyEnvelope) -> Self {
self.infos.push(envelope.summary());
self.accuracy = Some(envelope);
self
}

pub fn error(error_type: &str, error: &str) -> Self {
Self {
status: "error".to_string(),
data: None,
error_type: Some(error_type.to_string()),
error: Some(error.to_string()),
warnings: Vec::new(),
infos: Vec::new(),
accuracy: None,
}
}
}
Expand Down Expand Up @@ -225,11 +264,15 @@ impl QueryResponseAdapter for PrometheusHttpAdapter {
// so they land on the top-level `warnings` field, matching
// Prometheus's native API.
let warnings = result.query_result.warnings().to_vec();
let response = if warnings.is_empty() {
let accuracy = result.query_result.accuracy().cloned();
let mut response = if warnings.is_empty() {
PrometheusResponse::success(prometheus_data)
} else {
PrometheusResponse::success_with_warnings(prometheus_data, warnings)
};
if let Some(envelope) = accuracy {
response = response.with_accuracy(envelope);
}
Ok(Json(serde_json::to_value(response).unwrap()).into_response())
}

Expand All @@ -245,11 +288,15 @@ impl QueryResponseAdapter for PrometheusHttpAdapter {
StatusCode::INTERNAL_SERVER_ERROR
})?;
let warnings = result.warnings().to_vec();
let response = if warnings.is_empty() {
let accuracy = result.accuracy().cloned();
let mut response = if warnings.is_empty() {
PrometheusResponse::success(prometheus_data)
} else {
PrometheusResponse::success_with_warnings(prometheus_data, warnings)
};
if let Some(envelope) = accuracy {
response = response.with_accuracy(envelope);
}
Ok(Json(serde_json::to_value(response).unwrap()).into_response())
}

Expand Down
36 changes: 36 additions & 0 deletions asap-query-engine/src/engines/query_result.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use crate::data_model::KeyByLabelValues;
use crate::stores::sketch_db::AccuracyEnvelope;
use serde::{Deserialize, Serialize};

use promql_utilities::query_logics::enums::QueryResultType;
Expand All @@ -23,6 +24,7 @@ impl QueryResult {
values,
timestamp,
warnings: Vec::new(),
accuracy: None,
})
}

Expand All @@ -42,13 +44,15 @@ impl QueryResult {
values,
timestamp,
warnings,
accuracy: None,
})
}

pub fn matrix(values: Vec<RangeVectorElement>) -> Self {
QueryResult::Matrix(RangeVector {
values,
warnings: Vec::new(),
accuracy: None,
})
}

Expand All @@ -61,6 +65,29 @@ impl QueryResult {
QueryResult::Matrix(m) => &m.warnings,
}
}

/// Theoretical accuracy envelope for this answer (§6.4 of
/// the sketch-DB design). `None` when the engine couldn't
/// resolve a schema — legacy paths that haven't been wired
/// yet, or fallback-produced responses.
pub fn accuracy(&self) -> Option<&AccuracyEnvelope> {
match self {
QueryResult::Vector(iv) => iv.accuracy.as_ref(),
QueryResult::Matrix(m) => m.accuracy.as_ref(),
}
}

/// Attach an accuracy envelope. Chainable so engine paths
/// can build the bare result first and decorate once the
/// `agg_id → AggregationConfig → AccuracyProfile` lookup
/// has resolved.
pub fn with_accuracy(mut self, envelope: AccuracyEnvelope) -> Self {
match &mut self {
QueryResult::Vector(iv) => iv.accuracy = Some(envelope),
QueryResult::Matrix(m) => m.accuracy = Some(envelope),
}
self
}
}

/// Instant vector - a set of time series containing a single sample for each time series, all sharing the same timestamp
Expand All @@ -77,6 +104,12 @@ pub struct InstantVector {
/// missing from the current config).
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub warnings: Vec<String>,
/// §6.4 accuracy envelope. Attached by the engine once the
/// `agg_id` is resolved; surfaced as `PrometheusResponse`'s
/// top-level `accuracy` field and mirrored to `infos` for
/// Grafana 11+ inline display.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub accuracy: Option<AccuracyEnvelope>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
Expand All @@ -98,6 +131,9 @@ pub struct RangeVector {
/// See [`InstantVector::warnings`].
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub warnings: Vec<String>,
/// See [`InstantVector::accuracy`].
#[serde(default, skip_serializing_if = "Option::is_none")]
pub accuracy: Option<AccuracyEnvelope>,
}

/// Individual element in a range vector
Expand Down
53 changes: 46 additions & 7 deletions asap-query-engine/src/engines/simple_engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1921,16 +1921,35 @@ impl SimpleEngine {
context: QueryExecutionContext,
enable_topk: bool,
) -> Option<(KeyByLabelNames, QueryResult)> {
let agg_id = context.agg_info.aggregation_id_for_value;
let results = self
.execute_query_pipeline(&context, enable_topk)
.map_err(|e| {
warn!("Query execution failed: {}", e);
e
})
.ok()?;
Some((
context.metadata.query_output_labels,
QueryResult::vector(results, context.query_time),
let qr = QueryResult::vector(results, context.query_time);
let qr = match self.accuracy_envelope_for(agg_id) {
Some(env) => qr.with_accuracy(env),
None => qr,
};
Some((context.metadata.query_output_labels, qr))
}

/// Build an [`AccuracyEnvelope`] for a single resolved
/// `agg_id` by looking up the matching `AggregationConfig` in
/// the current streaming-config snapshot and deriving its
/// [`AccuracyProfile`]. Returns `None` when the agg isn't in
/// config (e.g. post-retire / test harness with empty config).
pub(crate) fn accuracy_envelope_for(
&self,
agg_id: u64,
) -> Option<crate::stores::sketch_db::AccuracyEnvelope> {
let snap = self.streaming_config_snapshot();
let cfg = snap.get_aggregation_config(agg_id)?;
Some(crate::stores::sketch_db::AccuracyEnvelope::single(
crate::stores::sketch_db::AccuracyProfile::derive(cfg),
))
}

Expand Down Expand Up @@ -3241,10 +3260,30 @@ impl SimpleEngine {
Vec::new()
};

Some((
probe_context.metadata.query_output_labels,
QueryResult::vector_with_warnings(output, probe_context.query_time, warnings),
))
// §6.4: build a per-segment accuracy envelope from each
// segment's resolved agg_id. Segments that don't resolve
// to an in-config agg drop out — their partial-ness is
// already reflected in `warnings` above.
let snap = self.streaming_config_snapshot();
let per_segment: Vec<crate::stores::sketch_db::PerSegmentAccuracy> = segments
.iter()
.filter_map(|seg| {
let cfg = snap.get_aggregation_config(seg.agg_id)?;
Some(crate::stores::sketch_db::PerSegmentAccuracy {
agg_id: seg.agg_id,
range_ms: [seg.start_ms as i64, seg.end_ms as i64],
profile: crate::stores::sketch_db::AccuracyProfile::derive(cfg),
})
})
.collect();
let envelope = crate::stores::sketch_db::AccuracyEnvelope::from_segments(per_segment);

let qr = QueryResult::vector_with_warnings(output, probe_context.query_time, warnings);
let qr = match envelope {
Some(e) => qr.with_accuracy(e),
None => qr,
};
Some((probe_context.metadata.query_output_labels, qr))
}

/// Merge precomputed outputs (extracts buckets from timestamped data)
Expand Down
105 changes: 105 additions & 0 deletions asap-query-engine/src/stores/sketch_db/accuracy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,24 @@ impl AccuracyProfile {
}
}

/// Human-readable one-liner. Surfaced in Prometheus-style
/// `infos` arrays so Grafana 11+ shows it inline without a
/// custom panel.
pub fn summary(&self) -> String {
format!(
"accuracy: ε={}, δ={}, kind={}",
self.epsilon,
self.delta,
match self.kind {
AccuracyKind::Exact => "exact",
AccuracyKind::AdditiveFrequency => "additive_frequency",
AccuracyKind::RelativeCardinality => "relative_cardinality",
AccuracyKind::RankQuantile => "rank_quantile",
AccuracyKind::RelativeQuantile => "relative_quantile",
}
)
}

/// Derive an [`AccuracyProfile`] from a pinned
/// [`AggregationConfig`]. Reads `aggregation_type` and any
/// necessary entries in `parameters`; falls back to exact for
Expand Down Expand Up @@ -254,6 +272,93 @@ fn ddsketch_alpha(config: &AggregationConfig) -> f64 {
.unwrap_or(0.01)
}

/// Per-segment accuracy record. Attached to a multi-segment
/// [`AccuracyEnvelope`] so clients can see the error bound for
/// each piece of the schema-timeline-crossing query.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct PerSegmentAccuracy {
pub agg_id: u64,
/// Half-open millisecond range `[start_ms, end_ms)` this
/// segment covered.
pub range_ms: [i64; 2],
#[serde(flatten)]
pub profile: AccuracyProfile,
}

/// Wire-side envelope emitted on PromQL responses as the
/// top-level `accuracy` field. Single-schema queries fill
/// `profile`; queries that span a schema-timeline boundary also
/// populate `per_segment` so the caller can see each piece's
/// bound. The top-level `profile` is the worst-case (max ε,
/// max δ) across segments — a conservative upper envelope.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct AccuracyEnvelope {
#[serde(flatten)]
pub profile: AccuracyProfile,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub per_segment: Vec<PerSegmentAccuracy>,
}

impl AccuracyEnvelope {
/// Envelope for a single resolved aggregation.
pub fn single(profile: AccuracyProfile) -> Self {
Self {
profile,
per_segment: Vec::new(),
}
}

/// Build an envelope from a slice of per-segment tuples.
/// Top-level `profile.epsilon` is `max(segment.epsilon)` and
/// same for δ — the conservative envelope across segments.
/// Returns `None` when the slice is empty.
pub fn from_segments(segs: Vec<PerSegmentAccuracy>) -> Option<Self> {
if segs.is_empty() {
return None;
}
let mut epsilon = 0.0_f64;
let mut delta = 0.0_f64;
// Pick the "most lossy" kind: any non-Exact wins over
// Exact; if mixed non-Exact kinds span segments we pick
// the first non-Exact and trust the per-segment data for
// the caller's finer needs.
let mut kind = AccuracyKind::Exact;
for s in &segs {
if s.profile.epsilon > epsilon {
epsilon = s.profile.epsilon;
}
if s.profile.delta > delta {
delta = s.profile.delta;
}
if matches!(kind, AccuracyKind::Exact) && !matches!(s.profile.kind, AccuracyKind::Exact)
{
kind = s.profile.kind;
}
}
Some(Self {
profile: AccuracyProfile {
epsilon,
delta,
kind,
},
per_segment: segs,
})
}

/// Summary line suitable for Prometheus `infos`.
pub fn summary(&self) -> String {
if self.per_segment.is_empty() {
self.profile.summary()
} else {
format!(
"{} (worst-case over {} schema-timeline segments)",
self.profile.summary(),
self.per_segment.len()
)
}
}
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
2 changes: 1 addition & 1 deletion asap-query-engine/src/stores/sketch_db/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ pub mod schema;
pub mod schema_eviction;
pub mod simple_map_store;

pub use accuracy::{AccuracyKind, AccuracyProfile};
pub use accuracy::{AccuracyEnvelope, AccuracyKind, AccuracyProfile, PerSegmentAccuracy};
pub use backfill::{
BackfillJob, BackfillRegistry, BackfillSource, BackfillStatus, Coverage, CreateError,
};
Expand Down
Loading