From ac0268c0fdfc6ba058b5190e844bbb5468bdf971 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 14:16:07 -0600 Subject: [PATCH 1/2] Evaluate ClickHouse SQL corpus on current main --- .../examples/audit_clickhouse_corpus.rs | 164 +++++++++ .../examples/audit_clickhouse_fallback.rs | 138 +++++++ tools/o11y-sql-main-eval/REPORT.md | 121 ++++++ .../o11y-sql-main-eval/artifacts/baseline.txt | 1 + .../artifacts/corpus.sha256 | 1 + .../artifacts/data-plane-routing.json | 139 +++++++ .../frontend-planner-publication.json | 298 +++++++++++++++ .../o11y-sql-main-eval/artifacts/matrix.json | 347 ++++++++++++++++++ tools/o11y-sql-main-eval/corpus.json | 342 +++++++++++++++++ .../evaluation_query_datasets.snapshot.yaml | 27 ++ tools/o11y-sql-main-eval/reproduce.sh | 24 ++ tools/o11y-sql-main-eval/summarize.py | 46 +++ 12 files changed, 1648 insertions(+) create mode 100644 control_plane/examples/audit_clickhouse_corpus.rs create mode 100644 data_plane/examples/audit_clickhouse_fallback.rs create mode 100644 tools/o11y-sql-main-eval/REPORT.md create mode 100644 tools/o11y-sql-main-eval/artifacts/baseline.txt create mode 100644 tools/o11y-sql-main-eval/artifacts/corpus.sha256 create mode 100644 tools/o11y-sql-main-eval/artifacts/data-plane-routing.json create mode 100644 tools/o11y-sql-main-eval/artifacts/frontend-planner-publication.json create mode 100644 tools/o11y-sql-main-eval/artifacts/matrix.json create mode 100644 tools/o11y-sql-main-eval/corpus.json create mode 100644 tools/o11y-sql-main-eval/evaluation_query_datasets.snapshot.yaml create mode 100755 tools/o11y-sql-main-eval/reproduce.sh create mode 100755 tools/o11y-sql-main-eval/summarize.py diff --git a/control_plane/examples/audit_clickhouse_corpus.rs b/control_plane/examples/audit_clickhouse_corpus.rs new file mode 100644 index 000000000..b5187a70f --- /dev/null +++ b/control_plane/examples/audit_clickhouse_corpus.rs @@ -0,0 +1,164 @@ +use asap_frontend_sql::SqlCatalog; +use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; +use control_plane::clickhouse::{ClickHouseSqlWorkload, ClickHouseSqlWorkloadEntry}; +use control_plane::physical::compiler::{ + PlanEnvelope, PrecomputePlan, TransmissionPlan, BACKEND_COMPAT, PLANNER_REVISION, +}; +use planner_types::pre_asap::{Column, DataType, Schema}; +use planner_types::types::AccuracyTarget; +use serde::Deserialize; +use serde_json::json; +use std::{fs, path::PathBuf}; + +#[derive(Deserialize)] +struct Corpus { + queries: Vec, +} + +#[derive(Deserialize)] +struct Row { + id: String, + clickhouse_sql: Option, + operators: Vec, +} + +fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { + let mut materialization = PrecomputeMaterialization::new( + AggregationType::MinMax, + String::new(), + std::collections::HashMap::from([("variant".into(), json!(2))]), + KeyByLabelNames::new(vec!["labels".into()]), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 60, + 60, + WindowKind::Tumbling, + String::new(), + "raw_samples.value".into(), + None, + Some("raw_samples".into()), + Some("value".into()), + ); + materialization.pane_origin_ms = Some(0); + let envelope = PlanEnvelope { + plan_id: 27, + plan_version: 1, + generated_at_unix_ms: 1_788_891_296_000, + activation_unix_ms: 1_788_891_296_000, + expiry_unix_ms: None, + backend_compat: BACKEND_COMPAT.into(), + planner_revision: PLANNER_REVISION.into(), + capability_snapshot_id: "sql27-main-eval".into(), + }; + let mut precompute_plan = + PrecomputePlan::build_backend_local(envelope.clone(), vec![materialization.clone()]) + .unwrap(); + let mut transmission_plan = + TransmissionPlan::build(envelope, &precompute_plan, &Default::default()).unwrap(); + let sds = asap_types::summary_catalog::SummaryCatalog::from_materializations( + 27, + 1, + &[materialization], + ) + .unwrap(); + let reference = sds.reference().unwrap(); + precompute_plan.summary_catalog = Some(reference.clone()); + transmission_plan.summary_catalog = Some(reference); + ClickHouseSqlWorkload { + sds, + precompute_plan, + transmission_plan, + tables: std::collections::HashMap::from([("raw_samples".into(), schema.clone())]), + accuracy: AccuracyTarget::Epsilon(0.01), + queries: vec![ClickHouseSqlWorkloadEntry { + sql, + start_ms: 1_788_848_096_000, + end_ms: 1_788_891_296_000, + cumulative: false, + }], + } +} + +#[tokio::main] +async fn main() { + let path = std::env::args_os() + .nth(1) + .map(PathBuf::from) + .expect("usage: audit_clickhouse_corpus CORPUS.json"); + let corpus: Corpus = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + let schema = Schema::with_time_index( + vec![ + Column::new("metric", DataType::Utf8, false), + Column::new("labels", DataType::Utf8, false), + Column::new("ts_ms", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ], + 2, + vec![], + ); + let catalog = SqlCatalog::new().with_table("raw_samples", schema.clone()); + let accuracy = AccuracyTarget::Epsilon(0.01); + let mut rows = Vec::new(); + for row in corpus.queries { + let Some(sql) = row.clickhouse_sql else { + rows.push(json!({ + "id": row.id, + "operators": row.operators, + "parser_planner": "failure", + "reason": "missing ClickHouse SQL mapping" + })); + continue; + }; + let sql = sql.replace("{eval_ms}", "1788891296000"); + match control_plane::clickhouse::plan_clickhouse_sql(&sql, &catalog, accuracy.clone()).await + { + Ok(planned) => { + let publication = control_plane::clickhouse::compile_clickhouse_workload( + &publication_inputs(&schema, sql.clone()), + ) + .await; + match publication { + Ok(publication) => rows.push(json!({ + "id": row.id, + "operators": row.operators, + "parser_planner": "pass", + "publication": "pass", + "classification": "publication_ready", + "query_plan_nodes": publication.query_plan.entries.values().next().map(|entry| &entry.nodes), + "canonical_sql": planned.canonical_sql, + "physical": format!("{:?}", planned.physical), + "sql": sql, + })), + Err(error) => rows.push(json!({ + "id": row.id, + "operators": row.operators, + "parser_planner": "pass", + "publication": "failure", + "classification": "failure", + "reason": error.to_string(), + "canonical_sql": planned.canonical_sql, + "physical": format!("{:?}", planned.physical), + "sql": sql, + })), + } + } + Err(error) => rows.push(json!({ + "id": row.id, + "operators": row.operators, + "parser_planner": "failure", + "reason": error.to_string(), + "sql": sql, + })), + } + } + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "baseline": env!("CARGO_PKG_VERSION"), + "query_count": rows.len(), + "queries": rows, + })) + .unwrap() + ); +} diff --git a/data_plane/examples/audit_clickhouse_fallback.rs b/data_plane/examples/audit_clickhouse_fallback.rs new file mode 100644 index 000000000..6255c4377 --- /dev/null +++ b/data_plane/examples/audit_clickhouse_fallback.rs @@ -0,0 +1,138 @@ +use axum::{ + body::Bytes, + http::{HeaderMap, Method}, +}; +use control_plane::{ + physical::compiler::{ + PlanEnvelope, PrecomputePlan, TransmissionPlan, BACKEND_COMPAT, PLANNER_REVISION, + }, + query_plan::{ClickHousePlanningContext, QueryPlan}, +}; +use data_plane::{ + drivers::query::servers::http::{build_active_physical_plan, PhysicalPlanInstallRequest}, + query_engines::asap_clickhouse_query_engine::{ + accelerator::CatalogClickHouseAccelerator, ClickHouseAccelerationOutcome, + ClickHouseAccelerator, + }, + storage_engines::{ + sketch_db::index::SketchStore, + types::{BackendStorageRouting, HotReloadActivePhysicalPlan}, + }, +}; +use planner_types::{ + pre_asap::{Column, DataType, Schema}, + types::AccuracyTarget, +}; +use serde::Deserialize; +use serde_json::json; +use std::{ + collections::{BTreeMap, HashMap}, + fs, + path::PathBuf, + sync::Arc, +}; + +#[derive(Deserialize)] +struct Corpus { + queries: Vec, +} + +#[derive(Deserialize)] +struct Row { + id: String, + clickhouse_sql: Option, +} + +#[tokio::main] +async fn main() { + let path = std::env::args_os() + .nth(1) + .map(PathBuf::from) + .expect("usage: audit_clickhouse_fallback CORPUS.json"); + let corpus: Corpus = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + let schema = Schema::with_time_index( + vec![ + Column::new("metric", DataType::Utf8, false), + Column::new("labels", DataType::Utf8, false), + Column::new("ts_ms", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ], + 2, + vec![], + ); + let envelope = PlanEnvelope { + plan_id: 27, + plan_version: 1, + generated_at_unix_ms: 1_788_891_296_000, + activation_unix_ms: 1_788_891_296_000, + expiry_unix_ms: None, + backend_compat: BACKEND_COMPAT.into(), + planner_revision: PLANNER_REVISION.into(), + capability_snapshot_id: "sql27-main-eval".into(), + }; + let catalog = + asap_types::summary_catalog::SummaryCatalog::from_materializations(27, 1, &[]).unwrap(); + let reference = catalog.reference().unwrap(); + let mut precompute_plan = + PrecomputePlan::build_backend_local(envelope.clone(), vec![]).unwrap(); + precompute_plan.summary_catalog = Some(reference.clone()); + let mut transmission_plan = + TransmissionPlan::build(envelope, &precompute_plan, &BTreeMap::new()).unwrap(); + transmission_plan.summary_catalog = Some(reference); + let query_plan = QueryPlan { + plan_id: 27, + plan_version: 1, + clickhouse_context: Some(ClickHousePlanningContext { + tables: HashMap::from([("raw_samples".into(), schema)]), + accuracy: AccuracyTarget::Epsilon(0.01), + }), + entries: BTreeMap::new(), + }; + let active = build_active_physical_plan( + PhysicalPlanInstallRequest { + summary_catalog: catalog, + collector_plans: vec![], + precompute_plan, + transmission_plan, + query_plan, + storage_routing: None, + adaptation_evidence: vec![], + }, + Arc::new(BackendStorageRouting::empty()), + ) + .unwrap(); + let accelerator = CatalogClickHouseAccelerator::with_active_physical_plan( + Arc::new(SketchStore::new()), + HotReloadActivePhysicalPlan::new(active), + ); + let mut rows = Vec::new(); + for row in corpus.queries { + let sql = row + .clickhouse_sql + .unwrap_or_default() + .replace("{eval_ms}", "1788891296000"); + let request = data_plane::query_engines::asap_clickhouse_query_engine::request::ClickHouseQueryRequest { + method: Method::GET, + sql, + body: Bytes::new(), + parameters: BTreeMap::new(), + headers: HeaderMap::new(), + }; + let outcome = accelerator.execute(&request).await; + rows.push(match outcome { + ClickHouseAccelerationOutcome::Fallback(reason) => json!({ + "id": row.id, + "route": "exact_fallback", + "reason": format!("{reason:?}"), + }), + ClickHouseAccelerationOutcome::Accelerated(_) => json!({ + "id": row.id, + "route": "warm", + }), + }); + } + println!( + "{}", + serde_json::to_string_pretty(&json!({"queries": rows})).unwrap() + ); +} diff --git a/tools/o11y-sql-main-eval/REPORT.md b/tools/o11y-sql-main-eval/REPORT.md new file mode 100644 index 000000000..8928f58f0 --- /dev/null +++ b/tools/o11y-sql-main-eval/REPORT.md @@ -0,0 +1,121 @@ +# ClickHouse SQL 27-query acceptance matrix on current `main` + +This report records a fresh run against backend commit +`791f7d7b0feab3827e7e6f5100e65ef29aec68de`. That commit contains the merged +shared-DAG ClickHouse path from #589. The corpus is the 27-query o11y corpus +previously used by #560, but none of #560's classifications or runtime output +is reused. Its SHA-256 is recorded in `artifacts/corpus.sha256`. + +## Result + +| Mode | Queries | +|---|---:| +| warm | 0 | +| partial hybrid | 0 | +| exact fallback | 27 | +| request failure | 0 | + +The parser and Planner accept q05, q06, and q23 and select an exact MinMax +summary followed by shared relational Project/Sort/Limit nodes. Publication +rejects all three because the `metric = ...` table predicate has no canonical +population binding in the summary catalog. The other 24 queries fail closed in +the SQL frontend. The data-plane accelerator was invoked for every query using +an installed, validated current-generation catalog. It returns `Planning` for +the 24 frontend misses and `CatalogMiss` for the three entries whose publication +was refused. The HTTP adapter maps both outcomes to the configured exact +ClickHouse backend, so these are exact fallbacks rather than failed requests. + +The machine-readable per-query evidence is in `artifacts/matrix.json`. Full +canonical AST, selected post-ASAP tree, publication result, and data-plane +routing reason are retained in the other JSON artifacts. + +## Operator and dialect gaps + +The first failing stage is more useful than assigning each query to its intended +PromQL operator: most SQL mappings encode Prometheus semantics with nested +ClickHouse-native expressions, so the frontend cannot reach the logical +aggregation operator. + +| First gap | Count | Queries | +|---|---:|---| +| Nested SELECT/alias parsing (`found: ,`) | 7 | q08, q13, q14, q19, q20, q22, q24 | +| Qualified derived-table columns (`found: .`) | 4 | q01, q02, q17, q25 | +| Empty `map()` / derived alias parsing (`found: )`) | 3 | q03, q04, q16 | +| Metric population predicate cannot bind to SDS | 3 | q05, q06, q23 | +| Alias parsing after derived relation (`found: labels`) | 3 | q11, q18, q26 | +| ClickHouse modulo operator/function | 2 | q10, q27 | +| `mapConcat` | 1 | q07 | +| Zero-argument `map()` | 1 | q09 | +| Map indexing | 1 | q12 | +| Array indexing | 1 | q15 | +| Tuple field access | 1 | q21 | + +The corpus contains 13 `sum`, 11 `rate`, 8 `increase`, 8 division, 8 `topk`, +6 `max_over_time`, 4 subquery, 2 offset, and one each of sorting, comparison, +histogram quantile, scalar, and `avg_over_time` workloads. These counts describe +the intended workload; they are not claims of SQL support. Only the three +MinMax workloads currently reach post-ASAP selection. + +The SDS predicate failure is the smallest production gap, but accepting it +requires a typed population identity for table predicates and matching +ClickHouse backfill/ingest routing. Encoding the predicate as an ad hoc string +inside the runner would produce a catalog entry that the producer cannot safely +maintain, so this evaluation keeps the production fail-closed behavior. + +## Data-structure drift audit + +The merged production path has one backend query-plan identity and node model: + +- `asap_types::QueryLanguage` is the backend-wide language tag and is re-exported + by `control_plane::query_plan`; storage adapters also re-export the same type. +- `QueryNodeId`, `QueryPlanNode`, `MaterializationBinding`, `PhysicalGrouping`, + `ExternalExactRequest`, and `FixedEvaluationRange` each have one definition in + `control_plane::query_plan`. ClickHouse compilation and both data-plane engines + consume those definitions directly. +- Summary identities are `asap_types::sds::SummaryDefinitionId` end to end. + `MaterializationBinding` references that ID and validation resolves it through + the authoritative `SummaryCatalog` before installation. +- Relational node schemas use ASAPPlanner's `SummarySchema` in the published + shared DAG and in data-plane execution. `ClickHouseRelation` is a runtime row + carrier and does not define a second published schema contract. +- `ClickHousePlanningContext` is embedded in the shared `QueryPlan`; there is no + ClickHouse sidecar plan catalog, activation generation, materialization ID, or + parallel execution-node enum on `main`. + +Two similarly named language types remain across the Planner boundary: +`planner_types::workload::QueryLanguage` describes Planner workload input and +`asap_types::QueryLanguage` identifies backend serving entries. Their variants +and roles differ, and conversion happens at compilation; they are not parallel +backend catalogs. The SQL-specific request/response, table row adapter, schema +catalog, and exact HTTP backend types are protocol adapters around the shared +plan. + +The `data_plane::query_engines::canonical` module is production code: the SQL +relational adapter uses its common executor and `RelationalAdapter` trait. Its +comment calls the underlying PromQL file location a compatibility location, +but the exposed types are shared and no stale SQL-only duplicate was found. +The large hand-built plans in ClickHouse accelerator tests are test fixtures; +they mirror wire values to exercise coverage, typed exact leaves, joins, and +relational nodes, but they are not a production planning path. + +## Reproduction + +From a checkout descended from the recorded baseline: + +```bash +tools/o11y-sql-main-eval/reproduce.sh +``` + +Artifacts: + +- `artifacts/frontend-planner-publication.json` +- `artifacts/data-plane-routing.json` +- `artifacts/matrix.json` +- `artifacts/baseline.txt` +- `artifacts/corpus.sha256` + +The run validates routing through the in-process data-plane accelerator. It does +not claim differential value equality against a live ClickHouse instance because +no query is published into a warm or hybrid state on this corpus. A live exact +server remains necessary for the separate HTTP differential E2E and latency +experiment. diff --git a/tools/o11y-sql-main-eval/artifacts/baseline.txt b/tools/o11y-sql-main-eval/artifacts/baseline.txt new file mode 100644 index 000000000..2dcc8d9a9 --- /dev/null +++ b/tools/o11y-sql-main-eval/artifacts/baseline.txt @@ -0,0 +1 @@ +791f7d7b0feab3827e7e6f5100e65ef29aec68de Add a strict multi-fit ERP Figure 1 runner (#592) diff --git a/tools/o11y-sql-main-eval/artifacts/corpus.sha256 b/tools/o11y-sql-main-eval/artifacts/corpus.sha256 new file mode 100644 index 000000000..5642a1564 --- /dev/null +++ b/tools/o11y-sql-main-eval/artifacts/corpus.sha256 @@ -0,0 +1 @@ +ec6db7ca425a3a77e8a810de5ed2928ca6d535476f8bb10f4e68a15b8887b24a /mydata/asapquery-sql27-main/tools/o11y-sql-main-eval/corpus.json diff --git a/tools/o11y-sql-main-eval/artifacts/data-plane-routing.json b/tools/o11y-sql-main-eval/artifacts/data-plane-routing.json new file mode 100644 index 000000000..d68a8b767 --- /dev/null +++ b/tools/o11y-sql-main-eval/artifacts/data-plane-routing.json @@ -0,0 +1,139 @@ +{ + "queries": [ + { + "id": "q01", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: .\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q02", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: .\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q03", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: identifier, found: )\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q04", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: identifier, found: )\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q05", + "reason": "CatalogMiss", + "route": "exact_fallback" + }, + { + "id": "q06", + "reason": "CatalogMiss", + "route": "exact_fallback" + }, + { + "id": "q07", + "reason": "Planning(\"SQL lowering failed: DataFusion error: Error during planning: Invalid function 'mapconcat'.\\nDid you mean 'concat'?\")", + "route": "exact_fallback" + }, + { + "id": "q08", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q09", + "reason": "Planning(\"SQL lowering failed: DataFusion error: Error during planning: Error during planning: map does not support zero arguments. No function matches the given name and argument types 'map()'. You might need to add explicit type casts.\\n\\tCandidate functions:\\n\\tmap(Any, .., Any)\")", + "route": "exact_fallback" + }, + { + "id": "q10", + "reason": "Planning(\"SQL lowering failed: DataFusion error: Error during planning: Invalid function 'modulo'.\\nDid you mean 'md5'?\")", + "route": "exact_fallback" + }, + { + "id": "q11", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: labels\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q12", + "reason": "Planning(\"SQL lowering failed: DataFusion error: This feature is not implemented: Map Access\")", + "route": "exact_fallback" + }, + { + "id": "q13", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q14", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q15", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: [\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q16", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: identifier, found: )\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q17", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: .\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q18", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: labels\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q19", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q20", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q21", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: ), found: .1\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q22", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q23", + "reason": "CatalogMiss", + "route": "exact_fallback" + }, + { + "id": "q24", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q25", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: .\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q26", + "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: labels\\\")\")", + "route": "exact_fallback" + }, + { + "id": "q27", + "reason": "Planning(\"SQL lowering failed: DataFusion error: Error during planning: Invalid function 'modulo'.\\nDid you mean 'today'?\")", + "route": "exact_fallback" + } + ] +} diff --git a/tools/o11y-sql-main-eval/artifacts/frontend-planner-publication.json b/tools/o11y-sql-main-eval/artifacts/frontend-planner-publication.json new file mode 100644 index 000000000..dfa9160bc --- /dev/null +++ b/tools/o11y-sql-main-eval/artifacts/frontend-planner-publication.json @@ -0,0 +1,298 @@ +{ + "baseline": "0.1.0", + "queries": [ + { + "id": "q01", + "operators": [ + "sort_desc", + "increase", + "division" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: .\")", + "sql": "SELECT * FROM (SELECT a.labels,a.value/b.value value FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=((1788891296000-0)-21600000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) a INNER JOIN (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_requests_total' AND ts_ms>=((1788891296000-0)-21600000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) b ON a.labels=b.labels) ORDER BY value DESC,labels" + }, + { + "id": "q02", + "operators": [ + "topk", + "increase", + "division" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: .\")", + "sql": "SELECT * FROM (SELECT a.labels,a.value/b.value value FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=((1788891296000-0)-21600000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) a INNER JOIN (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_requests_total' AND ts_ms>=((1788891296000-0)-21600000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) b ON a.labels=b.labels) ORDER BY value DESC,labels LIMIT 1" + }, + { + "id": "q03", + "operators": [ + "increase", + "sum", + "division" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: identifier, found: )\")", + "sql": "SELECT a.labels,a.value/b.value value FROM (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='payment_service_http_5xx_total' AND ts_ms>=((1788891296000-0)-3600000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) a INNER JOIN (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='payment_service_http_requests_total' AND ts_ms>=((1788891296000-0)-3600000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) b ON a.labels=b.labels" + }, + { + "id": "q04", + "operators": [ + "increase", + "sum", + "offset", + "division" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: identifier, found: )\")", + "sql": "SELECT a.labels,a.value/b.value value FROM (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='payment_service_http_5xx_total' AND ts_ms>=((1788891296000-21600000)-3600000) AND ts_ms<=(1788891296000-21600000) GROUP BY labels) WHERE n>=2))) a INNER JOIN (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='payment_service_http_requests_total' AND ts_ms>=((1788891296000-21600000)-3600000) AND ts_ms<=(1788891296000-21600000) GROUP BY labels) WHERE n>=2))) b ON a.labels=b.labels" + }, + { + "canonical_sql": "Sort { keys: [SortKey { expr: Column(0), ascending: true, nulls_first: false }], partition_by: GroupKeys { keys: [], without: false }, child: Project { cols: [ProjectItem { alias: None, expr: Column(0) }, ProjectItem { alias: Some(\"value\"), expr: Column(1) }], qualifier: None, child: Aggregate { reduction: Reduce(GroupKeys { keys: [1], without: false }), measures: [Max { col: Some(3) }], output_names: [\"max(raw_samples.value)\"], having: None, child: Scan { source: Table { table_ref: \"raw_samples\" }, predicates: [Predicate(BoolAnd([Compare { left: Column(0), op: Eq, right: Literal(Utf8(\"cache_refresh_lag_seconds\")) }, Compare { left: Column(2), op: Gt, right: Arithmetic { op: Sub, left: Literal(Int64(1788891296000)), right: Literal(Int64(43200000)) } }, Compare { left: Column(2), op: Le, right: Literal(Int64(1788891296000)) }]))], schema: Schema { columns: [Column { name: \"metric\", dtype: Utf8, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"labels\", dtype: Utf8, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"ts_ms\", dtype: Timestamp, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"value\", dtype: Float64, nullable: false, table: Some(\"raw_samples\") }], time_index: Some(2), unique_keys: [], closed: true } } } } }", + "classification": "failure", + "id": "q05", + "operators": [ + "max_over_time" + ], + "parser_planner": "pass", + "physical": "Committed(Summary(SummaryNode { expr: ValueOperation { child: SummaryNode { expr: ValueOperation { child: SummaryNode { expr: SummaryAgg { child: SummaryNode { expr: KeepPreAsap(Scan { source: Table { table_ref: \"raw_samples\" }, predicates: [Predicate(BoolAnd([Compare { left: Column(0), op: Eq, right: Literal(Utf8(\"cache_refresh_lag_seconds\")) }, Compare { left: Column(2), op: Gt, right: Arithmetic { op: Sub, left: Literal(Int64(1788891296000)), right: Literal(Int64(43200000)) } }, Compare { left: Column(2), op: Le, right: Literal(Int64(1788891296000)) }]))], schema: Schema { columns: [Column { name: \"metric\", dtype: Utf8, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"labels\", dtype: Utf8, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"ts_ms\", dtype: Timestamp, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"value\", dtype: Float64, nullable: false, table: Some(\"raw_samples\") }], time_index: Some(2), unique_keys: [], closed: true } }), schema: SummarySchema { fields: [SummaryField { name: \"metric\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"labels\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"ts_ms\", dtype: Plain(Timestamp), nullable: false }, SummaryField { name: \"value\", dtype: Plain(Float64), nullable: false }], time_index: Some(2) }, guarantee: Some(ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"KeepPreAsap\" }] }) }, family: ExactAggregate(MinMax, MinMax), input: SummaryUpdate { item: None, weight: Column(Qualified { table: \"raw_samples\", name: \"value\" }), weight_domain: UnknownOrSigned }, reduction: Reduce(GroupKeys { keys: [1], without: false }), grouping: PerSubpopulationInstance }, schema: SummarySchema { fields: [SummaryField { name: \"labels\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"max(raw_samples.value)\", dtype: ExactAggregate(MinMax, MinMax), nullable: false }], time_index: None }, guarantee: Some(ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"ExactAggregate(MinMax)\" }, ChildGuarantee { input_index: 0, guarantee: ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"KeepPreAsap\" }] } }, CompositionStep { operator: ExactExtremum, rule: \"exact_input\" }] }) }, operation: Project { cols: [ProjectItem { alias: None, expr: Column(0) }, ProjectItem { alias: Some(\"value\"), expr: Column(1) }], qualifier: None }, timing: ReadTime }, schema: SummarySchema { fields: [SummaryField { name: \"labels\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"value\", dtype: Plain(Float64), nullable: false }], time_index: None }, guarantee: Some(ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"ExactAggregate(MinMax)\" }, ChildGuarantee { input_index: 0, guarantee: ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"KeepPreAsap\" }] } }, CompositionStep { operator: ExactExtremum, rule: \"exact_input\" }] }) }, operation: Sort { keys: [SortKey { expr: Column(0), ascending: true, nulls_first: false }], partition_by: GroupKeys { keys: [], without: false } }, timing: ReadTime }, schema: SummarySchema { fields: [SummaryField { name: \"labels\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"value\", dtype: Plain(Float64), nullable: false }], time_index: None }, guarantee: Some(ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"ExactAggregate(MinMax)\" }, ChildGuarantee { input_index: 0, guarantee: ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"KeepPreAsap\" }] } }, CompositionStep { operator: ExactExtremum, rule: \"exact_input\" }] }) }))", + "publication": "failure", + "reason": "SQL lowering failed: invalid QueryPlan: SQL population predicate on metric needs a canonical catalog filter", + "sql": "SELECT labels, max(value) AS value FROM raw_samples WHERE metric='cache_refresh_lag_seconds' AND ts_ms>1788891296000-43200000 AND ts_ms<=1788891296000 GROUP BY labels ORDER BY labels" + }, + { + "canonical_sql": "Sort { keys: [SortKey { expr: Column(0), ascending: true, nulls_first: false }], partition_by: GroupKeys { keys: [], without: false }, child: Project { cols: [ProjectItem { alias: None, expr: Column(0) }, ProjectItem { alias: Some(\"value\"), expr: Column(1) }], qualifier: None, child: Aggregate { reduction: Reduce(GroupKeys { keys: [1], without: false }), measures: [Max { col: Some(3) }], output_names: [\"max(raw_samples.value)\"], having: None, child: Scan { source: Table { table_ref: \"raw_samples\" }, predicates: [Predicate(BoolAnd([Compare { left: Column(0), op: Eq, right: Literal(Utf8(\"user_service_cache_refresh_lag_seconds\")) }, Compare { left: Column(2), op: Gt, right: Arithmetic { op: Sub, left: Literal(Int64(1788891296000)), right: Literal(Int64(43200000)) } }, Compare { left: Column(2), op: Le, right: Literal(Int64(1788891296000)) }]))], schema: Schema { columns: [Column { name: \"metric\", dtype: Utf8, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"labels\", dtype: Utf8, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"ts_ms\", dtype: Timestamp, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"value\", dtype: Float64, nullable: false, table: Some(\"raw_samples\") }], time_index: Some(2), unique_keys: [], closed: true } } } } }", + "classification": "failure", + "id": "q06", + "operators": [ + "max_over_time" + ], + "parser_planner": "pass", + "physical": "Committed(Summary(SummaryNode { expr: ValueOperation { child: SummaryNode { expr: ValueOperation { child: SummaryNode { expr: SummaryAgg { child: SummaryNode { expr: KeepPreAsap(Scan { source: Table { table_ref: \"raw_samples\" }, predicates: [Predicate(BoolAnd([Compare { left: Column(0), op: Eq, right: Literal(Utf8(\"user_service_cache_refresh_lag_seconds\")) }, Compare { left: Column(2), op: Gt, right: Arithmetic { op: Sub, left: Literal(Int64(1788891296000)), right: Literal(Int64(43200000)) } }, Compare { left: Column(2), op: Le, right: Literal(Int64(1788891296000)) }]))], schema: Schema { columns: [Column { name: \"metric\", dtype: Utf8, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"labels\", dtype: Utf8, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"ts_ms\", dtype: Timestamp, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"value\", dtype: Float64, nullable: false, table: Some(\"raw_samples\") }], time_index: Some(2), unique_keys: [], closed: true } }), schema: SummarySchema { fields: [SummaryField { name: \"metric\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"labels\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"ts_ms\", dtype: Plain(Timestamp), nullable: false }, SummaryField { name: \"value\", dtype: Plain(Float64), nullable: false }], time_index: Some(2) }, guarantee: Some(ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"KeepPreAsap\" }] }) }, family: ExactAggregate(MinMax, MinMax), input: SummaryUpdate { item: None, weight: Column(Qualified { table: \"raw_samples\", name: \"value\" }), weight_domain: UnknownOrSigned }, reduction: Reduce(GroupKeys { keys: [1], without: false }), grouping: PerSubpopulationInstance }, schema: SummarySchema { fields: [SummaryField { name: \"labels\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"max(raw_samples.value)\", dtype: ExactAggregate(MinMax, MinMax), nullable: false }], time_index: None }, guarantee: Some(ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"ExactAggregate(MinMax)\" }, ChildGuarantee { input_index: 0, guarantee: ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"KeepPreAsap\" }] } }, CompositionStep { operator: ExactExtremum, rule: \"exact_input\" }] }) }, operation: Project { cols: [ProjectItem { alias: None, expr: Column(0) }, ProjectItem { alias: Some(\"value\"), expr: Column(1) }], qualifier: None }, timing: ReadTime }, schema: SummarySchema { fields: [SummaryField { name: \"labels\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"value\", dtype: Plain(Float64), nullable: false }], time_index: None }, guarantee: Some(ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"ExactAggregate(MinMax)\" }, ChildGuarantee { input_index: 0, guarantee: ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"KeepPreAsap\" }] } }, CompositionStep { operator: ExactExtremum, rule: \"exact_input\" }] }) }, operation: Sort { keys: [SortKey { expr: Column(0), ascending: true, nulls_first: false }], partition_by: GroupKeys { keys: [], without: false } }, timing: ReadTime }, schema: SummarySchema { fields: [SummaryField { name: \"labels\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"value\", dtype: Plain(Float64), nullable: false }], time_index: None }, guarantee: Some(ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"ExactAggregate(MinMax)\" }, ChildGuarantee { input_index: 0, guarantee: ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"KeepPreAsap\" }] } }, CompositionStep { operator: ExactExtremum, rule: \"exact_input\" }] }) }))", + "publication": "failure", + "reason": "SQL lowering failed: invalid QueryPlan: SQL population predicate on metric needs a canonical catalog filter", + "sql": "SELECT labels, max(value) AS value FROM raw_samples WHERE metric='user_service_cache_refresh_lag_seconds' AND ts_ms>1788891296000-43200000 AND ts_ms<=1788891296000 GROUP BY labels ORDER BY labels" + }, + { + "id": "q07", + "operators": [], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: Error during planning: Invalid function 'mapconcat'.\nDid you mean 'concat'?", + "sql": "SELECT mapConcat(labels,map('__name__','user_service_cache_refresh_lag_seconds')) AS labels, argMax(value,ts_ms) AS value FROM raw_samples WHERE metric='user_service_cache_refresh_lag_seconds' AND ts_ms>1788891296000-300000 AND ts_ms<=1788891296000 GROUP BY labels ORDER BY labels" + }, + { + "id": "q08", + "operators": [ + "rate", + "sum" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: ,\")", + "sql": "SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(3600000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_process_cpu_seconds_total' AND ts_ms>=((1788891296000-0)-3600000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))" + }, + { + "id": "q09", + "operators": [ + "sum" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: Error during planning: Error during planning: map does not support zero arguments. No function matches the given name and argument types 'map()'. You might need to add explicit type casts.\n\tCandidate functions:\n\tmap(Any, .., Any)", + "sql": "SELECT map() AS labels, sum(value) AS value FROM (SELECT labels,argMax(value,ts_ms) AS value FROM raw_samples WHERE metric='backend_process_resident_memory_bytes' AND ts_ms>1788891296000-300000 AND ts_ms<=1788891296000 GROUP BY labels)" + }, + { + "id": "q10", + "operators": [ + "max_over_time", + "sum", + "subquery" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: Error during planning: Invalid function 'modulo'.\nDid you mean 'today'?", + "sql": "SELECT map() labels,max(value) value FROM (SELECT ts_ms,sum(value) value FROM raw_samples WHERE metric='backend_process_resident_memory_bytes' AND ts_ms>=1788891296000-21600000 AND ts_ms<=1788891296000 AND modulo(1788891296000-ts_ms,60000)=0 GROUP BY ts_ms)" + }, + { + "id": "q11", + "operators": [ + "topk", + "rate" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: labels\")", + "sql": "SELECT * FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(3600000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_process_cpu_seconds_total' AND ts_ms>=((1788891296000-0)-3600000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) ORDER BY value DESC,labels LIMIT 2" + }, + { + "id": "q12", + "operators": [ + "topk" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: This feature is not implemented: Map Access", + "sql": "SELECT map('job',job) AS labels,sum(value) AS value FROM (SELECT labels['job'] AS job,labels,argMax(value,ts_ms) AS value FROM raw_samples WHERE metric='backend_process_resident_memory_bytes' AND ts_ms>1788891296000-300000 AND ts_ms<=1788891296000 GROUP BY job,labels) GROUP BY job ORDER BY value DESC,job LIMIT 2" + }, + { + "id": "q13", + "operators": [ + "rate", + "max_over_time", + "sum", + "subquery" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: ,\")", + "sql": "SELECT map() labels,max(value) value FROM (SELECT grid_ms,sum(value) value FROM (SELECT grid_ms,labels,corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/300 value FROM (SELECT grid_ms,labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2x.1,groupArray((ts_ms,value))) samples FROM raw_samples CROSS JOIN (SELECT arrayJoin(arrayMap(x->x*60000,range(toUInt64(intDiv(1788891296000-21600000+59999,60000)),toUInt64(intDiv(1788891296000,60000)+1)))) grid_ms) grids WHERE metric='backend_process_cpu_seconds_total' AND ts_ms>=grid_ms-300000 AND ts_ms<=grid_ms GROUP BY grid_ms,labels) WHERE n>=2)) GROUP BY grid_ms)" + }, + { + "id": "q14", + "operators": [ + "rate", + "sum" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: ,\")", + "sql": "SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(300000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_requests_total' AND ts_ms>=((1788891296000-0)-300000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))" + }, + { + "id": "q15", + "operators": [ + "rate" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: [\")", + "sql": "SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(300000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_requests_total' AND ts_ms>=((1788891296000-0)-300000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) GROUP BY job" + }, + { + "id": "q16", + "operators": [ + "increase", + "sum", + "division" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: identifier, found: )\")", + "sql": "SELECT a.labels,a.value/b.value value FROM (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=((1788891296000-0)-3600000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) a INNER JOIN (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_requests_total' AND ts_ms>=((1788891296000-0)-3600000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) b ON a.labels=b.labels" + }, + { + "id": "q17", + "operators": [ + "topk", + "increase", + "division" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: .\")", + "sql": "SELECT * FROM (SELECT a.labels,a.value/b.value value FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=((1788891296000-0)-21600000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) a INNER JOIN (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_requests_total' AND ts_ms>=((1788891296000-0)-21600000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) b ON a.labels=b.labels) ORDER BY value DESC,labels LIMIT 1" + }, + { + "id": "q18", + "operators": [ + "increase", + "comparison" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: labels\")", + "sql": "SELECT * FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=((1788891296000-0)-86400000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) WHERE value>0" + }, + { + "id": "q19", + "operators": [ + "rate", + "sum" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: ,\")", + "sql": "SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(300000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='order_service_http_requests_total' AND ts_ms>=((1788891296000-0)-300000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))" + }, + { + "id": "q20", + "operators": [ + "rate", + "sum", + "offset" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: ,\")", + "sql": "SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(300000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='order_service_http_requests_total' AND ts_ms>=((1788891296000-3600000)-300000) AND ts_ms<=(1788891296000-3600000) GROUP BY labels) WHERE n>=2))" + }, + { + "id": "q21", + "operators": [ + "histogram_quantile", + "rate" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: ), found: .1\")", + "sql": "SELECT map() labels,if(idx=length(buckets),buckets[idx-1].1,if(idx=1 AND buckets[idx].1<=0,buckets[idx].1,(if(idx=1,0.,buckets[idx-1].1)+(buckets[idx].1-if(idx=1,0.,buckets[idx-1].1))*(rank-if(idx=1,0.,buckets[idx-1].2))/(buckets[idx].2-if(idx=1,0.,buckets[idx-1].2))))) value FROM (SELECT buckets,0.95*buckets[length(buckets)].2 rank,arrayFirstIndex(x->x.2>=rank,buckets) idx FROM (SELECT arrayMap(i->(raw[i].1,arrayMax(arrayMap(x->x.2,arraySlice(raw,1,i)))),range(1,length(raw)+1)) buckets FROM (SELECT arraySort(x->x.1,groupArray((if(le='+Inf',inf,toFloat64(le)),value))) raw FROM (SELECT labels['le'] le,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(300000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='order_service_http_request_duration_seconds_bucket' AND ts_ms>=((1788891296000-0)-300000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2)) GROUP BY le)))) WHERE idx>0" + }, + { + "id": "q22", + "operators": [ + "rate", + "sum" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: ,\")", + "sql": "SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(300000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='order_service_http_requests_total' AND ts_ms>=((1788891296000-0)-300000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))" + }, + { + "canonical_sql": "Limit { n: 2, offset: 0, child: Sort { keys: [SortKey { expr: Column(1), ascending: false, nulls_first: true }, SortKey { expr: Column(0), ascending: true, nulls_first: false }], partition_by: GroupKeys { keys: [], without: false }, child: Project { cols: [ProjectItem { alias: None, expr: Column(0) }, ProjectItem { alias: Some(\"value\"), expr: Column(1) }], qualifier: None, child: Aggregate { reduction: Reduce(GroupKeys { keys: [1], without: false }), measures: [Max { col: Some(3) }], output_names: [\"max(raw_samples.value)\"], having: None, child: Scan { source: Table { table_ref: \"raw_samples\" }, predicates: [Predicate(BoolAnd([Compare { left: Column(0), op: Eq, right: Literal(Utf8(\"backend_retry_backlog_depth\")) }, Compare { left: Column(2), op: Gt, right: Arithmetic { op: Sub, left: Literal(Int64(1788891296000)), right: Literal(Int64(21600000)) } }, Compare { left: Column(2), op: Le, right: Literal(Int64(1788891296000)) }]))], schema: Schema { columns: [Column { name: \"metric\", dtype: Utf8, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"labels\", dtype: Utf8, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"ts_ms\", dtype: Timestamp, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"value\", dtype: Float64, nullable: false, table: Some(\"raw_samples\") }], time_index: Some(2), unique_keys: [], closed: true } } } } } }", + "classification": "failure", + "id": "q23", + "operators": [ + "topk", + "max_over_time" + ], + "parser_planner": "pass", + "physical": "Committed(Summary(SummaryNode { expr: ValueOperation { child: SummaryNode { expr: ValueOperation { child: SummaryNode { expr: ValueOperation { child: SummaryNode { expr: SummaryAgg { child: SummaryNode { expr: KeepPreAsap(Scan { source: Table { table_ref: \"raw_samples\" }, predicates: [Predicate(BoolAnd([Compare { left: Column(0), op: Eq, right: Literal(Utf8(\"backend_retry_backlog_depth\")) }, Compare { left: Column(2), op: Gt, right: Arithmetic { op: Sub, left: Literal(Int64(1788891296000)), right: Literal(Int64(21600000)) } }, Compare { left: Column(2), op: Le, right: Literal(Int64(1788891296000)) }]))], schema: Schema { columns: [Column { name: \"metric\", dtype: Utf8, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"labels\", dtype: Utf8, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"ts_ms\", dtype: Timestamp, nullable: false, table: Some(\"raw_samples\") }, Column { name: \"value\", dtype: Float64, nullable: false, table: Some(\"raw_samples\") }], time_index: Some(2), unique_keys: [], closed: true } }), schema: SummarySchema { fields: [SummaryField { name: \"metric\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"labels\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"ts_ms\", dtype: Plain(Timestamp), nullable: false }, SummaryField { name: \"value\", dtype: Plain(Float64), nullable: false }], time_index: Some(2) }, guarantee: Some(ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"KeepPreAsap\" }] }) }, family: ExactAggregate(MinMax, MinMax), input: SummaryUpdate { item: None, weight: Column(Qualified { table: \"raw_samples\", name: \"value\" }), weight_domain: UnknownOrSigned }, reduction: Reduce(GroupKeys { keys: [1], without: false }), grouping: PerSubpopulationInstance }, schema: SummarySchema { fields: [SummaryField { name: \"labels\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"max(raw_samples.value)\", dtype: ExactAggregate(MinMax, MinMax), nullable: false }], time_index: None }, guarantee: Some(ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"ExactAggregate(MinMax)\" }, ChildGuarantee { input_index: 0, guarantee: ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"KeepPreAsap\" }] } }, CompositionStep { operator: ExactExtremum, rule: \"exact_input\" }] }) }, operation: Project { cols: [ProjectItem { alias: None, expr: Column(0) }, ProjectItem { alias: Some(\"value\"), expr: Column(1) }], qualifier: None }, timing: ReadTime }, schema: SummarySchema { fields: [SummaryField { name: \"labels\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"value\", dtype: Plain(Float64), nullable: false }], time_index: None }, guarantee: Some(ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"ExactAggregate(MinMax)\" }, ChildGuarantee { input_index: 0, guarantee: ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"KeepPreAsap\" }] } }, CompositionStep { operator: ExactExtremum, rule: \"exact_input\" }] }) }, operation: Sort { keys: [SortKey { expr: Column(1), ascending: false, nulls_first: true }, SortKey { expr: Column(0), ascending: true, nulls_first: false }], partition_by: GroupKeys { keys: [], without: false } }, timing: ReadTime }, schema: SummarySchema { fields: [SummaryField { name: \"labels\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"value\", dtype: Plain(Float64), nullable: false }], time_index: None }, guarantee: Some(ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"ExactAggregate(MinMax)\" }, ChildGuarantee { input_index: 0, guarantee: ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"KeepPreAsap\" }] } }, CompositionStep { operator: ExactExtremum, rule: \"exact_input\" }] }) }, operation: Limit { n: 2, offset: 0 }, timing: ReadTime }, schema: SummarySchema { fields: [SummaryField { name: \"labels\", dtype: Plain(Utf8), nullable: false }, SummaryField { name: \"value\", dtype: Plain(Float64), nullable: false }], time_index: None }, guarantee: Some(ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"ExactAggregate(MinMax)\" }, ChildGuarantee { input_index: 0, guarantee: ResultGuarantee { metric: AbsoluteValue, bound: Zero, failure_probability: Zero, provenance: [Exact { reason: \"KeepPreAsap\" }] } }, CompositionStep { operator: ExactExtremum, rule: \"exact_input\" }] }) }))", + "publication": "failure", + "reason": "SQL lowering failed: invalid QueryPlan: SQL population predicate on metric needs a canonical catalog filter", + "sql": "SELECT labels,max(value) AS value FROM raw_samples WHERE metric='backend_retry_backlog_depth' AND ts_ms>1788891296000-21600000 AND ts_ms<=1788891296000 GROUP BY labels ORDER BY value DESC,labels LIMIT 2" + }, + { + "id": "q24", + "operators": [ + "rate", + "max_over_time", + "sum", + "subquery", + "division" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: ,\")", + "sql": "SELECT map() labels,max(a.value/b.value) value FROM (SELECT grid_ms,sum(value) value FROM (SELECT grid_ms,labels,corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/300 value FROM (SELECT grid_ms,labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2x.1,groupArray((ts_ms,value))) samples FROM raw_samples CROSS JOIN (SELECT arrayJoin(arrayMap(x->x*60000,range(toUInt64(intDiv(1788891296000-21600000+59999,60000)),toUInt64(intDiv(1788891296000,60000)+1)))) grid_ms) grids WHERE metric='order_service_http_5xx_total' AND ts_ms>=grid_ms-300000 AND ts_ms<=grid_ms GROUP BY grid_ms,labels) WHERE n>=2)) GROUP BY grid_ms) a INNER JOIN (SELECT grid_ms,sum(value) value FROM (SELECT grid_ms,labels,corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/300 value FROM (SELECT grid_ms,labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2x.1,groupArray((ts_ms,value))) samples FROM raw_samples CROSS JOIN (SELECT arrayJoin(arrayMap(x->x*60000,range(toUInt64(intDiv(1788891296000-21600000+59999,60000)),toUInt64(intDiv(1788891296000,60000)+1)))) grid_ms) grids WHERE metric='order_service_http_requests_total' AND ts_ms>=grid_ms-300000 AND ts_ms<=grid_ms GROUP BY grid_ms,labels) WHERE n>=2)) GROUP BY grid_ms) b ON a.grid_ms=b.grid_ms" + }, + { + "id": "q25", + "operators": [ + "topk", + "scalar", + "increase", + "sum", + "division" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: .\")", + "sql": "SELECT * FROM (SELECT a.labels,a.value/b.value value FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=((1788891296000-0)-86400000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) a CROSS JOIN (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=((1788891296000-0)-86400000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) b) ORDER BY value DESC,labels LIMIT 1" + }, + { + "id": "q26", + "operators": [ + "topk", + "rate" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: SQL error: ParserError(\"Expected: joined table, found: labels\")", + "sql": "SELECT * FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(21600000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_process_cpu_seconds_total' AND ts_ms>=((1788891296000-0)-21600000) AND ts_ms<=(1788891296000-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) ORDER BY value DESC,labels LIMIT 1" + }, + { + "id": "q27", + "operators": [ + "topk", + "avg_over_time", + "subquery" + ], + "parser_planner": "failure", + "reason": "SQL lowering failed: DataFusion error: Error during planning: Invalid function 'modulo'.\nDid you mean 'today'?", + "sql": "SELECT map('job',job) labels,avg(value) value FROM (SELECT labels['job'] job,ts_ms,sum(value) value FROM raw_samples WHERE metric='backend_process_resident_memory_bytes' AND ts_ms+4000>=1788891296000-21600000 AND ts_ms+4000<=1788891296000 AND modulo(ts_ms+4000,60000)=0 GROUP BY job,ts_ms) GROUP BY job ORDER BY value DESC,job LIMIT 3" + } + ], + "query_count": 27 +} diff --git a/tools/o11y-sql-main-eval/artifacts/matrix.json b/tools/o11y-sql-main-eval/artifacts/matrix.json new file mode 100644 index 000000000..7690b27ff --- /dev/null +++ b/tools/o11y-sql-main-eval/artifacts/matrix.json @@ -0,0 +1,347 @@ +{ + "schema_version": 1, + "baseline_commit": "791f7d7b0feab3827e7e6f5100e65ef29aec68de", + "corpus_count": 27, + "counts": { + "warm": 0, + "partial_hybrid": 0, + "exact_fallback": 27, + "failure": 0 + }, + "queries": [ + { + "id": "q01", + "operators": [ + "sort_desc", + "increase", + "division" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: .\")" + }, + { + "id": "q02", + "operators": [ + "topk", + "increase", + "division" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: .\")" + }, + { + "id": "q03", + "operators": [ + "increase", + "sum", + "division" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: identifier, found: )\")" + }, + { + "id": "q04", + "operators": [ + "increase", + "sum", + "offset", + "division" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: identifier, found: )\")" + }, + { + "id": "q05", + "operators": [ + "max_over_time" + ], + "parser_planner": "pass", + "publication": "failure", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL population predicate on metric needs a canonical catalog filter" + }, + { + "id": "q06", + "operators": [ + "max_over_time" + ], + "parser_planner": "pass", + "publication": "failure", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL population predicate on metric needs a canonical catalog filter" + }, + { + "id": "q07", + "operators": [], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "Error during planning: Invalid function 'mapconcat'." + }, + { + "id": "q08", + "operators": [ + "rate", + "sum" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" + }, + { + "id": "q09", + "operators": [ + "sum" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "Error during planning: Error during planning: map does not support zero arguments. No function matches the given name and argument types 'map()'. You might need to add explicit type casts." + }, + { + "id": "q10", + "operators": [ + "max_over_time", + "sum", + "subquery" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "Error during planning: Invalid function 'modulo'." + }, + { + "id": "q11", + "operators": [ + "topk", + "rate" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: labels\")" + }, + { + "id": "q12", + "operators": [ + "topk" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "This feature is not implemented: Map Access" + }, + { + "id": "q13", + "operators": [ + "rate", + "max_over_time", + "sum", + "subquery" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" + }, + { + "id": "q14", + "operators": [ + "rate", + "sum" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" + }, + { + "id": "q15", + "operators": [ + "rate" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: [\")" + }, + { + "id": "q16", + "operators": [ + "increase", + "sum", + "division" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: identifier, found: )\")" + }, + { + "id": "q17", + "operators": [ + "topk", + "increase", + "division" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: .\")" + }, + { + "id": "q18", + "operators": [ + "increase", + "comparison" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: labels\")" + }, + { + "id": "q19", + "operators": [ + "rate", + "sum" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" + }, + { + "id": "q20", + "operators": [ + "rate", + "sum", + "offset" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" + }, + { + "id": "q21", + "operators": [ + "histogram_quantile", + "rate" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: ), found: .1\")" + }, + { + "id": "q22", + "operators": [ + "rate", + "sum" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" + }, + { + "id": "q23", + "operators": [ + "topk", + "max_over_time" + ], + "parser_planner": "pass", + "publication": "failure", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL population predicate on metric needs a canonical catalog filter" + }, + { + "id": "q24", + "operators": [ + "rate", + "max_over_time", + "sum", + "subquery", + "division" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" + }, + { + "id": "q25", + "operators": [ + "topk", + "scalar", + "increase", + "sum", + "division" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: .\")" + }, + { + "id": "q26", + "operators": [ + "topk", + "rate" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "SQL error: ParserError(\"Expected: joined table, found: labels\")" + }, + { + "id": "q27", + "operators": [ + "topk", + "avg_over_time", + "subquery" + ], + "parser_planner": "failure", + "publication": "not_reached", + "data_plane_route": "exact_fallback", + "classification": "exact_fallback", + "reason": "Error during planning: Invalid function 'modulo'." + } + ] +} diff --git a/tools/o11y-sql-main-eval/corpus.json b/tools/o11y-sql-main-eval/corpus.json new file mode 100644 index 000000000..58ee835eb --- /dev/null +++ b/tools/o11y-sql-main-eval/corpus.json @@ -0,0 +1,342 @@ +{ + "schema_version": 1, + "source": "source-corpus.json", + "count": 27, + "queries": [ + { + "id": "q01", + "promql": "sort_desc(sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", + "metricsql": "sort_desc(sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", + "clickhouse_sql": "SELECT * FROM (SELECT a.labels,a.value/b.value value FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=(({eval_ms}-0)-21600000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) a INNER JOIN (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_requests_total' AND ts_ms>=(({eval_ms}-0)-21600000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) b ON a.labels=b.labels) ORDER BY value DESC,labels", + "operators": [ + "sort_desc", + "increase", + "division" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q02", + "promql": "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", + "metricsql": "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", + "clickhouse_sql": "SELECT * FROM (SELECT a.labels,a.value/b.value value FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=(({eval_ms}-0)-21600000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) a INNER JOIN (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_requests_total' AND ts_ms>=(({eval_ms}-0)-21600000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) b ON a.labels=b.labels) ORDER BY value DESC,labels LIMIT 1", + "operators": [ + "topk", + "increase", + "division" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q03", + "promql": "sum(increase(payment_service_http_5xx_total[1h])) / sum(increase(payment_service_http_requests_total[1h]))", + "metricsql": "sum(increase(payment_service_http_5xx_total[1h])) / sum(increase(payment_service_http_requests_total[1h]))", + "clickhouse_sql": "SELECT a.labels,a.value/b.value value FROM (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='payment_service_http_5xx_total' AND ts_ms>=(({eval_ms}-0)-3600000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) a INNER JOIN (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='payment_service_http_requests_total' AND ts_ms>=(({eval_ms}-0)-3600000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) b ON a.labels=b.labels", + "operators": [ + "increase", + "sum", + "division" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q04", + "promql": "sum(increase(payment_service_http_5xx_total[1h] offset 6h)) / sum(increase(payment_service_http_requests_total[1h] offset 6h))", + "metricsql": "sum(increase(payment_service_http_5xx_total[1h] offset 6h)) / sum(increase(payment_service_http_requests_total[1h] offset 6h))", + "clickhouse_sql": "SELECT a.labels,a.value/b.value value FROM (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='payment_service_http_5xx_total' AND ts_ms>=(({eval_ms}-21600000)-3600000) AND ts_ms<=({eval_ms}-21600000) GROUP BY labels) WHERE n>=2))) a INNER JOIN (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='payment_service_http_requests_total' AND ts_ms>=(({eval_ms}-21600000)-3600000) AND ts_ms<=({eval_ms}-21600000) GROUP BY labels) WHERE n>=2))) b ON a.labels=b.labels", + "operators": [ + "increase", + "sum", + "offset", + "division" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q05", + "promql": "max_over_time(cache_refresh_lag_seconds[12h])", + "metricsql": "max_over_time(cache_refresh_lag_seconds[12h])", + "clickhouse_sql": "SELECT labels, max(value) AS value FROM raw_samples WHERE metric='cache_refresh_lag_seconds' AND ts_ms>{eval_ms}-43200000 AND ts_ms<={eval_ms} GROUP BY labels ORDER BY labels", + "operators": [ + "max_over_time" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q06", + "promql": "max_over_time(user_service_cache_refresh_lag_seconds[12h])", + "metricsql": "max_over_time(user_service_cache_refresh_lag_seconds[12h])", + "clickhouse_sql": "SELECT labels, max(value) AS value FROM raw_samples WHERE metric='user_service_cache_refresh_lag_seconds' AND ts_ms>{eval_ms}-43200000 AND ts_ms<={eval_ms} GROUP BY labels ORDER BY labels", + "operators": [ + "max_over_time" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q07", + "promql": "user_service_cache_refresh_lag_seconds", + "metricsql": "user_service_cache_refresh_lag_seconds", + "clickhouse_sql": "SELECT mapConcat(labels,map('__name__','user_service_cache_refresh_lag_seconds')) AS labels, argMax(value,ts_ms) AS value FROM raw_samples WHERE metric='user_service_cache_refresh_lag_seconds' AND ts_ms>{eval_ms}-300000 AND ts_ms<={eval_ms} GROUP BY labels ORDER BY labels", + "operators": [], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q08", + "promql": "sum(rate(backend_process_cpu_seconds_total[1h]))", + "metricsql": "sum(rate(backend_process_cpu_seconds_total[1h]))", + "clickhouse_sql": "SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(3600000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_process_cpu_seconds_total' AND ts_ms>=(({eval_ms}-0)-3600000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))", + "operators": [ + "rate", + "sum" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q09", + "promql": "sum(backend_process_resident_memory_bytes)", + "metricsql": "sum(backend_process_resident_memory_bytes)", + "clickhouse_sql": "SELECT map() AS labels, sum(value) AS value FROM (SELECT labels,argMax(value,ts_ms) AS value FROM raw_samples WHERE metric='backend_process_resident_memory_bytes' AND ts_ms>{eval_ms}-300000 AND ts_ms<={eval_ms} GROUP BY labels)", + "operators": [ + "sum" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q10", + "promql": "max_over_time((sum(backend_process_resident_memory_bytes))[6h:])", + "metricsql": "max_over_time((sum(backend_process_resident_memory_bytes))[6h:])", + "clickhouse_sql": "SELECT map() labels,max(value) value FROM (SELECT ts_ms,sum(value) value FROM raw_samples WHERE metric='backend_process_resident_memory_bytes' AND ts_ms>={eval_ms}-21600000 AND ts_ms<={eval_ms} AND modulo({eval_ms}-ts_ms,60000)=0 GROUP BY ts_ms)", + "operators": [ + "max_over_time", + "sum", + "subquery" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q11", + "promql": "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", + "metricsql": "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", + "clickhouse_sql": "SELECT * FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(3600000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_process_cpu_seconds_total' AND ts_ms>=(({eval_ms}-0)-3600000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) ORDER BY value DESC,labels LIMIT 2", + "operators": [ + "topk", + "rate" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q12", + "promql": "topk(2, sum by (job) (backend_process_resident_memory_bytes))", + "metricsql": "topk(2, sum by (job) (backend_process_resident_memory_bytes))", + "clickhouse_sql": "SELECT map('job',job) AS labels,sum(value) AS value FROM (SELECT labels['job'] AS job,labels,argMax(value,ts_ms) AS value FROM raw_samples WHERE metric='backend_process_resident_memory_bytes' AND ts_ms>{eval_ms}-300000 AND ts_ms<={eval_ms} GROUP BY job,labels) GROUP BY job ORDER BY value DESC,job LIMIT 2", + "operators": [ + "topk" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q13", + "promql": "max_over_time((sum(rate(backend_process_cpu_seconds_total[5m])))[6h:])", + "metricsql": "max_over_time((sum(rate(backend_process_cpu_seconds_total[5m])))[6h:])", + "clickhouse_sql": "SELECT map() labels,max(value) value FROM (SELECT grid_ms,sum(value) value FROM (SELECT grid_ms,labels,corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/300 value FROM (SELECT grid_ms,labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2x.1,groupArray((ts_ms,value))) samples FROM raw_samples CROSS JOIN (SELECT arrayJoin(arrayMap(x->x*60000,range(toUInt64(intDiv({eval_ms}-21600000+59999,60000)),toUInt64(intDiv({eval_ms},60000)+1)))) grid_ms) grids WHERE metric='backend_process_cpu_seconds_total' AND ts_ms>=grid_ms-300000 AND ts_ms<=grid_ms GROUP BY grid_ms,labels) WHERE n>=2)) GROUP BY grid_ms)", + "operators": [ + "rate", + "max_over_time", + "sum", + "subquery" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q14", + "promql": "sum(rate(backend_http_requests_total[5m]))", + "metricsql": "sum(rate(backend_http_requests_total[5m]))", + "clickhouse_sql": "SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(300000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_requests_total' AND ts_ms>=(({eval_ms}-0)-300000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))", + "operators": [ + "rate", + "sum" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q15", + "promql": "sum by (job) (rate(backend_http_requests_total[5m]))", + "metricsql": "sum by (job) (rate(backend_http_requests_total[5m]))", + "clickhouse_sql": "SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(300000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_requests_total' AND ts_ms>=(({eval_ms}-0)-300000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) GROUP BY job", + "operators": [ + "rate" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q16", + "promql": "sum(increase(backend_http_5xx_total[60m])) / sum(increase(backend_http_requests_total[60m]))", + "metricsql": "sum(increase(backend_http_5xx_total[60m])) / sum(increase(backend_http_requests_total[60m]))", + "clickhouse_sql": "SELECT a.labels,a.value/b.value value FROM (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=(({eval_ms}-0)-3600000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) a INNER JOIN (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_requests_total' AND ts_ms>=(({eval_ms}-0)-3600000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) b ON a.labels=b.labels", + "operators": [ + "increase", + "sum", + "division" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q17", + "promql": "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", + "metricsql": "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", + "clickhouse_sql": "SELECT * FROM (SELECT a.labels,a.value/b.value value FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=(({eval_ms}-0)-21600000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) a INNER JOIN (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_requests_total' AND ts_ms>=(({eval_ms}-0)-21600000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) b ON a.labels=b.labels) ORDER BY value DESC,labels LIMIT 1", + "operators": [ + "topk", + "increase", + "division" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q18", + "promql": "sum by (job) (increase(backend_http_5xx_total[24h])) > 0", + "metricsql": "sum by (job) (increase(backend_http_5xx_total[24h])) > 0", + "clickhouse_sql": "SELECT * FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=(({eval_ms}-0)-86400000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) WHERE value>0", + "operators": [ + "increase", + "comparison" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q19", + "promql": "sum(rate(order_service_http_requests_total[5m]))", + "metricsql": "sum(rate(order_service_http_requests_total[5m]))", + "clickhouse_sql": "SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(300000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='order_service_http_requests_total' AND ts_ms>=(({eval_ms}-0)-300000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))", + "operators": [ + "rate", + "sum" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q20", + "promql": "sum(rate(order_service_http_requests_total[5m] offset 1h))", + "metricsql": "sum(rate(order_service_http_requests_total[5m] offset 1h))", + "clickhouse_sql": "SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(300000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='order_service_http_requests_total' AND ts_ms>=(({eval_ms}-3600000)-300000) AND ts_ms<=({eval_ms}-3600000) GROUP BY labels) WHERE n>=2))", + "operators": [ + "rate", + "sum", + "offset" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q21", + "promql": "histogram_quantile(0.95, sum by (le) (rate(order_service_http_request_duration_seconds_bucket[5m])))", + "metricsql": "histogram_quantile(0.95, sum by (le) (rate(order_service_http_request_duration_seconds_bucket[5m])))", + "clickhouse_sql": "SELECT map() labels,if(idx=length(buckets),buckets[idx-1].1,if(idx=1 AND buckets[idx].1<=0,buckets[idx].1,(if(idx=1,0.,buckets[idx-1].1)+(buckets[idx].1-if(idx=1,0.,buckets[idx-1].1))*(rank-if(idx=1,0.,buckets[idx-1].2))/(buckets[idx].2-if(idx=1,0.,buckets[idx-1].2))))) value FROM (SELECT buckets,0.95*buckets[length(buckets)].2 rank,arrayFirstIndex(x->x.2>=rank,buckets) idx FROM (SELECT arrayMap(i->(raw[i].1,arrayMax(arrayMap(x->x.2,arraySlice(raw,1,i)))),range(1,length(raw)+1)) buckets FROM (SELECT arraySort(x->x.1,groupArray((if(le='+Inf',inf,toFloat64(le)),value))) raw FROM (SELECT labels['le'] le,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(300000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='order_service_http_request_duration_seconds_bucket' AND ts_ms>=(({eval_ms}-0)-300000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2)) GROUP BY le)))) WHERE idx>0", + "operators": [ + "histogram_quantile", + "rate" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q22", + "promql": "sum(rate(order_service_http_requests_total[5m]))", + "metricsql": "sum(rate(order_service_http_requests_total[5m]))", + "clickhouse_sql": "SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(300000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='order_service_http_requests_total' AND ts_ms>=(({eval_ms}-0)-300000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))", + "operators": [ + "rate", + "sum" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q23", + "promql": "topk(2, max_over_time(backend_retry_backlog_depth[6h]))", + "metricsql": "topk(2, max_over_time(backend_retry_backlog_depth[6h]))", + "clickhouse_sql": "SELECT labels,max(value) AS value FROM raw_samples WHERE metric='backend_retry_backlog_depth' AND ts_ms>{eval_ms}-21600000 AND ts_ms<={eval_ms} GROUP BY labels ORDER BY value DESC,labels LIMIT 2", + "operators": [ + "topk", + "max_over_time" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q24", + "promql": "max_over_time((sum(rate(order_service_http_5xx_total[5m])) / sum(rate(order_service_http_requests_total[5m])))[6h:])", + "metricsql": "max_over_time((sum(rate(order_service_http_5xx_total[5m])) / sum(rate(order_service_http_requests_total[5m])))[6h:])", + "clickhouse_sql": "SELECT map() labels,max(a.value/b.value) value FROM (SELECT grid_ms,sum(value) value FROM (SELECT grid_ms,labels,corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/300 value FROM (SELECT grid_ms,labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2x.1,groupArray((ts_ms,value))) samples FROM raw_samples CROSS JOIN (SELECT arrayJoin(arrayMap(x->x*60000,range(toUInt64(intDiv({eval_ms}-21600000+59999,60000)),toUInt64(intDiv({eval_ms},60000)+1)))) grid_ms) grids WHERE metric='order_service_http_5xx_total' AND ts_ms>=grid_ms-300000 AND ts_ms<=grid_ms GROUP BY grid_ms,labels) WHERE n>=2)) GROUP BY grid_ms) a INNER JOIN (SELECT grid_ms,sum(value) value FROM (SELECT grid_ms,labels,corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/300 value FROM (SELECT grid_ms,labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2x.1,groupArray((ts_ms,value))) samples FROM raw_samples CROSS JOIN (SELECT arrayJoin(arrayMap(x->x*60000,range(toUInt64(intDiv({eval_ms}-21600000+59999,60000)),toUInt64(intDiv({eval_ms},60000)+1)))) grid_ms) grids WHERE metric='order_service_http_requests_total' AND ts_ms>=grid_ms-300000 AND ts_ms<=grid_ms GROUP BY grid_ms,labels) WHERE n>=2)) GROUP BY grid_ms) b ON a.grid_ms=b.grid_ms", + "operators": [ + "rate", + "max_over_time", + "sum", + "subquery", + "division" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q25", + "promql": "topk(1, sum by (job) (increase(backend_http_5xx_total[24h])) / scalar(sum(increase(backend_http_5xx_total[24h]))))", + "metricsql": "topk(1, sum by (job) (increase(backend_http_5xx_total[24h])) / scalar(sum(increase(backend_http_5xx_total[24h]))))", + "clickhouse_sql": "SELECT * FROM (SELECT a.labels,a.value/b.value value FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=(({eval_ms}-0)-86400000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) a CROSS JOIN (SELECT map() labels,sum(value) value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_http_5xx_total' AND ts_ms>=(({eval_ms}-0)-86400000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) b) ORDER BY value DESC,labels LIMIT 1", + "operators": [ + "topk", + "scalar", + "increase", + "sum", + "division" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q26", + "promql": "topk(1, sum by (job) (rate(backend_process_cpu_seconds_total[6h])))", + "metricsql": "topk(1, sum by (job) (rate(backend_process_cpu_seconds_total[6h])))", + "clickhouse_sql": "SELECT * FROM (SELECT map('job',job) labels,sum(value) value FROM (SELECT labels['job'] job,value FROM (SELECT labels, corrected*(sampled+least(start_extra,if(corrected>0,sampled*(first_value/corrected),start_extra))+end_extra)/sampled/(21600000/1000) AS value FROM (SELECT labels,samples,length(samples) n,samples[1].1 first_ts,samples[n].1 last_ts,samples[1].2 first_value,samples[n].2 last_value,(last_ts-first_ts)/1000 sampled,(last_value-first_value)+arraySum(i -> if(samples[i].2 x.1,groupArray((ts_ms,value))) samples FROM raw_samples WHERE metric='backend_process_cpu_seconds_total' AND ts_ms>=(({eval_ms}-0)-21600000) AND ts_ms<=({eval_ms}-0) GROUP BY labels) WHERE n>=2))) GROUP BY job) ORDER BY value DESC,labels LIMIT 1", + "operators": [ + "topk", + "rate" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + }, + { + "id": "q27", + "promql": "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", + "metricsql": "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", + "clickhouse_sql": "SELECT map('job',job) labels,avg(value) value FROM (SELECT labels['job'] job,ts_ms,sum(value) value FROM raw_samples WHERE metric='backend_process_resident_memory_bytes' AND ts_ms+4000>={eval_ms}-21600000 AND ts_ms+4000<={eval_ms} AND modulo(ts_ms+4000,60000)=0 GROUP BY job,ts_ms) GROUP BY job ORDER BY value DESC,job LIMIT 3", + "operators": [ + "topk", + "avg_over_time", + "subquery" + ], + "sql_mapping_contract": "Derived from identical raw samples by the checked Prometheus-semantics loader, never from baseline query answers.", + "sql_mapping_status": "oracle_valid" + } + ], + "source_sha256": "72c334fab18674746a2cef7c926e81d06a13dc9f11858d080460a99aa5b31ecf" +} diff --git a/tools/o11y-sql-main-eval/evaluation_query_datasets.snapshot.yaml b/tools/o11y-sql-main-eval/evaluation_query_datasets.snapshot.yaml new file mode 100644 index 000000000..95bb913bb --- /dev/null +++ b/tools/o11y-sql-main-eval/evaluation_query_datasets.snapshot.yaml @@ -0,0 +1,27 @@ +sort_desc(sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h]))) +topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h]))) +sum(increase(payment_service_http_5xx_total[1h])) / sum(increase(payment_service_http_requests_total[1h])) +sum(increase(payment_service_http_5xx_total[1h] offset 6h)) / sum(increase(payment_service_http_requests_total[1h] offset 6h)) +max_over_time(cache_refresh_lag_seconds[12h]) +max_over_time(user_service_cache_refresh_lag_seconds[12h]) +user_service_cache_refresh_lag_seconds +sum(rate(backend_process_cpu_seconds_total[1h])) +sum(backend_process_resident_memory_bytes) +max_over_time((sum(backend_process_resident_memory_bytes))[6h:]) +topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h]))) +topk(2, sum by (job) (backend_process_resident_memory_bytes)) +max_over_time((sum(rate(backend_process_cpu_seconds_total[5m])))[6h:]) +sum(rate(backend_http_requests_total[5m])) +sum by (job) (rate(backend_http_requests_total[5m])) +sum(increase(backend_http_5xx_total[60m])) / sum(increase(backend_http_requests_total[60m])) +topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h]))) +sum by (job) (increase(backend_http_5xx_total[24h])) > 0 +sum(rate(order_service_http_requests_total[5m])) +sum(rate(order_service_http_requests_total[5m] offset 1h)) +histogram_quantile(0.95, sum by (le) (rate(order_service_http_request_duration_seconds_bucket[5m]))) +sum(rate(order_service_http_requests_total[5m])) +topk(2, max_over_time(backend_retry_backlog_depth[6h])) +max_over_time((sum(rate(order_service_http_5xx_total[5m])) / sum(rate(order_service_http_requests_total[5m])))[6h:]) +topk(1, sum by (job) (increase(backend_http_5xx_total[24h])) / scalar(sum(increase(backend_http_5xx_total[24h])))) +topk(1, sum by (job) (rate(backend_process_cpu_seconds_total[6h]))) +topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:])) \ No newline at end of file diff --git a/tools/o11y-sql-main-eval/reproduce.sh b/tools/o11y-sql-main-eval/reproduce.sh new file mode 100755 index 000000000..d26f17225 --- /dev/null +++ b/tools/o11y-sql-main-eval/reproduce.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(git rev-parse --show-toplevel) +artifact_dir="$repo_root/tools/o11y-sql-main-eval/artifacts" +corpus="$repo_root/tools/o11y-sql-main-eval/corpus.json" +baseline=791f7d7b0feab3827e7e6f5100e65ef29aec68de + +if [[ $(git merge-base "$baseline" HEAD) != "$baseline" ]]; then + echo "HEAD does not descend from recorded main baseline $baseline" >&2 + exit 1 +fi + +mkdir -p "$artifact_dir" +cargo run -q -p control_plane --example audit_clickhouse_corpus -- "$corpus" \ + > "$artifact_dir/frontend-planner-publication.json" +cargo run -q -p data_plane --example audit_clickhouse_fallback -- "$corpus" \ + > "$artifact_dir/data-plane-routing.json" +python3 "$repo_root/tools/o11y-sql-main-eval/summarize.py" \ + --planner "$artifact_dir/frontend-planner-publication.json" \ + --runtime "$artifact_dir/data-plane-routing.json" \ + --output "$artifact_dir/matrix.json" +sha256sum "$corpus" > "$artifact_dir/corpus.sha256" +git -C "$repo_root" show -s --format='%H %s' "$baseline" > "$artifact_dir/baseline.txt" diff --git a/tools/o11y-sql-main-eval/summarize.py b/tools/o11y-sql-main-eval/summarize.py new file mode 100755 index 000000000..f807839a4 --- /dev/null +++ b/tools/o11y-sql-main-eval/summarize.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Join current-main stage and runtime evidence into the acceptance matrix.""" +import argparse +import json +from pathlib import Path + + +def short_reason(reason: str) -> str: + reason = reason.replace("SQL lowering failed: DataFusion error: ", "") + reason = reason.replace("SQL lowering failed: invalid QueryPlan: ", "") + return reason.split("\n", 1)[0] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--planner", type=Path, required=True) + parser.add_argument("--runtime", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + planner = json.loads(args.planner.read_text()) + runtime = {row["id"]: row for row in json.loads(args.runtime.read_text())["queries"]} + rows = [] + for stage in planner["queries"]: + route = runtime[stage["id"]] + rows.append({ + "id": stage["id"], + "operators": stage["operators"], + "parser_planner": stage["parser_planner"], + "publication": stage.get("publication", "not_reached"), + "data_plane_route": route["route"], + "classification": route["route"], + "reason": short_reason(stage.get("reason", route.get("reason", ""))), + }) + counts = {name: sum(row["classification"] == name for row in rows) + for name in ("warm", "partial_hybrid", "exact_fallback", "failure")} + args.output.write_text(json.dumps({ + "schema_version": 1, + "baseline_commit": "791f7d7b0feab3827e7e6f5100e65ef29aec68de", + "corpus_count": len(rows), + "counts": counts, + "queries": rows, + }, indent=2) + "\n") + + +if __name__ == "__main__": + main() From 40ba15d932ff9e6d41ec42f3a20b816ea2722ba8 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 14:25:04 -0600 Subject: [PATCH 2/2] Verify SQL fallback against live ClickHouse --- .../examples/audit_clickhouse_fallback.rs | 66 +++- tools/o11y-sql-main-eval/REPORT.md | 22 +- .../artifacts/data-plane-routing.json | 324 +++++++++++++++--- .../frontend-planner-publication.json | 2 +- .../o11y-sql-main-eval/artifacts/matrix.json | 189 ++++++++-- tools/o11y-sql-main-eval/reproduce.sh | 4 + tools/o11y-sql-main-eval/summarize.py | 15 +- 7 files changed, 520 insertions(+), 102 deletions(-) diff --git a/data_plane/examples/audit_clickhouse_fallback.rs b/data_plane/examples/audit_clickhouse_fallback.rs index 6255c4377..2ada8ebaf 100644 --- a/data_plane/examples/audit_clickhouse_fallback.rs +++ b/data_plane/examples/audit_clickhouse_fallback.rs @@ -12,7 +12,7 @@ use data_plane::{ drivers::query::servers::http::{build_active_physical_plan, PhysicalPlanInstallRequest}, query_engines::asap_clickhouse_query_engine::{ accelerator::CatalogClickHouseAccelerator, ClickHouseAccelerationOutcome, - ClickHouseAccelerator, + ClickHouseAccelerator, ClickHouseHttpFallback, ClickHouseHttpServer, }, storage_engines::{ sketch_db::index::SketchStore, @@ -101,10 +101,21 @@ async fn main() { Arc::new(BackendStorageRouting::empty()), ) .unwrap(); - let accelerator = CatalogClickHouseAccelerator::with_active_physical_plan( + let exact_url = + std::env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://127.0.0.1:18123".into()); + let exact = Arc::new(ClickHouseHttpFallback::new( + exact_url.clone(), + std::env::var("CLICKHOUSE_DATABASE").unwrap_or_else(|_| "default".into()), + )); + let accelerator = Arc::new(CatalogClickHouseAccelerator::with_active_physical_plan( Arc::new(SketchStore::new()), HotReloadActivePhysicalPlan::new(active), - ); + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = ClickHouseHttpServer::router_with_accelerator(exact, accelerator.clone()); + let server = tokio::spawn(async move { axum::serve(listener, app).await }); + let client = reqwest::Client::new(); let mut rows = Vec::new(); for row in corpus.queries { let sql = row @@ -118,16 +129,54 @@ async fn main() { parameters: BTreeMap::new(), headers: HeaderMap::new(), }; - let outcome = accelerator.execute(&request).await; - rows.push(match outcome { + let acceleration = accelerator.execute(&request).await; + let mut outbound = client + .post(format!("http://{address}/")) + .body(request.sql.clone()); + if let Ok(user) = std::env::var("CLICKHOUSE_USER") { + outbound = outbound.header("x-clickhouse-user", user); + } + if let Ok(password) = std::env::var("CLICKHOUSE_PASSWORD") { + outbound = outbound.header("x-clickhouse-key", password); + } + let response = outbound.send().await.unwrap(); + let status = response.status().as_u16(); + let headers = response.headers().clone(); + let body = response.text().await.unwrap(); + let mut direct = client.post(&exact_url).body(request.sql.clone()); + if let Ok(user) = std::env::var("CLICKHOUSE_USER") { + direct = direct.header("x-clickhouse-user", user); + } + if let Ok(password) = std::env::var("CLICKHOUSE_PASSWORD") { + direct = direct.header("x-clickhouse-key", password); + } + let direct = direct.send().await.unwrap(); + let direct_status = direct.status().as_u16(); + let direct_body = direct.text().await.unwrap(); + let matches_direct_exact = status == direct_status && body == direct_body; + rows.push(match acceleration { ClickHouseAccelerationOutcome::Fallback(reason) => json!({ "id": row.id, - "route": "exact_fallback", - "reason": format!("{reason:?}"), + "fallback_requested": true, + "acceleration_reason": format!("{reason:?}"), + "exact_executed": status != 502, + "exact_success": (200..300).contains(&status) && matches_direct_exact, + "exact_status": status, + "direct_exact_status": direct_status, + "matches_direct_exact": matches_direct_exact, + "clickhouse_summary": headers.get("x-clickhouse-summary").and_then(|v| v.to_str().ok()), + "clickhouse_exception_code": headers.get("x-clickhouse-exception-code").and_then(|v| v.to_str().ok()), + "result": body, }), ClickHouseAccelerationOutcome::Accelerated(_) => json!({ "id": row.id, - "route": "warm", + "fallback_requested": false, + "exact_executed": false, + "exact_success": false, + "exact_status": null, + "direct_exact_status": direct_status, + "matches_direct_exact": false, + "result": body, }), }); } @@ -135,4 +184,5 @@ async fn main() { "{}", serde_json::to_string_pretty(&json!({"queries": rows})).unwrap() ); + server.abort(); } diff --git a/tools/o11y-sql-main-eval/REPORT.md b/tools/o11y-sql-main-eval/REPORT.md index 8928f58f0..66b41f47f 100644 --- a/tools/o11y-sql-main-eval/REPORT.md +++ b/tools/o11y-sql-main-eval/REPORT.md @@ -19,11 +19,14 @@ The parser and Planner accept q05, q06, and q23 and select an exact MinMax summary followed by shared relational Project/Sort/Limit nodes. Publication rejects all three because the `metric = ...` table predicate has no canonical population binding in the summary catalog. The other 24 queries fail closed in -the SQL frontend. The data-plane accelerator was invoked for every query using -an installed, validated current-generation catalog. It returns `Planning` for -the 24 frontend misses and `CatalogMiss` for the three entries whose publication -was refused. The HTTP adapter maps both outcomes to the configured exact -ClickHouse backend, so these are exact fallbacks rather than failed requests. +the SQL frontend. Every query was sent through the production ClickHouse HTTP +listener with an installed, validated current-generation catalog and a real +ClickHouse 26.8.2.7 exact backend. The accelerator returns `Planning` for the 24 +frontend misses and `CatalogMiss` for the three entries whose publication was +refused. The listener then executed exact ClickHouse: all 27 responses were HTTP +200 and byte-for-byte equal to a direct request sent to the same exact backend. +The artifacts record `fallback_requested`, `exact_executed`, `exact_success`, +both HTTP statuses, response provenance headers, and the returned result. The machine-readable per-query evidence is in `artifacts/matrix.json`. Full canonical AST, selected post-ASAP tree, publication result, and data-plane @@ -114,8 +117,7 @@ Artifacts: - `artifacts/baseline.txt` - `artifacts/corpus.sha256` -The run validates routing through the in-process data-plane accelerator. It does -not claim differential value equality against a live ClickHouse instance because -no query is published into a warm or hybrid state on this corpus. A live exact -server remains necessary for the separate HTTP differential E2E and latency -experiment. +The run validates exact fallback execution and response equality through the +production HTTP listener against live ClickHouse. It does not claim accelerated +versus exact value equality or performance benefit because no query is published +into a warm or hybrid state on this corpus. diff --git a/tools/o11y-sql-main-eval/artifacts/data-plane-routing.json b/tools/o11y-sql-main-eval/artifacts/data-plane-routing.json index d68a8b767..e6f9ac172 100644 --- a/tools/o11y-sql-main-eval/artifacts/data-plane-routing.json +++ b/tools/o11y-sql-main-eval/artifacts/data-plane-routing.json @@ -1,139 +1,355 @@ { "queries": [ { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: .\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"40960\",\"read_bytes\":\"4192978\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"40960\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"124613711\",\"memory_usage\":\"5790933\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q01", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: .\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'payment-service'}\t0.09651474530831099\n{'job':'order-service'}\t0.04043126684636118\n{'job':'user-service'}\t0.03504043126684636\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: .\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"40960\",\"read_bytes\":\"4192978\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"40960\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"161313045\",\"memory_usage\":\"5793981\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q02", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: .\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'payment-service'}\t0.09651474530831099\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: identifier, found: )\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"24576\",\"read_bytes\":\"2065212\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"24576\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"65564208\",\"memory_usage\":\"1999791\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q03", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: identifier, found: )\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{}\t0.017857142857142856\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: identifier, found: )\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"24576\",\"read_bytes\":\"2065212\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"24576\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"47389095\",\"memory_usage\":\"2022831\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q04", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: identifier, found: )\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{}\t0.017241379310344827\n" }, { + "acceleration_reason": "CatalogMiss", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"24576\",\"read_bytes\":\"1737396\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"24576\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"9692977\",\"memory_usage\":\"2940118\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q05", - "reason": "CatalogMiss", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'api-gateway','instance':'api-gateway:8081'}\t0\n{'job':'order-service','instance':'order-service:8083'}\t0\n{'job':'payment-service','instance':'payment-service:8084'}\t0\n{'job':'user-service','instance':'user-service:8082'}\t530\n{'job':'webapp','instance':'webapp:8080'}\t0\n" }, { + "acceleration_reason": "CatalogMiss", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"6596\",\"read_bytes\":\"639068\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"6596\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"10163560\",\"memory_usage\":\"1267007\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q06", - "reason": "CatalogMiss", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'user-service','instance':'user-service:8082'}\t530\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: Error during planning: Invalid function 'mapconcat'.\\nDid you mean 'concat'?\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"6596\",\"read_bytes\":\"639068\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"6596\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"10933695\",\"memory_usage\":\"1306007\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q07", - "reason": "Planning(\"SQL lowering failed: DataFusion error: Error during planning: Invalid function 'mapconcat'.\\nDid you mean 'concat'?\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'user-service','instance':'user-service:8082','__name__':'user_service_cache_refresh_lag_seconds'}\t0\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"16384\",\"read_bytes\":\"1597946\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"16384\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"38599942\",\"memory_usage\":\"3091395\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q08", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{}\t0.007247222222222231\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: Error during planning: Error during planning: map does not support zero arguments. No function matches the given name and argument types 'map()'. You might need to add explicit type casts.\\n\\tCandidate functions:\\n\\tmap(Any, .., Any)\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"16384\",\"read_bytes\":\"1572468\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"16384\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"14678487\",\"memory_usage\":\"2788497\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q09", - "reason": "Planning(\"SQL lowering failed: DataFusion error: Error during planning: Error during planning: map does not support zero arguments. No function matches the given name and argument types 'map()'. You might need to add explicit type casts.\\n\\tCandidate functions:\\n\\tmap(Any, .., Any)\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{}\t570522659\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: Error during planning: Invalid function 'modulo'.\\nDid you mean 'today'?\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"16384\",\"read_bytes\":\"278528\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"16384\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"12765529\",\"memory_usage\":\"1337455\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q10", - "reason": "Planning(\"SQL lowering failed: DataFusion error: Error during planning: Invalid function 'modulo'.\\nDid you mean 'md5'?\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{}\t631591324\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: labels\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"16384\",\"read_bytes\":\"1597946\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"16384\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"58732518\",\"memory_usage\":\"3865123\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q11", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: labels\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'payment-service'}\t0.0014705555555555587\n{'job':'order-service'}\t0.0014694444444444462\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: This feature is not implemented: Map Access\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"16384\",\"read_bytes\":\"1572468\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"16384\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"22346675\",\"memory_usage\":\"3161721\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q12", - "reason": "Planning(\"SQL lowering failed: DataFusion error: This feature is not implemented: Map Access\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'webapp'}\t119294036\n{'job':'api-gateway'}\t116909806\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"16385\",\"read_bytes\":\"1597947\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"16385\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"717890827\",\"memory_usage\":\"6499881\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q13", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{}\t0.008770370370370298\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"24576\",\"read_bytes\":\"2212906\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"24576\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"41261718\",\"memory_usage\":\"3274558\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q14", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{}\t0.07\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: [\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"24576\",\"read_bytes\":\"2212906\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"24576\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"53541928\",\"memory_usage\":\"3781334\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q15", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: [\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'user-service'}\t0.02\n{'job':'payment-service'}\t0.006666666666666667\n{'job':'order-service'}\t0.043333333333333335\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: identifier, found: )\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"40960\",\"read_bytes\":\"3330802\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"40960\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"75231363\",\"memory_usage\":\"3762184\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q16", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: identifier, found: )\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{}\t0.04609929078014184\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: .\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"40960\",\"read_bytes\":\"4192978\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"40960\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"149860681\",\"memory_usage\":\"5793981\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q17", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: .\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'payment-service'}\t0.09651474530831099\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: labels\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"16384\",\"read_bytes\":\"1117896\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"16384\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"392401904\",\"memory_usage\":\"5178170\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q18", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: labels\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'user-service'}\t23.00422018348624\n{'job':'payment-service'}\t46.008164714235\n{'job':'order-service'}\t79.01389377418221\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"8192\",\"read_bytes\":\"1028096\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"8192\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"36567364\",\"memory_usage\":\"1867447\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q19", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{}\t0.043333333333333335\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"8192\",\"read_bytes\":\"1028096\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"8192\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"36440712\",\"memory_usage\":\"1867447\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q20", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{}\t0.06666666666666667\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: ), found: .1\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"0\",\"read_bytes\":\"0\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"0\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"74591970\",\"memory_usage\":\"6236100\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q21", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: ), found: .1\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"8192\",\"read_bytes\":\"1028096\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"8192\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"28747512\",\"memory_usage\":\"1867447\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q22", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{}\t0.043333333333333335\n" }, { + "acceleration_reason": "CatalogMiss", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"24576\",\"read_bytes\":\"1923892\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"24576\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"17787041\",\"memory_usage\":\"2829098\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q23", - "reason": "CatalogMiss", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'payment-service','instance':'payment-service:8084'}\t101\n{'job':'order-service','instance':'order-service:8083'}\t24\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"24578\",\"read_bytes\":\"3075298\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"24578\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"531152875\",\"memory_usage\":\"5381809\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q24", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: ,\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{}\t0.3333333333333333\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: .\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"32768\",\"read_bytes\":\"2235792\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"32768\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"463444920\",\"memory_usage\":\"6463074\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q25", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: .\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'order-service'}\t0.5337828828982084\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: labels\\\")\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"16384\",\"read_bytes\":\"1597946\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"16384\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"93888655\",\"memory_usage\":\"4047283\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q26", - "reason": "Planning(\"SQL lowering failed: DataFusion error: SQL error: ParserError(\\\"Expected: joined table, found: labels\\\")\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'order-service'}\t0.0015755555555555553\n" }, { + "acceleration_reason": "Planning(\"SQL lowering failed: DataFusion error: Error during planning: Invalid function 'modulo'.\\nDid you mean 'today'?\")", + "clickhouse_exception_code": null, + "clickhouse_summary": "{\"read_rows\":\"16384\",\"read_bytes\":\"515898\",\"written_rows\":\"0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"16384\",\"result_rows\":\"0\",\"result_bytes\":\"0\",\"elapsed_ns\":\"26775548\",\"memory_usage\":\"3428334\"}", + "direct_exact_status": 200, + "exact_executed": true, + "exact_status": 200, + "exact_success": true, + "fallback_requested": true, "id": "q27", - "reason": "Planning(\"SQL lowering failed: DataFusion error: Error during planning: Invalid function 'modulo'.\\nDid you mean 'today'?\")", - "route": "exact_fallback" + "matches_direct_exact": true, + "result": "{'job':'api-gateway'}\t118035226.79722223\n{'job':'user-service'}\t117853344.7638889\n{'job':'webapp'}\t117812478.69722222\n" } ] } diff --git a/tools/o11y-sql-main-eval/artifacts/frontend-planner-publication.json b/tools/o11y-sql-main-eval/artifacts/frontend-planner-publication.json index dfa9160bc..2deb95a2e 100644 --- a/tools/o11y-sql-main-eval/artifacts/frontend-planner-publication.json +++ b/tools/o11y-sql-main-eval/artifacts/frontend-planner-publication.json @@ -106,7 +106,7 @@ "subquery" ], "parser_planner": "failure", - "reason": "SQL lowering failed: DataFusion error: Error during planning: Invalid function 'modulo'.\nDid you mean 'today'?", + "reason": "SQL lowering failed: DataFusion error: Error during planning: Invalid function 'modulo'.\nDid you mean 'todate'?", "sql": "SELECT map() labels,max(value) value FROM (SELECT ts_ms,sum(value) value FROM raw_samples WHERE metric='backend_process_resident_memory_bytes' AND ts_ms>=1788891296000-21600000 AND ts_ms<=1788891296000 AND modulo(1788891296000-ts_ms,60000)=0 GROUP BY ts_ms)" }, { diff --git a/tools/o11y-sql-main-eval/artifacts/matrix.json b/tools/o11y-sql-main-eval/artifacts/matrix.json index 7690b27ff..6996e8499 100644 --- a/tools/o11y-sql-main-eval/artifacts/matrix.json +++ b/tools/o11y-sql-main-eval/artifacts/matrix.json @@ -18,7 +18,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: .\")" }, @@ -31,7 +36,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: .\")" }, @@ -44,7 +54,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: identifier, found: )\")" }, @@ -58,7 +73,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: identifier, found: )\")" }, @@ -69,7 +89,12 @@ ], "parser_planner": "pass", "publication": "failure", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL population predicate on metric needs a canonical catalog filter" }, @@ -80,7 +105,12 @@ ], "parser_planner": "pass", "publication": "failure", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL population predicate on metric needs a canonical catalog filter" }, @@ -89,7 +119,12 @@ "operators": [], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "Error during planning: Invalid function 'mapconcat'." }, @@ -101,7 +136,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" }, @@ -112,7 +152,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "Error during planning: Error during planning: map does not support zero arguments. No function matches the given name and argument types 'map()'. You might need to add explicit type casts." }, @@ -125,7 +170,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "Error during planning: Invalid function 'modulo'." }, @@ -137,7 +187,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: labels\")" }, @@ -148,7 +203,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "This feature is not implemented: Map Access" }, @@ -162,7 +222,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" }, @@ -174,7 +239,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" }, @@ -185,7 +255,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: [\")" }, @@ -198,7 +273,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: identifier, found: )\")" }, @@ -211,7 +291,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: .\")" }, @@ -223,7 +308,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: labels\")" }, @@ -235,7 +325,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" }, @@ -248,7 +343,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" }, @@ -260,7 +360,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: ), found: .1\")" }, @@ -272,7 +377,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" }, @@ -284,7 +394,12 @@ ], "parser_planner": "pass", "publication": "failure", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL population predicate on metric needs a canonical catalog filter" }, @@ -299,7 +414,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: ,\")" }, @@ -314,7 +434,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: .\")" }, @@ -326,7 +451,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "SQL error: ParserError(\"Expected: joined table, found: labels\")" }, @@ -339,7 +469,12 @@ ], "parser_planner": "failure", "publication": "not_reached", - "data_plane_route": "exact_fallback", + "fallback_requested": true, + "exact_executed": true, + "exact_success": true, + "exact_status": 200, + "direct_exact_status": 200, + "matches_direct_exact": true, "classification": "exact_fallback", "reason": "Error during planning: Invalid function 'modulo'." } diff --git a/tools/o11y-sql-main-eval/reproduce.sh b/tools/o11y-sql-main-eval/reproduce.sh index d26f17225..9aa34d042 100755 --- a/tools/o11y-sql-main-eval/reproduce.sh +++ b/tools/o11y-sql-main-eval/reproduce.sh @@ -6,6 +6,10 @@ artifact_dir="$repo_root/tools/o11y-sql-main-eval/artifacts" corpus="$repo_root/tools/o11y-sql-main-eval/corpus.json" baseline=791f7d7b0feab3827e7e6f5100e65ef29aec68de +export CLICKHOUSE_URL=${CLICKHOUSE_URL:-http://127.0.0.1:18123} +: "${CLICKHOUSE_USER:?set CLICKHOUSE_USER for the exact ClickHouse backend}" +: "${CLICKHOUSE_PASSWORD:?set CLICKHOUSE_PASSWORD for the exact ClickHouse backend}" + if [[ $(git merge-base "$baseline" HEAD) != "$baseline" ]]; then echo "HEAD does not descend from recorded main baseline $baseline" >&2 exit 1 diff --git a/tools/o11y-sql-main-eval/summarize.py b/tools/o11y-sql-main-eval/summarize.py index f807839a4..0cbd581f7 100755 --- a/tools/o11y-sql-main-eval/summarize.py +++ b/tools/o11y-sql-main-eval/summarize.py @@ -22,13 +22,24 @@ def main() -> None: rows = [] for stage in planner["queries"]: route = runtime[stage["id"]] + if route["fallback_requested"] and route["exact_success"]: + classification = "exact_fallback" + elif not route["fallback_requested"]: + classification = "warm" + else: + classification = "failure" rows.append({ "id": stage["id"], "operators": stage["operators"], "parser_planner": stage["parser_planner"], "publication": stage.get("publication", "not_reached"), - "data_plane_route": route["route"], - "classification": route["route"], + "fallback_requested": route["fallback_requested"], + "exact_executed": route["exact_executed"], + "exact_success": route["exact_success"], + "exact_status": route["exact_status"], + "direct_exact_status": route["direct_exact_status"], + "matches_direct_exact": route["matches_direct_exact"], + "classification": classification, "reason": short_reason(stage.get("reason", route.get("reason", ""))), }) counts = {name: sum(row["classification"] == name for row in rows)