diff --git a/control_plane/examples/audit_clickhouse_corpus.rs b/control_plane/examples/audit_clickhouse_corpus.rs index b5187a70f..71fedf79e 100644 --- a/control_plane/examples/audit_clickhouse_corpus.rs +++ b/control_plane/examples/audit_clickhouse_corpus.rs @@ -1,3 +1,5 @@ +#[path = "support/automatic_clickhouse_corpus.rs"] +mod automatic; use asap_frontend_sql::SqlCatalog; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; use control_plane::clickhouse::{ClickHouseSqlWorkload, ClickHouseSqlWorkloadEntry}; @@ -82,6 +84,10 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { #[tokio::main] async fn main() { + if std::env::args().nth(1).as_deref() == Some("--automatic") { + automatic::run().await; + return; + } let path = std::env::args_os() .nth(1) .map(PathBuf::from) diff --git a/control_plane/examples/support/automatic_clickhouse_corpus.rs b/control_plane/examples/support/automatic_clickhouse_corpus.rs new file mode 100644 index 000000000..71cd2dfd3 --- /dev/null +++ b/control_plane/examples/support/automatic_clickhouse_corpus.rs @@ -0,0 +1,139 @@ +use control_plane::clickhouse::{ClickHouseSqlAutomaticWorkload, ClickHouseSqlWorkloadEntry}; +use control_plane::physical::compiler::{PlanEnvelope, BACKEND_COMPAT, PLANNER_REVISION}; +use planner_types::pre_asap::{Column, DataType, Schema}; +use planner_types::types::AccuracyTarget; +use serde::Deserialize; +use serde_json::json; +use std::{fs, path::PathBuf}; + +#[derive(Deserialize)] +struct Corpus { + queries: Vec, +} + +#[derive(Deserialize)] +struct Row { + id: String, + clickhouse_sql: Option, + operators: Vec, + promql: String, +} + +fn publication_inputs( + schema: &Schema, + sql: String, + window_ms: u64, +) -> ClickHouseSqlAutomaticWorkload { + ClickHouseSqlAutomaticWorkload { + envelope: PlanEnvelope { + plan_id: 27, + plan_version: 1, + generated_at_unix_ms: 1_788_891_296_000, + activation_unix_ms: 1_788_891_296_000, + expiry_unix_ms: None, + backend_compat: BACKEND_COMPAT.into(), + planner_revision: PLANNER_REVISION.into(), + capability_snapshot_id: "sql27-automatic-eval".into(), + }, + tables: std::collections::HashMap::from([("raw_samples".into(), schema.clone())]), + accuracy: AccuracyTarget::Epsilon(0.01), + queries: vec![ClickHouseSqlWorkloadEntry { + sql, + start_ms: 1_788_891_296_001 - window_ms, + end_ms: 1_788_891_296_001, + cumulative: false, + }], + } +} + +pub async fn run() { + let path = std::env::args_os() + .nth(2) + .map(PathBuf::from) + .expect("usage: audit_clickhouse_corpus --automatic CORPUS.json"); + let corpus: Corpus = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + let schema = Schema::with_time_index( + vec![ + Column::new("metric", DataType::Utf8, false), + Column::new( + "labels", + DataType::Map { + key: Box::new(DataType::Utf8), + value: Box::new(DataType::Utf8), + value_nullable: false, + }, + false, + ), + Column::new("ts_ms", DataType::Int64, false), + Column::new("value", DataType::Float64, false), + ], + 2, + vec![], + ); + let mut rows = Vec::new(); + for row in corpus.queries { + let Some(sql) = row.clickhouse_sql else { + rows.push(json!({ + "id": row.id, + "operators": row.operators, + "parser_planner": "failure", + "reason": "missing ClickHouse SQL mapping" + })); + continue; + }; + let sql = sql.replace("{eval_ms}", "1788891296000"); + // Evaluation span only: SQL source boundaries are still validated by the compiler. + let window_ms = row + .promql + .split('[') + .skip(1) + .filter_map(|range| { + let digits = range.chars().take_while(char::is_ascii_digit).count(); + let number = range[..digits].parse::().ok()?; + let unit = range.as_bytes().get(digits)?; + let multiplier = match unit { + b's' => 1_000, + b'm' => 60_000, + b'h' => 3_600_000, + b'd' => 86_400_000, + _ => return None, + }; + Some(number * multiplier) + }) + .max() + .unwrap_or(300_000); + match control_plane::clickhouse::compile_automatic_clickhouse_workload(&publication_inputs( + &schema, + sql.clone(), + window_ms, + )) + .await + { + Ok((publication, traces)) => rows.push(json!({ + "id": row.id, "operators": row.operators, "sql": sql, + "publication": "pass", "execution": "not_run", + "selection_traces": traces, + "window_start_ms": 1_788_891_296_001u64 - window_ms, + "window_end_ms": 1_788_891_296_001u64, + "install": publication.install_request(None, Vec::new()).unwrap(), + "summary_catalog": publication.summary_catalog, + "precompute_plan": publication.precompute_plan, + "query_plan": publication.query_plan, + })), + Err(error) => rows.push(json!({ + "id": row.id, "operators": row.operators, "sql": sql, + "publication": "failure", "execution": "not_run", "reason": error.to_string(), + })), + } + } + + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "baseline": env!("CARGO_PKG_VERSION"), + "query_count": rows.len(), + "queries": rows, + })) + .unwrap() + ); +} diff --git a/tools/o11y-sql-main-eval/automatic-map-artifact-manifest.json b/tools/o11y-sql-main-eval/automatic-map-artifact-manifest.json new file mode 100644 index 000000000..b79d4a8c0 --- /dev/null +++ b/tools/o11y-sql-main-eval/automatic-map-artifact-manifest.json @@ -0,0 +1,247 @@ +[ + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/driver-checkout.txt", + "bytes": 41, + "sha256": "18a3dd592033f38652545dcaa36ab404a40c7a83768110c09d9b7321d7982789" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/inputs.sha256", + "bytes": 247, + "sha256": "fadb2b1c95f0d398eb2a116ed66b77a061561977dc589c1f447da7fcdd1c426e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/runtime-source-commit.txt", + "bytes": 9, + "sha256": "6b594b65f78fb97ed820c019b4644fe9d783ad3a9d17394a1748ee7571421adf" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-1-q05/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-1-q05/q05/requests.jsonl", + "bytes": 3548372, + "sha256": "6d6f7c36aa924d1c52d533a023d304169bf8179d09340f120d55337fcd3c7f99" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-1-q05/q05/result.json", + "bytes": 5260, + "sha256": "1ac6780718e6fb66b159c92a44901783ce2c7033327bcd8ea75d84dfa3f4bb2f" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-1-q06/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-1-q06/q06/requests.jsonl", + "bytes": 2604895, + "sha256": "f671c3032885973cb521e4b7af450f5698391dc004a82fbc6d611c1dd7037724" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-1-q06/q06/result.json", + "bytes": 4320, + "sha256": "de50b85a5325560ef63243a0bd7073d67a9a31afa574ce7f7e40bfb4f6da9816" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-1-q23/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-1-q23/q23/requests.jsonl", + "bytes": 2905541, + "sha256": "ee05883c651673b36df8608dc31d421d51e37c597f5d8a47cd0d2ea434bf53ba" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-1-q23/q23/result.json", + "bytes": 4620, + "sha256": "8100b7054f0c8f161bd986148c9fc661c7ded2ce90c33125f5aae6a6b944b8e7" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-2-q05/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-2-q05/q05/requests.jsonl", + "bytes": 3548547, + "sha256": "65fc261c415cc3485a58bf301e0a7ffef2bb2c8a4c22e69122ed63a25555c53f" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-2-q05/q05/result.json", + "bytes": 5260, + "sha256": "4ae2305987f4f538fd4da13f817aa11942ebbb3308ad9b29ebd83c28c2e94978" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-2-q06/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-2-q06/q06/requests.jsonl", + "bytes": 2606114, + "sha256": "c4345d24793e863c2e9af41901854e6ce0145c626c1e44f5e1e6df7310c37b33" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-2-q06/q06/result.json", + "bytes": 4320, + "sha256": "c9e2985833acbed36c264bbcbc8ca43d75b3629b402c73a9f642a63a4a760d24" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-2-q23/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-2-q23/q23/requests.jsonl", + "bytes": 2906164, + "sha256": "8fa00d84d1d2f5264d13168e39825cd97419cfa9c903a604d56ceb6651694d7c" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-2-q23/q23/result.json", + "bytes": 4620, + "sha256": "bd5f276af8c3c0cc30d7204924a926cb5c710c75d698f6e69a0572b3a1171d9b" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-3-q05/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-3-q05/q05/requests.jsonl", + "bytes": 3547588, + "sha256": "58046878910dbf4d93d4d3ff569e1f40423086260237cbaa23845993e44ff2a0" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-3-q05/q05/result.json", + "bytes": 5262, + "sha256": "04459d12a190f6a6fcb9bc4ef2eb460845df5759e9a629d68e21d20330e7ad41" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-3-q06/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-3-q06/q06/requests.jsonl", + "bytes": 2606081, + "sha256": "212c6e2c61640348dd362e3f9bc5744cff9e7ef54c10545f6221b7bae43c4e7d" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-3-q06/q06/result.json", + "bytes": 4320, + "sha256": "d754143c030aabd1be105dc46888c96dd0ad8a30b598db0f4202797e7358f337" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-3-q23/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-3-q23/q23/requests.jsonl", + "bytes": 2905867, + "sha256": "5a0554274d325f6ddec93830e7af7f28798e9e893ed4b5868e984cc67d87b1e7" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-3-q23/q23/result.json", + "bytes": 4619, + "sha256": "7269fd8c20b6277d96ed7ffbb9c5272e20e1cc219ecacedf9f8146be16be8c52" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/inputs.sha256", + "bytes": 397, + "sha256": "c70dc43ef6d8772e796a6f10c840bdd82a012b140ff9faa7af2e77a9961a9212" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-1-asap/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-1-asap/q05/requests.jsonl", + "bytes": 1227891, + "sha256": "d81cd271bdacd5f3f07c2cb3bee149d9e4ebac04768e0f9f5b818e8e19cccb98" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-1-asap/q05/result.json", + "bytes": 5414, + "sha256": "d5ecb41643d21f22eb812ffc2c85e5f4ede6518663d05fa47bc8033f71ff6695" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-1-native/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-1-native/requests.jsonl", + "bytes": 1250315, + "sha256": "df3329158c97f5ac95e125c153c19bcece3a4a59e8f93cf60939db905f23e32e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-1-native/result.json", + "bytes": 403, + "sha256": "d572fdfc34a5208af4c263f878084860d3c0ab469e8a78f9c2d6f0ffc5286f85" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-2-asap/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-2-asap/q05/requests.jsonl", + "bytes": 1227892, + "sha256": "53619e372243b06b374695712aedbbd009599107a2166b56743bcd66c1eb3725" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-2-asap/q05/result.json", + "bytes": 5414, + "sha256": "d397f396056ad8d433ca51c41b6dd7d65ee565ae8af9a0a847e35187310dd21d" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-2-native/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-2-native/requests.jsonl", + "bytes": 1249755, + "sha256": "9979477f6c16d9cf51c23fa46243b0a5c5ac0609684e44790806e947f3495bca" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-2-native/result.json", + "bytes": 402, + "sha256": "75e34dc04bd0ee64381cd9b99bd4c5e1c5dab1ddbeac013df169b88143739a97" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-3-asap/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-3-asap/q05/requests.jsonl", + "bytes": 1227890, + "sha256": "2f196c6f74c13ec8453ec2c1bbb0b1a58e3449aa307ad63edabb94e73a4d7295" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-3-asap/q05/result.json", + "bytes": 5414, + "sha256": "a58f6e47deae4a301399472d41b37983d81f751ee722820234654d5331295fa3" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-3-native/clickhouse-runtime.json", + "bytes": 162, + "sha256": "2d019b9f4b287881e48052ed6b581fa6e1624f5d3541b58d0d6ef3cd252c986e" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-3-native/requests.jsonl", + "bytes": 1249777, + "sha256": "f2956c2a5a70f16088dbc38b750d0ebe531ae8117313f83b0bc6b766edd35328" + }, + { + "path": "/mydata/clickhouse-o11y-main-results/original27-deployment-pairs/trial-3-native/result.json", + "bytes": 402, + "sha256": "d86b54ee3db980d3501e37c51f0e0bc7289b952866a6e9ac34798ab917827810" + } +] diff --git a/tools/o11y-sql-main-eval/automatic-map-coverage.json b/tools/o11y-sql-main-eval/automatic-map-coverage.json new file mode 100644 index 000000000..9b65a7082 --- /dev/null +++ b/tools/o11y-sql-main-eval/automatic-map-coverage.json @@ -0,0 +1,245 @@ +[ + { + "id": "q01", + "publication": "failure", + "execution": "failed", + "route": "exact_fallback", + "equal": false, + "backend_status": 500, + "native_status": 500 + }, + { + "id": "q02", + "publication": "failure", + "execution": "failed", + "route": "exact_fallback", + "equal": false, + "backend_status": 500, + "native_status": 500 + }, + { + "id": "q03", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q04", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q05", + "publication": "pass", + "execution": "warm", + "route": "warm", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q06", + "publication": "pass", + "execution": "warm", + "route": "warm", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q07", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q08", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q09", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q10", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q11", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q12", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q13", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q14", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q15", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q16", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q17", + "publication": "failure", + "execution": "failed", + "route": "exact_fallback", + "equal": false, + "backend_status": 500, + "native_status": 500 + }, + { + "id": "q18", + "publication": "failure", + "execution": "failed", + "route": "exact_fallback", + "equal": false, + "backend_status": 500, + "native_status": 500 + }, + { + "id": "q19", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q20", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q21", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q22", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q23", + "publication": "pass", + "execution": "warm", + "route": "warm", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q24", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + }, + { + "id": "q25", + "publication": "failure", + "execution": "failed", + "route": "exact_fallback", + "equal": false, + "backend_status": 500, + "native_status": 500 + }, + { + "id": "q26", + "publication": "failure", + "execution": "failed", + "route": "exact_fallback", + "equal": false, + "backend_status": 500, + "native_status": 500 + }, + { + "id": "q27", + "publication": "failure", + "execution": "exact_fallback", + "route": "exact_fallback", + "equal": true, + "backend_status": 200, + "native_status": 200 + } +] diff --git a/tools/o11y-sql-main-eval/automatic-map-deployment-measurements.json b/tools/o11y-sql-main-eval/automatic-map-deployment-measurements.json new file mode 100644 index 000000000..7063a2a53 --- /dev/null +++ b/tools/o11y-sql-main-eval/automatic-map-deployment-measurements.json @@ -0,0 +1,56 @@ +{ + "scope": "fresh process startup, summary construction and 1000 queries; excludes common source ingestion and offline planning", + "pairs": [ + { + "trial": 1, + "asap": { + "requests": { + "warm": 1000, + "equal": 1000, + "failed": 0 + }, + "cpu_seconds": 12.3, + "combined_end_rss_bytes": 906338304 + }, + "native": { + "requests_equal": 1000, + "cpu_seconds": 98.86, + "end_rss_bytes": 878465024 + } + }, + { + "trial": 2, + "asap": { + "requests": { + "warm": 1000, + "equal": 1000, + "failed": 0 + }, + "cpu_seconds": 11.16, + "combined_end_rss_bytes": 878211072 + }, + "native": { + "requests_equal": 1000, + "cpu_seconds": 97.49, + "end_rss_bytes": 837402624 + } + }, + { + "trial": 3, + "asap": { + "requests": { + "warm": 1000, + "equal": 1000, + "failed": 0 + }, + "cpu_seconds": 12.33, + "combined_end_rss_bytes": 909520896 + }, + "native": { + "requests_equal": 1000, + "cpu_seconds": 96.67, + "end_rss_bytes": 862224384 + } + } + ] +} diff --git a/tools/o11y-sql-main-eval/automatic-map-measurements.json b/tools/o11y-sql-main-eval/automatic-map-measurements.json new file mode 100644 index 000000000..ce7951159 --- /dev/null +++ b/tools/o11y-sql-main-eval/automatic-map-measurements.json @@ -0,0 +1,743 @@ +{ + "scope": "fixed-window repeated queries; excludes common source ingestion and offline planning cost", + "trials": [ + { + "query": "q05", + "trial": "trial-1-q05", + "numeric_pairs_equal": 1000, + "warm_requests": 1000, + "latency_speedup": 21.573089315157763, + "routes": { + "backend": { + "requests": 1000, + "median_ms": 2.3571194999999996, + "p95_ms": 2.797406, + "request_time_rate_per_second": 422.34594680611144, + "backend_cpu_s_during_requests": 1.31, + "clickhouse_cpu_s_during_requests": 0.18, + "max_observed_backend_rss_bytes": 113074176, + "max_observed_clickhouse_rss_bytes": 1029861376 + }, + "exact": { + "requests": 1000, + "median_ms": 50.8503495, + "p95_ms": 72.84543, + "request_time_rate_per_second": 18.953400238689348, + "backend_cpu_s_during_requests": 0.04, + "clickhouse_cpu_s_during_requests": 88.92, + "max_observed_backend_rss_bytes": 113074176, + "max_observed_clickhouse_rss_bytes": 1029861376 + } + }, + "install_and_backfill_s": 4.289933355, + "build_cpu_s": { + "backend": 2.75, + "clickhouse": 1.29 + }, + "process_after_build": { + "backend": { + "cpu_ticks": 278, + "clock_ticks_per_second": 100, + "VmHWM": 118611968, + "VmRSS": 93077504 + }, + "clickhouse": { + "cpu_ticks": 756, + "clock_ticks_per_second": 100, + "VmHWM": 928849920, + "VmRSS": 865157120 + } + }, + "process_after_queries": { + "backend": { + "cpu_ticks": 413, + "clock_ticks_per_second": 100, + "VmHWM": 118611968, + "VmRSS": 113074176 + }, + "clickhouse": { + "cpu_ticks": 9696, + "clock_ticks_per_second": 100, + "VmHWM": 1029861376, + "VmRSS": 868360192 + } + }, + "state_directory_bytes_at_build": 5238, + "state_directory_bytes_after_stop": 7008, + "first_post_build_request_ms": { + "backend": 5.247254, + "exact": 72.45841 + }, + "backend_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "clickhouse_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "raw_trace": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-1-q05/q05/requests.jsonl", + "raw_sha256": "6d6f7c36aa924d1c52d533a023d304169bf8179d09340f120d55337fcd3c7f99" + }, + { + "query": "q06", + "trial": "trial-1-q06", + "numeric_pairs_equal": 1000, + "warm_requests": 1000, + "latency_speedup": 8.966121744078176, + "routes": { + "backend": { + "requests": 1000, + "median_ms": 2.2706895, + "p95_ms": 2.743556, + "request_time_rate_per_second": 434.58155879595193, + "backend_cpu_s_during_requests": 1.26, + "clickhouse_cpu_s_during_requests": 0.39, + "max_observed_backend_rss_bytes": 63422464, + "max_observed_clickhouse_rss_bytes": 1027055616 + }, + "exact": { + "requests": 1000, + "median_ms": 20.359278500000002, + "p95_ms": 34.012472, + "request_time_rate_per_second": 45.640250449306194, + "backend_cpu_s_during_requests": 0.01, + "clickhouse_cpu_s_during_requests": 31.92, + "max_observed_backend_rss_bytes": 63422464, + "max_observed_clickhouse_rss_bytes": 1027338240 + } + }, + "install_and_backfill_s": 1.771135748, + "build_cpu_s": { + "backend": 0.55, + "clickhouse": 0.24 + }, + "process_after_build": { + "backend": { + "cpu_ticks": 59, + "clock_ticks_per_second": 100, + "VmHWM": 48328704, + "VmRSS": 43794432 + }, + "clickhouse": { + "cpu_ticks": 561, + "clock_ticks_per_second": 100, + "VmHWM": 825528320, + "VmRSS": 801665024 + } + }, + "process_after_queries": { + "backend": { + "cpu_ticks": 186, + "clock_ticks_per_second": 100, + "VmHWM": 63422464, + "VmRSS": 63422464 + }, + "clickhouse": { + "cpu_ticks": 3806, + "clock_ticks_per_second": 100, + "VmHWM": 1031806976, + "VmRSS": 1005379584 + } + }, + "state_directory_bytes_at_build": 5238, + "state_directory_bytes_after_stop": 5238, + "first_post_build_request_ms": { + "backend": 5.141876, + "exact": 35.229285 + }, + "backend_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "clickhouse_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "raw_trace": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-1-q06/q06/requests.jsonl", + "raw_sha256": "f671c3032885973cb521e4b7af450f5698391dc004a82fbc6d611c1dd7037724" + }, + { + "query": "q23", + "trial": "trial-1-q23", + "numeric_pairs_equal": 1000, + "warm_requests": 1000, + "latency_speedup": 13.844018231429525, + "routes": { + "backend": { + "requests": 1000, + "median_ms": 2.4299795, + "p95_ms": 2.933214, + "request_time_rate_per_second": 400.44073885234246, + "backend_cpu_s_during_requests": 1.33, + "clickhouse_cpu_s_during_requests": 0.35, + "max_observed_backend_rss_bytes": 85598208, + "max_observed_clickhouse_rss_bytes": 1049919488 + }, + "exact": { + "requests": 1000, + "median_ms": 33.6406805, + "p95_ms": 48.905244, + "request_time_rate_per_second": 27.98093622395067, + "backend_cpu_s_during_requests": 0.07, + "clickhouse_cpu_s_during_requests": 52.32, + "max_observed_backend_rss_bytes": 85598208, + "max_observed_clickhouse_rss_bytes": 1049587712 + } + }, + "install_and_backfill_s": 2.782158745, + "build_cpu_s": { + "backend": 1.35, + "clickhouse": 0.84 + }, + "process_after_build": { + "backend": { + "cpu_ticks": 139, + "clock_ticks_per_second": 100, + "VmHWM": 74645504, + "VmRSS": 66084864 + }, + "clickhouse": { + "cpu_ticks": 597, + "clock_ticks_per_second": 100, + "VmHWM": 868454400, + "VmRSS": 867385344 + } + }, + "process_after_queries": { + "backend": { + "cpu_ticks": 279, + "clock_ticks_per_second": 100, + "VmHWM": 85598208, + "VmRSS": 85598208 + }, + "clickhouse": { + "cpu_ticks": 5889, + "clock_ticks_per_second": 100, + "VmHWM": 1050730496, + "VmRSS": 838840320 + } + }, + "state_directory_bytes_at_build": 5238, + "state_directory_bytes_after_stop": 6122, + "first_post_build_request_ms": { + "backend": 6.238476, + "exact": 37.060582 + }, + "backend_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "clickhouse_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "raw_trace": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-1-q23/q23/requests.jsonl", + "raw_sha256": "ee05883c651673b36df8608dc31d421d51e37c597f5d8a47cd0d2ea434bf53ba" + }, + { + "query": "q05", + "trial": "trial-2-q05", + "numeric_pairs_equal": 1000, + "warm_requests": 1000, + "latency_speedup": 21.037893332050977, + "routes": { + "backend": { + "requests": 1000, + "median_ms": 2.433027, + "p95_ms": 2.870542, + "request_time_rate_per_second": 399.68444625195576, + "backend_cpu_s_during_requests": 1.38, + "clickhouse_cpu_s_during_requests": 0.39, + "max_observed_backend_rss_bytes": 112373760, + "max_observed_clickhouse_rss_bytes": 1034268672 + }, + "exact": { + "requests": 1000, + "median_ms": 51.185762499999996, + "p95_ms": 77.391198, + "request_time_rate_per_second": 18.836641733699338, + "backend_cpu_s_during_requests": 0.03, + "clickhouse_cpu_s_during_requests": 89.05, + "max_observed_backend_rss_bytes": 112373760, + "max_observed_clickhouse_rss_bytes": 1034608640 + } + }, + "install_and_backfill_s": 4.291561459, + "build_cpu_s": { + "backend": 2.8, + "clickhouse": 1.0 + }, + "process_after_build": { + "backend": { + "cpu_ticks": 284, + "clock_ticks_per_second": 100, + "VmHWM": 119218176, + "VmRSS": 93425664 + }, + "clickhouse": { + "cpu_ticks": 640, + "clock_ticks_per_second": 100, + "VmHWM": 920530944, + "VmRSS": 868999168 + } + }, + "process_after_queries": { + "backend": { + "cpu_ticks": 427, + "clock_ticks_per_second": 100, + "VmHWM": 119218176, + "VmRSS": 112373760 + }, + "clickhouse": { + "cpu_ticks": 9629, + "clock_ticks_per_second": 100, + "VmHWM": 1034608640, + "VmRSS": 844017664 + } + }, + "state_directory_bytes_at_build": 5238, + "state_directory_bytes_after_stop": 7008, + "first_post_build_request_ms": { + "backend": 5.668682, + "exact": 84.462201 + }, + "backend_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "clickhouse_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "raw_trace": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-2-q05/q05/requests.jsonl", + "raw_sha256": "65fc261c415cc3485a58bf301e0a7ffef2bb2c8a4c22e69122ed63a25555c53f" + }, + { + "query": "q06", + "trial": "trial-2-q06", + "numeric_pairs_equal": 1000, + "warm_requests": 1000, + "latency_speedup": 8.373888984752382, + "routes": { + "backend": { + "requests": 1000, + "median_ms": 2.3789615, + "p95_ms": 2.744391, + "request_time_rate_per_second": 419.6522049931427, + "backend_cpu_s_during_requests": 1.29, + "clickhouse_cpu_s_during_requests": 0.39, + "max_observed_backend_rss_bytes": 63205376, + "max_observed_clickhouse_rss_bytes": 1022054400 + }, + "exact": { + "requests": 1000, + "median_ms": 19.9211595, + "p95_ms": 32.514515, + "request_time_rate_per_second": 46.399180384610496, + "backend_cpu_s_during_requests": 0.01, + "clickhouse_cpu_s_during_requests": 30.98, + "max_observed_backend_rss_bytes": 63205376, + "max_observed_clickhouse_rss_bytes": 1022054400 + } + }, + "install_and_backfill_s": 1.770374337, + "build_cpu_s": { + "backend": 0.55, + "clickhouse": 0.42 + }, + "process_after_build": { + "backend": { + "cpu_ticks": 58, + "clock_ticks_per_second": 100, + "VmHWM": 48218112, + "VmRSS": 43552768 + }, + "clickhouse": { + "cpu_ticks": 598, + "clock_ticks_per_second": 100, + "VmHWM": 811405312, + "VmRSS": 798826496 + } + }, + "process_after_queries": { + "backend": { + "cpu_ticks": 188, + "clock_ticks_per_second": 100, + "VmHWM": 63205376, + "VmRSS": 63205376 + }, + "clickhouse": { + "cpu_ticks": 3765, + "clock_ticks_per_second": 100, + "VmHWM": 1022054400, + "VmRSS": 916492288 + } + }, + "state_directory_bytes_at_build": 5238, + "state_directory_bytes_after_stop": 5238, + "first_post_build_request_ms": { + "backend": 4.364995, + "exact": 40.993606 + }, + "backend_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "clickhouse_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "raw_trace": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-2-q06/q06/requests.jsonl", + "raw_sha256": "c4345d24793e863c2e9af41901854e6ce0145c626c1e44f5e1e6df7310c37b33" + }, + { + "query": "q23", + "trial": "trial-2-q23", + "numeric_pairs_equal": 1000, + "warm_requests": 1000, + "latency_speedup": 14.663263235615045, + "routes": { + "backend": { + "requests": 1000, + "median_ms": 2.414867, + "p95_ms": 2.883722, + "request_time_rate_per_second": 407.0072293824753, + "backend_cpu_s_during_requests": 1.37, + "clickhouse_cpu_s_during_requests": 0.34, + "max_observed_backend_rss_bytes": 84406272, + "max_observed_clickhouse_rss_bytes": 1030189056 + }, + "exact": { + "requests": 1000, + "median_ms": 35.4098305, + "p95_ms": 51.677955, + "request_time_rate_per_second": 27.23270500509498, + "backend_cpu_s_during_requests": 0.04, + "clickhouse_cpu_s_during_requests": 53.73, + "max_observed_backend_rss_bytes": 84406272, + "max_observed_clickhouse_rss_bytes": 1030373376 + } + }, + "install_and_backfill_s": 2.777689036, + "build_cpu_s": { + "backend": 1.38, + "clickhouse": 0.6 + }, + "process_after_build": { + "backend": { + "cpu_ticks": 142, + "clock_ticks_per_second": 100, + "VmHWM": 75399168, + "VmRSS": 65105920 + }, + "clickhouse": { + "cpu_ticks": 611, + "clock_ticks_per_second": 100, + "VmHWM": 861368320, + "VmRSS": 844075008 + } + }, + "process_after_queries": { + "backend": { + "cpu_ticks": 283, + "clock_ticks_per_second": 100, + "VmHWM": 84406272, + "VmRSS": 84406272 + }, + "clickhouse": { + "cpu_ticks": 6045, + "clock_ticks_per_second": 100, + "VmHWM": 1030373376, + "VmRSS": 854810624 + } + }, + "state_directory_bytes_at_build": 5238, + "state_directory_bytes_after_stop": 6122, + "first_post_build_request_ms": { + "backend": 5.632831, + "exact": 36.710725 + }, + "backend_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "clickhouse_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "raw_trace": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-2-q23/q23/requests.jsonl", + "raw_sha256": "8fa00d84d1d2f5264d13168e39825cd97419cfa9c903a604d56ceb6651694d7c" + }, + { + "query": "q05", + "trial": "trial-3-q05", + "numeric_pairs_equal": 1000, + "warm_requests": 1000, + "latency_speedup": 21.741075496175057, + "routes": { + "backend": { + "requests": 1000, + "median_ms": 2.3618680000000003, + "p95_ms": 2.857925, + "request_time_rate_per_second": 417.6474853349689, + "backend_cpu_s_during_requests": 1.36, + "clickhouse_cpu_s_during_requests": 0.25, + "max_observed_backend_rss_bytes": 112996352, + "max_observed_clickhouse_rss_bytes": 1061302272 + }, + "exact": { + "requests": 1000, + "median_ms": 51.3495505, + "p95_ms": 77.297097, + "request_time_rate_per_second": 18.608955117751, + "backend_cpu_s_during_requests": 0.06, + "clickhouse_cpu_s_during_requests": 89.79, + "max_observed_backend_rss_bytes": 112996352, + "max_observed_clickhouse_rss_bytes": 1061408768 + } + }, + "install_and_backfill_s": 4.29229429, + "build_cpu_s": { + "backend": 2.73, + "clickhouse": 1.07 + }, + "process_after_build": { + "backend": { + "cpu_ticks": 277, + "clock_ticks_per_second": 100, + "VmHWM": 119361536, + "VmRSS": 93249536 + }, + "clickhouse": { + "cpu_ticks": 724, + "clock_ticks_per_second": 100, + "VmHWM": 991547392, + "VmRSS": 802988032 + } + }, + "process_after_queries": { + "backend": { + "cpu_ticks": 419, + "clock_ticks_per_second": 100, + "VmHWM": 119361536, + "VmRSS": 112996352 + }, + "clickhouse": { + "cpu_ticks": 9789, + "clock_ticks_per_second": 100, + "VmHWM": 1061408768, + "VmRSS": 891162624 + } + }, + "state_directory_bytes_at_build": 5238, + "state_directory_bytes_after_stop": 7008, + "first_post_build_request_ms": { + "backend": 4.985173, + "exact": 146.821824 + }, + "backend_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "clickhouse_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "raw_trace": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-3-q05/q05/requests.jsonl", + "raw_sha256": "58046878910dbf4d93d4d3ff569e1f40423086260237cbaa23845993e44ff2a0" + }, + { + "query": "q06", + "trial": "trial-3-q06", + "numeric_pairs_equal": 1000, + "warm_requests": 1000, + "latency_speedup": 8.72838994110124, + "routes": { + "backend": { + "requests": 1000, + "median_ms": 2.2988600000000003, + "p95_ms": 2.744593, + "request_time_rate_per_second": 428.3359654378522, + "backend_cpu_s_during_requests": 1.25, + "clickhouse_cpu_s_during_requests": 0.5, + "max_observed_backend_rss_bytes": 64716800, + "max_observed_clickhouse_rss_bytes": 1071005696 + }, + "exact": { + "requests": 1000, + "median_ms": 20.0653465, + "p95_ms": 33.411354, + "request_time_rate_per_second": 46.06321649971232, + "backend_cpu_s_during_requests": 0.02, + "clickhouse_cpu_s_during_requests": 31.53, + "max_observed_backend_rss_bytes": 64716800, + "max_observed_clickhouse_rss_bytes": 1071005696 + } + }, + "install_and_backfill_s": 1.772314251, + "build_cpu_s": { + "backend": 0.52, + "clickhouse": 0.3 + }, + "process_after_build": { + "backend": { + "cpu_ticks": 55, + "clock_ticks_per_second": 100, + "VmHWM": 48406528, + "VmRSS": 43905024 + }, + "clickhouse": { + "cpu_ticks": 586, + "clock_ticks_per_second": 100, + "VmHWM": 828514304, + "VmRSS": 806227968 + } + }, + "process_after_queries": { + "backend": { + "cpu_ticks": 183, + "clock_ticks_per_second": 100, + "VmHWM": 64716800, + "VmRSS": 64716800 + }, + "clickhouse": { + "cpu_ticks": 3817, + "clock_ticks_per_second": 100, + "VmHWM": 1071005696, + "VmRSS": 866656256 + } + }, + "state_directory_bytes_at_build": 5238, + "state_directory_bytes_after_stop": 5238, + "first_post_build_request_ms": { + "backend": 4.753846, + "exact": 36.319163 + }, + "backend_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "clickhouse_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "raw_trace": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-3-q06/q06/requests.jsonl", + "raw_sha256": "212c6e2c61640348dd362e3f9bc5744cff9e7ef54c10545f6221b7bae43c4e7d" + }, + { + "query": "q23", + "trial": "trial-3-q23", + "numeric_pairs_equal": 1000, + "warm_requests": 1000, + "latency_speedup": 14.699514293456742, + "routes": { + "backend": { + "requests": 1000, + "median_ms": 2.369126, + "p95_ms": 2.858031, + "request_time_rate_per_second": 414.0231017330702, + "backend_cpu_s_during_requests": 1.36, + "clickhouse_cpu_s_during_requests": 0.37, + "max_observed_backend_rss_bytes": 84619264, + "max_observed_clickhouse_rss_bytes": 1056153600 + }, + "exact": { + "requests": 1000, + "median_ms": 34.8250015, + "p95_ms": 54.463904, + "request_time_rate_per_second": 27.23789216168408, + "backend_cpu_s_during_requests": 0.03, + "clickhouse_cpu_s_during_requests": 53.69, + "max_observed_backend_rss_bytes": 84619264, + "max_observed_clickhouse_rss_bytes": 1056423936 + } + }, + "install_and_backfill_s": 2.782497867, + "build_cpu_s": { + "backend": 1.37, + "clickhouse": 0.65 + }, + "process_after_build": { + "backend": { + "cpu_ticks": 141, + "clock_ticks_per_second": 100, + "VmHWM": 74371072, + "VmRSS": 65110016 + }, + "clickhouse": { + "cpu_ticks": 623, + "clock_ticks_per_second": 100, + "VmHWM": 868835328, + "VmRSS": 863690752 + } + }, + "process_after_queries": { + "backend": { + "cpu_ticks": 281, + "clock_ticks_per_second": 100, + "VmHWM": 84619264, + "VmRSS": 84619264 + }, + "clickhouse": { + "cpu_ticks": 6053, + "clock_ticks_per_second": 100, + "VmHWM": 1056423936, + "VmRSS": 862023680 + } + }, + "state_directory_bytes_at_build": 5238, + "state_directory_bytes_after_stop": 6122, + "first_post_build_request_ms": { + "backend": 4.921386, + "exact": 47.286535 + }, + "backend_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "clickhouse_budget": { + "image": "sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb", + "cpu_affinity": "60,61", + "nano_cpus": 2000000000, + "memory_limit_bytes": 4294967296 + }, + "raw_trace": "/mydata/clickhouse-o11y-main-results/original27-matched/trial-3-q23/q23/requests.jsonl", + "raw_sha256": "5a0554274d325f6ddec93830e7af7f28798e9e893ed4b5868e984cc67d87b1e7" + } + ] +} diff --git a/tools/o11y-sql-main-eval/automatic-map-results.md b/tools/o11y-sql-main-eval/automatic-map-results.md new file mode 100644 index 000000000..64fe7a306 --- /dev/null +++ b/tools/o11y-sql-main-eval/automatic-map-results.md @@ -0,0 +1,77 @@ +# Original o11y SQL execution through automatic summary plans + +The original 27-query corpus now has a real execution matrix: 3 warm queries, 18 successful exact fallbacks, and 6 failures on both the backend fallback and native ClickHouse. All 21 successful responses have equal decoded JSON metadata and data. This is per-query execution coverage; it does not establish acceleration for all 27 queries or one jointly installed dashboard workload. + +The warm queries are q05 (12-hour maximum grouped by the native labels Map), q06 (12-hour maximum for the user-service metric), and q23 (top 2 groups by the 6-hour retry-backlog maximum). The compiler receives the original SQL, table schema, and evaluation range. It calls Planner, records its actual selection, builds the catalog and executable plans, and installs the selected exact MinMax summaries. No materialization family or winner is supplied by the evaluation script. + +## Data and provenance + +- Existing o11y-derived capture with deterministic metric aliases: `/mydata/query-benefit-evaluation/o11y-evaluation-1s-24h.prom`. +- Source SHA256: `527efecab22f935b7aa00a29d7ed8a187da902a223c4fef3991d433268b528f6`. +- Samples: 12,236,002 across 142 series, at 1-second sampling granularity. +- Observed timestamp span: 1788804926000–1788891296000 ms (86,370 seconds, approximately 24 hours). +- Imported audit copy: `/mydata/clickhouse-o11y-main-results/o11y-27-1s-24h.jsonl.gz`, 85,682,422 bytes. Its `.jsonl.manifest.json` records counts per metric and the unchanged source hash. +- Isolated ClickHouse destination: `asap_o11y27_eval.raw_samples` on port 28123. + +The importer preserves metric names, labels, Float64 sample values, and integral-millisecond timestamps. It rejects duplicate/malformed labels and nonfinite samples instead of silently changing them. Metric aliases were already present in the source file; this run does not claim those names are emitted directly by upstream o11y-bench. + +## Execution coverage + +The original arithmetic timestamp bounds `(evaluation_time - window, evaluation_time]` remain unchanged in SQL. The compiler evaluates checked integer constants to bind their equivalent half-open source interval. Native Map labels stay one typed grouping column through backfill, SummaryStore, and query readout. + +The six failures are q01, q02, q17, q18, q25, and q26. Both native ClickHouse and forwarded execution returned HTTP 500 with memory-limit errors under the 4 GiB server limit. The existing counter SQL uses indexed `arraySum` expressions that trigger large intermediate allocations. These failures are not counted as successful fallback or as an ASAP benefit. A separate diagnostic isolates the indexed array capture with two groups: ClickHouse query-log peak memory rises from 33.45 MB at 1,000 samples/group to 129.55 MB at 2,000, 513.77 MB at 4,000, and 2.05 GB at 8,000. Doubling samples approximately quadruples peak memory. This explains why the original long counter windows exhaust the limit; the diagnostic does not replace the original queries or count as workload acceleration. [Exact diagnostic SQL and query-log measurements](captured-array-memory-diagnostic.json) are included. HTTP summary memory is not used as a peak-memory measurement. + +The other 24 queries still fail the Planner SQL frontend because of ClickHouse-specific syntax or functions, including lambda/tuple syntax, Map access, `mapConcat`, empty `map()`, and `modulo`. Eighteen nevertheless execute correctly through whole-query native fallback. Their fallback success demonstrates routing correctness, not local DAG support or acceleration. + +## Measurement scope + +Repeated-query results are recorded separately from the correctness matrix. Each selected query uses a fresh backend and restarted native ClickHouse process, with 2 CPU cores (60–61) and a 4 GiB memory cap per process. Query cache is disabled and ClickHouse `max_threads=2`. The raw dataset is unchanged. Three independent trials contain 1000 backend and 1000 native requests each, alternating route order. + +The measurements include publication/backfill wall time, backend and source ClickHouse CPU during construction, process RSS/high-water marks, state directory bytes, and per-request latency/CPU. They exclude original data ingestion and offline planning costs. Estimated numeric Planner cost is not returned by the current SQL selection trace. No complete deployment cost reduction, moving-window behavior, or memory reduction is inferred from these fixed-window trials. The backend continues to require the external ClickHouse service for fallback. + +## Measured results + +All 9,000 backend requests were warm, and all 9,000 matched native/backend pairs had equal decoded JSON metadata and data. HTTP response bytes differ in formatting; this is not byte equality. + +| Original query | Backend median latency | Native median latency | Speedup across three trials | Summary directory after shutdown | +| --- | ---: | ---: | ---: | ---: | +| q05: 12h maximum, five groups | 2.36–2.43 ms | 50.85–51.35 ms | 21.04–21.74× | 7,008 B | +| q06: 12h user-service maximum | 2.27–2.38 ms | 19.92–20.36 ms | 8.37–8.97× | 5,238 B | +| q23: TopK over 6h retry-backlog maximum | 2.37–2.43 ms | 33.64–35.41 ms | 13.84–14.70× | 6,122 B | + +The serial request-time rate (requests divided by summed HTTP request durations, excluding controller gaps) was 399.7–422.3 versus 18.6–19.0 requests/s for q05, 419.7–434.6 versus 45.6–46.4 for q06, and 400.4–414.0 versus 27.2–28.0 for q23. This is not a concurrent saturation-throughput benchmark. + +First requests after state construction took 4.99–5.67 versus 72.46–146.82 ms for q05, 4.36–5.14 versus 35.23–40.99 ms for q06, and 4.92–6.24 versus 36.71–47.29 ms for q23. They are separate from the repeated-query medians and do not represent empty filesystem-cache cold starts: construction has already read the source data. + +Publication and backfill took approximately 4.29 seconds for q05, 1.77 seconds for q06, and 2.78 seconds for q23. Backend construction CPU was 2.73–2.80, 0.52–0.55, and 1.35–1.38 seconds respectively; the source ClickHouse process additionally used 1.00–1.29, 0.24–0.42, and 0.60–0.84 seconds. State-directory bytes include metadata and are not a comparison against the complete source database. + +A separate q05 experiment restarted ClickHouse before each of three ASAP-only and native-only phases. The ASAP phase used a previously validated response only as a correctness oracle, never to construct state, and issued no native query during its 1,000 warm requests. This avoids attributing native query allocations to the ASAP deployment. + +| Fresh deployment, startup through 1,000 queries | Backend plus retained ClickHouse | Native ClickHouse alone | +| --- | ---: | ---: | +| Process CPU, including summary construction | 11.16–12.33 s | 96.67–98.86 s | +| End-of-query combined RSS | 837.5–867.4 MiB | 798.6–837.8 MiB | + +The CPU reduction survives including summary construction for this repeated workload. **Total deployment memory is higher with ASAP in all three pairs.** Summing individual process high-water marks is only an upper bound on simultaneous peak memory, so it is not reported as a measured deployment peak. Common initial raw-data ingestion and offline planning remain excluded. These results establish benefits for the three warm queries and do not establish benefits for the remaining 24 queries. + +The release runtime was built from `c825e804`; driver checkout `eff81f48` adds formatting changes only. Binary and input hashes are recorded with the raw trials. The correctness matrix was produced before the final conservative output-format guards; the three warm paths were rerun with the final release runtime. + +## Reproduction + +Compile publications with `cargo run -p control_plane --example audit_clickhouse_corpus -- --automatic tools/o11y-sql-main-eval/corpus.json`. The automatic mode shares the existing audit executable, avoiding an additional large evaluation binary during workspace checks. + +Use [the importer](import_openmetrics.py) with the source capture above, then [the automatic corpus compiler](../../control_plane/examples/support/automatic_clickhouse_corpus.rs). [The process driver](run_automatic.py) stages and activates each compiled publication and submits typed backfill jobs. [The matched runner](run_matched.sh) pins the container image, resource limits, executable hash, and input matrix, and restarts ClickHouse before each measured query/trial. + +Raw execution artifacts remain at `/mydata/clickhouse-o11y-main-results/original27-map-runtime/`; raw timing traces remain at `/mydata/clickhouse-o11y-main-results/original27-matched/`. Reproducing the exact capture requires that source file; a fresh upstream capture can have different values and cardinality. + +The separate [deployment runner](run_deployment_pair.sh) compares fresh ASAP-plus-ClickHouse and native-only phases. With `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD` already set in the environment, run: + +```bash +export MATRIX=/mydata/clickhouse-o11y-main-results/automatic-map-corpus.json +export BACKEND_BIN=/dev/shm/asap-clickhouse-release/release/data_plane +export RUNTIME_SOURCE_COMMIT=c825e804 +RESULT_ROOT=/mydata/clickhouse-o11y-main-results/new-matched bash tools/o11y-sql-main-eval/run_matched.sh +EXACT_REFERENCE=/mydata/clickhouse-o11y-main-results/new-matched/trial-1-q05/q05/result.json RESULT_ROOT=/mydata/clickhouse-o11y-main-results/new-deployment bash tools/o11y-sql-main-eval/run_deployment_pair.sh +``` + +The runners require the dedicated source container and populated table described above and verify its pinned image and resource budget before restarting it. Use new output paths; do not point them at production services. Compact [coverage](automatic-map-coverage.json), [latency measurements](automatic-map-measurements.json), [deployment measurements](automatic-map-deployment-measurements.json), and [raw artifact hashes](automatic-map-artifact-manifest.json) accompany this report. diff --git a/tools/o11y-sql-main-eval/captured-array-memory-diagnostic.json b/tools/o11y-sql-main-eval/captured-array-memory-diagnostic.json new file mode 100644 index 000000000..5bc66ddd9 --- /dev/null +++ b/tools/o11y-sql-main-eval/captured-array-memory-diagnostic.json @@ -0,0 +1,22 @@ +[ + { + "query_duration_ms": 1863, + "memory_usage": 2050188539, + "query": "SELECT g,arraySum(i -> if(samples[i].2 < samples[i-1].2, samples[i-1].2, 0), range(2,length(samples)+1)) FROM (SELECT number%2 g,groupArray(tuple(number,toFloat64(number))) samples FROM numbers(16000) GROUP BY g) FORMAT JSON" + }, + { + "query_duration_ms": 442, + "memory_usage": 513765627, + "query": "SELECT g,arraySum(i -> if(samples[i].2 < samples[i-1].2, samples[i-1].2, 0), range(2,length(samples)+1)) FROM (SELECT number%2 g,groupArray(tuple(number,toFloat64(number))) samples FROM numbers(8000) GROUP BY g) FORMAT JSON" + }, + { + "query_duration_ms": 124, + "memory_usage": 129554299, + "query": "SELECT g,arraySum(i -> if(samples[i].2 < samples[i-1].2, samples[i-1].2, 0), range(2,length(samples)+1)) FROM (SELECT number%2 g,groupArray(tuple(number,toFloat64(number))) samples FROM numbers(4000) GROUP BY g) FORMAT JSON" + }, + { + "query_duration_ms": 47, + "memory_usage": 33448587, + "query": "SELECT g,arraySum(i -> if(samples[i].2 < samples[i-1].2, samples[i-1].2, 0), range(2,length(samples)+1)) FROM (SELECT number%2 g,groupArray(tuple(number,toFloat64(number))) samples FROM numbers(2000) GROUP BY g) FORMAT JSON" + } +] diff --git a/tools/o11y-sql-main-eval/import_openmetrics.py b/tools/o11y-sql-main-eval/import_openmetrics.py new file mode 100644 index 000000000..bb6165bf8 --- /dev/null +++ b/tools/o11y-sql-main-eval/import_openmetrics.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Import unchanged OpenMetrics samples into an isolated ClickHouse table.""" +import argparse +import base64 +import collections +from decimal import Decimal +import gzip +import hashlib +import json +import math +import os +from pathlib import Path +import re +import urllib.parse +import urllib.request + +LABEL = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)=("(?:\\.|[^"\\])*")(?:,|$)') + + +def parse_series(token): + if '{' not in token: + return token, {} + metric, tail = token.split('{', 1) + if not tail.endswith('}'): + raise ValueError('unterminated series labels') + labels, offset = {}, 0 + for match in LABEL.finditer(tail[:-1]): + if match.start() != offset or match[1] in labels: + raise ValueError('invalid or duplicate series label') + labels[match[1]] = json.loads(match[2]) + offset = match.end() + if offset != len(tail) - 1: + raise ValueError('unparsed series labels') + return metric, labels + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('source', type=Path) + ap.add_argument('output', type=Path, help='lossless JSONEachRow gzip artifact') + ap.add_argument('--url', default='http://127.0.0.1:28123') + ap.add_argument('--database', default='asap_o11y27_eval') + args = ap.parse_args() + if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', args.database): + ap.error('database must be a simple identifier') + credentials = f"{os.environ['CLICKHOUSE_USER']}:{os.environ['CLICKHOUSE_PASSWORD']}" + auth = 'Basic ' + base64.b64encode(credentials.encode()).decode() + + def request(body, database=None, compressed=False): + url = args.url + '/?' + urllib.parse.urlencode({'database': database or 'default'}) + headers = {'Authorization': auth} + if compressed: + headers['Content-Encoding'] = 'gzip' + req = urllib.request.Request(url, data=body, headers=headers) + with urllib.request.urlopen(req, timeout=120) as response: + return response.read() + + request(f'CREATE DATABASE IF NOT EXISTS `{args.database}`'.encode()) + request(b'CREATE TABLE IF NOT EXISTS raw_samples (metric String, labels Map(String,String), ts_ms Int64, value Float64) ENGINE=MergeTree ORDER BY (metric,ts_ms)', args.database) + if int(request(b'SELECT count() FROM raw_samples', args.database)): + raise RuntimeError('destination contains samples; use a fresh database') + if args.output.exists(): + raise RuntimeError('output artifact already exists; choose a new path') + source_sha = hashlib.sha256() + counts, series, pending = collections.Counter(), {}, [] + lower = upper = None + args.output.parent.mkdir(parents=True, exist_ok=True) + + def flush(): + if pending: + request(gzip.compress(b'INSERT INTO raw_samples FORMAT JSONEachRow\n' + b''.join(pending), compresslevel=1), args.database, True) + pending.clear() + + with args.source.open('rb') as source, gzip.open(args.output, 'wb', compresslevel=1) as output: + for raw in source: + source_sha.update(raw) + if not raw.strip() or raw.startswith(b'#'): + continue + token, value, timestamp = raw.decode().strip().rsplit(None, 2) + if token not in series: + series[token] = parse_series(token) + metric, labels = series[token] + milliseconds = Decimal(timestamp) * 1000 + if milliseconds != milliseconds.to_integral_value(): + raise ValueError('sub-millisecond timestamp cannot enter the Int64 millisecond schema') + ts = int(milliseconds) + number = float(value) + if not math.isfinite(number): + raise ValueError('nonfinite sample requires an explicit ClickHouse transport contract') + record = (json.dumps({'metric': metric, 'labels': labels, 'ts_ms': ts, 'value': number}, separators=(',', ':'), allow_nan=False) + '\n').encode() + output.write(record) + pending.append(record) + counts[metric] += 1 + lower = ts if lower is None else min(lower, ts) + upper = ts if upper is None else max(upper, ts) + if len(pending) >= 50_000: + flush() + flush() + actual = int(request(b'SELECT count() FROM raw_samples', args.database)) + if actual != sum(counts.values()): + raise RuntimeError(f'row count differs: source {sum(counts.values())}, ClickHouse {actual}') + manifest = {'source': str(args.source.resolve()), 'source_sha256': source_sha.hexdigest(), 'output': str(args.output.resolve()), 'output_bytes': args.output.stat().st_size, 'database': args.database, 'table': 'raw_samples', 'samples': actual, 'series': len(series), 'start_ms': lower, 'end_ms': upper, 'metric_samples': counts, 'transformation': 'unchanged metric names, labels, Float64 samples, and integral-millisecond timestamps; no query-answer ingestion'} + args.output.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2) + '\n') + print(json.dumps(manifest)) + +if __name__ == '__main__': + main() diff --git a/tools/o11y-sql-main-eval/measure_native.py b/tools/o11y-sql-main-eval/measure_native.py new file mode 100644 index 000000000..4ff5e05e8 --- /dev/null +++ b/tools/o11y-sql-main-eval/measure_native.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Fresh native-only replay for deployment CPU/RSS comparison.""" +import argparse +import base64 +import json +import os +from pathlib import Path +import time +import urllib.parse +import urllib.request +from run_automatic import process_metrics + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('matrix', type=Path) + ap.add_argument('output', type=Path) + ap.add_argument('--query', default='q05') + ap.add_argument('--pid', type=int, required=True) + ap.add_argument('--expected', type=Path, required=True) + args = ap.parse_args() + query = next(q for q in json.loads(args.matrix.read_text())['queries'] if q['id'] == args.query) + reference = json.loads(args.expected.read_text()) + if reference['id'] != args.query: + raise ValueError('reference identity mismatch') + expected = json.loads(reference['exact']['body']) + args.output.mkdir(parents=True, exist_ok=False) + auth = 'Basic ' + base64.b64encode(f"{os.environ['CLICKHOUSE_USER']}:{os.environ['CLICKHOUSE_PASSWORD']}".encode()).decode() + url = 'http://127.0.0.1:28123/?' + urllib.parse.urlencode({'database': 'asap_o11y27_eval', 'default_format': 'JSON', 'max_threads': 2, 'use_query_cache': 0}) + before = process_metrics(args.pid) + with (args.output / 'requests.jsonl').open('w') as trace: + for repeat in range(1000): + start = time.perf_counter_ns() + with urllib.request.urlopen(urllib.request.Request(url, data=query['sql'].encode(), headers={'Authorization': auth}), timeout=180) as response: + body = response.read().decode() + status = response.status + elapsed = time.perf_counter_ns() - start + actual = json.loads(body) + if status != 200 or actual['meta'] != expected['meta'] or actual['data'] != expected['data']: + raise ValueError('native result mismatch') + trace.write(json.dumps({'repeat': repeat, 'status': status, 'elapsed_ns': elapsed, 'body': body, 'process': process_metrics(args.pid)}) + '\n') + result = {'query': args.query, 'requests_equal': 1000, 'process_before': before, 'process_after': process_metrics(args.pid), 'scope': 'fresh native process, persisted source data; common initial ingestion excluded'} + (args.output / 'result.json').write_text(json.dumps(result, indent=2) + '\n') + print(args.query, 'native-only', 1000, flush=True) + +if __name__ == '__main__': + main() diff --git a/tools/o11y-sql-main-eval/run_automatic.py b/tools/o11y-sql-main-eval/run_automatic.py new file mode 100644 index 000000000..4e87bf87d --- /dev/null +++ b/tools/o11y-sql-main-eval/run_automatic.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Execute the original SQL corpus against automatic publications and native CH.""" +import argparse +import base64 +import json +import os +from pathlib import Path +import socket +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request + + +def port(): + with socket.socket() as sock: + sock.bind(('127.0.0.1', 0)) + return sock.getsockname()[1] + + +def process_metrics(pid): + if not pid: + return None + stat = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split() + status = {} + for line in Path(f'/proc/{pid}/status').read_text().splitlines(): + if line.startswith(('VmRSS:', 'VmHWM:')): + name, value, _ = line.split() + status[name[:-1]] = int(value) * 1024 + return {'cpu_ticks': int(stat[11]) + int(stat[12]), 'clock_ticks_per_second': os.sysconf('SC_CLK_TCK'), **status} + + +def cpu_delta(before, after): + return None if before is None or after is None else after['cpu_ticks'] - before['cpu_ticks'] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('matrix', type=Path) + ap.add_argument('backend', type=Path) + ap.add_argument('output', type=Path) + ap.add_argument('--clickhouse', default='http://127.0.0.1:28123') + ap.add_argument('--database', default='asap_o11y27_eval') + ap.add_argument('--ids', help='comma-separated subset; the installed Planner choice is unchanged') + ap.add_argument('--repetitions', type=int, default=0) + ap.add_argument('--container-image', help='pin an image digest to enforce matched2CPU/4GiB budgets') + ap.add_argument('--clickhouse-pid', type=int) + ap.add_argument('--only-backend', action='store_true') + ap.add_argument('--expected-result', type=Path) + args = ap.parse_args() + if args.only_backend and (not args.expected_result or not args.ids or ',' in args.ids): + ap.error('--only-backend requires one --ids value and --expected-result') + args.output.mkdir(parents=True, exist_ok=False) + user, password = os.environ['CLICKHOUSE_USER'], os.environ['CLICKHOUSE_PASSWORD'] + auth = 'Basic ' + base64.b64encode(f'{user}:{password}'.encode()).decode() + + def request(url, body=None, data=None): + headers = {'Authorization': auth} + if data is not None: + body = json.dumps(data).encode() + headers['Content-Type'] = 'application/json' + if isinstance(body, str): + body = body.encode() + start = time.perf_counter_ns() + try: + response = urllib.request.urlopen(urllib.request.Request(url, data=body, headers=headers), timeout=180) + except urllib.error.HTTPError as error: + response = error + with response: + result = {'status': response.status, 'headers': {key.lower(): value for key, value in response.headers.items()}, 'body': response.read().decode()} + result['elapsed_ns'] = time.perf_counter_ns() - start + return result + + def require(result): + if not 200 <= result['status'] < 300: + raise RuntimeError(str(result)) + return json.loads(result['body']) if result['body'] else None + + results = [] + for query in json.loads(args.matrix.read_text())['queries']: + if args.ids and query['id'] not in args.ids.split(','): + continue + directory = args.output / query['id'] + directory.mkdir() + bootstrap = directory / 'bootstrap.yaml' + bootstrap.write_text('aggregations: []\n') + api, sql = port(), port() + command = [str(args.backend.resolve()), '--streaming-config', str(bootstrap), '--http-port', str(api), '--clickhouse-http-port', str(sql), '--clickhouse-url', args.clickhouse, '--clickhouse-database', args.database, '--clickhouse-backfill-table', 'raw_samples', '--clickhouse-backfill-database', args.database, '--clickhouse-user', user, '--clickhouse-password', password, '--enable-backfill-worker', '--precompute-allowed-lateness-ms', '0', '--precompute-flush-interval-ms', '50', '--persistence-delete-older-than-secs', '0', '--output-dir', str(directory / 'state')] + container = None + if args.container_image: + container = f'asap-o11y27-{args.output.name}-{query["id"]}' + runtime = args.backend.resolve().parent.parent + command = ['docker', 'run', '--name', container, '--user', f'{os.getuid()}:{os.getgid()}', '--network', 'host', '--pid', 'host', '--cpuset-cpus', '60,61', '--cpus', '2', '--memory', '4g', '-v', f'{runtime}:{runtime}:ro', '-v', f'{args.output.resolve()}:{args.output.resolve()}', '--entrypoint', command[0], args.container_image, *command[1:]] + with (directory / 'backend.log').open('w') as log: + process = subprocess.Popen(command, stdout=log, stderr=log, env=dict(os.environ, RUST_LOG='data_plane=info')) + try: + base = f'http://127.0.0.1:{api}' + for _ in range(200): + if process.poll() is not None: + raise RuntimeError('backend exited: inspect backend.log') + try: + if request(base + '/api/v1/health')['status'] == 200: + break + except OSError: + pass + time.sleep(.05) + else: + raise RuntimeError('backend did not start') + backend_pid = int(subprocess.check_output(['docker', 'inspect', container, '--format', '{{.State.Pid}}'])) if container else process.pid + build_before = {'backend': process_metrics(backend_pid), 'clickhouse': process_metrics(args.clickhouse_pid)} + build_started = time.perf_counter_ns() + jobs = None + if query['publication'] == 'pass': + install = query['install'] + require(request(base + '/api/v1/physical-plan', data=install)) + envelope = install['precompute_plan']['envelope'] + require(request(base + '/api/v1/physical-plan/activate', data={'plan_id': envelope['plan_id'], 'plan_version': envelope['plan_version']})) + for identity in install['summary_catalog']['materializations']: + require(request(base + '/api/v1/db/backfill', data={'agg_id': int(identity), 'start_ms': query['window_start_ms'], 'end_ms': query['window_end_ms'], 'source': {'ClickHouse': {'database': args.database, 'table': 'raw_samples'}}, 'windows_total': 1})) + for _ in range(2400): + jobs = require(request(base + '/api/v1/db/backfill/jobs')) + statuses = [job['status'] for job in jobs['jobs']] + if statuses and all(status == 'complete' for status in statuses): + break + if 'failed' in statuses: + raise RuntimeError(str(jobs)) + time.sleep(.25) + else: + raise RuntimeError('backfill timed out') + build_ns = time.perf_counter_ns() - build_started + build_after = {'backend': process_metrics(backend_pid), 'clickhouse': process_metrics(args.clickhouse_pid)} + params = urllib.parse.urlencode({'database': args.database, 'default_format': 'JSON', 'max_threads': 2, 'use_query_cache': 0}) + if args.only_backend: + reference = json.loads(args.expected_result.read_text()) + if reference['id'] != query['id']: + raise ValueError('reference query identity mismatch') + exact = dict(reference['exact'], elapsed_ns=None) + else: + exact = request(args.clickhouse + '/?' + params, query['sql']) + actual = request(f'http://127.0.0.1:{sql}/?' + params, query['sql']) + equal = False + if actual['status'] == exact['status'] == 200: + left, right = json.loads(actual['body']), json.loads(exact['body']) + equal = left['meta'] == right['meta'] and left['data'] == right['data'] + result = {'id': query['id'], 'publication': query['publication'], 'route': actual['headers'].get('x-asap-execution', 'unmarked'), 'execution': actual['headers'].get('x-asap-execution', 'unmarked') if actual['status'] == 200 else 'failed', 'equal': equal, 'install_and_backfill_ns': build_ns, 'backfill_jobs': jobs, 'backend': actual, 'exact': exact, 'timing_scope': 'debug correctness run, not performance evidence'} + result['build_resources'] = {'before': build_before, 'after': build_after} + result['state_directory_bytes'] = sum(path.stat().st_size for path in (directory / 'state').rglob('*') if path.is_file()) + if args.repetitions: + result['timing_scope'] = ('ASAP-only deployment replay against a prior validated reference; ' if args.only_backend else 'alternating backend/native replay; ') + 'source-data ingestion and offline planning costs are not measured here' + if args.only_backend: + result['exact_reference_path'] = str(args.expected_result.resolve()) + counts = {'warm': 0, 'equal': 0, 'failed': 0} + with (directory / 'requests.jsonl').open('w') as trace: + for repeat in range(args.repetitions): + pair = {} + for route in (['backend'] if args.only_backend else (['backend', 'exact'] if repeat % 2 == 0 else ['exact', 'backend'])): + before_backend, before_ch = process_metrics(backend_pid), process_metrics(args.clickhouse_pid) + endpoint = f'http://127.0.0.1:{sql}/' if route == 'backend' else args.clickhouse + '/' + measured = request(endpoint + '?' + params, query['sql']) + after_backend, after_ch = process_metrics(backend_pid), process_metrics(args.clickhouse_pid) + trace.write(json.dumps({'repeat': repeat, 'route': route, 'response': measured, 'backend_cpu_ticks': cpu_delta(before_backend, after_backend), 'clickhouse_cpu_ticks': cpu_delta(before_ch, after_ch), 'backend_process': after_backend, 'clickhouse_process': after_ch}) + '\n') + pair[route] = measured + if args.only_backend: + pair['exact'] = exact + counts['warm'] += pair['backend']['headers'].get('x-asap-execution') == 'warm' + if pair['backend']['status'] == pair['exact']['status'] == 200: + left, right = json.loads(pair['backend']['body']), json.loads(pair['exact']['body']) + counts['equal'] += left['meta'] == right['meta'] and left['data'] == right['data'] + else: + counts['failed'] += 1 + result['repeated'] = counts + result['query_resources_after'] = {'backend': process_metrics(backend_pid), 'clickhouse': process_metrics(args.clickhouse_pid)} + if container: + inspection = json.loads(subprocess.check_output(['docker', 'inspect', container]))[0] + result['container'] = {'image': inspection['Image'], 'cpu_affinity': inspection['HostConfig']['CpusetCpus'], 'nano_cpus': inspection['HostConfig']['NanoCpus'], 'memory_limit_bytes': inspection['HostConfig']['Memory']} + (directory / 'result.json').write_text(json.dumps(result, indent=2) + '\n') + results.append(result) + print(query['id'], result['execution'], equal, flush=True) + except Exception as error: + result = {'id': query['id'], 'publication': query['publication'], 'execution': 'failed', 'error': str(error)} + results.append(result) + (directory / 'result.json').write_text(json.dumps(result, indent=2) + '\n') + print(query['id'], 'failed', str(error)[:200], flush=True) + finally: + if container: + subprocess.run(['docker', 'stop', '-t', '10', container], stdout=subprocess.DEVNULL, check=False) + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + (args.output / 'results.json').write_text(json.dumps(results, indent=2) + '\n') + if args.repetitions and (not results or any(result.get('repeated') != {'warm': args.repetitions, 'equal': args.repetitions, 'failed': 0} for result in results)): + raise SystemExit('Formal replay failed: every requested repetition must be warm and equal') + +if __name__ == '__main__': + main() diff --git a/tools/o11y-sql-main-eval/run_deployment_pair.sh b/tools/o11y-sql-main-eval/run_deployment_pair.sh new file mode 100644 index 000000000..4e9d38938 --- /dev/null +++ b/tools/o11y-sql-main-eval/run_deployment_pair.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail +: "${CLICKHOUSE_USER:?set the evaluation account}" +: "${CLICKHOUSE_PASSWORD:?set the evaluation password}" +: "${MATRIX:?set the automatic publication matrix}" +: "${BACKEND_BIN:?set the frozen release executable}" +: "${RESULT_ROOT:?set a new result root}" +: "${EXACT_REFERENCE:?set a previously validated q05 paired result}" +image=clickhouse/clickhouse-server@sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb +server=asap-clickhouse-current-eval +repo_root=$(git rev-parse --show-toplevel) +mkdir "$RESULT_ROOT" +sha256sum "$BACKEND_BIN" "$MATRIX" "$EXACT_REFERENCE" > "$RESULT_ROOT/inputs.sha256" +for trial in 1 2 3; do + for route in asap native; do + [[ $(docker inspect "$server" --format '{{.HostConfig.CpusetCpus}}/{{.HostConfig.NanoCpus}}/{{.HostConfig.Memory}}') == '60,61/2000000000/4294967296' ]] + [[ $(docker inspect "$server" --format '{{.Image}}') == $(docker image inspect "$image" --format '{{.Id}}') ]] + docker restart "$server" >/dev/null + for _ in $(seq 1 100); do + if curl --silent --fail http://127.0.0.1:28123/ping >/dev/null; then break; fi + sleep 0.1 + done + curl --silent --fail http://127.0.0.1:28123/ping >/dev/null + server_pid=$(docker inspect "$server" --format '{{.State.Pid}}') + if [[ "$route" == asap ]]; then + python3 -B "$repo_root/tools/o11y-sql-main-eval/run_automatic.py" "$MATRIX" "$BACKEND_BIN" "$RESULT_ROOT/trial-$trial-asap" --ids q05 --repetitions 1000 --container-image "$image" --clickhouse-pid "$server_pid" --only-backend --expected-result "$EXACT_REFERENCE" + else + python3 -B "$repo_root/tools/o11y-sql-main-eval/measure_native.py" "$MATRIX" "$RESULT_ROOT/trial-$trial-native" --pid "$server_pid" --expected "$EXACT_REFERENCE" + fi + docker inspect "$server" --format '{"image":"{{.Image}}","cpu_affinity":"{{.HostConfig.CpusetCpus}}","nano_cpus":{{.HostConfig.NanoCpus}},"memory_limit_bytes":{{.HostConfig.Memory}}}' > "$RESULT_ROOT/trial-$trial-$route/clickhouse-runtime.json" + done +done diff --git a/tools/o11y-sql-main-eval/run_matched.sh b/tools/o11y-sql-main-eval/run_matched.sh new file mode 100644 index 000000000..322b61020 --- /dev/null +++ b/tools/o11y-sql-main-eval/run_matched.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail +: "${CLICKHOUSE_USER:?set the evaluation account}" +: "${CLICKHOUSE_PASSWORD:?set the evaluation password}" +: "${MATRIX:?set the automatic publication matrix}" +: "${BACKEND_BIN:?set the frozen release backend executable}" +: "${RESULT_ROOT:?set a new result root}" +: "${RUNTIME_SOURCE_COMMIT:?set the commit used to build BACKEND_BIN}" +image=clickhouse/clickhouse-server@sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb +server=asap-clickhouse-current-eval +repo_root=$(git rev-parse --show-toplevel) +mkdir "$RESULT_ROOT" +sha256sum "$BACKEND_BIN" "$MATRIX" > "$RESULT_ROOT/inputs.sha256" +git rev-parse HEAD > "$RESULT_ROOT/driver-checkout.txt" +printf '%s\n' "$RUNTIME_SOURCE_COMMIT" > "$RESULT_ROOT/runtime-source-commit.txt" +for trial in 1 2 3; do + for query in q05 q06 q23; do + [[ $(docker inspect "$server" --format '{{.Image}}') == $(docker image inspect "$image" --format '{{.Id}}') ]] + [[ $(docker inspect "$server" --format '{{.HostConfig.CpusetCpus}}/{{.HostConfig.NanoCpus}}/{{.HostConfig.Memory}}') == '60,61/2000000000/4294967296' ]] + docker restart "$server" >/dev/null + for _ in $(seq 1 100); do + if curl --silent --fail http://127.0.0.1:28123/ping >/dev/null; then break; fi + sleep 0.1 + done + curl --silent --fail http://127.0.0.1:28123/ping >/dev/null + server_pid=$(docker inspect "$server" --format '{{.State.Pid}}') + python3 -B "$repo_root/tools/o11y-sql-main-eval/run_automatic.py" "$MATRIX" "$BACKEND_BIN" "$RESULT_ROOT/trial-$trial-$query" --ids "$query" --repetitions 1000 --container-image "$image" --clickhouse-pid "$server_pid" + docker inspect "$server" --format '{"image":"{{.Image}}","cpu_affinity":"{{.HostConfig.CpusetCpus}}","nano_cpus":{{.HostConfig.NanoCpus}},"memory_limit_bytes":{{.HostConfig.Memory}}}' > "$RESULT_ROOT/trial-$trial-$query/clickhouse-runtime.json" + done +done diff --git a/tools/o11y-sql-main-eval/summarize_automatic.py b/tools/o11y-sql-main-eval/summarize_automatic.py new file mode 100644 index 000000000..a3cc5f1aa --- /dev/null +++ b/tools/o11y-sql-main-eval/summarize_automatic.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Validate paired raw requests and summarize fixed-window release trials.""" +import argparse +import hashlib +import json +from pathlib import Path +import statistics + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('root', type=Path) + ap.add_argument('output', type=Path) + args = ap.parse_args() + summaries = [] + for trace in sorted(args.root.glob('trial-*/q*/requests.jsonl')): + result = json.loads(trace.with_name('result.json').read_text()) + rows = [json.loads(line) for line in trace.read_text().splitlines()] + pairs = {} + for row in rows: + pair = pairs.setdefault(row['repeat'], {}) + if row['route'] in pair: + raise ValueError('duplicate route/repetition') + pair[row['route']] = row + for pair in pairs.values(): + if set(pair) != {'backend', 'exact'}: + raise ValueError('incomplete pair') + left, right = pair['backend']['response'], pair['exact']['response'] + if left['status'] != 200 or right['status'] != 200: + raise ValueError('unsuccessful response') + if left['headers'].get('x-asap-execution') != 'warm': + raise ValueError('backend did not execute warm') + left, right = json.loads(left['body']), json.loads(right['body']) + if left['meta'] != right['meta'] or left['data'] != right['data']: + raise ValueError('decoded result mismatch') + if len(pairs) != 1000 or set(pairs) != set(range(1000)): + raise ValueError('expected1000 complete repetitions') + routes = {} + for route in ['backend', 'exact']: + selected = [row for row in rows if row['route'] == route] + latency = [row['response']['elapsed_ns'] / 1e6 for row in selected] + hz = selected[0]['backend_process']['clock_ticks_per_second'] + routes[route] = { + 'requests': len(selected), 'median_ms': statistics.median(latency), + 'p95_ms': sorted(latency)[949], + 'request_time_rate_per_second': len(selected) / (sum(latency) / 1000), + 'backend_cpu_s_during_requests': sum(row['backend_cpu_ticks'] for row in selected) / hz, + 'clickhouse_cpu_s_during_requests': sum(row['clickhouse_cpu_ticks'] for row in selected) / hz, + 'max_observed_backend_rss_bytes': max(row['backend_process']['VmRSS'] for row in selected), + 'max_observed_clickhouse_rss_bytes': max(row['clickhouse_process']['VmRSS'] for row in selected), + } + before, after = result['build_resources']['before'], result['build_resources']['after'] + build_cpu = {name: (after[name]['cpu_ticks'] - before[name]['cpu_ticks']) / after[name]['clock_ticks_per_second'] for name in before} + summary = {'query': result['id'], 'trial': trace.parent.parent.name, + 'numeric_pairs_equal': len(pairs), 'warm_requests': len(pairs), + 'latency_speedup': routes['exact']['median_ms'] / routes['backend']['median_ms'], + 'routes': routes, 'install_and_backfill_s': result['install_and_backfill_ns'] / 1e9, + 'build_cpu_s': build_cpu, 'process_after_build': after, + 'process_after_queries': result['query_resources_after'], + 'state_directory_bytes_at_build': result['state_directory_bytes'], + 'state_directory_bytes_after_stop': sum(path.stat().st_size for path in trace.parent.joinpath('state').rglob('*') if path.is_file()), + 'first_post_build_request_ms': {name: result[name]['elapsed_ns'] / 1e6 for name in ['backend', 'exact']}, + 'backend_budget': result['container'], + 'clickhouse_budget': json.loads(trace.parent.parent.joinpath('clickhouse-runtime.json').read_text()), + 'raw_trace': str(trace.resolve()), 'raw_sha256': hashlib.sha256(trace.read_bytes()).hexdigest()} + for name in ['backend_budget', 'clickhouse_budget']: + budget = summary[name] + if (budget['cpu_affinity'], budget['nano_cpus'], budget['memory_limit_bytes']) != ('60,61',2000000000,4294967296): + raise ValueError('resource budget mismatch') + summaries.append(summary) + if len(summaries) != 9: + raise ValueError('expected9 completed trials') + args.output.write_text(json.dumps({'scope': 'fixed-window repeated queries; excludes common source ingestion and offline planning cost', 'trials': summaries}, indent=2) + '\n') + for q in ['q05','q06','q23']: + group = [s for s in summaries if s['query'] == q] + print(q, 'backend_ms', [round(s['routes']['backend']['median_ms'],3) for s in group], 'exact_ms', [round(s['routes']['exact']['median_ms'],3) for s in group], 'speedup', [round(s['latency_speedup'],2) for s in group]) + +if __name__ == '__main__': + main() diff --git a/tools/o11y-sql-main-eval/test_import.py b/tools/o11y-sql-main-eval/test_import.py new file mode 100644 index 000000000..a046db4c8 --- /dev/null +++ b/tools/o11y-sql-main-eval/test_import.py @@ -0,0 +1,23 @@ +import unittest +from import_openmetrics import parse_series + + +class LabelTransportTests(unittest.TestCase): + def test_escaped_labels_remain_distinct_and_ordered(self): + metric, labels = parse_series(r'm{job="a,b",member="x\"y",path="a\\b",line="a\nb"}') + self.assertEqual(metric, 'm') + self.assertEqual(list(labels), ['job', 'member', 'path', 'line']) + self.assertEqual(labels, {'job': 'a,b', 'member': 'x"y', 'path': 'a\\b', 'line': 'a\nb'}) + + def test_empty_label_set(self): + self.assertEqual(parse_series('m{}'), ('m', {})) + self.assertEqual(parse_series('m'), ('m', {})) + + def test_invalid_or_duplicate_labels_are_rejected(self): + for token in ['m{x="a",x="b"}', 'm{x="a",garbage}', 'm{x="a"', 'm{garbage}']: + with self.assertRaises(ValueError): + parse_series(token) + + +if __name__ == '__main__': + unittest.main()