From a5dd4b3cd757b67055ac5ca365a428f6b8b3bd69 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 19:36:44 -0600 Subject: [PATCH 1/2] test(clickhouse): measure automatically selected aggregate plans --- .../examples/compile_clickhouse_workload.rs | 22 +++++ .../tests/clickhouse_q05_process_e2e.rs | 87 +++++++------------ tools/clickhouse-benefit/run_matched_trial.sh | 1 + 3 files changed, 54 insertions(+), 56 deletions(-) create mode 100644 control_plane/examples/compile_clickhouse_workload.rs diff --git a/control_plane/examples/compile_clickhouse_workload.rs b/control_plane/examples/compile_clickhouse_workload.rs new file mode 100644 index 000000000..dccc30157 --- /dev/null +++ b/control_plane/examples/compile_clickhouse_workload.rs @@ -0,0 +1,22 @@ +//! Compile a SQL workload through the real Planner without preset materializations. +use std::io::Read; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let mut input = String::new(); + std::io::stdin().read_to_string(&mut input)?; + let workload: control_plane::clickhouse::ClickHouseSqlAutomaticWorkload = + serde_json::from_str(&input)?; + let (publication, selection_trace) = + control_plane::clickhouse::compile_automatic_clickhouse_workload(&workload).await?; + let install = publication.install_request(None, Vec::new())?; + serde_json::to_writer_pretty( + std::io::stdout(), + &serde_json::json!({ + "publication": publication, + "selection_trace": selection_trace, + "install": install, + }), + )?; + Ok(()) +} diff --git a/data_plane/tests/clickhouse_q05_process_e2e.rs b/data_plane/tests/clickhouse_q05_process_e2e.rs index 3d0a8cd84..f358dea8c 100644 --- a/data_plane/tests/clickhouse_q05_process_e2e.rs +++ b/data_plane/tests/clickhouse_q05_process_e2e.rs @@ -1,8 +1,7 @@ //! Optional OS-process E2E for a q05-class bounded SQL max query. //! Run with `CLICKHOUSE_URL=http://127.0.0.1:8123 cargo test -p data_plane --test clickhouse_q05_process_e2e`. -use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; -use control_plane::physical::compiler::{PlanEnvelope, PrecomputePlan, TransmissionPlan}; +use control_plane::physical::compiler::{PlanEnvelope, BACKEND_COMPAT, PLANNER_REVISION}; use planner_types::pre_asap::{Column, DataType, Schema}; use std::io::Read; use std::{collections::HashMap, process::Stdio, time::Duration}; @@ -146,49 +145,21 @@ async fn q05_sql_is_planned_backfilled_and_served_warm_by_backend_process() { let source_load_elapsed_ns = experiment_started.elapsed().as_nanos(); let clickhouse_post_load = clickhouse_pid.map(process_snapshot); + let aggregate = std::env::var("CLICKHOUSE_BENCH_AGGREGATE").unwrap_or_else(|_| "max".into()); + assert!(matches!(aggregate.as_str(), "max" | "sum")); let sql = format!( - "SELECT max(value) AS value FROM q05_samples WHERE ts_ms>={start_ms} AND ts_ms<{end_ms}" + "SELECT {aggregate}(value) AS value FROM q05_samples WHERE ts_ms>={start_ms} AND ts_ms<{end_ms}" ); - let mut config = PrecomputeMaterialization::new( - AggregationType::MinMax, - "max".into(), - Default::default(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 43_200, - 43_200, - WindowKind::Tumbling, - String::new(), - metric.clone(), - None, - Some("q05_samples".into()), - Some("value".into()), - ); - config.pane_origin_ms = Some(start_ms as i64); - let sds = control_plane::physical::summary_catalog::SummaryCatalog::from_materializations( - 73, - 1, - &[config.clone()], - ) - .unwrap(); let envelope = PlanEnvelope { plan_id: 73, plan_version: 1, generated_at_unix_ms: 0, activation_unix_ms: 0, expiry_unix_ms: None, - backend_compat: "test".into(), - planner_revision: "test".into(), + backend_compat: BACKEND_COMPAT.into(), + planner_revision: PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }; - let mut precompute = - PrecomputePlan::build_backend_local(envelope.clone(), vec![config.clone()]).unwrap(); - precompute.summary_catalog = Some(sds.reference().unwrap()); - let mut transmission = - TransmissionPlan::build(envelope, &precompute, &Default::default()).unwrap(); - transmission.summary_catalog = precompute.summary_catalog.clone(); let schema = Schema::with_time_index( vec![ Column::new("metric", DataType::Utf8, false), @@ -199,24 +170,27 @@ async fn q05_sql_is_planned_backfilled_and_served_warm_by_backend_process() { 2, vec![], ); - let compiled = control_plane::clickhouse::compile_clickhouse_workload( - &control_plane::clickhouse::ClickHouseSqlWorkload { - sds: sds.clone(), - precompute_plan: precompute.clone(), - transmission_plan: transmission.clone(), - tables: HashMap::from([("q05_samples".into(), schema)]), - accuracy: planner_types::types::AccuracyTarget::Exact, - queries: vec![control_plane::clickhouse::ClickHouseSqlWorkloadEntry { - sql: sql.clone(), - start_ms, - end_ms, - cumulative: true, - }], - }, - ) - .await - .unwrap(); + let (compiled, selection_trace) = + control_plane::clickhouse::compile_automatic_clickhouse_workload( + &control_plane::clickhouse::ClickHouseSqlAutomaticWorkload { + envelope, + tables: HashMap::from([("q05_samples".into(), schema)]), + accuracy: planner_types::types::AccuracyTarget::Exact, + queries: vec![control_plane::clickhouse::ClickHouseSqlWorkloadEntry { + sql: sql.clone(), + start_ms, + end_ms, + cumulative: true, + }], + }, + ) + .await + .unwrap(); + assert_eq!(compiled.precompute_plan.materializations.len(), 1); + let config = compiled.precompute_plan.materializations[0].clone(); + let planning = + serde_json::json!({"selection_trace": selection_trace, "publication": &compiled}); let http_port = free_port(); let mut sql_port = free_port(); while sql_port == http_port { @@ -380,13 +354,13 @@ async fn q05_sql_is_planned_backfilled_and_served_warm_by_backend_process() { let first_warm_ns = first_warm_started.elapsed().as_nanos(); let first_exact_started = std::time::Instant::now(); let exact_value: f64 = clickhouse_auth(client.post(&clickhouse)) - .body(format!("SELECT max(value) FROM asap_q05_e2e.q05_samples WHERE ts_ms>={start_ms} AND ts_ms<{end_ms} FORMAT TabSeparated")) + .body(format!("SELECT {aggregate}(value) FROM asap_q05_e2e.q05_samples WHERE ts_ms>={start_ms} AND ts_ms<{end_ms} FORMAT TabSeparated")) .send().await.unwrap().text().await.unwrap().trim().parse().unwrap(); let first_exact_ns = first_exact_started.elapsed().as_nanos(); assert_eq!(warm_value, exact_value); if let Ok(output) = std::env::var("CLICKHOUSE_BENCH_OUTPUT") { - let exact_sql = format!("SELECT max(value) FROM asap_q05_e2e.q05_samples WHERE ts_ms>={start_ms} AND ts_ms<{end_ms} FORMAT TabSeparated"); + let exact_sql = format!("SELECT {aggregate}(value) FROM asap_q05_e2e.q05_samples WHERE ts_ms>={start_ms} AND ts_ms<{end_ms} FORMAT TabSeparated"); for _ in 0..10 { let _ = client .post(format!("http://127.0.0.1:{sql_port}/")) @@ -474,7 +448,8 @@ async fn q05_sql_is_planned_backfilled_and_served_warm_by_backend_process() { "clock_ticks_per_second": std::process::Command::new("getconf").arg("CLK_TCK").output().ok().and_then(|value| String::from_utf8(value.stdout).ok()).and_then(|value| value.trim().parse::().ok()), "query": sql, "input": {"path":input_path,"metric":metric,"samples":input_samples,"series":series,"start_ms":start_ms,"end_ms":end_ms}, - "planning_scope": "Planner chooses query DAG against a predeclared MinMax catalog; materialization sizing/selection is not measured", + "planning_scope": "Real Planner selects the DAG and automatic control-plane binding constructs the catalog; fixed-window layout is not cost-optimized", + "planning": planning, "classification_required": "warm", "build_phase": {"elapsed_ns":build_elapsed_ns,"backend":post_build,"backend_output_bytes":directory_bytes(output_dir.path()),"clickhouse_before":clickhouse_initial,"clickhouse_after":clickhouse_post_build}, "source_load_phase": {"elapsed_ns":source_load_elapsed_ns,"clickhouse_before":clickhouse_initial,"clickhouse_after":clickhouse_post_load}, @@ -483,7 +458,7 @@ async fn q05_sql_is_planned_backfilled_and_served_warm_by_backend_process() { "query_phase": {"elapsed_ns":query_elapsed_ns,"backend_before":pre_query,"backend_after":post_query,"backend_output_bytes":directory_bytes(output_dir.path()),"clickhouse_before":clickhouse_pre_query,"clickhouse_after":clickhouse_post_query,"clickhouse_storage_bytes":clickhouse_storage_bytes}, "clickhouse_table": table_stats, "requests": requests, - "limitations": ["single q05 max workload", "CPU uses Linux scheduler ticks", "RSS is whole-process", "ClickHouse server is externally managed"], + "limitations": ["bounded single-series aggregate sensitivity; not full original o11y workload coverage", "CPU uses Linux scheduler ticks", "RSS is whole-process", "ClickHouse server is externally managed"], }); std::fs::write(output, serde_json::to_vec_pretty(&artifact).unwrap()).unwrap(); } diff --git a/tools/clickhouse-benefit/run_matched_trial.sh b/tools/clickhouse-benefit/run_matched_trial.sh index 6bfbe0683..28461fe86 100644 --- a/tools/clickhouse-benefit/run_matched_trial.sh +++ b/tools/clickhouse-benefit/run_matched_trial.sh @@ -53,6 +53,7 @@ docker run --name "$backend" --user "$(id -u):$(id -g)" \ -w "$repo_root" -e CLICKHOUSE_URL=http://127.0.0.1:28123 \ -e CLICKHOUSE_USER -e CLICKHOUSE_PASSWORD -e CLICKHOUSE_PID="$server_pid" \ -e CLICKHOUSE_BENCH_INPUT -e CLICKHOUSE_BENCH_METRIC -e CLICKHOUSE_BENCH_END_MS \ + -e CLICKHOUSE_BENCH_AGGREGATE="${CLICKHOUSE_BENCH_AGGREGATE:-max}" \ -e CLICKHOUSE_BENCH_REPETITIONS="${CLICKHOUSE_BENCH_REPETITIONS:-1000}" \ -e CLICKHOUSE_BENCH_OUTPUT="$output.json" \ --entrypoint "$TEST_BINARY" "$image" --nocapture From 17fe74c97fdb57f482f70162bffc4860f4db1938 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 19:43:49 -0600 Subject: [PATCH 2/2] docs(clickhouse): report automatic aggregate latency and lifecycle costs --- ...matched-trialcontrolled-max-1-summary.json | 164 ++++ ...trialcontrolled-max-1.backend-runtime.json | 1 + ...alcontrolled-max-1.clickhouse-runtime.json | 1 + ...alcontrolled-max-1.clickhouse-settings.tsv | 2 + ...d-trialcontrolled-max-1.executables.sha256 | 2 + ...-trialcontrolled-max-1.source-checkout.txt | 1 + ...matched-trialcontrolled-max-2-summary.json | 164 ++++ ...trialcontrolled-max-2.backend-runtime.json | 1 + ...alcontrolled-max-2.clickhouse-runtime.json | 1 + ...alcontrolled-max-2.clickhouse-settings.tsv | 2 + ...d-trialcontrolled-max-2.executables.sha256 | 2 + ...-trialcontrolled-max-2.source-checkout.txt | 1 + ...matched-trialcontrolled-max-3-summary.json | 164 ++++ ...trialcontrolled-max-3.backend-runtime.json | 1 + ...alcontrolled-max-3.clickhouse-runtime.json | 1 + ...alcontrolled-max-3.clickhouse-settings.tsv | 2 + ...d-trialcontrolled-max-3.executables.sha256 | 2 + ...-trialcontrolled-max-3.source-checkout.txt | 1 + ...matched-trialcontrolled-sum-1-summary.json | 164 ++++ ...trialcontrolled-sum-1.backend-runtime.json | 1 + ...alcontrolled-sum-1.clickhouse-runtime.json | 1 + ...alcontrolled-sum-1.clickhouse-settings.tsv | 2 + ...d-trialcontrolled-sum-1.executables.sha256 | 2 + ...-trialcontrolled-sum-1.source-checkout.txt | 1 + ...matched-trialcontrolled-sum-2-summary.json | 164 ++++ ...trialcontrolled-sum-2.backend-runtime.json | 1 + ...alcontrolled-sum-2.clickhouse-runtime.json | 1 + ...alcontrolled-sum-2.clickhouse-settings.tsv | 2 + ...d-trialcontrolled-sum-2.executables.sha256 | 2 + ...-trialcontrolled-sum-2.source-checkout.txt | 1 + ...matched-trialcontrolled-sum-3-summary.json | 164 ++++ ...trialcontrolled-sum-3.backend-runtime.json | 1 + ...alcontrolled-sum-3.clickhouse-runtime.json | 1 + ...alcontrolled-sum-3.clickhouse-settings.tsv | 2 + ...d-trialcontrolled-sum-3.executables.sha256 | 2 + ...-trialcontrolled-sum-3.source-checkout.txt | 1 + .../artifacts/automatic/max-planning.json | 733 ++++++++++++++++++ .../artifacts/automatic/raw-traces.json | 26 + .../artifacts/automatic/sum-planning.json | 733 ++++++++++++++++++ .../automatic-aggregate-results.md | 99 +++ 40 files changed, 2617 insertions(+) create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1-summary.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.backend-runtime.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.clickhouse-runtime.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.clickhouse-settings.tsv create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.executables.sha256 create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.source-checkout.txt create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2-summary.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.backend-runtime.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.clickhouse-runtime.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.clickhouse-settings.tsv create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.executables.sha256 create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.source-checkout.txt create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3-summary.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.backend-runtime.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.clickhouse-runtime.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.clickhouse-settings.tsv create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.executables.sha256 create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.source-checkout.txt create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1-summary.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.backend-runtime.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.clickhouse-runtime.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.clickhouse-settings.tsv create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.executables.sha256 create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.source-checkout.txt create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2-summary.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.backend-runtime.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.clickhouse-runtime.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.clickhouse-settings.tsv create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.executables.sha256 create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.source-checkout.txt create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3-summary.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.backend-runtime.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.clickhouse-runtime.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.clickhouse-settings.tsv create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.executables.sha256 create mode 100644 tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.source-checkout.txt create mode 100644 tools/clickhouse-benefit/artifacts/automatic/max-planning.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/raw-traces.json create mode 100644 tools/clickhouse-benefit/artifacts/automatic/sum-planning.json create mode 100644 tools/clickhouse-benefit/automatic-aggregate-results.md diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1-summary.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1-summary.json new file mode 100644 index 000000000..b86978314 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1-summary.json @@ -0,0 +1,164 @@ +{ + "artifact": "/mydata/clickhouse-automatic-benefit-study/fine-1s-matched-trialcontrolled-max-1.json", + "git_head": null, + "input": { + "end_ms": 1788891296001, + "metric": "service_cache_refresh_lag_seconds", + "path": "/mydata/clickhouse-o11y-main-results/fine-1s-24h-series.jsonl", + "samples": 86371, + "series": [ + "service_cache_refresh_lag_seconds{job=\"user-service\",instance=\"user-service:8082\"}" + ], + "start_ms": 1788848096001 + }, + "planning_scope": "Real Planner selects the DAG and automatic control-plane binding constructs the catalog; fixed-window layout is not cost-optimized", + "routes": { + "warm": { + "requests": 1000, + "median_ms": 0.85268, + "p95_ms": 0.947403, + "serial_service_requests_per_second": 1190.33156519102, + "all_http_200": true, + "execution_modes": [ + "warm" + ], + "backend_cpu_ms": 770.0, + "clickhouse_cpu_ms": 130.0 + }, + "exact": { + "requests": 1000, + "median_ms": 6.775694, + "p95_ms": 15.607599, + "serial_service_requests_per_second": 133.2508830524694, + "all_http_200": true, + "execution_modes": [ + "None" + ], + "backend_cpu_ms": 0.0, + "clickhouse_cpu_ms": 8240.0 + } + }, + "latency_speedup": 7.946350330721959, + "first_query": { + "exact_elapsed_ns": 9086382, + "warm_elapsed_ns": 7444864 + }, + "build_phase": { + "backend": { + "cpu_ticks": 26, + "pid": 3317, + "rss_pages": 9554, + "vm_hwm": "VmHWM:\t 42172 kB", + "vm_rss": "VmRSS:\t 38216 kB" + }, + "backend_output_bytes": 5795, + "clickhouse_after": { + "cpu_ticks": 635, + "pid": 2693, + "rss_pages": 199390, + "vm_hwm": "VmHWM:\t 834532 kB", + "vm_rss": "VmRSS:\t 797560 kB" + }, + "clickhouse_before": { + "cpu_ticks": 584, + "pid": 2693, + "rss_pages": 190958, + "vm_hwm": "VmHWM:\t 794548 kB", + "vm_rss": "VmRSS:\t 763832 kB" + }, + "elapsed_ns": 2078524302 + }, + "source_load_phase": { + "clickhouse_after": { + "cpu_ticks": 615, + "pid": 2693, + "rss_pages": 207345, + "vm_hwm": "VmHWM:\t 829380 kB", + "vm_rss": "VmRSS:\t 829380 kB" + }, + "clickhouse_before": { + "cpu_ticks": 584, + "pid": 2693, + "rss_pages": 190958, + "vm_hwm": "VmHWM:\t 794548 kB", + "vm_rss": "VmRSS:\t 763832 kB" + }, + "elapsed_ns": 600844868 + }, + "backfill_phase": { + "backend_after": { + "cpu_ticks": 26, + "pid": 3317, + "rss_pages": 9554, + "vm_hwm": "VmHWM:\t 42172 kB", + "vm_rss": "VmRSS:\t 38216 kB" + }, + "backend_before": { + "cpu_ticks": 0, + "pid": 3317, + "rss_pages": 6557, + "vm_hwm": "VmHWM:\t 26228 kB", + "vm_rss": "VmRSS:\t 26228 kB" + }, + "clickhouse_after": { + "cpu_ticks": 635, + "pid": 2693, + "rss_pages": 199390, + "vm_hwm": "VmHWM:\t 834532 kB", + "vm_rss": "VmRSS:\t 797560 kB" + }, + "clickhouse_before": { + "cpu_ticks": 615, + "pid": 2693, + "rss_pages": 207345, + "vm_hwm": "VmHWM:\t 829380 kB", + "vm_rss": "VmRSS:\t 829380 kB" + }, + "elapsed_ns": 1419429320 + }, + "query_phase": { + "backend_after": { + "cpu_ticks": 105, + "pid": 3317, + "rss_pages": 13857, + "vm_hwm": "VmHWM:\t 55428 kB", + "vm_rss": "VmRSS:\t 55428 kB" + }, + "backend_before": { + "cpu_ticks": 28, + "pid": 3317, + "rss_pages": 13857, + "vm_hwm": "VmHWM:\t 55428 kB", + "vm_rss": "VmRSS:\t 55428 kB" + }, + "backend_output_bytes": 5795, + "clickhouse_after": { + "cpu_ticks": 1488, + "pid": 2693, + "rss_pages": 204239, + "vm_hwm": "VmHWM:\t 844840 kB", + "vm_rss": "VmRSS:\t 816956 kB" + }, + "clickhouse_before": { + "cpu_ticks": 648, + "pid": 2693, + "rss_pages": 201089, + "vm_hwm": "VmHWM:\t 834532 kB", + "vm_rss": "VmRSS:\t 804356 kB" + }, + "clickhouse_storage_bytes": null, + "elapsed_ns": 8850136205 + }, + "clickhouse_table": { + "bytes_on_disk": 507146, + "compressed_bytes": 499358, + "rows": 86371 + }, + "limitations": [ + "bounded single-series aggregate sensitivity; not full original o11y workload coverage", + "CPU uses Linux scheduler ticks", + "RSS is whole-process", + "ClickHouse server is externally managed", + "CPU counters include server background work and have scheduler-tick precision" + ] +} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.backend-runtime.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.backend-runtime.json new file mode 100644 index 000000000..99aeffea7 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.backend-runtime.json @@ -0,0 +1 @@ +{"cpu_affinity":"60,61","nano_cpus":2000000000,"memory_limit_bytes":4294967296,"image":"sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb"} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.clickhouse-runtime.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.clickhouse-runtime.json new file mode 100644 index 000000000..99aeffea7 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.clickhouse-runtime.json @@ -0,0 +1 @@ +{"cpu_affinity":"60,61","nano_cpus":2000000000,"memory_limit_bytes":4294967296,"image":"sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb"} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.clickhouse-settings.tsv b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.clickhouse-settings.tsv new file mode 100644 index 000000000..7aa34f2d5 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.clickhouse-settings.tsv @@ -0,0 +1,2 @@ +max_threads auto(2) +use_query_cache 0 diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.executables.sha256 b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.executables.sha256 new file mode 100644 index 000000000..193ef8b1f --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.executables.sha256 @@ -0,0 +1,2 @@ +2a1a1c568994ff787c170e165681693be96d97b46a8d9b178fc836cd79dfcaf8 /dev/shm/asap-clickhouse-release/release/deps/clickhouse_q05_process_e2e-c20ee66394ae01e2 +dac9b1428be3f702e99980a15af8a3e524ebce31099cb4d5c82e5214cedc8a7d /dev/shm/asap-clickhouse-release/release/data_plane diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.source-checkout.txt b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.source-checkout.txt new file mode 100644 index 000000000..e9afe33b6 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-1.source-checkout.txt @@ -0,0 +1 @@ +a5dd4b3cd757b67055ac5ca365a428f6b8b3bd69 diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2-summary.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2-summary.json new file mode 100644 index 000000000..6f75ed745 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2-summary.json @@ -0,0 +1,164 @@ +{ + "artifact": "/mydata/clickhouse-automatic-benefit-study/fine-1s-matched-trialcontrolled-max-2.json", + "git_head": null, + "input": { + "end_ms": 1788891296001, + "metric": "service_cache_refresh_lag_seconds", + "path": "/mydata/clickhouse-o11y-main-results/fine-1s-24h-series.jsonl", + "samples": 86371, + "series": [ + "service_cache_refresh_lag_seconds{job=\"user-service\",instance=\"user-service:8082\"}" + ], + "start_ms": 1788848096001 + }, + "planning_scope": "Real Planner selects the DAG and automatic control-plane binding constructs the catalog; fixed-window layout is not cost-optimized", + "routes": { + "warm": { + "requests": 1000, + "median_ms": 0.8538295, + "p95_ms": 0.914374, + "serial_service_requests_per_second": 1194.1049218057829, + "all_http_200": true, + "execution_modes": [ + "warm" + ], + "backend_cpu_ms": 770.0, + "clickhouse_cpu_ms": 50.0 + }, + "exact": { + "requests": 1000, + "median_ms": 6.7164684999999995, + "p95_ms": 14.64908, + "serial_service_requests_per_second": 133.68239673326647, + "all_http_200": true, + "execution_modes": [ + "None" + ], + "backend_cpu_ms": 0.0, + "clickhouse_cpu_ms": 8360.0 + } + }, + "latency_speedup": 7.8662877073233, + "first_query": { + "exact_elapsed_ns": 5258459, + "warm_elapsed_ns": 7469621 + }, + "build_phase": { + "backend": { + "cpu_ticks": 30, + "pid": 4259, + "rss_pages": 9243, + "vm_hwm": "VmHWM:\t 41772 kB", + "vm_rss": "VmRSS:\t 36972 kB" + }, + "backend_output_bytes": 5795, + "clickhouse_after": { + "cpu_ticks": 758, + "pid": 3589, + "rss_pages": 211698, + "vm_hwm": "VmHWM:\t 866604 kB", + "vm_rss": "VmRSS:\t 846792 kB" + }, + "clickhouse_before": { + "cpu_ticks": 717, + "pid": 3589, + "rss_pages": 198466, + "vm_hwm": "VmHWM:\t 804668 kB", + "vm_rss": "VmRSS:\t 793864 kB" + }, + "elapsed_ns": 2028679777 + }, + "source_load_phase": { + "clickhouse_after": { + "cpu_ticks": 740, + "pid": 3589, + "rss_pages": 216651, + "vm_hwm": "VmHWM:\t 866604 kB", + "vm_rss": "VmRSS:\t 866604 kB" + }, + "clickhouse_before": { + "cpu_ticks": 717, + "pid": 3589, + "rss_pages": 198466, + "vm_hwm": "VmHWM:\t 804668 kB", + "vm_rss": "VmRSS:\t 793864 kB" + }, + "elapsed_ns": 549677466 + }, + "backfill_phase": { + "backend_after": { + "cpu_ticks": 30, + "pid": 4259, + "rss_pages": 9243, + "vm_hwm": "VmHWM:\t 41772 kB", + "vm_rss": "VmRSS:\t 36972 kB" + }, + "backend_before": { + "cpu_ticks": 0, + "pid": 4259, + "rss_pages": 6825, + "vm_hwm": "VmHWM:\t 27300 kB", + "vm_rss": "VmRSS:\t 27300 kB" + }, + "clickhouse_after": { + "cpu_ticks": 758, + "pid": 3589, + "rss_pages": 211698, + "vm_hwm": "VmHWM:\t 866604 kB", + "vm_rss": "VmRSS:\t 846792 kB" + }, + "clickhouse_before": { + "cpu_ticks": 741, + "pid": 3589, + "rss_pages": 214825, + "vm_hwm": "VmHWM:\t 866604 kB", + "vm_rss": "VmRSS:\t 859300 kB" + }, + "elapsed_ns": 1420511502 + }, + "query_phase": { + "backend_after": { + "cpu_ticks": 108, + "pid": 4259, + "rss_pages": 12911, + "vm_hwm": "VmHWM:\t 51644 kB", + "vm_rss": "VmRSS:\t 51644 kB" + }, + "backend_before": { + "cpu_ticks": 31, + "pid": 4259, + "rss_pages": 12911, + "vm_hwm": "VmHWM:\t 51644 kB", + "vm_rss": "VmRSS:\t 51644 kB" + }, + "backend_output_bytes": 5795, + "clickhouse_after": { + "cpu_ticks": 1617, + "pid": 3589, + "rss_pages": 208774, + "vm_hwm": "VmHWM:\t 881544 kB", + "vm_rss": "VmRSS:\t 835096 kB" + }, + "clickhouse_before": { + "cpu_ticks": 771, + "pid": 3589, + "rss_pages": 213578, + "vm_hwm": "VmHWM:\t 866604 kB", + "vm_rss": "VmRSS:\t 854312 kB" + }, + "clickhouse_storage_bytes": null, + "elapsed_ns": 8824909933 + }, + "clickhouse_table": { + "bytes_on_disk": 507146, + "compressed_bytes": 499358, + "rows": 86371 + }, + "limitations": [ + "bounded single-series aggregate sensitivity; not full original o11y workload coverage", + "CPU uses Linux scheduler ticks", + "RSS is whole-process", + "ClickHouse server is externally managed", + "CPU counters include server background work and have scheduler-tick precision" + ] +} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.backend-runtime.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.backend-runtime.json new file mode 100644 index 000000000..99aeffea7 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.backend-runtime.json @@ -0,0 +1 @@ +{"cpu_affinity":"60,61","nano_cpus":2000000000,"memory_limit_bytes":4294967296,"image":"sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb"} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.clickhouse-runtime.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.clickhouse-runtime.json new file mode 100644 index 000000000..99aeffea7 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.clickhouse-runtime.json @@ -0,0 +1 @@ +{"cpu_affinity":"60,61","nano_cpus":2000000000,"memory_limit_bytes":4294967296,"image":"sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb"} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.clickhouse-settings.tsv b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.clickhouse-settings.tsv new file mode 100644 index 000000000..7aa34f2d5 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.clickhouse-settings.tsv @@ -0,0 +1,2 @@ +max_threads auto(2) +use_query_cache 0 diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.executables.sha256 b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.executables.sha256 new file mode 100644 index 000000000..193ef8b1f --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.executables.sha256 @@ -0,0 +1,2 @@ +2a1a1c568994ff787c170e165681693be96d97b46a8d9b178fc836cd79dfcaf8 /dev/shm/asap-clickhouse-release/release/deps/clickhouse_q05_process_e2e-c20ee66394ae01e2 +dac9b1428be3f702e99980a15af8a3e524ebce31099cb4d5c82e5214cedc8a7d /dev/shm/asap-clickhouse-release/release/data_plane diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.source-checkout.txt b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.source-checkout.txt new file mode 100644 index 000000000..e9afe33b6 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-2.source-checkout.txt @@ -0,0 +1 @@ +a5dd4b3cd757b67055ac5ca365a428f6b8b3bd69 diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3-summary.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3-summary.json new file mode 100644 index 000000000..40e3ae3cc --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3-summary.json @@ -0,0 +1,164 @@ +{ + "artifact": "/mydata/clickhouse-automatic-benefit-study/fine-1s-matched-trialcontrolled-max-3.json", + "git_head": null, + "input": { + "end_ms": 1788891296001, + "metric": "service_cache_refresh_lag_seconds", + "path": "/mydata/clickhouse-o11y-main-results/fine-1s-24h-series.jsonl", + "samples": 86371, + "series": [ + "service_cache_refresh_lag_seconds{job=\"user-service\",instance=\"user-service:8082\"}" + ], + "start_ms": 1788848096001 + }, + "planning_scope": "Real Planner selects the DAG and automatic control-plane binding constructs the catalog; fixed-window layout is not cost-optimized", + "routes": { + "warm": { + "requests": 1000, + "median_ms": 0.854446, + "p95_ms": 0.943564, + "serial_service_requests_per_second": 1177.1869868424878, + "all_http_200": true, + "execution_modes": [ + "warm" + ], + "backend_cpu_ms": 770.0, + "clickhouse_cpu_ms": 150.0 + }, + "exact": { + "requests": 1000, + "median_ms": 7.590913, + "p95_ms": 16.708811, + "serial_service_requests_per_second": 112.36784183132968, + "all_http_200": true, + "execution_modes": [ + "None" + ], + "backend_cpu_ms": 10.0, + "clickhouse_cpu_ms": 12130.0 + } + }, + "latency_speedup": 8.88401724626249, + "first_query": { + "exact_elapsed_ns": 6165972, + "warm_elapsed_ns": 4058572 + }, + "build_phase": { + "backend": { + "cpu_ticks": 26, + "pid": 5095, + "rss_pages": 9076, + "vm_hwm": "VmHWM:\t 40740 kB", + "vm_rss": "VmRSS:\t 36304 kB" + }, + "backend_output_bytes": 5795, + "clickhouse_after": { + "cpu_ticks": 662, + "pid": 4472, + "rss_pages": 207161, + "vm_hwm": "VmHWM:\t 855836 kB", + "vm_rss": "VmRSS:\t 828644 kB" + }, + "clickhouse_before": { + "cpu_ticks": 613, + "pid": 4472, + "rss_pages": 196706, + "vm_hwm": "VmHWM:\t 793148 kB", + "vm_rss": "VmRSS:\t 786824 kB" + }, + "elapsed_ns": 2059081468 + }, + "source_load_phase": { + "clickhouse_after": { + "cpu_ticks": 641, + "pid": 4472, + "rss_pages": 213959, + "vm_hwm": "VmHWM:\t 855836 kB", + "vm_rss": "VmRSS:\t 855836 kB" + }, + "clickhouse_before": { + "cpu_ticks": 613, + "pid": 4472, + "rss_pages": 196706, + "vm_hwm": "VmHWM:\t 793148 kB", + "vm_rss": "VmRSS:\t 786824 kB" + }, + "elapsed_ns": 580256117 + }, + "backfill_phase": { + "backend_after": { + "cpu_ticks": 26, + "pid": 5095, + "rss_pages": 9076, + "vm_hwm": "VmHWM:\t 40740 kB", + "vm_rss": "VmRSS:\t 36304 kB" + }, + "backend_before": { + "cpu_ticks": 0, + "pid": 5095, + "rss_pages": 5912, + "vm_hwm": "VmHWM:\t 23648 kB", + "vm_rss": "VmRSS:\t 23648 kB" + }, + "clickhouse_after": { + "cpu_ticks": 662, + "pid": 4472, + "rss_pages": 207161, + "vm_hwm": "VmHWM:\t 855836 kB", + "vm_rss": "VmRSS:\t 828644 kB" + }, + "clickhouse_before": { + "cpu_ticks": 641, + "pid": 4472, + "rss_pages": 213959, + "vm_hwm": "VmHWM:\t 855836 kB", + "vm_rss": "VmRSS:\t 855836 kB" + }, + "elapsed_ns": 1420822274 + }, + "query_phase": { + "backend_after": { + "cpu_ticks": 106, + "pid": 5095, + "rss_pages": 13156, + "vm_hwm": "VmHWM:\t 52624 kB", + "vm_rss": "VmRSS:\t 52624 kB" + }, + "backend_before": { + "cpu_ticks": 28, + "pid": 5095, + "rss_pages": 13156, + "vm_hwm": "VmHWM:\t 52624 kB", + "vm_rss": "VmRSS:\t 52624 kB" + }, + "backend_output_bytes": 5795, + "clickhouse_after": { + "cpu_ticks": 1907, + "pid": 4472, + "rss_pages": 217427, + "vm_hwm": "VmHWM:\t 977328 kB", + "vm_rss": "VmRSS:\t 869708 kB" + }, + "clickhouse_before": { + "cpu_ticks": 671, + "pid": 4472, + "rss_pages": 207898, + "vm_hwm": "VmHWM:\t 855836 kB", + "vm_rss": "VmRSS:\t 831592 kB" + }, + "clickhouse_storage_bytes": null, + "elapsed_ns": 10256904415 + }, + "clickhouse_table": { + "bytes_on_disk": 507146, + "compressed_bytes": 499358, + "rows": 86371 + }, + "limitations": [ + "bounded single-series aggregate sensitivity; not full original o11y workload coverage", + "CPU uses Linux scheduler ticks", + "RSS is whole-process", + "ClickHouse server is externally managed", + "CPU counters include server background work and have scheduler-tick precision" + ] +} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.backend-runtime.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.backend-runtime.json new file mode 100644 index 000000000..99aeffea7 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.backend-runtime.json @@ -0,0 +1 @@ +{"cpu_affinity":"60,61","nano_cpus":2000000000,"memory_limit_bytes":4294967296,"image":"sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb"} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.clickhouse-runtime.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.clickhouse-runtime.json new file mode 100644 index 000000000..99aeffea7 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.clickhouse-runtime.json @@ -0,0 +1 @@ +{"cpu_affinity":"60,61","nano_cpus":2000000000,"memory_limit_bytes":4294967296,"image":"sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb"} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.clickhouse-settings.tsv b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.clickhouse-settings.tsv new file mode 100644 index 000000000..7aa34f2d5 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.clickhouse-settings.tsv @@ -0,0 +1,2 @@ +max_threads auto(2) +use_query_cache 0 diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.executables.sha256 b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.executables.sha256 new file mode 100644 index 000000000..193ef8b1f --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.executables.sha256 @@ -0,0 +1,2 @@ +2a1a1c568994ff787c170e165681693be96d97b46a8d9b178fc836cd79dfcaf8 /dev/shm/asap-clickhouse-release/release/deps/clickhouse_q05_process_e2e-c20ee66394ae01e2 +dac9b1428be3f702e99980a15af8a3e524ebce31099cb4d5c82e5214cedc8a7d /dev/shm/asap-clickhouse-release/release/data_plane diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.source-checkout.txt b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.source-checkout.txt new file mode 100644 index 000000000..e9afe33b6 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-max-3.source-checkout.txt @@ -0,0 +1 @@ +a5dd4b3cd757b67055ac5ca365a428f6b8b3bd69 diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1-summary.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1-summary.json new file mode 100644 index 000000000..ab8211646 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1-summary.json @@ -0,0 +1,164 @@ +{ + "artifact": "/mydata/clickhouse-automatic-benefit-study/fine-1s-matched-trialcontrolled-sum-1.json", + "git_head": null, + "input": { + "end_ms": 1788891296001, + "metric": "service_cache_refresh_lag_seconds", + "path": "/mydata/clickhouse-o11y-main-results/fine-1s-24h-series.jsonl", + "samples": 86371, + "series": [ + "service_cache_refresh_lag_seconds{job=\"user-service\",instance=\"user-service:8082\"}" + ], + "start_ms": 1788848096001 + }, + "planning_scope": "Real Planner selects the DAG and automatic control-plane binding constructs the catalog; fixed-window layout is not cost-optimized", + "routes": { + "warm": { + "requests": 1000, + "median_ms": 0.8606505, + "p95_ms": 1.098009, + "serial_service_requests_per_second": 1124.3672725772078, + "all_http_200": true, + "execution_modes": [ + "warm" + ], + "backend_cpu_ms": 810.0, + "clickhouse_cpu_ms": 100.0 + }, + "exact": { + "requests": 1000, + "median_ms": 6.8911245, + "p95_ms": 15.357338, + "serial_service_requests_per_second": 127.26332816558143, + "all_http_200": true, + "execution_modes": [ + "None" + ], + "backend_cpu_ms": 0.0, + "clickhouse_cpu_ms": 9600.0 + } + }, + "latency_speedup": 8.006879099007088, + "first_query": { + "exact_elapsed_ns": 11073046, + "warm_elapsed_ns": 7462164 + }, + "build_phase": { + "backend": { + "cpu_ticks": 31, + "pid": 606, + "rss_pages": 9240, + "vm_hwm": "VmHWM:\t 41760 kB", + "vm_rss": "VmRSS:\t 36960 kB" + }, + "backend_output_bytes": 5795, + "clickhouse_after": { + "cpu_ticks": 564, + "pid": 4193881, + "rss_pages": 205101, + "vm_hwm": "VmHWM:\t 844444 kB", + "vm_rss": "VmRSS:\t 820404 kB" + }, + "clickhouse_before": { + "cpu_ticks": 514, + "pid": 4193881, + "rss_pages": 191081, + "vm_hwm": "VmHWM:\t 780064 kB", + "vm_rss": "VmRSS:\t 764324 kB" + }, + "elapsed_ns": 2052333532 + }, + "source_load_phase": { + "clickhouse_after": { + "cpu_ticks": 546, + "pid": 4193881, + "rss_pages": 211111, + "vm_hwm": "VmHWM:\t 844444 kB", + "vm_rss": "VmRSS:\t 844444 kB" + }, + "clickhouse_before": { + "cpu_ticks": 514, + "pid": 4193881, + "rss_pages": 191081, + "vm_hwm": "VmHWM:\t 780064 kB", + "vm_rss": "VmRSS:\t 764324 kB" + }, + "elapsed_ns": 572166269 + }, + "backfill_phase": { + "backend_after": { + "cpu_ticks": 31, + "pid": 606, + "rss_pages": 9240, + "vm_hwm": "VmHWM:\t 41760 kB", + "vm_rss": "VmRSS:\t 36960 kB" + }, + "backend_before": { + "cpu_ticks": 1, + "pid": 606, + "rss_pages": 5917, + "vm_hwm": "VmHWM:\t 23668 kB", + "vm_rss": "VmRSS:\t 23668 kB" + }, + "clickhouse_after": { + "cpu_ticks": 564, + "pid": 4193881, + "rss_pages": 205101, + "vm_hwm": "VmHWM:\t 844444 kB", + "vm_rss": "VmRSS:\t 820404 kB" + }, + "clickhouse_before": { + "cpu_ticks": 546, + "pid": 4193881, + "rss_pages": 211111, + "vm_hwm": "VmHWM:\t 844444 kB", + "vm_rss": "VmRSS:\t 844444 kB" + }, + "elapsed_ns": 1421970926 + }, + "query_phase": { + "backend_after": { + "cpu_ticks": 113, + "pid": 606, + "rss_pages": 12938, + "vm_hwm": "VmHWM:\t 51752 kB", + "vm_rss": "VmRSS:\t 51752 kB" + }, + "backend_before": { + "cpu_ticks": 32, + "pid": 606, + "rss_pages": 12938, + "vm_hwm": "VmHWM:\t 51752 kB", + "vm_rss": "VmRSS:\t 51752 kB" + }, + "backend_output_bytes": 5795, + "clickhouse_after": { + "cpu_ticks": 1554, + "pid": 4193881, + "rss_pages": 204146, + "vm_hwm": "VmHWM:\t 956748 kB", + "vm_rss": "VmRSS:\t 816584 kB" + }, + "clickhouse_before": { + "cpu_ticks": 580, + "pid": 4193881, + "rss_pages": 206861, + "vm_hwm": "VmHWM:\t 844444 kB", + "vm_rss": "VmRSS:\t 827444 kB" + }, + "clickhouse_storage_bytes": null, + "elapsed_ns": 9270179610 + }, + "clickhouse_table": { + "bytes_on_disk": 507146, + "compressed_bytes": 499358, + "rows": 86371 + }, + "limitations": [ + "bounded single-series aggregate sensitivity; not full original o11y workload coverage", + "CPU uses Linux scheduler ticks", + "RSS is whole-process", + "ClickHouse server is externally managed", + "CPU counters include server background work and have scheduler-tick precision" + ] +} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.backend-runtime.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.backend-runtime.json new file mode 100644 index 000000000..99aeffea7 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.backend-runtime.json @@ -0,0 +1 @@ +{"cpu_affinity":"60,61","nano_cpus":2000000000,"memory_limit_bytes":4294967296,"image":"sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb"} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.clickhouse-runtime.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.clickhouse-runtime.json new file mode 100644 index 000000000..99aeffea7 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.clickhouse-runtime.json @@ -0,0 +1 @@ +{"cpu_affinity":"60,61","nano_cpus":2000000000,"memory_limit_bytes":4294967296,"image":"sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb"} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.clickhouse-settings.tsv b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.clickhouse-settings.tsv new file mode 100644 index 000000000..7aa34f2d5 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.clickhouse-settings.tsv @@ -0,0 +1,2 @@ +max_threads auto(2) +use_query_cache 0 diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.executables.sha256 b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.executables.sha256 new file mode 100644 index 000000000..193ef8b1f --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.executables.sha256 @@ -0,0 +1,2 @@ +2a1a1c568994ff787c170e165681693be96d97b46a8d9b178fc836cd79dfcaf8 /dev/shm/asap-clickhouse-release/release/deps/clickhouse_q05_process_e2e-c20ee66394ae01e2 +dac9b1428be3f702e99980a15af8a3e524ebce31099cb4d5c82e5214cedc8a7d /dev/shm/asap-clickhouse-release/release/data_plane diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.source-checkout.txt b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.source-checkout.txt new file mode 100644 index 000000000..e9afe33b6 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-1.source-checkout.txt @@ -0,0 +1 @@ +a5dd4b3cd757b67055ac5ca365a428f6b8b3bd69 diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2-summary.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2-summary.json new file mode 100644 index 000000000..bd5bca7f8 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2-summary.json @@ -0,0 +1,164 @@ +{ + "artifact": "/mydata/clickhouse-automatic-benefit-study/fine-1s-matched-trialcontrolled-sum-2.json", + "git_head": null, + "input": { + "end_ms": 1788891296001, + "metric": "service_cache_refresh_lag_seconds", + "path": "/mydata/clickhouse-o11y-main-results/fine-1s-24h-series.jsonl", + "samples": 86371, + "series": [ + "service_cache_refresh_lag_seconds{job=\"user-service\",instance=\"user-service:8082\"}" + ], + "start_ms": 1788848096001 + }, + "planning_scope": "Real Planner selects the DAG and automatic control-plane binding constructs the catalog; fixed-window layout is not cost-optimized", + "routes": { + "warm": { + "requests": 1000, + "median_ms": 0.8539585000000001, + "p95_ms": 0.925258, + "serial_service_requests_per_second": 1178.2060510954382, + "all_http_200": true, + "execution_modes": [ + "warm" + ], + "backend_cpu_ms": 770.0, + "clickhouse_cpu_ms": 140.0 + }, + "exact": { + "requests": 1000, + "median_ms": 6.670814, + "p95_ms": 14.947635, + "serial_service_requests_per_second": 131.33290859123719, + "all_http_200": true, + "execution_modes": [ + "None" + ], + "backend_cpu_ms": 0.0, + "clickhouse_cpu_ms": 8360.0 + } + }, + "latency_speedup": 7.811637216562631, + "first_query": { + "exact_elapsed_ns": 6100366, + "warm_elapsed_ns": 7512927 + }, + "build_phase": { + "backend": { + "cpu_ticks": 23, + "pid": 1574, + "rss_pages": 9441, + "vm_hwm": "VmHWM:\t 42532 kB", + "vm_rss": "VmRSS:\t 37764 kB" + }, + "backend_output_bytes": 5795, + "clickhouse_after": { + "cpu_ticks": 570, + "pid": 913, + "rss_pages": 208818, + "vm_hwm": "VmHWM:\t 844804 kB", + "vm_rss": "VmRSS:\t 835272 kB" + }, + "clickhouse_before": { + "cpu_ticks": 526, + "pid": 913, + "rss_pages": 192807, + "vm_hwm": "VmHWM:\t 785372 kB", + "vm_rss": "VmRSS:\t 771228 kB" + }, + "elapsed_ns": 1958081076 + }, + "source_load_phase": { + "clickhouse_after": { + "cpu_ticks": 553, + "pid": 913, + "rss_pages": 211201, + "vm_hwm": "VmHWM:\t 844804 kB", + "vm_rss": "VmRSS:\t 844804 kB" + }, + "clickhouse_before": { + "cpu_ticks": 526, + "pid": 913, + "rss_pages": 192807, + "vm_hwm": "VmHWM:\t 785372 kB", + "vm_rss": "VmRSS:\t 771228 kB" + }, + "elapsed_ns": 581441143 + }, + "backfill_phase": { + "backend_after": { + "cpu_ticks": 23, + "pid": 1574, + "rss_pages": 9441, + "vm_hwm": "VmHWM:\t 42532 kB", + "vm_rss": "VmRSS:\t 37764 kB" + }, + "backend_before": { + "cpu_ticks": 1, + "pid": 1574, + "rss_pages": 6393, + "vm_hwm": "VmHWM:\t 25572 kB", + "vm_rss": "VmRSS:\t 25572 kB" + }, + "clickhouse_after": { + "cpu_ticks": 570, + "pid": 913, + "rss_pages": 208818, + "vm_hwm": "VmHWM:\t 844804 kB", + "vm_rss": "VmRSS:\t 835272 kB" + }, + "clickhouse_before": { + "cpu_ticks": 554, + "pid": 913, + "rss_pages": 211201, + "vm_hwm": "VmHWM:\t 844804 kB", + "vm_rss": "VmRSS:\t 844804 kB" + }, + "elapsed_ns": 1318013326 + }, + "query_phase": { + "backend_after": { + "cpu_ticks": 103, + "pid": 1574, + "rss_pages": 14061, + "vm_hwm": "VmHWM:\t 56244 kB", + "vm_rss": "VmRSS:\t 56244 kB" + }, + "backend_before": { + "cpu_ticks": 26, + "pid": 1574, + "rss_pages": 14061, + "vm_hwm": "VmHWM:\t 56244 kB", + "vm_rss": "VmRSS:\t 56244 kB" + }, + "backend_output_bytes": 5795, + "clickhouse_after": { + "cpu_ticks": 1437, + "pid": 913, + "rss_pages": 199569, + "vm_hwm": "VmHWM:\t 844804 kB", + "vm_rss": "VmRSS:\t 798276 kB" + }, + "clickhouse_before": { + "cpu_ticks": 582, + "pid": 913, + "rss_pages": 208731, + "vm_hwm": "VmHWM:\t 844804 kB", + "vm_rss": "VmRSS:\t 834924 kB" + }, + "clickhouse_storage_bytes": null, + "elapsed_ns": 8977135160 + }, + "clickhouse_table": { + "bytes_on_disk": 507146, + "compressed_bytes": 499358, + "rows": 86371 + }, + "limitations": [ + "bounded single-series aggregate sensitivity; not full original o11y workload coverage", + "CPU uses Linux scheduler ticks", + "RSS is whole-process", + "ClickHouse server is externally managed", + "CPU counters include server background work and have scheduler-tick precision" + ] +} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.backend-runtime.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.backend-runtime.json new file mode 100644 index 000000000..99aeffea7 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.backend-runtime.json @@ -0,0 +1 @@ +{"cpu_affinity":"60,61","nano_cpus":2000000000,"memory_limit_bytes":4294967296,"image":"sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb"} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.clickhouse-runtime.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.clickhouse-runtime.json new file mode 100644 index 000000000..99aeffea7 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.clickhouse-runtime.json @@ -0,0 +1 @@ +{"cpu_affinity":"60,61","nano_cpus":2000000000,"memory_limit_bytes":4294967296,"image":"sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb"} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.clickhouse-settings.tsv b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.clickhouse-settings.tsv new file mode 100644 index 000000000..7aa34f2d5 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.clickhouse-settings.tsv @@ -0,0 +1,2 @@ +max_threads auto(2) +use_query_cache 0 diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.executables.sha256 b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.executables.sha256 new file mode 100644 index 000000000..193ef8b1f --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.executables.sha256 @@ -0,0 +1,2 @@ +2a1a1c568994ff787c170e165681693be96d97b46a8d9b178fc836cd79dfcaf8 /dev/shm/asap-clickhouse-release/release/deps/clickhouse_q05_process_e2e-c20ee66394ae01e2 +dac9b1428be3f702e99980a15af8a3e524ebce31099cb4d5c82e5214cedc8a7d /dev/shm/asap-clickhouse-release/release/data_plane diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.source-checkout.txt b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.source-checkout.txt new file mode 100644 index 000000000..e9afe33b6 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-2.source-checkout.txt @@ -0,0 +1 @@ +a5dd4b3cd757b67055ac5ca365a428f6b8b3bd69 diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3-summary.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3-summary.json new file mode 100644 index 000000000..67be3077b --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3-summary.json @@ -0,0 +1,164 @@ +{ + "artifact": "/mydata/clickhouse-automatic-benefit-study/fine-1s-matched-trialcontrolled-sum-3.json", + "git_head": null, + "input": { + "end_ms": 1788891296001, + "metric": "service_cache_refresh_lag_seconds", + "path": "/mydata/clickhouse-o11y-main-results/fine-1s-24h-series.jsonl", + "samples": 86371, + "series": [ + "service_cache_refresh_lag_seconds{job=\"user-service\",instance=\"user-service:8082\"}" + ], + "start_ms": 1788848096001 + }, + "planning_scope": "Real Planner selects the DAG and automatic control-plane binding constructs the catalog; fixed-window layout is not cost-optimized", + "routes": { + "warm": { + "requests": 1000, + "median_ms": 0.8532839999999999, + "p95_ms": 0.936681, + "serial_service_requests_per_second": 1188.9259315539932, + "all_http_200": true, + "execution_modes": [ + "warm" + ], + "backend_cpu_ms": 750.0, + "clickhouse_cpu_ms": 120.0 + }, + "exact": { + "requests": 1000, + "median_ms": 4.702767, + "p95_ms": 15.371925, + "serial_service_requests_per_second": 136.28889310623663, + "all_http_200": true, + "execution_modes": [ + "None" + ], + "backend_cpu_ms": 10.0, + "clickhouse_cpu_ms": 9460.0 + } + }, + "latency_speedup": 5.511373704417287, + "first_query": { + "exact_elapsed_ns": 9203775, + "warm_elapsed_ns": 7340392 + }, + "build_phase": { + "backend": { + "cpu_ticks": 26, + "pid": 2444, + "rss_pages": 9454, + "vm_hwm": "VmHWM:\t 42360 kB", + "vm_rss": "VmRSS:\t 37816 kB" + }, + "backend_output_bytes": 5795, + "clickhouse_after": { + "cpu_ticks": 680, + "pid": 1834, + "rss_pages": 208311, + "vm_hwm": "VmHWM:\t 888156 kB", + "vm_rss": "VmRSS:\t 833244 kB" + }, + "clickhouse_before": { + "cpu_ticks": 641, + "pid": 1834, + "rss_pages": 214459, + "vm_hwm": "VmHWM:\t 887996 kB", + "vm_rss": "VmRSS:\t 857836 kB" + }, + "elapsed_ns": 1851402434 + }, + "source_load_phase": { + "clickhouse_after": { + "cpu_ticks": 661, + "pid": 1834, + "rss_pages": 222039, + "vm_hwm": "VmHWM:\t 888156 kB", + "vm_rss": "VmRSS:\t 888156 kB" + }, + "clickhouse_before": { + "cpu_ticks": 641, + "pid": 1834, + "rss_pages": 214459, + "vm_hwm": "VmHWM:\t 887996 kB", + "vm_rss": "VmRSS:\t 857836 kB" + }, + "elapsed_ns": 475146555 + }, + "backfill_phase": { + "backend_after": { + "cpu_ticks": 26, + "pid": 2444, + "rss_pages": 9454, + "vm_hwm": "VmHWM:\t 42360 kB", + "vm_rss": "VmRSS:\t 37816 kB" + }, + "backend_before": { + "cpu_ticks": 1, + "pid": 2444, + "rss_pages": 7158, + "vm_hwm": "VmHWM:\t 28632 kB", + "vm_rss": "VmRSS:\t 28632 kB" + }, + "clickhouse_after": { + "cpu_ticks": 680, + "pid": 1834, + "rss_pages": 208311, + "vm_hwm": "VmHWM:\t 888156 kB", + "vm_rss": "VmRSS:\t 833244 kB" + }, + "clickhouse_before": { + "cpu_ticks": 662, + "pid": 1834, + "rss_pages": 216715, + "vm_hwm": "VmHWM:\t 888156 kB", + "vm_rss": "VmRSS:\t 866860 kB" + }, + "elapsed_ns": 1318331602 + }, + "query_phase": { + "backend_after": { + "cpu_ticks": 104, + "pid": 2444, + "rss_pages": 13549, + "vm_hwm": "VmHWM:\t 54196 kB", + "vm_rss": "VmRSS:\t 54196 kB" + }, + "backend_before": { + "cpu_ticks": 28, + "pid": 2444, + "rss_pages": 13549, + "vm_hwm": "VmHWM:\t 54196 kB", + "vm_rss": "VmRSS:\t 54196 kB" + }, + "backend_output_bytes": 5795, + "clickhouse_after": { + "cpu_ticks": 1660, + "pid": 1834, + "rss_pages": 208511, + "vm_hwm": "VmHWM:\t 912208 kB", + "vm_rss": "VmRSS:\t 834044 kB" + }, + "clickhouse_before": { + "cpu_ticks": 689, + "pid": 1834, + "rss_pages": 209894, + "vm_hwm": "VmHWM:\t 888156 kB", + "vm_rss": "VmRSS:\t 839576 kB" + }, + "clickhouse_storage_bytes": null, + "elapsed_ns": 8685801242 + }, + "clickhouse_table": { + "bytes_on_disk": 507146, + "compressed_bytes": 499358, + "rows": 86371 + }, + "limitations": [ + "bounded single-series aggregate sensitivity; not full original o11y workload coverage", + "CPU uses Linux scheduler ticks", + "RSS is whole-process", + "ClickHouse server is externally managed", + "CPU counters include server background work and have scheduler-tick precision" + ] +} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.backend-runtime.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.backend-runtime.json new file mode 100644 index 000000000..99aeffea7 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.backend-runtime.json @@ -0,0 +1 @@ +{"cpu_affinity":"60,61","nano_cpus":2000000000,"memory_limit_bytes":4294967296,"image":"sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb"} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.clickhouse-runtime.json b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.clickhouse-runtime.json new file mode 100644 index 000000000..99aeffea7 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.clickhouse-runtime.json @@ -0,0 +1 @@ +{"cpu_affinity":"60,61","nano_cpus":2000000000,"memory_limit_bytes":4294967296,"image":"sha256:fa394da808cc53f76d0344429421d6c422a6ee85fe7450135c0e3cff4df9bcbb"} diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.clickhouse-settings.tsv b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.clickhouse-settings.tsv new file mode 100644 index 000000000..7aa34f2d5 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.clickhouse-settings.tsv @@ -0,0 +1,2 @@ +max_threads auto(2) +use_query_cache 0 diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.executables.sha256 b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.executables.sha256 new file mode 100644 index 000000000..193ef8b1f --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.executables.sha256 @@ -0,0 +1,2 @@ +2a1a1c568994ff787c170e165681693be96d97b46a8d9b178fc836cd79dfcaf8 /dev/shm/asap-clickhouse-release/release/deps/clickhouse_q05_process_e2e-c20ee66394ae01e2 +dac9b1428be3f702e99980a15af8a3e524ebce31099cb4d5c82e5214cedc8a7d /dev/shm/asap-clickhouse-release/release/data_plane diff --git a/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.source-checkout.txt b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.source-checkout.txt new file mode 100644 index 000000000..e9afe33b6 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/fine-1s-matched-trialcontrolled-sum-3.source-checkout.txt @@ -0,0 +1 @@ +a5dd4b3cd757b67055ac5ca365a428f6b8b3bd69 diff --git a/tools/clickhouse-benefit/artifacts/automatic/max-planning.json b/tools/clickhouse-benefit/artifacts/automatic/max-planning.json new file mode 100644 index 000000000..9e052f122 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/max-planning.json @@ -0,0 +1,733 @@ +{ + "publication": { + "collector_plans": [], + "precompute_plan": { + "envelope": { + "activation_unix_ms": 0, + "backend_compat": "asap-query-backend.v1", + "capability_snapshot_id": "test", + "expiry_unix_ms": null, + "generated_at_unix_ms": 0, + "plan_id": 73, + "plan_version": 1, + "planner_revision": "0deceda3e776216c5542d638d958b159f22e27ce" + }, + "executable_dags": { + "SELECT max(value) AS value FROM q05_samples WHERE ts_ms>=1788848096001 AND ts_ms<1788891296001": { + "binding": { + "nodes": { + "0": { + "placement": "maintenance_input" + }, + "1": { + "placement": "materialization", + "summary_definition": 11497701252531036795 + }, + "2": { + "placement": "query", + "query_node": 0 + } + }, + "precompute_sinks": [ + 1 + ], + "query_plan_sink": 0, + "query_sink": 2 + }, + "document": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "MaintenanceTime" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "metric", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "labels", + "nullable": false + }, + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts_ms", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + } + ], + "time_index": 2 + }, + "producer": 0, + "role": "Input", + "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual" + }, + { + "consumer": 2, + "data_state": { + "primitive": "SummaryState", + "timing": "MaintenanceTime" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "fields": [ + { + "dtype": { + "ExactAggregate": [ + "MinMax", + "MinMax" + ] + }, + "name": "max(q05_samples.value)", + "nullable": false + } + ], + "time_index": null + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "KeepPreAsap" + } + ] + }, + "id": 0, + "operator": "Fallback", + "output_schema": { + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "metric", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "labels", + "nullable": false + }, + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts_ms", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + } + ], + "time_index": 2 + }, + "output_state": { + "primitive": "Raw", + "timing": "MaintenanceTime" + }, + "payload": { + "expression": { + "Scan": { + "predicates": [ + { + "BoolAnd": [ + { + "Compare": { + "left": { + "Column": 2 + }, + "op": "Ge", + "right": { + "Literal": { + "Int64": 1788848096001 + } + } + } + }, + { + "Compare": { + "left": { + "Column": 2 + }, + "op": "Lt", + "right": { + "Literal": { + "Int64": 1788891296001 + } + } + } + } + ] + } + ], + "schema": { + "closed": true, + "columns": [ + { + "dtype": "utf8", + "name": "metric", + "nullable": false, + "table": "q05_samples" + }, + { + "dtype": "utf8", + "name": "labels", + "nullable": false, + "table": "q05_samples" + }, + { + "dtype": "timestamp", + "name": "ts_ms", + "nullable": false, + "table": "q05_samples" + }, + { + "dtype": "float64", + "name": "value", + "nullable": false, + "table": "q05_samples" + } + ], + "time_index": 2, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "q05_samples" + } + } + } + }, + "kind": "fallback" + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(MinMax)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "KeepPreAsap" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "exact_extremum" + }, + "rule": "exact_input" + } + ] + }, + "id": 1, + "operator": "SummaryAgg", + "output_schema": { + "fields": [ + { + "dtype": { + "ExactAggregate": [ + "MinMax", + "MinMax" + ] + }, + "name": "max(q05_samples.value)", + "nullable": false + } + ], + "time_index": null + }, + "output_state": { + "primitive": "SummaryState", + "timing": "MaintenanceTime" + }, + "payload": { + "family": { + "ExactAggregate": [ + "MinMax", + "MinMax" + ] + }, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": { + "Qualified": { + "name": "value", + "table": "q05_samples" + } + } + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "kind": "summary_agg", + "reduction": { + "Reduce": [] + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(MinMax)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "KeepPreAsap" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "exact_extremum" + }, + "rule": "exact_input" + } + ] + }, + "id": 2, + "operator": "Value", + "output_schema": { + "fields": [ + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + } + ], + "time_index": null + }, + "output_state": { + "primitive": "Raw", + "timing": "ReadTime" + }, + "payload": { + "kind": "value", + "operation": { + "Project": { + "cols": [ + { + "alias": "value", + "expr": { + "Column": 0 + } + } + ], + "qualifier": null + } + }, + "timing": "ReadTime" + } + } + ], + "query_id": "SELECT max(value) AS value FROM q05_samples WHERE ts_ms>=1788848096001 AND ts_ms<1788891296001", + "root": 2, + "schema_version": 1 + } + } + }, + "ingest": { + "endpoint_path": "/api/v1/write", + "protocol": "prometheus_remote_write_v1", + "require_plan_identity": false, + "require_registered_producer": false, + "require_summary_definition_identity": false, + "timestamp_unit": "unix_milliseconds" + }, + "materializations": [ + { + "aggregated_labels": { + "labels": [] + }, + "aggregation_sub_type": "max", + "aggregation_type": "MinMax", + "grouping_labels": { + "labels": [] + }, + "metric": "q05_samples.value", + "num_aggregates_to_retain": 2, + "original_yaml": "", + "pane_origin_ms": 1788848096001, + "parameters": {}, + "partitioning": "grouped", + "rollup_labels": { + "labels": [] + }, + "slide_interval": 43200, + "spatial_filter": "", + "spatial_filter_normalized": "", + "table_name": "q05_samples", + "table_population": { + "predicates": [] + }, + "table_timestamp_column": "ts_ms", + "value_column": "value", + "window_layout": { + "kind": "pane", + "pane_secs": 43200 + }, + "window_size": 43200, + "window_type": "tumbling" + } + ], + "producers": [], + "schemas": [ + { + "encodings": [ + "exact_accumulator_v1" + ], + "family": { + "family": "exact", + "kind": "min_max" + }, + "group_by": [], + "materialization": 11497701252531036795, + "schema_id": "asap-query-backend.v1:summary-state:v1:11497701252531036795", + "schema_version": 1, + "source": { + "Table": { + "table_ref": "q05_samples" + } + }, + "value_column": { + "Named": "value" + }, + "window": { + "kind": "tumbling", + "pane_origin_ms": 1788848096001, + "size_ms": 43200000, + "slide_ms": null + } + } + ], + "summary_catalog": { + "plan_id": 73, + "plan_version": 1, + "schema_version": 2, + "snapshot_sha256": "642bd91f4dd5c5bb54f59fe86705a01ec2691b825730a36a5ac878ee7dfa621b" + } + }, + "query_plan": { + "clickhouse_context": { + "accuracy": "Exact", + "tables": { + "q05_samples": { + "closed": false, + "columns": [ + { + "dtype": "utf8", + "name": "metric", + "nullable": false, + "table": null + }, + { + "dtype": "utf8", + "name": "labels", + "nullable": false, + "table": null + }, + { + "dtype": "timestamp", + "name": "ts_ms", + "nullable": false, + "table": null + }, + { + "dtype": "float64", + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 2, + "unique_keys": [] + } + } + }, + "entries": { + "clickhouse:Project { cols: [ProjectItem { alias: Some(\"value\"), expr: Column(0) }], qualifier: None, child: Aggregate { reduction: Reduce(GroupKeys { keys: [], without: false }), measures: [Max { col: Some(3) }], output_names: [\"max(q05_samples.value)\"], having: None, child: Scan { source: Table { table_ref: \"q05_samples\" }, predicates: [Predicate(BoolAnd([Compare { left: Column(2), op: Ge, right: Literal(Int64(1788848096001)) }, Compare { left: Column(2), op: Lt, right: Literal(Int64(1788891296001)) }]))], schema: Schema { columns: [Column { name: \"metric\", dtype: Utf8, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"labels\", dtype: Utf8, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"ts_ms\", dtype: Timestamp, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"value\", dtype: Float64, nullable: false, table: Some(\"q05_samples\") }], time_index: Some(2), unique_keys: [], closed: true } } } }": { + "canonical_query": "Project { cols: [ProjectItem { alias: Some(\"value\"), expr: Column(0) }], qualifier: None, child: Aggregate { reduction: Reduce(GroupKeys { keys: [], without: false }), measures: [Max { col: Some(3) }], output_names: [\"max(q05_samples.value)\"], having: None, child: Scan { source: Table { table_ref: \"q05_samples\" }, predicates: [Predicate(BoolAnd([Compare { left: Column(2), op: Ge, right: Literal(Int64(1788848096001)) }, Compare { left: Column(2), op: Lt, right: Literal(Int64(1788891296001)) }]))], schema: Schema { columns: [Column { name: \"metric\", dtype: Utf8, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"labels\", dtype: Utf8, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"ts_ms\", dtype: Timestamp, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"value\", dtype: Float64, nullable: false, table: Some(\"q05_samples\") }], time_index: Some(2), unique_keys: [], closed: true } } } }", + "fallback": "exact_backend", + "fixed_evaluation": { + "cumulative": true, + "end_ms": 1788891296001, + "start_ms": 1788848096001 + }, + "instant": { + "cumulative_readout": true, + "full_history": false, + "lookback_ms": 43200000 + }, + "language": "click_house_sql", + "nodes": { + "0": { + "input": 1, + "input_schema": { + "fields": [ + { + "dtype": { + "ExactAggregate": [ + "MinMax", + "MinMax" + ] + }, + "name": "max(q05_samples.value)", + "nullable": false + } + ], + "time_index": null + }, + "op": "relational", + "operation": { + "Project": { + "cols": [ + { + "alias": "value", + "expr": { + "Column": 0 + } + } + ], + "qualifier": null + } + }, + "output_schema": { + "fields": [ + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + } + ], + "time_index": null + } + }, + "1": { + "input": 2, + "op": "exact_readout", + "readout": "max" + }, + "2": { + "binding": { + "materialization": 11497701252531036795, + "output_grouping": { + "keys": [], + "mode": "reduce" + }, + "pane_origin_ms": 1788848096001, + "readout_lookback_ms": 43200000, + "window_ms": 43200000 + }, + "op": "read_materialization" + } + }, + "query_id": "SELECT max(value) AS value FROM q05_samples WHERE ts_ms>=1788848096001 AND ts_ms<1788891296001", + "root": 0 + } + }, + "plan_id": 73, + "plan_version": 1 + }, + "summary_catalog": { + "data_descriptors": { + "data:v2|37:{\"table\":{\"table_ref\":\"q05_samples\"}}|27:{\"column\":{\"name\":\"value\"}}|0:|partition:Grouped|timestamp-ms:5:ts_ms|32:asap.timestamped-observations.v2": { + "group_by_keys": [], + "id": "data:v2|37:{\"table\":{\"table_ref\":\"q05_samples\"}}|27:{\"column\":{\"name\":\"value\"}}|0:|partition:Grouped|timestamp-ms:5:ts_ms|32:asap.timestamped-observations.v2", + "observation_semantics": "asap.timestamped-observations.v2", + "partitioning": "grouped", + "population_filter_canonical": "", + "source": { + "table": { + "table_ref": "q05_samples" + } + }, + "timestamp_column": "ts_ms", + "value_projection": { + "column": { + "name": "value" + } + } + } + }, + "materializations": { + "11497701252531036795": { + "data_descriptor_id": "data:v2|37:{\"table\":{\"table_ref\":\"q05_samples\"}}|27:{\"column\":{\"name\":\"value\"}}|0:|partition:Grouped|timestamp-ms:5:ts_ms|32:asap.timestamped-observations.v2", + "pane_origin_ms": 1788848096001, + "summary_descriptor_id": "summary:v2:{\"fidelity\":{\"kind\":\"exact\"},\"operator\":{\"aggregation_sub_type\":\"max\",\"aggregation_type\":\"MinMax\",\"kind\":\"configured\",\"parameters\":{}},\"state_schema_version\":1}", + "window_layout": { + "kind": "pane", + "pane_secs": 43200 + } + } + }, + "plan_id": 73, + "plan_version": 1, + "schema_version": 2, + "summary_descriptors": { + "summary:v2:{\"fidelity\":{\"kind\":\"exact\"},\"operator\":{\"aggregation_sub_type\":\"max\",\"aggregation_type\":\"MinMax\",\"kind\":\"configured\",\"parameters\":{}},\"state_schema_version\":1}": { + "fidelity": { + "kind": "exact" + }, + "id": "summary:v2:{\"fidelity\":{\"kind\":\"exact\"},\"operator\":{\"aggregation_sub_type\":\"max\",\"aggregation_type\":\"MinMax\",\"kind\":\"configured\",\"parameters\":{}},\"state_schema_version\":1}", + "operator": { + "aggregation_sub_type": "max", + "aggregation_type": "MinMax", + "kind": "configured", + "parameters": {} + }, + "state_schema_version": 1 + } + } + }, + "transmission_plan": { + "envelope": { + "activation_unix_ms": 0, + "backend_compat": "asap-query-backend.v1", + "capability_snapshot_id": "test", + "expiry_unix_ms": null, + "generated_at_unix_ms": 0, + "plan_id": 73, + "plan_version": 1, + "planner_revision": "0deceda3e776216c5542d638d958b159f22e27ce" + }, + "frame_identity": { + "identity_version": 1, + "require_base_checkpoint_for_delta": true, + "require_checkpoint_for_full": true, + "sequence_scope": "materialization_series_producer_epoch" + }, + "rules": [], + "summary_catalog": { + "plan_id": 73, + "plan_version": 1, + "schema_version": 2, + "snapshot_sha256": "642bd91f4dd5c5bb54f59fe86705a01ec2691b825730a36a5ac878ee7dfa621b" + } + } + }, + "selection_trace": { + "Project { cols: [ProjectItem { alias: Some(\"value\"), expr: Column(0) }], qualifier: None, child: Aggregate { reduction: Reduce(GroupKeys { keys: [], without: false }), measures: [Max { col: Some(3) }], output_names: [\"max(q05_samples.value)\"], having: None, child: Scan { source: Table { table_ref: \"q05_samples\" }, predicates: [Predicate(BoolAnd([Compare { left: Column(2), op: Ge, right: Literal(Int64(1788848096001)) }, Compare { left: Column(2), op: Lt, right: Literal(Int64(1788891296001)) }]))], schema: Schema { columns: [Column { name: \"metric\", dtype: Utf8, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"labels\", dtype: Utf8, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"ts_ms\", dtype: Timestamp, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"value\", dtype: Float64, nullable: false, table: Some(\"q05_samples\") }], time_index: Some(2), unique_keys: [], closed: true } } } }": { + "group_id_scope": "this_selection", + "groups": [ + { + "candidates": [], + "consumer_count": 1, + "group_id": 0 + }, + { + "candidates": [ + { + "estimated_cost": null, + "estimated_cost_status": "not_reported_by_cost_model", + "provenance": "SummaryImplementation", + "rank": 0, + "rationale": "Max { col: Some(3) } realizes as an exact MinMax accumulator \u2014 the only realization implementations_for_with produces for this intent (no approximate candidate applies)", + "replacement_kind": "summary", + "selected": true, + "strategy": "SketchAlgorithmStrategy" + } + ], + "consumer_count": 1, + "group_id": 1 + }, + { + "candidates": [], + "consumer_count": 1, + "group_id": 2 + } + ], + "schema_version": 1 + } + } +} diff --git a/tools/clickhouse-benefit/artifacts/automatic/raw-traces.json b/tools/clickhouse-benefit/artifacts/automatic/raw-traces.json new file mode 100644 index 000000000..1c37a57a7 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/raw-traces.json @@ -0,0 +1,26 @@ +[ + { + "path": "/mydata/clickhouse-automatic-benefit-study/fine-1s-matched-trialcontrolled-sum-1.json", + "sha256": "a957d60fec6132ea843ef8a3548740e4287dee1e22968a89609a903a6c7de52a" + }, + { + "path": "/mydata/clickhouse-automatic-benefit-study/fine-1s-matched-trialcontrolled-sum-2.json", + "sha256": "d7256007455ab6a25672e2d9970e717e9fec5cc8014463f7faa7f0f8d8a40886" + }, + { + "path": "/mydata/clickhouse-automatic-benefit-study/fine-1s-matched-trialcontrolled-sum-3.json", + "sha256": "db529845c5e1fd91c0080e3b7ced9649dbf3ec84c99469692dce46fd792c0aa2" + }, + { + "path": "/mydata/clickhouse-automatic-benefit-study/fine-1s-matched-trialcontrolled-max-1.json", + "sha256": "8735b24def194437e90301667e9d2b806021a92724dc6d87350cf3ebe0ecc2b6" + }, + { + "path": "/mydata/clickhouse-automatic-benefit-study/fine-1s-matched-trialcontrolled-max-2.json", + "sha256": "6c1b86e861bbc16cf1ed44b1297ca8b985163802f135c06790c86653aa4b0c49" + }, + { + "path": "/mydata/clickhouse-automatic-benefit-study/fine-1s-matched-trialcontrolled-max-3.json", + "sha256": "cd00d30dd2d3ab4c17cbba223d1b6837c7ce6492fb2ecbee84fc85d59dd6428c" + } +] diff --git a/tools/clickhouse-benefit/artifacts/automatic/sum-planning.json b/tools/clickhouse-benefit/artifacts/automatic/sum-planning.json new file mode 100644 index 000000000..54c0ddda9 --- /dev/null +++ b/tools/clickhouse-benefit/artifacts/automatic/sum-planning.json @@ -0,0 +1,733 @@ +{ + "publication": { + "collector_plans": [], + "precompute_plan": { + "envelope": { + "activation_unix_ms": 0, + "backend_compat": "asap-query-backend.v1", + "capability_snapshot_id": "test", + "expiry_unix_ms": null, + "generated_at_unix_ms": 0, + "plan_id": 73, + "plan_version": 1, + "planner_revision": "0deceda3e776216c5542d638d958b159f22e27ce" + }, + "executable_dags": { + "SELECT sum(value) AS value FROM q05_samples WHERE ts_ms>=1788848096001 AND ts_ms<1788891296001": { + "binding": { + "nodes": { + "0": { + "placement": "maintenance_input" + }, + "1": { + "placement": "materialization", + "summary_definition": 10897179430287057821 + }, + "2": { + "placement": "query", + "query_node": 0 + } + }, + "precompute_sinks": [ + 1 + ], + "query_plan_sink": 0, + "query_sink": 2 + }, + "document": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "MaintenanceTime" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "metric", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "labels", + "nullable": false + }, + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts_ms", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + } + ], + "time_index": 2 + }, + "producer": 0, + "role": "Input", + "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual" + }, + { + "consumer": 2, + "data_state": { + "primitive": "SummaryState", + "timing": "MaintenanceTime" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "fields": [ + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "sum(q05_samples.value)", + "nullable": false + } + ], + "time_index": null + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "KeepPreAsap" + } + ] + }, + "id": 0, + "operator": "Fallback", + "output_schema": { + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "metric", + "nullable": false + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "labels", + "nullable": false + }, + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts_ms", + "nullable": false + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + } + ], + "time_index": 2 + }, + "output_state": { + "primitive": "Raw", + "timing": "MaintenanceTime" + }, + "payload": { + "expression": { + "Scan": { + "predicates": [ + { + "BoolAnd": [ + { + "Compare": { + "left": { + "Column": 2 + }, + "op": "Ge", + "right": { + "Literal": { + "Int64": 1788848096001 + } + } + } + }, + { + "Compare": { + "left": { + "Column": 2 + }, + "op": "Lt", + "right": { + "Literal": { + "Int64": 1788891296001 + } + } + } + } + ] + } + ], + "schema": { + "closed": true, + "columns": [ + { + "dtype": "utf8", + "name": "metric", + "nullable": false, + "table": "q05_samples" + }, + { + "dtype": "utf8", + "name": "labels", + "nullable": false, + "table": "q05_samples" + }, + { + "dtype": "timestamp", + "name": "ts_ms", + "nullable": false, + "table": "q05_samples" + }, + { + "dtype": "float64", + "name": "value", + "nullable": false, + "table": "q05_samples" + } + ], + "time_index": 2, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "q05_samples" + } + } + } + }, + "kind": "fallback" + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Sum)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "KeepPreAsap" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "exact_sum" + }, + "rule": "exact_input" + } + ] + }, + "id": 1, + "operator": "SummaryAgg", + "output_schema": { + "fields": [ + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "sum(q05_samples.value)", + "nullable": false + } + ], + "time_index": null + }, + "output_state": { + "primitive": "SummaryState", + "timing": "MaintenanceTime" + }, + "payload": { + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": { + "Qualified": { + "name": "value", + "table": "q05_samples" + } + } + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "kind": "summary_agg", + "reduction": { + "Reduce": [] + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Sum)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "KeepPreAsap" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "exact_sum" + }, + "rule": "exact_input" + } + ] + }, + "id": 2, + "operator": "Value", + "output_schema": { + "fields": [ + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + } + ], + "time_index": null + }, + "output_state": { + "primitive": "Raw", + "timing": "ReadTime" + }, + "payload": { + "kind": "value", + "operation": { + "Project": { + "cols": [ + { + "alias": "value", + "expr": { + "Column": 0 + } + } + ], + "qualifier": null + } + }, + "timing": "ReadTime" + } + } + ], + "query_id": "SELECT sum(value) AS value FROM q05_samples WHERE ts_ms>=1788848096001 AND ts_ms<1788891296001", + "root": 2, + "schema_version": 1 + } + } + }, + "ingest": { + "endpoint_path": "/api/v1/write", + "protocol": "prometheus_remote_write_v1", + "require_plan_identity": false, + "require_registered_producer": false, + "require_summary_definition_identity": false, + "timestamp_unit": "unix_milliseconds" + }, + "materializations": [ + { + "aggregated_labels": { + "labels": [] + }, + "aggregation_sub_type": "", + "aggregation_type": "Sum", + "grouping_labels": { + "labels": [] + }, + "metric": "q05_samples.value", + "num_aggregates_to_retain": 2, + "original_yaml": "", + "pane_origin_ms": 1788848096001, + "parameters": {}, + "partitioning": "grouped", + "rollup_labels": { + "labels": [] + }, + "slide_interval": 43200, + "spatial_filter": "", + "spatial_filter_normalized": "", + "table_name": "q05_samples", + "table_population": { + "predicates": [] + }, + "table_timestamp_column": "ts_ms", + "value_column": "value", + "window_layout": { + "kind": "pane", + "pane_secs": 43200 + }, + "window_size": 43200, + "window_type": "tumbling" + } + ], + "producers": [], + "schemas": [ + { + "encodings": [ + "exact_accumulator_v1" + ], + "family": { + "family": "exact", + "kind": "sum" + }, + "group_by": [], + "materialization": 10897179430287057821, + "schema_id": "asap-query-backend.v1:summary-state:v1:10897179430287057821", + "schema_version": 1, + "source": { + "Table": { + "table_ref": "q05_samples" + } + }, + "value_column": { + "Named": "value" + }, + "window": { + "kind": "tumbling", + "pane_origin_ms": 1788848096001, + "size_ms": 43200000, + "slide_ms": null + } + } + ], + "summary_catalog": { + "plan_id": 73, + "plan_version": 1, + "schema_version": 2, + "snapshot_sha256": "4ace75399a41e02d77391a26b75d2d6a501342ed73cb8ec3a1eb855fad188b67" + } + }, + "query_plan": { + "clickhouse_context": { + "accuracy": "Exact", + "tables": { + "q05_samples": { + "closed": false, + "columns": [ + { + "dtype": "utf8", + "name": "metric", + "nullable": false, + "table": null + }, + { + "dtype": "utf8", + "name": "labels", + "nullable": false, + "table": null + }, + { + "dtype": "timestamp", + "name": "ts_ms", + "nullable": false, + "table": null + }, + { + "dtype": "float64", + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 2, + "unique_keys": [] + } + } + }, + "entries": { + "clickhouse:Project { cols: [ProjectItem { alias: Some(\"value\"), expr: Column(0) }], qualifier: None, child: Aggregate { reduction: Reduce(GroupKeys { keys: [], without: false }), measures: [Sum { col: Some(3) }], output_names: [\"sum(q05_samples.value)\"], having: None, child: Scan { source: Table { table_ref: \"q05_samples\" }, predicates: [Predicate(BoolAnd([Compare { left: Column(2), op: Ge, right: Literal(Int64(1788848096001)) }, Compare { left: Column(2), op: Lt, right: Literal(Int64(1788891296001)) }]))], schema: Schema { columns: [Column { name: \"metric\", dtype: Utf8, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"labels\", dtype: Utf8, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"ts_ms\", dtype: Timestamp, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"value\", dtype: Float64, nullable: false, table: Some(\"q05_samples\") }], time_index: Some(2), unique_keys: [], closed: true } } } }": { + "canonical_query": "Project { cols: [ProjectItem { alias: Some(\"value\"), expr: Column(0) }], qualifier: None, child: Aggregate { reduction: Reduce(GroupKeys { keys: [], without: false }), measures: [Sum { col: Some(3) }], output_names: [\"sum(q05_samples.value)\"], having: None, child: Scan { source: Table { table_ref: \"q05_samples\" }, predicates: [Predicate(BoolAnd([Compare { left: Column(2), op: Ge, right: Literal(Int64(1788848096001)) }, Compare { left: Column(2), op: Lt, right: Literal(Int64(1788891296001)) }]))], schema: Schema { columns: [Column { name: \"metric\", dtype: Utf8, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"labels\", dtype: Utf8, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"ts_ms\", dtype: Timestamp, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"value\", dtype: Float64, nullable: false, table: Some(\"q05_samples\") }], time_index: Some(2), unique_keys: [], closed: true } } } }", + "fallback": "exact_backend", + "fixed_evaluation": { + "cumulative": true, + "end_ms": 1788891296001, + "start_ms": 1788848096001 + }, + "instant": { + "cumulative_readout": true, + "full_history": false, + "lookback_ms": 43200000 + }, + "language": "click_house_sql", + "nodes": { + "0": { + "input": 1, + "input_schema": { + "fields": [ + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "sum(q05_samples.value)", + "nullable": false + } + ], + "time_index": null + }, + "op": "relational", + "operation": { + "Project": { + "cols": [ + { + "alias": "value", + "expr": { + "Column": 0 + } + } + ], + "qualifier": null + } + }, + "output_schema": { + "fields": [ + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false + } + ], + "time_index": null + } + }, + "1": { + "input": 2, + "op": "exact_readout", + "readout": "sum" + }, + "2": { + "binding": { + "materialization": 10897179430287057821, + "output_grouping": { + "keys": [], + "mode": "reduce" + }, + "pane_origin_ms": 1788848096001, + "readout_lookback_ms": 43200000, + "window_ms": 43200000 + }, + "op": "read_materialization" + } + }, + "query_id": "SELECT sum(value) AS value FROM q05_samples WHERE ts_ms>=1788848096001 AND ts_ms<1788891296001", + "root": 0 + } + }, + "plan_id": 73, + "plan_version": 1 + }, + "summary_catalog": { + "data_descriptors": { + "data:v2|37:{\"table\":{\"table_ref\":\"q05_samples\"}}|27:{\"column\":{\"name\":\"value\"}}|0:|partition:Grouped|timestamp-ms:5:ts_ms|32:asap.timestamped-observations.v2": { + "group_by_keys": [], + "id": "data:v2|37:{\"table\":{\"table_ref\":\"q05_samples\"}}|27:{\"column\":{\"name\":\"value\"}}|0:|partition:Grouped|timestamp-ms:5:ts_ms|32:asap.timestamped-observations.v2", + "observation_semantics": "asap.timestamped-observations.v2", + "partitioning": "grouped", + "population_filter_canonical": "", + "source": { + "table": { + "table_ref": "q05_samples" + } + }, + "timestamp_column": "ts_ms", + "value_projection": { + "column": { + "name": "value" + } + } + } + }, + "materializations": { + "10897179430287057821": { + "data_descriptor_id": "data:v2|37:{\"table\":{\"table_ref\":\"q05_samples\"}}|27:{\"column\":{\"name\":\"value\"}}|0:|partition:Grouped|timestamp-ms:5:ts_ms|32:asap.timestamped-observations.v2", + "pane_origin_ms": 1788848096001, + "summary_descriptor_id": "summary:v2:{\"fidelity\":{\"kind\":\"exact\"},\"operator\":{\"aggregation_sub_type\":\"\",\"aggregation_type\":\"Sum\",\"kind\":\"configured\",\"parameters\":{}},\"state_schema_version\":1}", + "window_layout": { + "kind": "pane", + "pane_secs": 43200 + } + } + }, + "plan_id": 73, + "plan_version": 1, + "schema_version": 2, + "summary_descriptors": { + "summary:v2:{\"fidelity\":{\"kind\":\"exact\"},\"operator\":{\"aggregation_sub_type\":\"\",\"aggregation_type\":\"Sum\",\"kind\":\"configured\",\"parameters\":{}},\"state_schema_version\":1}": { + "fidelity": { + "kind": "exact" + }, + "id": "summary:v2:{\"fidelity\":{\"kind\":\"exact\"},\"operator\":{\"aggregation_sub_type\":\"\",\"aggregation_type\":\"Sum\",\"kind\":\"configured\",\"parameters\":{}},\"state_schema_version\":1}", + "operator": { + "aggregation_sub_type": "", + "aggregation_type": "Sum", + "kind": "configured", + "parameters": {} + }, + "state_schema_version": 1 + } + } + }, + "transmission_plan": { + "envelope": { + "activation_unix_ms": 0, + "backend_compat": "asap-query-backend.v1", + "capability_snapshot_id": "test", + "expiry_unix_ms": null, + "generated_at_unix_ms": 0, + "plan_id": 73, + "plan_version": 1, + "planner_revision": "0deceda3e776216c5542d638d958b159f22e27ce" + }, + "frame_identity": { + "identity_version": 1, + "require_base_checkpoint_for_delta": true, + "require_checkpoint_for_full": true, + "sequence_scope": "materialization_series_producer_epoch" + }, + "rules": [], + "summary_catalog": { + "plan_id": 73, + "plan_version": 1, + "schema_version": 2, + "snapshot_sha256": "4ace75399a41e02d77391a26b75d2d6a501342ed73cb8ec3a1eb855fad188b67" + } + } + }, + "selection_trace": { + "Project { cols: [ProjectItem { alias: Some(\"value\"), expr: Column(0) }], qualifier: None, child: Aggregate { reduction: Reduce(GroupKeys { keys: [], without: false }), measures: [Sum { col: Some(3) }], output_names: [\"sum(q05_samples.value)\"], having: None, child: Scan { source: Table { table_ref: \"q05_samples\" }, predicates: [Predicate(BoolAnd([Compare { left: Column(2), op: Ge, right: Literal(Int64(1788848096001)) }, Compare { left: Column(2), op: Lt, right: Literal(Int64(1788891296001)) }]))], schema: Schema { columns: [Column { name: \"metric\", dtype: Utf8, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"labels\", dtype: Utf8, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"ts_ms\", dtype: Timestamp, nullable: false, table: Some(\"q05_samples\") }, Column { name: \"value\", dtype: Float64, nullable: false, table: Some(\"q05_samples\") }], time_index: Some(2), unique_keys: [], closed: true } } } }": { + "group_id_scope": "this_selection", + "groups": [ + { + "candidates": [], + "consumer_count": 1, + "group_id": 0 + }, + { + "candidates": [ + { + "estimated_cost": null, + "estimated_cost_status": "not_reported_by_cost_model", + "provenance": "SummaryImplementation", + "rank": 0, + "rationale": "Sum { col: Some(3) } realizes as an exact Sum accumulator \u2014 the only realization implementations_for_with produces for this intent (no approximate candidate applies)", + "replacement_kind": "summary", + "selected": true, + "strategy": "SketchAlgorithmStrategy" + } + ], + "consumer_count": 1, + "group_id": 1 + }, + { + "candidates": [], + "consumer_count": 1, + "group_id": 2 + } + ], + "schema_version": 1 + } + } +} diff --git a/tools/clickhouse-benefit/automatic-aggregate-results.md b/tools/clickhouse-benefit/automatic-aggregate-results.md new file mode 100644 index 000000000..992b64adb --- /dev/null +++ b/tools/clickhouse-benefit/automatic-aggregate-results.md @@ -0,0 +1,99 @@ +# Automatically selected SUM and MAX over a bounded o11y series + +Measured 2026-09-11, source commit `a5dd4b3c`. The control plane now calls +ASAPPlanner and constructs the catalog from its selected DAG. The probe supplies +SQL, source schema, accuracy and time bounds; it does not supply a winning family +or a predeclared materialization. Planner selected exact `Sum` and +`MinMax(max)`, respectively. Full selected DAGs, catalog identities and candidate +traces are in [the artifacts](artifacts/automatic/). + +This is a bounded aggregate sensitivity experiment using original o11y values. +It is not evidence that the original 27-query SQL workload is accelerated. +Grouped labels, general residual operators and original workload coverage still +need their own validation. A quantile probe failed frontend function registration; +there is no quantile benefit result here. + +## Input and execution + +The unchanged source series is +`service_cache_refresh_lag_seconds{job="user-service",instance="user-service:8082"}`. +Its 24-hour input has 86,371 samples at approximately one sample/second, +16,086,775 bytes, at +`/mydata/clickhouse-o11y-main-results/fine-1s-24h-series.jsonl`. +The extraction provenance and original full-dataset hash are retained in +[the prior source manifest](artifacts/fine-1s-24h-series.provenance.json). +The half-open query interval is `[1788848096001,1788891296001)`: 12 hours and +43,200 samples. Both engines query the same interval and values. + +Each SQL query is `SELECT max(value) AS value ...` or `SELECT sum(value) AS value ...` +over that bounded interval. Three fresh-server trials per aggregate each alternate +1,000 backend and 1,000 ClickHouse requests. Both processes receive two CPUs, +CPU affinity 60–61 and a 4 GiB limit. The ClickHouse image is pinned, `max_threads` +is two and query caching is disabled. Other agents paused builds and workload +probes for the controlled measurements. An earlier interrupted, uncontrolled +SUM attempt was excluded and is recorded outside the committed results. + +All 6,000 backend requests reported `warm`; all 12,000 requests returned HTTP 200. +Every parsed backend result exactly equals its ClickHouse baseline: SUM is +1,328,070 and MAX is 530. This is exact numeric equality, with no approximation +or tolerance. The installed plan and completed backfill are exercised through +separate production backend processes. + +## Repeated queries + +| Aggregate/trial | Backend median ms | ClickHouse median ms | Speedup | Backend CPU ms / 1,000 | ClickHouse exact CPU ms / 1,000 | +| --- | ---: | ---: | ---: | ---: | ---: | +| SUM 1 | 0.861 | 6.891 | 8.01x | 810 | 9,600 | +| SUM 2 | 0.854 | 6.671 | 7.81x | 770 | 8,360 | +| SUM 3 | 0.853 | 4.703 | 5.51x | 750 | 9,460 | +| MAX 1 | 0.853 | 6.776 | 7.95x | 770 | 8,240 | +| MAX 2 | 0.854 | 6.716 | 7.87x | 770 | 8,360 | +| MAX 3 | 0.854 | 7.591 | 8.88x | 770 | 12,130 | + +Counting ClickHouse background CPU during the backend requests as well gives +820–920 ms per 1,000 warm requests. CPU uses Linux scheduler ticks, so individual +sub-millisecond CPU values are not claimed. Per-trial p95 latency and serial +service throughput are retained in the summaries; these are serial request +measurements, not saturated concurrent throughput. + +## Build, memory and storage + +Backfill took 1.32–1.42 seconds. Its backend CPU was 220–300 ms and its additional +ClickHouse source-read CPU was 160–210 ms. The enclosing setup-to-built interval +was 1.85–2.08 seconds and includes source loading, planning, installation and +startup. Source loading and backfill are separately recorded; planning, +installation and startup are not isolated measurements. First-query +latency was not consistently better: backend 4.06–7.51 ms versus ClickHouse +5.26–11.07 ms. The repeated-query gains do not erase construction costs. + +Backend query-phase RSS was 50.4–54.9 MiB; retained ClickHouse RSS was +779.6–849.3 MiB. Per-phase RSS and observed process high-water marks are in the +artifacts. These are whole-process observations, not isolated allocation sizes. +The backend output directory occupied 5,795 bytes including persisted metadata; +ClickHouse reported 507,146 bytes for its 86,371-row table. Backend summaries cover +12 hours while that table retains 24 hours, so these sizes are **not a compression +ratio**. + +The deployment still retains ClickHouse for exact queries and backfill. Adding +the backend therefore adds memory and state storage to that deployment. These +measurements demonstrate lower repeated-query latency and CPU for the served +summaries, **not lower combined deployment memory or storage**. Control-plane +planning memory is not included in backend RSS. The current fixed-window layout +is not cost-optimized; candidate traces explicitly report estimated cost as +unavailable rather than inventing a measured/estimated cost comparison. + +## Reproduction and evidence + +Build the release process probe, then use +[the matched-budget runner](run_matched_trial.sh) with +`CLICKHOUSE_BENCH_AGGREGATE=sum` or `max` and a unique trial suffix. Keep all +required environment inputs described by the script. The committed runtime +snapshots, executable hashes and checkout files identify these runs. The raw +traces remain under `/mydata/clickhouse-automatic-benefit-study`; their hashes are +in [the raw-trace manifest](artifacts/automatic/raw-traces.json). + +For independent plan inspection, +`cargo run --release -p control_plane --example compile_clickhouse_workload` +reads an automatic SQL workload JSON from stdin and emits the publication, +install request and actual selection trace. Example workload files and failed +quantile diagnostics are in `/mydata/clickhouse-automatic-benefit-study`.