From c5541d1dfe472e8629d0b9ed74d12e79ea8abb4f Mon Sep 17 00:00:00 2001 From: Zeying Zhu Date: Sat, 9 May 2026 16:46:20 -0400 Subject: [PATCH] fix(mvp): probe predicates use labels the fake-exporter actually emits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three places in the MVP demo hardcoded predicates against `service`, `status` labels that the fake-exporter never produces; those queries returned empty (but HTTP 200) and masked any real cold-path / postings behaviour they were meant to exercise. The fake-exporter (`deploy/fake-exporter/main.go::attrSetsZRNP`) emits `zone, rack, node, pod` labels. There is no `service` and no `status` label in any series. Previous predicates: - `service="payments"` / `service="api"` → matched 0 series - `status=~"5.."` → matched 0 series The cold-path probe verdict was technically PASS (HTTP 200 + the `data_source: gorilla_archive` marker is present) but the actual result was always `[]` — the demo was confirming "the archive engine accepts queries", not "the archive engine returns data." Repointed predicates to labels that exist: - `deploy/configs/mvp-workload.yaml` entry 4 (criterion ⑤ cold-fallback probe): `service="payments"` → `zone="z0"` - `deploy/scripts/run_mvp_demo.sh` Phase 5 cold-fallback curl + log line: same change - `deploy/scripts/run_mvp_demo.sh` Phase 4 postings probes: `count(http_requests_total{service="api"})` → `count(http_requests_total{zone="z0"})`; `rate(http_requests_total{status=~"5.."}[5m])` → `rate(http_requests_total{rack=~"r0[0-3]"}[5m])` - `deploy/configs/backend-storage-routing.yaml` doc comment Verified end-to-end (separate diagnostic run): with `zone="z0"`, thanos-query returns non-empty `count` results — the cold path actually serves data instead of returning HTTP-200-but-empty. Output file basenames (`cold_payments.*`) kept for backwards compatibility with `mvp_report.py`'s loader. Verification: - `pytest deploy/scripts/tests/test_mvp_report.py deploy/configs/tests/` → 35 passed. Co-Authored-By: Claude Opus 4.7 (1M context) --- deploy/configs/backend-storage-routing.yaml | 6 +++++- deploy/configs/mvp-workload.yaml | 10 ++++++++- deploy/scripts/run_mvp_demo.sh | 24 +++++++++++++++------ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/deploy/configs/backend-storage-routing.yaml b/deploy/configs/backend-storage-routing.yaml index 3e328cf2..83d49fdc 100644 --- a/deploy/configs/backend-storage-routing.yaml +++ b/deploy/configs/backend-storage-routing.yaml @@ -54,10 +54,14 @@ default: sketch_warm_tier # windows). # - The `[count, topk, rate_post_hoc]` slot routes ad-hoc / post-hoc # queries to the cold archive. `count(http_requests_total{ -# service="payments"})` is the criterion ⑤ probe — it surfaces +# zone="z0"})` is the criterion ⑤ probe — it surfaces # `data_source: gorilla_archive` because the cold engine answers # it. `topk(...)` and `rate(...)` shapes the warm tier doesn't # precompute also drop into the archive's exact streaming path. +# (Predicate uses `zone="z0"` rather than `service="payments"` +# because the fake-exporter emits `zone, rack, node, pod` — +# no `service` label is produced, so the older selector always +# matched zero series and masked the cold-engine traversal.) # # Per-query-shape dispatch is the right axis: a single-target form # would force ④ and ⑤ to be mutually exclusive (quantile-on-warm diff --git a/deploy/configs/mvp-workload.yaml b/deploy/configs/mvp-workload.yaml index fa89ac85..5ef97bf6 100644 --- a/deploy/configs/mvp-workload.yaml +++ b/deploy/configs/mvp-workload.yaml @@ -75,8 +75,16 @@ # backend-storage-routing.yaml so the warm tier doesn't cover # it; the query forces routing to GorillaQueryEngine. Drives # criterion ⑤ verification (data_source: gorilla_archive). +# +# Predicate uses `zone="z0"` because the fake-exporter emits +# `zone, rack, node, pod` labels (see `deploy/fake-exporter/main.go` +# `attrSetsZRNP`); there is NO `service` label on the produced +# series, so the previous `service="payments"` selector matched +# zero series and surfaced as an empty (but HTTP-200) cold-tier +# response. `zone="z0"` matches roughly 1/4 of the produced +# series and exercises the same routing+engine code path. - metric_name: http_requests_total - query_string: "count(http_requests_total{service=\"payments\"})" + query_string: "count(http_requests_total{zone=\"z0\"})" accuracy_sla: 0.0 assign_to_role: archive diff --git a/deploy/scripts/run_mvp_demo.sh b/deploy/scripts/run_mvp_demo.sh index 41bc0caf..c789a4ca 100755 --- a/deploy/scripts/run_mvp_demo.sh +++ b/deploy/scripts/run_mvp_demo.sh @@ -48,7 +48,7 @@ # - label-at-instant (sum by zone, gateway fan-in) # - combined (rate over 5m + sum by zone) # plus a fourth ad-hoc cold-fallback probe -# (`http_requests_total{service="payments"}` — assigned +# (`http_requests_total{zone="z0"}` — assigned # to role "archive"). # # 4. Freshness phase calls `run_freshness_phase.sh` which emits @@ -653,11 +653,16 @@ ad_hoc_postings_phase() { || log " [warn] curl exited non-zero for ${label}" } - # The two postings-exercise queries from the spec. + # The two postings-exercise queries from the spec. Predicates + # match labels the fake-exporter actually emits (`zone, rack, + # node, pod` per `deploy/fake-exporter/main.go::attrSetsZRNP`) — + # the previous `service="api"` and `status=~"5.."` selectors + # match zero series since neither label exists in the produced + # data, masking the postings-filter exercise with empty results. fire_query "count_api_series" \ - 'count(http_requests_total{service="api"})' + 'count(http_requests_total{zone="z0"})' fire_query "topk_5xx_by_zone" \ - 'topk(5, sum by (zone) (rate(http_requests_total{status=~"5.."}[5m])))' + 'topk(5, sum by (zone) (rate(http_requests_total{rack=~"r0[0-3]"}[5m])))' # Step 2.4: archive-only PromQL surface (Path A2 / Thanos engine). # These queries were rejected by the legacy curated-subset @@ -695,10 +700,17 @@ cold_fallback_phase() { local adir="${PIPELINE_OUT_BASE}/ad-hoc" local backend_url="http://localhost:${PIPELINE_QUERY_PORT}" - log " cold[payments]: count(http_requests_total{service=\"payments\"})" + # Probe predicate uses `zone="z0"` because the fake-exporter + # emits zone/rack/node/pod labels — no `service` label exists, + # so the previous `service="payments"` selector always matched + # zero series and surfaced an empty (but HTTP-200) response that + # masked any real cold-path data (deploy/fake-exporter/main.go + # `attrSetsZRNP`). Keeping the file basename `cold_payments.*` + # for backwards compatibility with mvp_report.py's loader. + log " cold[zone=z0]: count(http_requests_total{zone=\"z0\"})" curl -sG -m 10 \ "${backend_url}/api/v1/query" \ - --data-urlencode 'query=count(http_requests_total{service="payments"})' \ + --data-urlencode 'query=count(http_requests_total{zone="z0"})' \ -o "${adir}/cold_payments.json" \ -w '{"http_code":%{http_code},"time_total":%{time_total}}\n' \ > "${adir}/cold_payments.curlstats" \