test(integration): TSDB cold-path rewrite + warm-path e2e + parity unify

deploy/configs/asap-otel-agent-gorillas3-tier.yaml

@@ -1,9 +1,14 @@
-# Gorilla-S3 cold-engine tier (Phase 2 of the cold-engine integration).
+# Cold-path tier — gorillas3processor in `gateway_raw` role.
 #
-# Mirrors the b5-gorilla baseline shape but swaps the local-dir sink
-# for the S3-cold-engine `gorillas3` processor: encode raw samples
-# with Gorilla XOR-delta, write 60s chunks to MinIO, and DROP the
-# original metric so the agent does not OTLP-forward the bytes.
+# Direct raw-metric → Prometheus TSDB block emit. Suitable for the
+# single-collector cold-path layout (no upstream fragment edge agent;
+# this collector consumes raw OTLP and finalizes blocks itself).
+#
+# Output objects in MinIO:
+#
+#   <tsdb_bucket>/<ULID>/chunks/000001
+#   <tsdb_bucket>/<ULID>/index
+#   <tsdb_bucket>/<ULID>/meta.json   (uploaded last; readiness signal)
 #
 # Reuses the existing `minio` container in deploy/docker-compose/base.yml
 # (no overlay change needed). Bucket `asap-gorilla` must exist; the
@@ -28,18 +33,19 @@ receivers:
 
 processors:
   gorillas3:
+    role: gateway_raw
+    delivery_mode: durable_raw
     window_interval: 60s
     drop_original: true
     endpoint: http://minio:9000
     bucket: asap-gorilla
+    tsdb_bucket: asap-gorilla
     region: us-east-1
     use_ssl: false
     # MinIO root creds from base.yml. Replace with env-resolution in
     # production deploys (e.g. ${env:MINIO_ACCESS_KEY}).
     access_key_id: asap
     secret_access_key: asap-local-only
-    prefix_template: "{tenant}/{metric}/{YYYY}/{MM}/{DD}/{HH}/"
-    tenant: default
     max_retries: 3
     retry_backoff: 1s
     upload_timeout: 30s

deploy/docker-compose/cross-host-parity.yml

@@ -46,11 +46,11 @@ services:
     command: ["--config=/etc/otel/config.yaml"]
     volumes:
       # Agent configs live next to the test driver under
-      # integration/cross_host_parity/configs/ — keeps Phase 5E's
+      # integration/parity/agent-binary/configs/ — keeps Phase 5E's
       # per-agent shape co-located with the harness that drives them
       # rather than scattered through deploy/configs/, which is owned
       # by the eval pipeline (agent #13).
-      - ../../integration/cross_host_parity/configs/asap-otel.yaml:/etc/otel/config.yaml:ro
+      - ../../integration/parity/agent-binary/configs/asap-otel.yaml:/etc/otel/config.yaml:ro
     environment:
       - ASAP_AGENT_ID=asap-otel
     networks:
@@ -64,7 +64,7 @@ services:
     container_name: parity-asap-otap
     command: ["--config=/etc/otap/config.yaml"]
     volumes:
-      - ../../integration/cross_host_parity/configs/asap-otap.yaml:/etc/otap/config.yaml:ro
+      - ../../integration/parity/agent-binary/configs/asap-otap.yaml:/etc/otap/config.yaml:ro
     environment:
       - ASAP_AGENT_ID=asap-otap
     networks:
@@ -78,7 +78,7 @@ services:
     container_name: parity-asap-telegraf
     command: ["--config=/etc/telegraf/telegraf.conf"]
     volumes:
-      - ../../integration/cross_host_parity/configs/asap-telegraf.toml:/etc/telegraf/telegraf.conf:ro
+      - ../../integration/parity/agent-binary/configs/asap-telegraf.toml:/etc/telegraf/telegraf.conf:ro
     environment:
       - ASAP_AGENT_ID=asap-telegraf
     networks:
@@ -106,8 +106,8 @@ services:
     container_name: parity-envelope-tap
     command: ["--mode=cross-host-parity-tap", "--capture-dir=/captures"]
     volumes:
-      - ../../integration/cross_host_parity/captures:/captures
-      - ../../integration/cross_host_parity/golden_input:/golden_input:ro
+      - ../../integration/parity/agent-binary/captures:/captures
+      - ../../integration/parity/agent-binary/golden_input:/golden_input:ro
       - ../scripts/queries-e2e.json:/queries-e2e.json:ro
     networks:
       - parity-net

integration/gorilla_s3_e2e/Makefile → integration/e2e-cold-path/Makefile

@@ -1,8 +1,8 @@
-# Phase 6 Gorilla-S3 e2e shorthand. See README.md for full docs.
+# Cold-path e2e shorthand. See README.md for full docs.
 #
 # Targets:
 #   test       — fixture-only run (no docker, no sweep coordination)
-#   test-live  — live run (gated on GORILLA_E2E_LIVE=1; skips if sweep alive)
+#   test-live  — live run (gated on COLD_E2E_LIVE=1; skips if sweep alive)
 #   test-force — live run, ignore sweep busy check (use sparingly)
 #   vet        — go vet
 #   down       — tear down a stuck e2e compose stack
@@ -13,14 +13,14 @@ test:
 	go test -v ./...
 
 test-live:
-	GORILLA_E2E_LIVE=1 go test -v -run TestGorillaS3End2End ./...
+	COLD_E2E_LIVE=1 go test -v -run TestColdPathEnd2End ./...
 
 test-force:
-	GORILLA_E2E_LIVE=1 GORILLA_E2E_FORCE=1 go test -v -run TestGorillaS3End2End ./...
+	COLD_E2E_LIVE=1 COLD_E2E_FORCE=1 go test -v -run TestColdPathEnd2End ./...
 
 vet:
 	go vet ./...
 
 down:
-	docker compose -p asap-gorilla-e2e \
+	docker compose -p asap-cold-e2e \
 		-f docker-compose/e2e-overlay.yml down -v

integration/e2e-cold-path/README.md

@@ -0,0 +1,124 @@
+# integration/e2e-cold-path
+
+End-to-end integration test for the ASAP cold-path pipeline:
+
+```
+fake-driver  ─OTLP─▶  asap-otel + gorillas3processor (gateway_raw)
+                          │  Prometheus TSDB block builder
+                          ▼
+                       MinIO   (<ulid>/chunks/000001 + index + meta.json)
+                          │
+                          ▼
+                       Thanos store-gateway / backend cold-path
+```
+
+This is the cold-path e2e gate. After PR #359 the
+`gorillas3processor` runtime emits Prometheus TSDB blocks (not the
+legacy `GORILLA1` magic-prefixed XOR-delta chunks). The legacy in-test
+decoder + cross-language Rust subtest were retired with the rename
+from `integration/gorilla_s3_e2e/` to this directory.
+
+## Layout
+
+```
+integration/e2e-cold-path/
+├── e2e_test.go                  # main Go test — TestColdPathEnd2End
+├── golden/
+│   └── input_samples.json       # 100 deterministic samples
+├── docker-compose/
+│   └── e2e-overlay.yml          # ephemeral stack (separate compose
+│                                #   project + 29xxx port range)
+├── go.mod                       # depends on prometheus/prometheus
+├── Makefile                     # `make test` / `make test-live`
+└── README.md
+```
+
+## How the assertion works
+
+The runtime side of the cold path is the `gorillas3processor` in
+`gateway_raw` role + `delivery_mode=durable_raw`. On each
+`window_interval` tick it finalizes a Prometheus TSDB block to
+S3/MinIO with three files under `<tsdb_bucket>/<ULID>/`:
+
+- `chunks/000001` — XOR-encoded sample chunks
+- `index` — Prometheus inverted index
+- `meta.json` — block metadata (uploaded LAST, so Thanos doesn't see
+  half-written blocks)
+
+The test:
+
+1. Lists every `meta.json` marker in the `asap-gorilla` MinIO bucket
+   via `mc find ... --name 'meta.json'`. Each match identifies one
+   finalized block ULID.
+2. Downloads each block's three files into a per-block temp dir.
+3. Opens the block with `prometheus/tsdb`'s `tsdb.OpenBlock(nil, dir,
+   nil, nil)`, builds a `BlockQuerier` over the full time range, and
+   selects every series matching `__name__=~.+`.
+4. Iterates the resulting `SeriesSet`, flattens to (label_set, ts,
+   value) tuples, and asserts the multi-set of values equals the
+   input fixture.
+
+## Run
+
+### Default (no docker, fixture-only)
+
+```
+cd integration/e2e-cold-path
+go test -v ./...
+```
+
+The live e2e test (`TestColdPathEnd2End`) is gated on
+`COLD_E2E_LIVE=1` and SKIPs in this mode; the
+`TestGoldenInputFixtureWellFormed` test still runs and pins the
+shape of the input fixture.
+
+### Live e2e (requires docker, sweep idle)
+
+```
+COLD_E2E_LIVE=1 go test -v -run TestColdPathEnd2End ./...
+```
+
+The live test:
+
+1. Brings up `minio-e2e + asap-otel-e2e` via
+   `docker compose -p asap-cold-e2e -f docker-compose/e2e-overlay.yml up -d`.
+2. Pushes the 100-sample fixture via OTLP/HTTP to `127.0.0.1:24318/v1/metrics`.
+3. Waits 75s for the gorillas3processor's 60s window to flush.
+4. Runs `HappyPath` and `MultiBlockRange` subtests against the live stack.
+5. Tears the stack down with `down -v`.
+
+### Coordination with the host-wide sweep
+
+The host runs a long sweep (`run_e2e_sweep.sh`) that owns the default
+compose project. The e2e test takes the **defensive** posture:
+
+- The overlay uses a separate compose project name (`asap-cold-e2e`)
+  and a separate port range (29xxx) so it would NOT collide on
+  ports — but
+- Docker memory + CPU + network bandwidth are still shared with the
+  sweep. If the sweep PID (default `2862786`, override via
+  `COLD_E2E_SWEEP_PID=<pid>`) is alive, the live test SKIPs unless
+  forced via `COLD_E2E_FORCE=1`.
+- Disable the busy-check entirely with `COLD_E2E_SWEEP_PID=0` (or
+  `=none`) — useful in CI where there's no sweep concept.
+
+## Subtests
+
+| Subtest                            | What it asserts                                                                                          |
+| ---------------------------------- | -------------------------------------------------------------------------------------------------------- |
+| `HappyPath`                        | MinIO contains ≥1 finalized block (a `meta.json` exists); the block decodes back to the input value-set. |
+| `MultiBlockRange`                  | If the fixture spread crossed a window boundary, ≥2 blocks land; their union covers the full input.     |
+| `TestGoldenInputFixtureWellFormed` | (always runs, no docker) The fixture's `expected_sum` invariant matches `sum(values)`.                   |
+
+## Notes
+
+- The bucket name `asap-gorilla` matches the legacy MVP demo.
+- Block download uses `mc cat` inside a transient `minio/mc:latest`
+  container; no S3 SDK in `go.mod`.
+- OTLP push uses raw HTTP/JSON to the agent's `/v1/metrics` endpoint;
+  no OTLP client in `go.mod`.
+- The PromQL accuracy / data-source-marker subtests from the legacy
+  Gorilla-S3 e2e are NOT carried over: TSDB is upstream Prometheus's
+  native format, not an ASAP-specific wire format. PromQL response
+  shape is now exercised by the warm-path e2e + ASAPQuery-backend's
+  own tests.

integration/e2e-cold-path/docker-compose/e2e-overlay.yml

@@ -0,0 +1,113 @@
+# Cold-path e2e overlay.
+#
+# Brings up an EPHEMERAL stack on a separate compose project
+# (`asap-cold-e2e`) and a separate port range (29xxx) so it can run
+# side-by-side with the host-wide sweep stack
+# (`docker-compose-{minio,backend,...}` on 19xxx) without port
+# collisions.
+#
+# Coordination: the e2e Go test calls `docker compose -p asap-cold-e2e
+# -f integration/e2e-cold-path/docker-compose/e2e-overlay.yml up -d`
+# and tears it down with `down -v` on test completion. Sweep is left
+# untouched.
+#
+# Services brought up:
+#
+#   minio-e2e          MinIO S3-compatible store, ports 29000/29001
+#   minio-setup-e2e    Seeds the `asap-gorilla` bucket the
+#                      gorillas3processor writes Prometheus TSDB
+#                      blocks (`<ulid>/chunks/000001 + index +
+#                      meta.json`) into.
+#   asap-otel-e2e      Single asap-otel agent running
+#                      asap-otel-agent-gorillas3-tier.yaml in
+#                      role=gateway_raw + delivery_mode=durable_raw —
+#                      consumes raw OTLP and finalizes Prometheus
+#                      TSDB blocks to MinIO once per window_interval.
+#
+# fake-exporter is intentionally NOT in the compose stack; the Go
+# test pushes the deterministic 100-sample fixture directly via OTLP
+# to asap-otel-e2e. This is the only way to drive a fixed, repeatable
+# input — fake-exporter generates random data.
+#
+# The backend container is also intentionally NOT in this overlay.
+# The cold-path test asserts byte-equality of the recovered TSDB
+# samples vs. the input fixture; PromQL response shape is exercised
+# elsewhere (warm-path e2e + ASAPQuery-backend tests).
+
+version: "3.9"
+
+x-logging: &default-logging
+  driver: json-file
+  options:
+    max-size: "5m"
+    max-file: "2"
+
+services:
+  minio-e2e:
+    image: minio/minio:latest
+    command: server /data --console-address ":9001"
+    environment:
+      MINIO_ROOT_USER: asap
+      MINIO_ROOT_PASSWORD: asap-local-only
+    ports:
+      - "29000:9000"
+      - "29001:9001"
+    volumes:
+      - minio-e2e-data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 5s
+      timeout: 3s
+      retries: 20
+    logging: *default-logging
+
+  # Pre-create the asap-gorilla bucket so the gorillas3processor's
+  # PutTSDBBlock calls land somewhere. The default base.yml only seeds
+  # `raw` for the JSONL cold-fallback adapter; the TSDB blocks need a
+  # separate bucket per the gorillas3processor README.
+  minio-setup-e2e:
+    image: minio/mc:latest
+    depends_on:
+      minio-e2e:
+        condition: service_healthy
+    entrypoint: /bin/sh
+    command:
+      - -c
+      - |
+        mc alias set asap http://minio-e2e:9000 asap asap-local-only &&
+        mc mb --ignore-existing asap/asap-gorilla &&
+        mc anonymous set download asap/asap-gorilla
+    logging: *default-logging
+
+  # Single asap-otel agent running the gorillas3 cold-path tier
+  # config. The OTLP receiver listens on 4317 (gRPC) + 4318 (HTTP);
+  # we expose both on host 24317/24318 so the Go test can drive
+  # samples in without going through a fake-exporter container.
+  asap-otel-e2e:
+    image: asap/asap-otel:dev
+    depends_on:
+      minio-setup-e2e:
+        condition: service_completed_successfully
+    command:
+      - "--config=/etc/otel/config.yaml"
+    volumes:
+      # The cold-path config — drop_original=true, encode + emit TSDB
+      # blocks to MinIO. Path is relative to the compose file's dir.
+      - ../../../deploy/configs/asap-otel-agent-gorillas3-tier.yaml:/etc/otel/config.yaml:ro
+    environment:
+      RUST_LOG: "info"
+    networks:
+      default:
+        aliases:
+          - minio
+          # The agent config hardcodes endpoint http://minio:9000;
+          # alias the e2e MinIO under that name so no YAML rewrite
+          # is needed.
+    ports:
+      - "24317:4317"
+      - "24318:4318"
+      - "28890:8890"
+    logging: *default-logging
+
+volumes:
+  minio-e2e-data: