feat(gorilla-merger): decode-on-read for cold intchunk value chunks

gorilla-merger/.gitignore

@@ -2,3 +2,8 @@
 /data/
 /gorilla-merger
 *.test
+
+# Local-only go workspace used during development to point the asap-gorilla-go
+# replace at a specific intchunk-containing checkout; never committed.
+/go.work
+/go.work.sum

gorilla-merger/Dockerfile

@@ -2,65 +2,75 @@
 #
 # Multi-stage build for the gorilla-merger (Thanos-Receive-style component).
 #
-# ## Why this needs a build secret
+# ## asap-gorilla-go is local source, not the published module
 #
 # gorilla-merger imports the PRIVATE Go module
-# `github.com/ProjectASAP/asap-gorilla-go` (the shared edge<->merger
-# ASAPFRG1 wire codec). A naive `go build` inside Docker/CI cannot fetch it:
-# the module proxy + git fetch get a 404/auth prompt for the private repo.
+# `github.com/ProjectASAP/asap-gorilla-go` (the shared edge<->merger ASAPFRG1
+# wire codec) AND its `intchunk` SUBPACKAGE (the cold value-chunk codec the
+# internal/coldchunk decode-on-read helper builds on). intchunk does NOT exist
+# in the published asap-gorilla-go tag, so the module cannot come from the proxy.
 #
-# We solve this with a BuildKit *secret* mount carrying a GitHub token. The
-# secret is mounted ONLY for the duration of the build RUN that needs it and
-# is NOT baked into any image layer (unlike a build-arg or COPY of a token
-# file). Inside that RUN we set a transient `url.insteadOf` git rewrite so
-# `go` fetches the private module over HTTPS with the token. The git config
-# lives only in the container build layer, never on the host.
-#
-# `GOPRIVATE=github.com/ProjectASAP/*` keeps the fetch off the public proxy
-# and checksum DB; `GIT_TERMINAL_PROMPT=0` makes a missing/incorrect token
-# fail fast instead of hanging on an interactive credential prompt.
+# Instead we COPY the in-repo monorepo `asap-gorilla-go` checkout into the build
+# as a BuildKit *build-context* and rewrite go.mod's `replace` to point at that
+# in-image path. This is the container analogue of the relative
+# `../../ASAPCollector/asap-gorilla-go` replace committed in go.mod, and mirrors
+# how build_asap_otel.sh injects the same asap-gorilla-go replace for asap-otel:
+# the image always compiles against the intchunk-containing source WITHOUT a
+# published tag. Because asap-gorilla-go is now local source, no GitHub token is
+# required to fetch it; a `gh_token` secret is still accepted (and used if
+# mounted) so any OTHER private fetch keeps working, but it is optional.
 #
 # ## Build invocation
 #
-# Write a GitHub token (a PAT or `gh auth token`) to a file, then:
-#
-#   gh auth token > /tmp/gh_token            # or: echo "$GITHUB_TOKEN" > /tmp/gh_token
 #   DOCKER_BUILDKIT=1 docker build \
-#       --secret id=gh_token,src=/tmp/gh_token \
+#       --build-context asap-gorilla-go=/path/ASAPCollector/asap-gorilla-go \
 #       -t asap/gorilla-merger:dev \
 #       gorilla-merger/
-#   rm -f /tmp/gh_token
 #
-# The build context is the `gorilla-merger/` module directory (this file's
-# directory). The same GOPRIVATE + token requirement applies to
-# ASAPQuery-backend CI before PR #310 can be merged (the CI runner must
-# expose a `gh_token` secret / configure `url.insteadOf` the same way).
+# The primary build context is the `gorilla-merger/` module directory (this
+# file's directory); the `asap-gorilla-go` build-context supplies the monorepo
+# checkout that contains the intchunk subpackage. GOPRIVATE keeps any fetch off
+# the public proxy/sumdb; GIT_TERMINAL_PROMPT=0 fails fast instead of hanging.
 
 FROM golang:1.25-bookworm AS build
-WORKDIR /src
 
-# git is needed for the private-module fetch (insteadOf rewrite below).
+# git/ca-certificates for any private-module fetch (insteadOf rewrite below).
 RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates \
     && rm -rf /var/lib/apt/lists/*
 
-# Prime the module graph first so dependency downloads cache independently
-# of source edits. go.sum is committed, so the public deps resolve normally;
-# only the private module needs the token.
+# Lay the asap-gorilla-go monorepo checkout (with the intchunk subpackage) at
+# /ASAPCollector/asap-gorilla-go so it is the sibling the committed relative
+# replace `../../ASAPCollector/asap-gorilla-go` resolves to from the module dir
+# at /src/gorilla-merger (/src/gorilla-merger/../../ASAPCollector/... == that).
+COPY --from=asap-gorilla-go . /ASAPCollector/asap-gorilla-go
+
+WORKDIR /src/gorilla-merger
+
+# Prime the module graph first so dependency downloads cache independently of
+# source edits. go.sum is committed, so the public deps resolve normally; with
+# the directory replace, asap-gorilla-go itself resolves from the local source.
 COPY go.mod go.sum ./
+# Repoint the committed relative replace at the in-image checkout. The relative
+# path already resolves given the layout above; making it absolute keeps the
+# build independent of the WORKDIR depth.
 RUN --mount=type=secret,id=gh_token \
     GOPRIVATE=github.com/ProjectASAP/* \
     GONOSUMCHECK=github.com/ProjectASAP/* \
     GOFLAGS=-mod=mod \
     GIT_TERMINAL_PROMPT=0 \
-    sh -c 'git config --global url."https://x-access-token:$(cat /run/secrets/gh_token)@github.com/".insteadOf "/" && go mod download'
+    sh -c 'go mod edit -replace github.com/ProjectASAP/asap-gorilla-go=/ASAPCollector/asap-gorilla-go; \
+           if [ -f /run/secrets/gh_token ]; then git config --global url."https://x-access-token:$(cat /run/secrets/gh_token)@github.com/".insteadOf "/"; fi; \
+           go mod download'
 
 # Now copy the rest of the module and build.
 COPY . .
 RUN --mount=type=secret,id=gh_token \
     GOPRIVATE=github.com/ProjectASAP/* \
     GONOSUMCHECK=github.com/ProjectASAP/* \
+    GOFLAGS=-mod=mod \
     GIT_TERMINAL_PROMPT=0 \
-    sh -c 'git config --global url."https://x-access-token:$(cat /run/secrets/gh_token)@github.com/".insteadOf "/" && CGO_ENABLED=0 go build -ldflags="-s -w" -o /out/gorilla-merger ./cmd/gorilla-merger'
+    sh -c 'go mod edit -replace github.com/ProjectASAP/asap-gorilla-go=/ASAPCollector/asap-gorilla-go; \
+           CGO_ENABLED=0 go build -ldflags="-s -w" -o /out/gorilla-merger ./cmd/gorilla-merger'
 
 FROM gcr.io/distroless/static-debian12:nonroot
 COPY --from=build /out/gorilla-merger /usr/local/bin/gorilla-merger

gorilla-merger/go.mod

@@ -232,6 +232,20 @@ require (
 // standalone private module (github.com/ProjectASAP/asap-gorilla-go). Building
 // requires GOPRIVATE=github.com/ProjectASAP/* + git auth to fetch it.
 //
+// The cold value-chunk codec used by the decode-on-read helper
+// (internal/coldchunk) lives in the asap-gorilla-go/intchunk SUBPACKAGE, which
+// the published tag predates. To compile against intchunk WITHOUT cutting a new
+// release we point asap-gorilla-go at the in-repo monorepo checkout via a local
+// `replace`. The relative path mirrors the data_plane crate's sibling path-deps
+// (`../../ASAPCollector/...`): clone ASAPCollector next to ASAPQuery-backend so
+// `<repo>/gorilla-merger/../../ASAPCollector/asap-gorilla-go` resolves. The
+// container build supplies that same checkout as a BuildKit build-context and
+// rewrites this replace to the in-image path (see Dockerfile + run_demo.sh), so
+// the merger image always compiles against the intchunk-containing
+// asap-gorilla-go. intchunk only pulls in prometheus/tsdb/chunkenc (already a
+// merger dependency), so no new module is added to the graph.
+replace github.com/ProjectASAP/asap-gorilla-go => ../../ASAPCollector/asap-gorilla-go
+
 // Thanos v0.41.0 declares these replace directives in its own go.mod. Go does
 // NOT inherit a dependency's replace directives into the main module, so MVS
 // otherwise picks upstream versions whose APIs differ from what Thanos v0.41.0

gorilla-merger/internal/coldchunk/coldchunk.go

@@ -0,0 +1,94 @@
+// Package coldchunk provides the decode-on-read inverse of the cold ingest
+// path: it turns an intchunk-format value chunk (the best-of-N lossless cold
+// codec from asap-gorilla-go/intchunk) into a standard Prometheus XOR
+// (Gorilla) chunk that the rest of the merger — and a future decode-on-read
+// Thanos StoreAPI — can iterate with plain chunkenc.
+//
+// This is the read-side mirror of internal/merger/ingest.go, which decodes
+// ASAPFRG1 XOR fragments and appends them to the embedded tsdb.DB. Where ingest
+// goes (XOR bytes -> samples -> tsdb), coldchunk goes (intchunk bytes ->
+// samples -> XOR chunk). The edge agent does not emit intchunk yet, so this is
+// not wired end-to-end; it is a tested, importable capability that the
+// StoreAPI read path will build on.
+//
+// intchunk's value codecs (GORILLA_XOR, INT_FOR_DELTA/_DOD and their varint
+// variants) are all bit-exact lossless, so re-encoding the decoded samples as a
+// Prometheus XOR chunk reproduces the original float64 values exactly.
+package coldchunk
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/ProjectASAP/asap-gorilla-go/intchunk"
+	"github.com/prometheus/prometheus/tsdb/chunkenc"
+)
+
+// ErrNoSamples is returned when an intchunk decodes to zero samples; an empty
+// chunk has no XOR representation worth producing.
+var ErrNoSamples = errors.New("coldchunk: chunk decoded to zero samples")
+
+// Sample is a re-export of intchunk.Sample so callers of this package do not
+// have to import intchunk directly to read the decoded points.
+type Sample = intchunk.Sample
+
+// DecodeToSamples decodes a single self-contained intchunk-format chunk back to
+// its (timestamp, value) samples in time order. It is a thin wrapper over
+// intchunk.DecodeChunk kept here so the merger has one cold-read entry point.
+func DecodeToSamples(chunk []byte) ([]Sample, error) {
+	samples, err := intchunk.DecodeChunk(chunk)
+	if err != nil {
+		return nil, fmt.Errorf("decode intchunk: %w", err)
+	}
+	return samples, nil
+}
+
+// DecodeChunksToSamples decodes a sequence of concatenated intchunk chunks (e.g.
+// the multiple chunks an overflow re-base cut can produce for one block) back to
+// the full ordered sample stream.
+func DecodeChunksToSamples(chunks [][]byte) ([]Sample, error) {
+	samples, err := intchunk.DecodeChunks(chunks)
+	if err != nil {
+		return nil, fmt.Errorf("decode intchunks: %w", err)
+	}
+	return samples, nil
+}
+
+// SamplesToXORChunk re-encodes decoded samples as a standard Prometheus XOR
+// (Gorilla) chunk by appending each point through the XOR Appender — the exact
+// inverse of the iterate-the-XOR-chunk loop in ingest.go. The returned chunk is
+// a chunkenc.Chunk (EncXOR) that iterates back to the same samples.
+//
+// Samples must be in non-decreasing timestamp order, which is the order both
+// intchunk.DecodeChunk and the cold block layout already guarantee.
+func SamplesToXORChunk(samples []Sample) (chunkenc.Chunk, error) {
+	if len(samples) == 0 {
+		return nil, ErrNoSamples
+	}
+	c := chunkenc.NewXORChunk()
+	app, err := c.Appender()
+	if err != nil {
+		return nil, fmt.Errorf("xor appender: %w", err)
+	}
+	for _, s := range samples {
+		app.Append(s.T, s.V)
+	}
+	return c, nil
+}
+
+// DecodeToXORChunk is the headline helper: it decodes an intchunk-format chunk
+// and returns both the decoded samples and an equivalent Prometheus XOR chunk.
+// Returning the samples alongside the chunk lets callers that only need the
+// points skip re-iterating the XOR chunk, while callers feeding a chunk-based
+// API (Thanos StoreAPI, tsdb append) get a ready-to-use EncXOR chunk.
+func DecodeToXORChunk(chunk []byte) ([]Sample, chunkenc.Chunk, error) {
+	samples, err := DecodeToSamples(chunk)
+	if err != nil {
+		return nil, nil, err
+	}
+	xc, err := SamplesToXORChunk(samples)
+	if err != nil {
+		return nil, nil, err
+	}
+	return samples, xc, nil
+}