diff --git a/gorilla-merger/Dockerfile b/gorilla-merger/Dockerfile new file mode 100644 index 000000000..78912b5e7 --- /dev/null +++ b/gorilla-merger/Dockerfile @@ -0,0 +1,69 @@ +# syntax=docker/dockerfile:1.7 +# +# Multi-stage build for the gorilla-merger (Thanos-Receive-style component). +# +# ## Why this needs a build secret +# +# gorilla-merger imports the PRIVATE Go module +# `github.com/ProjectASAP/asap-gorilla-go` (the shared edge<->merger +# ASAPFRG1 wire codec). A naive `go build` inside Docker/CI cannot fetch it: +# the module proxy + git fetch get a 404/auth prompt for the private repo. +# +# We solve this with a BuildKit *secret* mount carrying a GitHub token. The +# secret is mounted ONLY for the duration of the build RUN that needs it and +# is NOT baked into any image layer (unlike a build-arg or COPY of a token +# file). Inside that RUN we set a transient `url.insteadOf` git rewrite so +# `go` fetches the private module over HTTPS with the token. The git config +# lives only in the container build layer, never on the host. +# +# `GOPRIVATE=github.com/ProjectASAP/*` keeps the fetch off the public proxy +# and checksum DB; `GIT_TERMINAL_PROMPT=0` makes a missing/incorrect token +# fail fast instead of hanging on an interactive credential prompt. +# +# ## Build invocation +# +# Write a GitHub token (a PAT or `gh auth token`) to a file, then: +# +# gh auth token > /tmp/gh_token # or: echo "$GITHUB_TOKEN" > /tmp/gh_token +# DOCKER_BUILDKIT=1 docker build \ +# --secret id=gh_token,src=/tmp/gh_token \ +# -t asap/gorilla-merger:dev \ +# gorilla-merger/ +# rm -f /tmp/gh_token +# +# The build context is the `gorilla-merger/` module directory (this file's +# directory). The same GOPRIVATE + token requirement applies to +# ASAPQuery-backend CI before PR #310 can be merged (the CI runner must +# expose a `gh_token` secret / configure `url.insteadOf` the same way). + +FROM golang:1.25-bookworm AS build +WORKDIR /src + +# git is needed for the private-module fetch (insteadOf rewrite below). +RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Prime the module graph first so dependency downloads cache independently +# of source edits. go.sum is committed, so the public deps resolve normally; +# only the private module needs the token. +COPY go.mod go.sum ./ +RUN --mount=type=secret,id=gh_token \ + GOPRIVATE=github.com/ProjectASAP/* \ + GONOSUMCHECK=github.com/ProjectASAP/* \ + GOFLAGS=-mod=mod \ + GIT_TERMINAL_PROMPT=0 \ + sh -c 'git config --global url."https://x-access-token:$(cat /run/secrets/gh_token)@github.com/".insteadOf "/" && go mod download' + +# Now copy the rest of the module and build. +COPY . . +RUN --mount=type=secret,id=gh_token \ + GOPRIVATE=github.com/ProjectASAP/* \ + GONOSUMCHECK=github.com/ProjectASAP/* \ + GIT_TERMINAL_PROMPT=0 \ + sh -c 'git config --global url."https://x-access-token:$(cat /run/secrets/gh_token)@github.com/".insteadOf "/" && CGO_ENABLED=0 go build -ldflags="-s -w" -o /out/gorilla-merger ./cmd/gorilla-merger' + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/gorilla-merger /usr/local/bin/gorilla-merger +# HTTP ingest (/ingest/gorilla, /metrics) + Thanos StoreAPI gRPC. +EXPOSE 10908 10907 +ENTRYPOINT ["/usr/local/bin/gorilla-merger"] diff --git a/gorilla-merger/README.md b/gorilla-merger/README.md new file mode 100644 index 000000000..f2ceabb65 --- /dev/null +++ b/gorilla-merger/README.md @@ -0,0 +1,69 @@ +# gorilla-merger + +A Thanos-Receive-style component for ASAP edge agents. It: + +1. **Ingests** Gorilla XOR-chunk fragments (the `asap-gorilla-go` `ASAPFRG1` + wire codec) over HTTP `POST /ingest/gorilla` (gzip-aware). +2. **Appends** the decoded samples to an embedded Prometheus `tsdb.DB` + (2h block range + WAL). +3. **Serves** a Thanos **StoreAPI** (gRPC) over the open (`<2h` pending) + window so `thanos-query` can union recent data with the `>=2h` S3 blocks + that `thanos-store-gateway` serves. +4. **Ships** completed 2h blocks to object storage via the Thanos shipper + (one PUT set per block), into the same bucket the store-gateway watches. + +## Ports / flags + +| Flag | Env | Default | Purpose | +|------|-----|---------|---------| +| `-http-address` | `MERGER_HTTP_ADDRESS` | `:10908` | `/ingest/gorilla`, `/metrics`, `/-/healthy`, `/-/ready` | +| `-grpc-address` | `MERGER_GRPC_ADDRESS` | `:10907` | Thanos StoreAPI (the query surface `thanos-query --store=` points at) | +| `-tsdb.path` | `MERGER_TSDB_PATH` | `./data` | embedded tsdb dir (WAL + unshipped blocks) | +| `-objstore.config-file` | `MERGER_OBJSTORE_CONFIG_FILE` | _empty_ | Thanos objstore YAML; empty disables the shipper | +| `-external-labels` | `MERGER_EXTERNAL_LABELS` | _empty_ | `k=v,k=v` applied to every series + uploaded block; distinct mergers MUST carry a distinguishing label | +| `-shipper.interval` | `MERGER_SHIPPER_INTERVAL` | `1m` | block-scan / upload cadence | +| `-tsdb.retention` | `MERGER_TSDB_RETENTION` | `6h` | local on-disk retention (blocks live in S3 once shipped) | + +## Building the container + +The merger imports the **private** Go module +`github.com/ProjectASAP/asap-gorilla-go`, so a naive `go build` in +Docker/CI cannot fetch it (404/auth-prompt on the private repo). The +`Dockerfile` solves this with a **BuildKit secret** carrying a GitHub +token — the token is mounted only for the build `RUN`s that need it and is +never baked into an image layer (unlike a build-arg or a `COPY`ed token +file). The git `url.insteadOf` rewrite happens *inside* the container +build, never on the host. + +```sh +# Write a GitHub token to a file. With a modern gh: gh auth token > /tmp/gh_token +# With gh < 2.x (no `gh auth token` subcommand) read it from the gh config: +python3 -c "import yaml; d=yaml.safe_load(open('$HOME/.config/gh/hosts.yml')); print(d['github.com'].get('oauth_token') or d['github.com'].get('token'), end='')" > /tmp/gh_token +# ...or just: echo "$GITHUB_TOKEN" > /tmp/gh_token + +DOCKER_BUILDKIT=1 docker build \ + --secret id=gh_token,src=/tmp/gh_token \ + -t asap/gorilla-merger:dev \ + . # build context = this directory + +rm -f /tmp/gh_token +``` + +The token needs `repo` read scope on `github.com/ProjectASAP/asap-gorilla-go`. + +### CI note (PR #310) + +The **same** `GOPRIVATE=github.com/ProjectASAP/*` + token requirement +applies to ASAPQuery-backend CI before PR #310 can merge: the CI runner +must expose a `gh_token` secret (or set `url.insteadOf` with a token) so +`go build` / `go test` of `gorilla-merger/` can fetch the private module. + +## Local dev (no container) + +`go vet` / `go build` on a host that already has the module in its +`GOMODCACHE` (or with `git` configured for the private repo): + +```sh +GOPRIVATE=github.com/ProjectASAP/* go vet ./... +GOPRIVATE=github.com/ProjectASAP/* go build ./cmd/gorilla-merger +```