From 65b943515ba54a376b5d11f144d71da7d37ced82 Mon Sep 17 00:00:00 2001 From: Tim Van Steenburgh Date: Thu, 28 May 2026 15:57:28 -0400 Subject: [PATCH 1/3] fix: patch logger and Docker build to emit query-completed logs (ENG-7523) [ENG-7523](https://stacklet.atlassian.net/browse/ENG-7523) ### what - Patch `prodLogger` in `cubejs-server-core` to unconditionally emit "Performing query completed" messages regardless of log level - Introduce `scripts/build-docker.sh` to compile locally patched packages and stage their `dist/` output into `packages/cubejs-docker/stacklet-patches/` before `docker build` - Update `packages/cubejs-docker/latest.Dockerfile` to overlay any `*-dist/` directories found in `stacklet-patches/` onto the npm-installed package versions - Add `packages/cubejs-docker/DEVELOPMENT.md` documenting the Stacklet build workflow ### why The CloudWatch log metric filter in `platform-cubejs-schemas` relies on "Performing query completed" messages to track pre-aggregation runs, but the upstream logger gates them on log level. Our goal is to preserve these specific log lines needed for metrics without the expense of running at trace-level logging across the board. The Docker image installs packages from npm via `yarn install --prod`, so local TypeScript changes compiled into `packages/*/dist/` are silently ignored. The build script and Dockerfile overlay ensure the patched logger is always present in the resulting image. ### testing Run `./scripts/build-docker.sh ` and verify: docker run --rm grep -c "Performing query completed" \ /cube/node_modules/@cubejs-backend/server-core/dist/src/core/logger.js Expected output: 2 ### docs Updated `packages/cubejs-docker/DEVELOPMENT.md` with Stacklet build instructions and pattern for adding future patched packages. Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 1 + packages/cubejs-docker/DEVELOPMENT.md | 41 ++++++++++++++ packages/cubejs-docker/latest.Dockerfile | 8 +++ .../cubejs-docker/stacklet-patches/.gitkeep | 0 .../cubejs-server-core/src/core/logger.ts | 6 +++ scripts/build-docker.sh | 54 +++++++++++++++++++ 6 files changed, 110 insertions(+) create mode 100644 packages/cubejs-docker/stacklet-patches/.gitkeep create mode 100755 scripts/build-docker.sh diff --git a/.gitignore b/.gitignore index 98db478c5b8f2..d6c10c4e1fe7f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ node_modules .yarn +packages/cubejs-docker/stacklet-patches/*-dist/ .yarnrc /.idea /.vscode diff --git a/packages/cubejs-docker/DEVELOPMENT.md b/packages/cubejs-docker/DEVELOPMENT.md index 2342e933ed723..745e330ad44c7 100644 --- a/packages/cubejs-docker/DEVELOPMENT.md +++ b/packages/cubejs-docker/DEVELOPMENT.md @@ -24,3 +24,44 @@ docker build -t cubejs/cube:latest-jdk -f latest-debian-jdk.Dockerfile . docker build -t cubejs/cube:dev -f dev.Dockerfile ../../ docker buildx build --platform linux/amd64 -t cubejs/cube:dev -f dev.Dockerfile ../../ ``` + +## Stacklet builds + +The Stacklet fork patches certain Cube packages at the TypeScript source level. +Because `latest.Dockerfile` installs packages from npm (not from the local +workspace), a wrapper script is required to compile modified packages and stage +their compiled output before `docker build` runs. + +**Always use the build script — never `docker build` directly:** + +```sh +# From the repo root: +./scripts/build-docker.sh cubejs/cube:v1.6.6-stacklet-$(git rev-parse --short HEAD) +``` + +The script: +1. Runs `yarn workspace @cubejs-backend/server-core build` (and any other + patched packages) +2. Copies compiled `dist/` output into `stacklet-patches/-dist/` +3. Runs `docker build`; the Dockerfile overlays those dist files on the + npm-installed versions +4. Cleans up the staged files on exit + +### Adding a new patched package + +1. Make source changes and compile: `yarn workspace @cubejs-backend/ build` +2. Add a compile + stage block to `scripts/build-docker.sh`: + ```bash + mkdir -p "$PATCHES_DIR/-dist" + cp -r "$REPO_ROOT/packages/cubejs-/dist/." "$PATCHES_DIR/-dist/" + ``` +3. No Dockerfile change is needed — `latest.Dockerfile` automatically applies + any `*-dist/` directories it finds in `stacklet-patches/`. + +### Verifying a patch is in the image + +```sh +docker run --rm grep -c "Performing query completed" \ + /cube/node_modules/@cubejs-backend/server-core/dist/src/core/logger.js +# expected: 2 +``` diff --git a/packages/cubejs-docker/latest.Dockerfile b/packages/cubejs-docker/latest.Dockerfile index 0088df1b68a95..8802e49c99334 100644 --- a/packages/cubejs-docker/latest.Dockerfile +++ b/packages/cubejs-docker/latest.Dockerfile @@ -33,6 +33,14 @@ RUN yarn install --prod \ && rm -rf /cube/node_modules/duckdb/src \ && yarn cache clean +# Stacklet: overlay locally compiled dist files over npm-installed packages. +# stacklet-patches/-dist/ dirs are populated by scripts/build-docker.sh. +COPY stacklet-patches/ /cube/stacklet-patches/ +RUN find /cube/stacklet-patches -maxdepth 1 -name '*-dist' -type d | while read d; do \ + pkg=$(basename "$d" -dist); \ + cp -r "$d/." "/cube/node_modules/@cubejs-backend/$pkg/dist/"; \ + done && rm -rf /cube/stacklet-patches + FROM node:22.22.0-bookworm-slim ARG IMAGE_VERSION=unknown diff --git a/packages/cubejs-docker/stacklet-patches/.gitkeep b/packages/cubejs-docker/stacklet-patches/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/cubejs-server-core/src/core/logger.ts b/packages/cubejs-server-core/src/core/logger.ts index c7c4844e2b0dc..3f7f5a977517d 100644 --- a/packages/cubejs-server-core/src/core/logger.ts +++ b/packages/cubejs-server-core/src/core/logger.ts @@ -126,6 +126,12 @@ export const prodLogger = (level?: LogLevel) => (msg: string, params: ProdLogPar const logMessage = () => console.log(JSON.stringify({ message: msg, ...params })); + // Always emit regardless of log level — used by CloudWatch log metric filter + if (msg === 'Performing query completed') { + logMessage(); + return; + } + // eslint-disable-next-line default-case switch ((level || 'warn').toLowerCase()) { case 'trace': { diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh new file mode 100755 index 0000000000000..a79255ecf889c --- /dev/null +++ b/scripts/build-docker.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Build a Stacklet Cube Docker image. +# +# Usage: ./scripts/build-docker.sh [TAG] +# TAG defaults to cubejs/cube:dev +# +# This script compiles Stacklet-patched packages and stages their dist output +# into packages/cubejs-docker/stacklet-patches/ before invoking docker build. +# The Dockerfile picks up whatever *-dist/ directories it finds there and +# overlays them on the npm-installed versions of those packages. +# +# To add a new patched package in the future: +# 1. Make your source changes and run: yarn workspace @cubejs-backend/ build +# 2. Add a compile + stage block below (following the server-core example) +# 3. No Dockerfile change is needed. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DOCKER_DIR="$REPO_ROOT/packages/cubejs-docker" +PATCHES_DIR="$DOCKER_DIR/stacklet-patches" +TAG="${1:-cubejs/cube:dev}" + +cleanup() { + echo "Cleaning up staged patches..." + find "$PATCHES_DIR" -maxdepth 1 -name '*-dist' -type d -exec rm -rf {} + 2>/dev/null || true +} +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# Compile and stage Stacklet-patched packages +# --------------------------------------------------------------------------- + +echo "==> Building @cubejs-backend/server-core..." +yarn --cwd "$REPO_ROOT" workspace @cubejs-backend/server-core build + +echo "==> Staging server-core dist..." +mkdir -p "$PATCHES_DIR/server-core-dist" +cp -r "$REPO_ROOT/packages/cubejs-server-core/dist/." \ + "$PATCHES_DIR/server-core-dist/" + +# --------------------------------------------------------------------------- +# Docker build +# --------------------------------------------------------------------------- + +echo "==> Building Docker image $TAG..." +docker build -t "$TAG" -f "$DOCKER_DIR/latest.Dockerfile" "$DOCKER_DIR" + +echo "==> Done: $TAG" +echo "" +echo "Verify the logger patch:" +echo " docker run --rm $TAG grep -c 'Performing query completed' \\" +echo " /cube/node_modules/@cubejs-backend/server-core/dist/src/core/logger.js" +echo " (expected: 2)" From 3887984089ea011dc89378989f0e481b1da1f7cd Mon Sep 17 00:00:00 2001 From: Tim Van Steenburgh Date: Mon, 1 Jun 2026 12:43:47 -0400 Subject: [PATCH 2/3] trigger CI checks rerun From b75f580d992762de3585b9552d1f0da20750789c Mon Sep 17 00:00:00 2001 From: Tim Van Steenburgh Date: Mon, 1 Jun 2026 13:02:01 -0400 Subject: [PATCH 3/3] address review feedback Have script verify logger patch instead of telling user to do it. --- scripts/build-docker.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh index a79255ecf889c..0703d8960b8ab 100755 --- a/scripts/build-docker.sh +++ b/scripts/build-docker.sh @@ -47,8 +47,12 @@ echo "==> Building Docker image $TAG..." docker build -t "$TAG" -f "$DOCKER_DIR/latest.Dockerfile" "$DOCKER_DIR" echo "==> Done: $TAG" -echo "" -echo "Verify the logger patch:" -echo " docker run --rm $TAG grep -c 'Performing query completed' \\" -echo " /cube/node_modules/@cubejs-backend/server-core/dist/src/core/logger.js" -echo " (expected: 2)" + +echo "==> Verifying logger patch..." +COUNT=$(docker run --rm "$TAG" grep -c 'Performing query completed' \ + /cube/node_modules/@cubejs-backend/server-core/dist/src/core/logger.js) +if [ "$COUNT" -ne 2 ]; then + echo "ERROR: logger patch check failed (expected 2 matches, got $COUNT)" + exit 1 +fi +echo "Logger patch OK ($COUNT matches)"