From a79f18a3f4946c9998552c046cfe9bc6bcf8a515 Mon Sep 17 00:00:00 2001 From: Sami Alajrami Date: Thu, 20 Aug 2026 10:14:38 +0200 Subject: [PATCH 01/12] refactor(ci): make docker image smoke tests table-driven Extract docker image smoke tests into scripts/docker-smoke-tests.sh so new checks can be added as a function + one run_case call instead of a new workflow step pair each time. Also adds a smoke test for the `attest artifact` dir command. Kosli attestation reporting stays in the workflow, driven off the script's JSON results file. Postmortem follow-up from kosli-dev/server#6508. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/docker.yml | 37 ++++++++++--------- scripts/docker-smoke-tests.sh | 69 +++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 17 deletions(-) create mode 100755 scripts/docker-smoke-tests.sh diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 67276dce5..a7f18651f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -333,26 +333,29 @@ jobs: --scan-results snyk-docker.json --org ${{ inputs.kosli_org }} - - name: Smoke test the docker image to be sure it can connect to Kosli - id: smoke-test + - name: Run docker image smoke tests + id: smoke-tests env: - KOSLI_ORG: cyber-dojo - KOSLI_API_TOKEN: any-token-will-do - run: - docker run -e KOSLI_API_TOKEN - -e KOSLI_ORG --rm ${{ env.IMAGE }}:${{ inputs.tag }} - list environments + IMAGE: ${{ env.IMAGE }} + TAG: ${{ inputs.tag }} + RESULTS_FILE: smoke-test-results.json + run: ./scripts/docker-smoke-tests.sh - - name: Report Docker smoke test attestation to Kosli + - name: Report docker smoke test attestations to Kosli if: ${{ inputs.report_to_kosli != 'none' && (success() || failure()) }} env: KOSLI_API_TOKEN: ${{ secrets.kosli_api_token }} - SMOKE_TEST_OUTCOME: ${{ steps.smoke-test.outcome }} - run: - kosli attest generic - --flow ${{ inputs.flow_name }} - --trail ${{ inputs.trail_name }} - --fingerprint ${{ env.FINGERPRINT }} - --name smoke-test - --compliant=${{ steps.smoke-test.outcome == 'success' }} + run: | + jq -c '.[]' smoke-test-results.json | while read -r result; do + NAME=$(jq -r '.name' <<< "$result") + OUTCOME=$(jq -r '.outcome' <<< "$result") + COMPLIANT=false + [ "$OUTCOME" = "success" ] && COMPLIANT=true + kosli attest generic \ + --flow ${{ inputs.flow_name }} \ + --trail ${{ inputs.trail_name }} \ + --fingerprint ${{ env.FINGERPRINT }} \ + --name "$NAME" \ + --compliant="$COMPLIANT" \ --org ${{ inputs.kosli_org }} + done diff --git a/scripts/docker-smoke-tests.sh b/scripts/docker-smoke-tests.sh new file mode 100755 index 000000000..40bad655c --- /dev/null +++ b/scripts/docker-smoke-tests.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -uo pipefail + +IMAGE="${IMAGE:?IMAGE is required}" +TAG="${TAG:?TAG is required}" +RESULTS_FILE="${RESULTS_FILE:?RESULTS_FILE is required}" + +REPO_ROOT="$(git rev-parse --show-toplevel)" +EXIT_CODE=0 +RESULTS="[]" + +# Runs a smoke test case and records its outcome in $RESULTS_FILE, keyed by +# name, so the CI workflow can report a Kosli attestation for it. +# Usage: run_case +run_case() { + local name="$1" + local test_fn="$2" + + echo "::group::Smoke test: ${name}" + local outcome="success" + if ! "$test_fn"; then + outcome="failure" + EXIT_CODE=1 + fi + echo "::endgroup::" + echo "Smoke test ${name}: ${outcome}" + + RESULTS="$(jq --arg name "$name" --arg outcome "$outcome" \ + '. + [{name: $name, outcome: $outcome}]' <<< "$RESULTS")" +} + +# --- Smoke test cases ------------------------------------------------- +# Add a new smoke test by writing a test_* function below and adding one +# run_case call for it — no CI workflow changes needed. + +test_list_environments() { + docker run --rm \ + -e KOSLI_API_TOKEN=any-token-will-do \ + -e KOSLI_ORG=cyber-dojo \ + "${IMAGE}:${TAG}" \ + list environments +} + +test_attest_artifact_dir() { + docker run --rm \ + -v "${REPO_ROOT}":/workspace:ro \ + -w /workspace \ + -e KOSLI_API_TOKEN=DRY_RUN \ + -e KOSLI_ORG=test-org \ + "${IMAGE}:${TAG}" \ + attest artifact /workspace/internal/utils \ + --artifact-type dir \ + --flow test-flow \ + --trail test-trail \ + --name test-artifact \ + --build-url https://example.com/build/1 \ + --commit-url https://github.com/kosli-dev/cli/commit/HEAD \ + --repo-root /workspace \ + --dry-run \ + --debug +} + +# --- Run all cases ------------------------------------------------------ + +run_case "smoke-test" test_list_environments +run_case "smoke-test-attest-artifact-dir" test_attest_artifact_dir + +echo "$RESULTS" > "$RESULTS_FILE" +exit $EXIT_CODE From 4e172decb31fa0b792e220e7f29455ff9cc3fe80 Mon Sep 17 00:00:00 2001 From: Sami Alajrami Date: Thu, 20 Aug 2026 10:17:42 +0200 Subject: [PATCH 02/12] fix(ci): report a single aggregate smoke test attestation Report one smoke-tests attestation instead of one per test case, with the compliance verdict derived from whether all cases passed, and the per-test results attached as evidence. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/docker.yml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a7f18651f..6f60ca93d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -341,21 +341,18 @@ jobs: RESULTS_FILE: smoke-test-results.json run: ./scripts/docker-smoke-tests.sh - - name: Report docker smoke test attestations to Kosli + - name: Report docker smoke test attestation to Kosli if: ${{ inputs.report_to_kosli != 'none' && (success() || failure()) }} env: KOSLI_API_TOKEN: ${{ secrets.kosli_api_token }} run: | - jq -c '.[]' smoke-test-results.json | while read -r result; do - NAME=$(jq -r '.name' <<< "$result") - OUTCOME=$(jq -r '.outcome' <<< "$result") - COMPLIANT=false - [ "$OUTCOME" = "success" ] && COMPLIANT=true - kosli attest generic \ - --flow ${{ inputs.flow_name }} \ - --trail ${{ inputs.trail_name }} \ - --fingerprint ${{ env.FINGERPRINT }} \ - --name "$NAME" \ - --compliant="$COMPLIANT" \ - --org ${{ inputs.kosli_org }} - done + COMPLIANT=true + jq -e 'all(.[]; .outcome == "success")' smoke-test-results.json > /dev/null || COMPLIANT=false + kosli attest generic \ + --flow ${{ inputs.flow_name }} \ + --trail ${{ inputs.trail_name }} \ + --fingerprint ${{ env.FINGERPRINT }} \ + --name smoke-tests \ + --compliant="$COMPLIANT" \ + --attachments smoke-test-results.json \ + --org ${{ inputs.kosli_org }} From da227660f36b66d7fc781613021807d28d91df9a Mon Sep 17 00:00:00 2001 From: Sami Alajrami Date: Thu, 20 Aug 2026 10:18:26 +0200 Subject: [PATCH 03/12] restore attestation name to smoke-test --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6f60ca93d..2abdbba19 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -352,7 +352,7 @@ jobs: --flow ${{ inputs.flow_name }} \ --trail ${{ inputs.trail_name }} \ --fingerprint ${{ env.FINGERPRINT }} \ - --name smoke-tests \ + --name smoke-test \ --compliant="$COMPLIANT" \ --attachments smoke-test-results.json \ --org ${{ inputs.kosli_org }} From 19caa20cc504f5b7e57f44a36c705f98a6fea8eb Mon Sep 17 00:00:00 2001 From: Marko Bevc Date: Thu, 20 Aug 2026 09:24:04 +0100 Subject: [PATCH 04/12] chore: Apply suggestion from @mbevc1 --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2abdbba19..10088ae1b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -333,7 +333,7 @@ jobs: --scan-results snyk-docker.json --org ${{ inputs.kosli_org }} - - name: Run docker image smoke tests + - name: Run Docker image smoke tests id: smoke-tests env: IMAGE: ${{ env.IMAGE }} From 5a852c2b6d4df7e8df8c77305fee128c2f1c4a3b Mon Sep 17 00:00:00 2001 From: Marko Bevc Date: Thu, 20 Aug 2026 09:24:15 +0100 Subject: [PATCH 05/12] chore: Apply suggestion from @mbevc1 --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 10088ae1b..4e00a554d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -341,7 +341,7 @@ jobs: RESULTS_FILE: smoke-test-results.json run: ./scripts/docker-smoke-tests.sh - - name: Report docker smoke test attestation to Kosli + - name: Report Docker smoke test attestation to Kosli if: ${{ inputs.report_to_kosli != 'none' && (success() || failure()) }} env: KOSLI_API_TOKEN: ${{ secrets.kosli_api_token }} From d63041784615c62407bfb53064adb2f4211f1715 Mon Sep 17 00:00:00 2001 From: Sami Alajrami Date: Thu, 20 Aug 2026 10:32:49 +0200 Subject: [PATCH 06/12] fix(ci): harden smoke test reporting against silent failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply the outstanding review findings on the docker smoke tests refactor: - Report step now falls back to a failing result and rejects a vacuous empty results file, so a script crash before it writes results.json can no longer produce a missing or falsely-compliant attestation — the exact failure class this refactor is a postmortem for. - REPO_ROOT falls back from GITHUB_WORKSPACE and hard-fails if empty, instead of silently mounting an empty path into the container. - jq failures during result accumulation now abort instead of writing an empty, non-JSON results file; results are flushed after every case so a hung/cancelled run still reports what already passed. - Renamed run_case labels (list-environments, attest-artifact-dir) so they read as case labels and don't collide with the aggregate attestation's name; updated the stale comment describing them. - attest-artifact-dir case now uses the real HEAD sha for --commit-url and drops the redundant KOSLI_API_TOKEN=DRY_RUN (the --dry-run flag alone is sufficient), exercising the real input path more honestly. - Dropped the now-unused step id, ignored the local results file. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/docker.yml | 9 ++++++--- .gitignore | 1 + scripts/docker-smoke-tests.sh | 32 +++++++++++++++++++++++--------- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4e00a554d..36930e552 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -334,7 +334,6 @@ jobs: --org ${{ inputs.kosli_org }} - name: Run Docker image smoke tests - id: smoke-tests env: IMAGE: ${{ env.IMAGE }} TAG: ${{ inputs.tag }} @@ -345,14 +344,18 @@ jobs: if: ${{ inputs.report_to_kosli != 'none' && (success() || failure()) }} env: KOSLI_API_TOKEN: ${{ secrets.kosli_api_token }} + RESULTS_FILE: smoke-test-results.json run: | + if [ ! -s "$RESULTS_FILE" ]; then + echo '[{"name":"smoke-tests","outcome":"failure","note":"runner produced no results"}]' > "$RESULTS_FILE" + fi COMPLIANT=true - jq -e 'all(.[]; .outcome == "success")' smoke-test-results.json > /dev/null || COMPLIANT=false + jq -e 'length > 0 and all(.[]; .outcome == "success")' "$RESULTS_FILE" > /dev/null || COMPLIANT=false kosli attest generic \ --flow ${{ inputs.flow_name }} \ --trail ${{ inputs.trail_name }} \ --fingerprint ${{ env.FINGERPRINT }} \ --name smoke-test \ --compliant="$COMPLIANT" \ - --attachments smoke-test-results.json \ + --attachments "$RESULTS_FILE" \ --org ${{ inputs.kosli_org }} diff --git a/.gitignore b/.gitignore index 2b7d16bc6..37892df08 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ TODO.md merkely.yaml kosli.yaml pipe.json +smoke-test-results.json dist/ coverage.out cover.out diff --git a/scripts/docker-smoke-tests.sh b/scripts/docker-smoke-tests.sh index 40bad655c..5e2762068 100755 --- a/scripts/docker-smoke-tests.sh +++ b/scripts/docker-smoke-tests.sh @@ -1,17 +1,28 @@ #!/bin/bash +# Runs smoke tests against the built docker image and records each case's +# outcome to $RESULTS_FILE for the CI workflow to report as a Kosli +# attestation. +# +# Usage: IMAGE=... TAG=... RESULTS_FILE=... ./scripts/docker-smoke-tests.sh set -uo pipefail IMAGE="${IMAGE:?IMAGE is required}" TAG="${TAG:?TAG is required}" RESULTS_FILE="${RESULTS_FILE:?RESULTS_FILE is required}" -REPO_ROOT="$(git rev-parse --show-toplevel)" +REPO_ROOT="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}" +if [ -z "$REPO_ROOT" ]; then + echo "could not determine repo root" >&2 + exit 1 +fi + EXIT_CODE=0 RESULTS="[]" # Runs a smoke test case and records its outcome in $RESULTS_FILE, keyed by -# name, so the CI workflow can report a Kosli attestation for it. -# Usage: run_case +# name. The CI workflow reports a single aggregate attestation, compliant only +# if every recorded case succeeded, with the results file attached. +# Usage: run_case run_case() { local name="$1" local test_fn="$2" @@ -26,7 +37,8 @@ run_case() { echo "Smoke test ${name}: ${outcome}" RESULTS="$(jq --arg name "$name" --arg outcome "$outcome" \ - '. + [{name: $name, outcome: $outcome}]' <<< "$RESULTS")" + '. + [{name: $name, outcome: $outcome}]' <<< "$RESULTS")" || { echo "jq failed" >&2; exit 1; } + printf '%s\n' "$RESULTS" > "$RESULTS_FILE" } # --- Smoke test cases ------------------------------------------------- @@ -42,10 +54,13 @@ test_list_environments() { } test_attest_artifact_dir() { + local commit_sha + commit_sha="$(git -C "$REPO_ROOT" rev-parse HEAD)" + docker run --rm \ -v "${REPO_ROOT}":/workspace:ro \ -w /workspace \ - -e KOSLI_API_TOKEN=DRY_RUN \ + -e KOSLI_API_TOKEN=any-token-will-do \ -e KOSLI_ORG=test-org \ "${IMAGE}:${TAG}" \ attest artifact /workspace/internal/utils \ @@ -54,7 +69,7 @@ test_attest_artifact_dir() { --trail test-trail \ --name test-artifact \ --build-url https://example.com/build/1 \ - --commit-url https://github.com/kosli-dev/cli/commit/HEAD \ + --commit-url "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/kosli-dev/cli/commit/${commit_sha}" \ --repo-root /workspace \ --dry-run \ --debug @@ -62,8 +77,7 @@ test_attest_artifact_dir() { # --- Run all cases ------------------------------------------------------ -run_case "smoke-test" test_list_environments -run_case "smoke-test-attest-artifact-dir" test_attest_artifact_dir +run_case "list-environments" test_list_environments +run_case "attest-artifact-dir" test_attest_artifact_dir -echo "$RESULTS" > "$RESULTS_FILE" exit $EXIT_CODE From ab991682a5cf69c98db1566046596079263c0ef4 Mon Sep 17 00:00:00 2001 From: Sami Alajrami Date: Thu, 20 Aug 2026 10:49:12 +0200 Subject: [PATCH 07/12] fix(ci): don't let a Snyk failure mask smoke test results - Run the smoke tests step regardless of upstream job status (success() || failure()), matching the Snyk/report steps' own guards. Previously a Snyk finding (non-zero exit, no continue-on-error) would skip the smoke tests step entirely, and the reporting step's new "no results" fallback would then report a false non-compliant smoke-test attestation for tests that never ran. - Hoist RESULTS_FILE to a job-level env on merge: instead of declaring it identically on both steps, so a rename is a one-line change. - Guard the results-file write in the script with || exit, matching the jq call above it, so a failed write isn't silently swallowed. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/docker.yml | 5 +++-- scripts/docker-smoke-tests.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 36930e552..e543d9a48 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -184,6 +184,8 @@ jobs: attestations: write packages: write artifact-metadata: write + env: + RESULTS_FILE: smoke-test-results.json steps: - name: Harden Runner @@ -334,17 +336,16 @@ jobs: --org ${{ inputs.kosli_org }} - name: Run Docker image smoke tests + if: ${{ success() || failure() }} env: IMAGE: ${{ env.IMAGE }} TAG: ${{ inputs.tag }} - RESULTS_FILE: smoke-test-results.json run: ./scripts/docker-smoke-tests.sh - name: Report Docker smoke test attestation to Kosli if: ${{ inputs.report_to_kosli != 'none' && (success() || failure()) }} env: KOSLI_API_TOKEN: ${{ secrets.kosli_api_token }} - RESULTS_FILE: smoke-test-results.json run: | if [ ! -s "$RESULTS_FILE" ]; then echo '[{"name":"smoke-tests","outcome":"failure","note":"runner produced no results"}]' > "$RESULTS_FILE" diff --git a/scripts/docker-smoke-tests.sh b/scripts/docker-smoke-tests.sh index 5e2762068..795028820 100755 --- a/scripts/docker-smoke-tests.sh +++ b/scripts/docker-smoke-tests.sh @@ -38,7 +38,7 @@ run_case() { RESULTS="$(jq --arg name "$name" --arg outcome "$outcome" \ '. + [{name: $name, outcome: $outcome}]' <<< "$RESULTS")" || { echo "jq failed" >&2; exit 1; } - printf '%s\n' "$RESULTS" > "$RESULTS_FILE" + printf '%s\n' "$RESULTS" > "$RESULTS_FILE" || { echo "failed to write $RESULTS_FILE" >&2; exit 1; } } # --- Smoke test cases ------------------------------------------------- From abfc0302b8deefeb05969da47d8c11427a98c2b5 Mon Sep 17 00:00:00 2001 From: Sami Alajrami Date: Thu, 20 Aug 2026 11:01:15 +0200 Subject: [PATCH 08/12] fix(ci): stop a partially-executed smoke test run reporting compliant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seed every case as "not-run" in the results file before any case executes, and have run_case update entries in place instead of appending. Previously an abort after a passing case (jq failure, a step timeout on a hung docker run, a future test_* that kills the script instead of returning non-zero) left a results file that was non-empty and all-success, so the aggregate attestation went green for a run that never finished — the same failure shape as the postmortem this PR follows up on. The workflow's `all(.outcome == "success")` check needs no change, since "not-run" already isn't "success". Also: guard the git rev-parse for commit_sha with || return 1 (was silently producing a malformed but non-empty --commit-url on failure), and drop the now-provably-unused KOSLI_API_TOKEN from the attest-artifact-dir case, since --dry-run short-circuits before any request needs it. Co-Authored-By: Claude Sonnet 5 --- scripts/docker-smoke-tests.sh | 51 +++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/scripts/docker-smoke-tests.sh b/scripts/docker-smoke-tests.sh index 795028820..1aadfbd64 100755 --- a/scripts/docker-smoke-tests.sh +++ b/scripts/docker-smoke-tests.sh @@ -19,9 +19,23 @@ fi EXIT_CODE=0 RESULTS="[]" -# Runs a smoke test case and records its outcome in $RESULTS_FILE, keyed by -# name. The CI workflow reports a single aggregate attestation, compliant only -# if every recorded case succeeded, with the results file attached. +# Updates the named entry's outcome in $RESULTS and flushes to $RESULTS_FILE. +write_result() { + local name="$1" + local outcome="$2" + + RESULTS="$(jq --arg name "$name" --arg outcome "$outcome" \ + 'map(if .name == $name then .outcome = $outcome else . end)' <<< "$RESULTS")" \ + || { echo "jq failed" >&2; exit 1; } + printf '%s\n' "$RESULTS" > "$RESULTS_FILE" || { echo "failed to write $RESULTS_FILE" >&2; exit 1; } +} + +# Runs a smoke test case and records its outcome, keyed by name. The CI +# workflow reports a single aggregate attestation, compliant only if every +# recorded case succeeded, with the results file attached. Every case is +# seeded as "not-run" before any case executes (see below), so a run that +# aborts part-way leaves a results file that reads as incomplete rather than +# as a clean pass. # Usage: run_case run_case() { local name="$1" @@ -36,14 +50,12 @@ run_case() { echo "::endgroup::" echo "Smoke test ${name}: ${outcome}" - RESULTS="$(jq --arg name "$name" --arg outcome "$outcome" \ - '. + [{name: $name, outcome: $outcome}]' <<< "$RESULTS")" || { echo "jq failed" >&2; exit 1; } - printf '%s\n' "$RESULTS" > "$RESULTS_FILE" || { echo "failed to write $RESULTS_FILE" >&2; exit 1; } + write_result "$name" "$outcome" } # --- Smoke test cases ------------------------------------------------- # Add a new smoke test by writing a test_* function below and adding one -# run_case call for it — no CI workflow changes needed. +# entry to the CASES array further down — no CI workflow changes needed. test_list_environments() { docker run --rm \ @@ -55,12 +67,13 @@ test_list_environments() { test_attest_artifact_dir() { local commit_sha - commit_sha="$(git -C "$REPO_ROOT" rev-parse HEAD)" + commit_sha="$(git -C "$REPO_ROOT" rev-parse HEAD)" || return 1 + # --dry-run short-circuits before any request is sent, so no API token is + # needed here (unlike test_list_environments, which does talk to the API). docker run --rm \ -v "${REPO_ROOT}":/workspace:ro \ -w /workspace \ - -e KOSLI_API_TOKEN=any-token-will-do \ -e KOSLI_ORG=test-org \ "${IMAGE}:${TAG}" \ attest artifact /workspace/internal/utils \ @@ -76,8 +89,24 @@ test_attest_artifact_dir() { } # --- Run all cases ------------------------------------------------------ +# Add a case by adding one entry here alongside its test_* function above. + +CASES=( + "list-environments:test_list_environments" + "attest-artifact-dir:test_attest_artifact_dir" +) + +# Seed every case as not-run and flush before running any of them, so an +# abort part-way through (crash, timeout, hung docker run) leaves a results +# file that visibly distinguishes "didn't run" from "passed". +for entry in "${CASES[@]}"; do + RESULTS="$(jq --arg name "${entry%%:*}" '. + [{name: $name, outcome: "not-run"}]' <<< "$RESULTS")" \ + || { echo "jq failed" >&2; exit 1; } +done +printf '%s\n' "$RESULTS" > "$RESULTS_FILE" || { echo "failed to write $RESULTS_FILE" >&2; exit 1; } -run_case "list-environments" test_list_environments -run_case "attest-artifact-dir" test_attest_artifact_dir +for entry in "${CASES[@]}"; do + run_case "${entry%%:*}" "${entry##*:}" +done exit $EXIT_CODE From ddd7df3faf82b91b69453517464d6d0bb93548ae Mon Sep 17 00:00:00 2001 From: Sami Alajrami Date: Thu, 20 Aug 2026 11:24:11 +0200 Subject: [PATCH 09/12] fix(ci): restore api-token for attest-artifact-dir smoke test CI failed with "--api-token is not set" for this case. --api-token is a required flag checked before --dry-run's own request short-circuit, so dropping it (done in a prior commit on the mistaken belief that --dry-run made the token unused) broke the command outright. Worse, main.go turns any command error into a logged warning plus exit 0 when --dry-run is set, so the missing-token failure was silently reported as this smoke test passing, without ever reaching the fingerprinting/git-resolution code this case exists to exercise. Adds an explicit check for that warning string so a future required-flag regression fails the smoke test instead of being swallowed by --dry-run's exit-0 behaviour. Co-Authored-By: Claude Sonnet 5 --- scripts/docker-smoke-tests.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/scripts/docker-smoke-tests.sh b/scripts/docker-smoke-tests.sh index 1aadfbd64..b2951de5f 100755 --- a/scripts/docker-smoke-tests.sh +++ b/scripts/docker-smoke-tests.sh @@ -69,11 +69,18 @@ test_attest_artifact_dir() { local commit_sha commit_sha="$(git -C "$REPO_ROOT" rev-parse HEAD)" || return 1 - # --dry-run short-circuits before any request is sent, so no API token is - # needed here (unlike test_list_environments, which does talk to the API). - docker run --rm \ + # --api-token is a required flag, validated before --dry-run's own + # short-circuit is ever reached — omitting it fails the command, not the + # request. And under --dry-run, main.go turns ANY command error into a + # logged warning plus exit 0 ("Encountered an error but --dry-run is + # enabled"), so a missing/invalid token here would silently report this + # case as a pass without exercising fingerprinting or git resolution at + # all. Guard against that below rather than trusting the exit code alone. + local output + output="$(docker run --rm \ -v "${REPO_ROOT}":/workspace:ro \ -w /workspace \ + -e KOSLI_API_TOKEN=any-token-will-do \ -e KOSLI_ORG=test-org \ "${IMAGE}:${TAG}" \ attest artifact /workspace/internal/utils \ @@ -85,7 +92,11 @@ test_attest_artifact_dir() { --commit-url "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/kosli-dev/cli/commit/${commit_sha}" \ --repo-root /workspace \ --dry-run \ - --debug + --debug 2>&1)" + local status=$? + echo "$output" + + [ "$status" -eq 0 ] && ! grep -q "Encountered an error but --dry-run is enabled" <<< "$output" } # --- Run all cases ------------------------------------------------------ From 727e0a641790eb0f01507a7801031b9b464775d9 Mon Sep 17 00:00:00 2001 From: Sami Alajrami Date: Thu, 20 Aug 2026 11:28:09 +0200 Subject: [PATCH 10/12] cleanup unnecessary long comment --- scripts/docker-smoke-tests.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/scripts/docker-smoke-tests.sh b/scripts/docker-smoke-tests.sh index b2951de5f..2fb358c9c 100755 --- a/scripts/docker-smoke-tests.sh +++ b/scripts/docker-smoke-tests.sh @@ -69,13 +69,8 @@ test_attest_artifact_dir() { local commit_sha commit_sha="$(git -C "$REPO_ROOT" rev-parse HEAD)" || return 1 - # --api-token is a required flag, validated before --dry-run's own - # short-circuit is ever reached — omitting it fails the command, not the - # request. And under --dry-run, main.go turns ANY command error into a - # logged warning plus exit 0 ("Encountered an error but --dry-run is - # enabled"), so a missing/invalid token here would silently report this - # case as a pass without exercising fingerprinting or git resolution at - # all. Guard against that below rather than trusting the exit code alone. + # --dry-run here would silently report this + # case as a pass even if the command fails. Guard against that below rather than trusting the exit code alone. local output output="$(docker run --rm \ -v "${REPO_ROOT}":/workspace:ro \ From 31230afbeeaf4f6ae9bbe753fccfe65408fd9e00 Mon Sep 17 00:00:00 2001 From: Sami Alajrami Date: Thu, 20 Aug 2026 11:42:22 +0200 Subject: [PATCH 11/12] fix(ci): assert the dry-run banner instead of absence of an error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous guard asserted the *absence* of the "--dry-run is enabled" warning, which fails open: reword that message in main.go, or die before reaching it at all, and the negation makes the assertion true again — a green result for a case that never did its work. Assert the dry-run banner is present instead. requests.go:254 only prints it once the request is built, which is downstream of dir fingerprinting, git resolution and payload assembly, so it is positive evidence this case exercised what it exists to exercise, and it fails closed. Verified both directions locally: with a token the banner appears and the payload carries real git info; without one the command still exits 0 (swallowed by --dry-run) but the banner is absent and the case correctly fails. Also restores the note that --api-token is required by PreRunE even under --dry-run, since that is what a cleanup pass removed last time. Co-Authored-By: Claude Sonnet 5 --- scripts/docker-smoke-tests.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/docker-smoke-tests.sh b/scripts/docker-smoke-tests.sh index 2fb358c9c..63899ff69 100755 --- a/scripts/docker-smoke-tests.sh +++ b/scripts/docker-smoke-tests.sh @@ -69,8 +69,10 @@ test_attest_artifact_dir() { local commit_sha commit_sha="$(git -C "$REPO_ROOT" rev-parse HEAD)" || return 1 - # --dry-run here would silently report this - # case as a pass even if the command fails. Guard against that below rather than trusting the exit code alone. + # --api-token is required by PreRunE even under --dry-run + # (attestArtifact.go:115) — it looks redundant below, but removing it fails + # the command. And --dry-run turns any error into a warning plus exit 0 + # (main.go:202-206), so the exit code alone can't be trusted here. local output output="$(docker run --rm \ -v "${REPO_ROOT}":/workspace:ro \ @@ -91,7 +93,12 @@ test_attest_artifact_dir() { local status=$? echo "$output" - [ "$status" -eq 0 ] && ! grep -q "Encountered an error but --dry-run is enabled" <<< "$output" + # The dry-run banner is only printed once the dir is fingerprinted, git info + # resolved and the payload built (requests.go:254), so asserting it is + # present proves this case did its work — and fails closed if the command + # dies earlier or the message is reworded. + grep -q "THIS IS A DRY-RUN" <<< "$output" || return 1 + [ "$status" -eq 0 ] } # --- Run all cases ------------------------------------------------------ From 14e7b80606814ce066b48cb60fdf07f3e1235027 Mon Sep 17 00:00:00 2001 From: Sami Alajrami Date: Thu, 20 Aug 2026 14:01:20 +0200 Subject: [PATCH 12/12] remove redundant test case and cleanup comments --- scripts/docker-smoke-tests.sh | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/scripts/docker-smoke-tests.sh b/scripts/docker-smoke-tests.sh index 63899ff69..737fa8109 100755 --- a/scripts/docker-smoke-tests.sh +++ b/scripts/docker-smoke-tests.sh @@ -57,22 +57,12 @@ run_case() { # Add a new smoke test by writing a test_* function below and adding one # entry to the CASES array further down — no CI workflow changes needed. -test_list_environments() { - docker run --rm \ - -e KOSLI_API_TOKEN=any-token-will-do \ - -e KOSLI_ORG=cyber-dojo \ - "${IMAGE}:${TAG}" \ - list environments -} - test_attest_artifact_dir() { local commit_sha commit_sha="$(git -C "$REPO_ROOT" rev-parse HEAD)" || return 1 - # --api-token is required by PreRunE even under --dry-run - # (attestArtifact.go:115) — it looks redundant below, but removing it fails - # the command. And --dry-run turns any error into a warning plus exit 0 - # (main.go:202-206), so the exit code alone can't be trusted here. + #--dry-run turns any error into a warning plus exit 0, + # so the exit code alone can't be trusted here. local output output="$(docker run --rm \ -v "${REPO_ROOT}":/workspace:ro \ @@ -105,7 +95,6 @@ test_attest_artifact_dir() { # Add a case by adding one entry here alongside its test_* function above. CASES=( - "list-environments:test_list_environments" "attest-artifact-dir:test_attest_artifact_dir" )