diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 67276dce5..e543d9a48 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -184,6 +184,8 @@ jobs: attestations: write packages: write artifact-metadata: write + env: + RESULTS_FILE: smoke-test-results.json steps: - name: Harden Runner @@ -333,26 +335,28 @@ jobs: --scan-results snyk-docker.json --org ${{ inputs.kosli_org }} - - name: Smoke test the docker image to be sure it can connect to Kosli - id: smoke-test + - name: Run Docker image smoke tests + if: ${{ success() || failure() }} env: - KOSLI_ORG: cyber-dojo - KOSLI_API_TOKEN: any-token-will-do - run: - docker run -e KOSLI_API_TOKEN - -e KOSLI_ORG --rm ${{ env.IMAGE }}:${{ inputs.tag }} - list environments + IMAGE: ${{ env.IMAGE }} + TAG: ${{ inputs.tag }} + run: ./scripts/docker-smoke-tests.sh - name: Report Docker smoke test attestation to Kosli if: ${{ inputs.report_to_kosli != 'none' && (success() || failure()) }} env: KOSLI_API_TOKEN: ${{ secrets.kosli_api_token }} - SMOKE_TEST_OUTCOME: ${{ steps.smoke-test.outcome }} - run: - kosli attest generic - --flow ${{ inputs.flow_name }} - --trail ${{ inputs.trail_name }} - --fingerprint ${{ env.FINGERPRINT }} - --name smoke-test - --compliant=${{ steps.smoke-test.outcome == 'success' }} - --org ${{ inputs.kosli_org }} + run: | + if [ ! -s "$RESULTS_FILE" ]; then + echo '[{"name":"smoke-tests","outcome":"failure","note":"runner produced no results"}]' > "$RESULTS_FILE" + fi + COMPLIANT=true + jq -e 'length > 0 and all(.[]; .outcome == "success")' "$RESULTS_FILE" > /dev/null || COMPLIANT=false + kosli attest generic \ + --flow ${{ inputs.flow_name }} \ + --trail ${{ inputs.trail_name }} \ + --fingerprint ${{ env.FINGERPRINT }} \ + --name smoke-test \ + --compliant="$COMPLIANT" \ + --attachments "$RESULTS_FILE" \ + --org ${{ inputs.kosli_org }} diff --git a/.gitignore b/.gitignore index 2b7d16bc6..37892df08 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ TODO.md merkely.yaml kosli.yaml pipe.json +smoke-test-results.json dist/ coverage.out cover.out diff --git a/scripts/docker-smoke-tests.sh b/scripts/docker-smoke-tests.sh new file mode 100755 index 000000000..737fa8109 --- /dev/null +++ b/scripts/docker-smoke-tests.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Runs smoke tests against the built docker image and records each case's +# outcome to $RESULTS_FILE for the CI workflow to report as a Kosli +# attestation. +# +# Usage: IMAGE=... TAG=... RESULTS_FILE=... ./scripts/docker-smoke-tests.sh +set -uo pipefail + +IMAGE="${IMAGE:?IMAGE is required}" +TAG="${TAG:?TAG is required}" +RESULTS_FILE="${RESULTS_FILE:?RESULTS_FILE is required}" + +REPO_ROOT="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}" +if [ -z "$REPO_ROOT" ]; then + echo "could not determine repo root" >&2 + exit 1 +fi + +EXIT_CODE=0 +RESULTS="[]" + +# Updates the named entry's outcome in $RESULTS and flushes to $RESULTS_FILE. +write_result() { + local name="$1" + local outcome="$2" + + RESULTS="$(jq --arg name "$name" --arg outcome "$outcome" \ + 'map(if .name == $name then .outcome = $outcome else . end)' <<< "$RESULTS")" \ + || { echo "jq failed" >&2; exit 1; } + printf '%s\n' "$RESULTS" > "$RESULTS_FILE" || { echo "failed to write $RESULTS_FILE" >&2; exit 1; } +} + +# Runs a smoke test case and records its outcome, keyed by name. The CI +# workflow reports a single aggregate attestation, compliant only if every +# recorded case succeeded, with the results file attached. Every case is +# seeded as "not-run" before any case executes (see below), so a run that +# aborts part-way leaves a results file that reads as incomplete rather than +# as a clean pass. +# Usage: run_case +run_case() { + local name="$1" + local test_fn="$2" + + echo "::group::Smoke test: ${name}" + local outcome="success" + if ! "$test_fn"; then + outcome="failure" + EXIT_CODE=1 + fi + echo "::endgroup::" + echo "Smoke test ${name}: ${outcome}" + + write_result "$name" "$outcome" +} + +# --- Smoke test cases ------------------------------------------------- +# Add a new smoke test by writing a test_* function below and adding one +# entry to the CASES array further down — no CI workflow changes needed. + +test_attest_artifact_dir() { + local commit_sha + commit_sha="$(git -C "$REPO_ROOT" rev-parse HEAD)" || return 1 + + #--dry-run turns any error into a warning plus exit 0, + # so the exit code alone can't be trusted here. + local output + output="$(docker run --rm \ + -v "${REPO_ROOT}":/workspace:ro \ + -w /workspace \ + -e KOSLI_API_TOKEN=any-token-will-do \ + -e KOSLI_ORG=test-org \ + "${IMAGE}:${TAG}" \ + attest artifact /workspace/internal/utils \ + --artifact-type dir \ + --flow test-flow \ + --trail test-trail \ + --name test-artifact \ + --build-url https://example.com/build/1 \ + --commit-url "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/kosli-dev/cli/commit/${commit_sha}" \ + --repo-root /workspace \ + --dry-run \ + --debug 2>&1)" + local status=$? + echo "$output" + + # The dry-run banner is only printed once the dir is fingerprinted, git info + # resolved and the payload built (requests.go:254), so asserting it is + # present proves this case did its work — and fails closed if the command + # dies earlier or the message is reworded. + grep -q "THIS IS A DRY-RUN" <<< "$output" || return 1 + [ "$status" -eq 0 ] +} + +# --- Run all cases ------------------------------------------------------ +# Add a case by adding one entry here alongside its test_* function above. + +CASES=( + "attest-artifact-dir:test_attest_artifact_dir" +) + +# Seed every case as not-run and flush before running any of them, so an +# abort part-way through (crash, timeout, hung docker run) leaves a results +# file that visibly distinguishes "didn't run" from "passed". +for entry in "${CASES[@]}"; do + RESULTS="$(jq --arg name "${entry%%:*}" '. + [{name: $name, outcome: "not-run"}]' <<< "$RESULTS")" \ + || { echo "jq failed" >&2; exit 1; } +done +printf '%s\n' "$RESULTS" > "$RESULTS_FILE" || { echo "failed to write $RESULTS_FILE" >&2; exit 1; } + +for entry in "${CASES[@]}"; do + run_case "${entry%%:*}" "${entry##*:}" +done + +exit $EXIT_CODE