diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml new file mode 100644 index 000000000..03eaf19c3 --- /dev/null +++ b/.github/workflows/publish-images.yml @@ -0,0 +1,105 @@ +name: Publish core images to GHCR + +# Builds and publishes the 6 core Scope app images (NOT the two ACP +# coding-agent workers, which are built/pushed to the Bicep-provisioned ACR +# as a separate step — see deploy/helm/scope/README.md "Phase 3") to GHCR, +# so the Helm chart under deploy/helm/scope/ can install the app straight +# from ghcr.io//scope- with no private registry required. +on: + push: + branches: [main] + paths: + - "apps/api/**" + - "apps/judge/**" + - "apps/token-manager/**" + - "apps/scheduler/**" + - "apps/portal/**" + - "apps/workers/report-generator/**" + - "packages/shared/**" + - "packages/telemetry/**" + - "packages/db-migrations/**" + - "tsconfig.base.json" + - "pnpm-workspace.yaml" + - "pnpm-lock.yaml" + - ".github/workflows/publish-images.yml" + workflow_dispatch: {} + +permissions: + contents: read + packages: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + REGISTRY: ghcr.io + +jobs: + publish: + # Fork PRs never trigger this workflow (push to main / manual dispatch + # only), so there's no need for the ci.yml TRUSTED_CODE gate here. + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - name: scope-api + dockerfile: apps/api/Dockerfile + - name: scope-judge + dockerfile: apps/judge/Dockerfile + - name: scope-token-manager + dockerfile: apps/token-manager/Dockerfile + - name: scope-scheduler + dockerfile: apps/scheduler/Dockerfile + - name: scope-portal + dockerfile: apps/portal/Dockerfile + - name: scope-report-generator + dockerfile: apps/workers/report-generator/Dockerfile + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4.0.0 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push (production image) + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.image.dockerfile }} + push: true + tags: | + ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.image.name }}:latest + ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.image.name }}:${{ github.sha }} + build-args: | + GIT_COMMIT=${{ github.sha }} + BUILD_TIME=${{ github.event.head_commit.timestamp || '' }} + GIT_BRANCH=${{ github.ref_name }} + cache-from: type=gha,scope=${{ matrix.image.name }} + cache-to: type=gha,mode=min,scope=${{ matrix.image.name }} + + # The db-migrate Helm Job runs migrations via the api Dockerfile's + # `builder` stage (full TS source + tsx) — the slim production image + # above only ships packages/db-migrations/dist, which mongo-migrate-ts's + # CLI can't use (see deploy/helm/scope/templates/jobs/db-migrate.yaml). + - name: Build and push (api builder stage, for db-migrate job) + if: matrix.image.name == 'scope-api' + uses: docker/build-push-action@v6 + with: + context: . + file: apps/api/Dockerfile + target: builder + push: true + tags: | + ${{ env.REGISTRY }}/${{ github.repository_owner }}/scope-api:builder + ${{ env.REGISTRY }}/${{ github.repository_owner }}/scope-api:builder-${{ github.sha }} + cache-from: type=gha,scope=scope-api-builder + cache-to: type=gha,mode=min,scope=scope-api-builder diff --git a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap index 656b9031c..d2938fe48 100644 --- a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap +++ b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap @@ -171,6 +171,7 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "import": { "options": [ "--dry-run", + "--project", "--url", ], }, @@ -533,6 +534,7 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "import": { "options": [ "--dry-run", + "--project", "--url", ], }, diff --git a/apps/cli/src/commands/criteria.ts b/apps/cli/src/commands/criteria.ts index e59d0d79e..467f91eed 100644 --- a/apps/cli/src/commands/criteria.ts +++ b/apps/cli/src/commands/criteria.ts @@ -427,8 +427,10 @@ criteria .argument("", "Path to a .yaml file or a directory of .yaml files") .option("--dry-run", "Preview what would be imported without sending to API") .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .option("--project ", "Project ID for scoped operations (overrides SCOPE_PROJECT and the saved selection)") .action(async (inputPath: string, options) => { try { + const projectId = requireProjectId(options.project); const absPath = resolve(process.env.INIT_CWD || process.cwd(), inputPath); if (!existsSync(absPath)) { console.error(errorText(`Path not found: ${absPath}`)); @@ -512,6 +514,7 @@ criteria method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ criteria: allCriteria }), + projectId, }); if (!response.ok) { diff --git a/apps/cli/src/commands/prompt-feature.ts b/apps/cli/src/commands/prompt-feature.ts index 24b3a8339..56d1e4a07 100644 --- a/apps/cli/src/commands/prompt-feature.ts +++ b/apps/cli/src/commands/prompt-feature.ts @@ -242,8 +242,10 @@ promptFeature .argument("", "Path to a .yaml file or a directory of .yaml files") .option("--dry-run", "Preview what would be imported without sending to API") .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .option("--project ", "Project ID for scoped operations (overrides SCOPE_PROJECT and the saved selection)") .action(async (inputPath: string, options) => { try { + const projectId = requireProjectId(options.project); const absPath = resolve(inputPath); if (!existsSync(absPath)) { console.error(errorText(`Path not found: ${absPath}`)); @@ -320,6 +322,7 @@ promptFeature method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ features: allFeatures }), + projectId, }); if (!response.ok) { diff --git a/deploy/helm/scope/.helmignore b/deploy/helm/scope/.helmignore new file mode 100644 index 000000000..0b4ffec36 --- /dev/null +++ b/deploy/helm/scope/.helmignore @@ -0,0 +1,6 @@ +.git/ +.gitignore +*.swp +*.bak +*.tmp +.DS_Store diff --git a/deploy/helm/scope/CONFIGURATION.md b/deploy/helm/scope/CONFIGURATION.md new file mode 100644 index 000000000..bb4842c60 --- /dev/null +++ b/deploy/helm/scope/CONFIGURATION.md @@ -0,0 +1,74 @@ +# Configuration reference + +Details on values.yaml conventions and image registries. For the install +steps, see the [README](./README.md). + +## Bring your own secrets + +Port-forward the Portal if it isn't already running (see README "Verify"): + +```bash +kubectl -n scope port-forward svc/scope-portal 5100:80 +``` + +Then, at : + +1. Go to the **Secrets** tab. +2. Click **Register key**. +3. Add your GitHub token and Anthropic API key. + +> See [Workers](./WORKERS.md#registering-credentials-another-way) for other +> ways to register these credentials (a raw Key Vault secret, or the API +> directly). + +### GitHub OAuth token for the Copilot worker + +The `coder-acp-copilot` worker needs a scopeless GitHub OAuth token (not a +personal access token). Generate one with: + +```bash +pnpm get-copilot-token +``` + +Follow the device-code prompt to approve access, then register the printed +token as the GitHub secret key in the Portal's **Secrets** tab above. + +## Values conventions + +See [`values.yaml`](./values.yaml) for the full set of configurable values. +Every service section supports: + +- `enabled` +- `image.repository`, `image.tag`, `image.registry` +- `replicas` +- `resources` + +`image.registry` overrides `global.imageRegistry` for a single image. The +workers use this (they pull from ACR, not GHCR), as does `jobs.dbMigrate` +(which needs a separate `:builder`-tagged image — see +`templates/jobs/db-migrate.yaml`). + +## Image registry strategy + +By default, the six core services (api, portal, judge, token-manager, +scheduler, report-generator) pull from `global.imageRegistry` +(`ghcr.io/microsoft` by default — public images published by +[`.github/workflows/publish-images.yml`](../../../.github/workflows/publish-images.yml)). +No registry build step is required for these. + +If you publish your own fork's images, override the registry: + +```bash +helm install scope deploy/helm/scope -n scope \ + -f my-values.yaml \ + --set global.imageRegistry=ghcr.io/ +``` + +The two ACP worker images are different — they're never published to GHCR. +`scripts/bootstrap-workers.sh` builds and pushes them to the ACR instance +`deploy/azure/` provisions. See [Workers](./WORKERS.md). + +## Not covered + +Bring your own ingress controller, TLS/cert-manager, and DNS — this chart +doesn't configure any of them. diff --git a/deploy/helm/scope/Chart.yaml b/deploy/helm/scope/Chart.yaml new file mode 100644 index 000000000..fa43fbf7d --- /dev/null +++ b/deploy/helm/scope/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +name: scope +description: >- + Helm chart for deploying the Scope application (API, Portal, Judge, Token + Manager, Scheduler, and coding-agent workers) onto an AKS cluster — pairs + with the infrastructure provisioned by deploy/azure/. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://microsoft.github.io/scope/ +sources: + - https://github.com/microsoft/scope +keywords: + - scope + - ai-benchmarking + - coding-agents +maintainers: + - name: Scope maintainers + url: https://github.com/microsoft/scope diff --git a/deploy/helm/scope/README.md b/deploy/helm/scope/README.md new file mode 100644 index 000000000..a5d3baf96 --- /dev/null +++ b/deploy/helm/scope/README.md @@ -0,0 +1,79 @@ +# Scope Helm chart + +Deploys the Scope application onto the AKS cluster that +[`deploy/azure/`](../azure/README.md) provisions. + +| Phase | What it does | Tooling | +|---|---|---| +| 1. Infrastructure | AKS, Cosmos DB, Key Vault, Redis, Storage, ACR, networking | `deploy/azure/` (Bicep) | +| 2. Core app | api, portal, judge, token-manager, scheduler, report-generator | This chart | +| 3. Coding-agent workers | Worker images, agent registration | `scripts/bootstrap-workers.sh` | + +This README covers phases 2 and 3. + +## Prerequisites + +- Phase 1 infrastructure already deployed (AKS with Key Vault Secrets + Provider CSI driver + workload identity, Key Vault, Storage, Cosmos DB, + Redis, ACR). +- `helm` and `kubectl`, configured against the target cluster. + +## Phase 2: Deploy the core app + +**1. Get a values file.** Deployed via `deploy/azure/`'s Bicep: + +```bash +./scripts/generate-values-from-bicep.sh --resource-group +``` + +Otherwise, hand-author `my-values.yaml` — see [Configuration](./CONFIGURATION.md). + +**2. Install the chart:** + +```bash +helm upgrade --install scope deploy/helm/scope \ + --namespace scope --create-namespace \ + -f my-values.yaml \ + --wait --timeout 5m +``` + +**3. Add any bring-your-own secrets** (GitHub token, Anthropic API key) +Bicep doesn't provision — see +[Bring your own secrets](./CONFIGURATION.md#bring-your-own-secrets). + +## Phase 3: Bring workers online + +ACP coding-agent workers are disabled by default: + +```bash +./scripts/bootstrap-workers.sh --acr +``` + +Builds + pushes both worker images, enables their Deployments, and +registers both agents with the API. Register the GitHub token/Anthropic +API key they need yourself via the Portal's **Secrets** tab (or +`POST /api/v1/keys`) — see [Workers](./WORKERS.md) for flags and details. + +## Verify + +```bash +kubectl get pods -n scope + +# No ingress yet — reach services via port-forward. +kubectl -n scope port-forward svc/api 18080:80 & +kubectl -n scope port-forward svc/scope-portal 5100:80 & + +scope agent list --url http://localhost:18080 # or open http://localhost:5100 +``` + +Then try an end-to-end run — see [Run a sample scenario](./SAMPLE-SCENARIO.md). + +## Learn more + +- [Configuration reference](./CONFIGURATION.md) — values.yaml conventions, + image registry strategy, secrets options +- [Workers](./WORKERS.md) — bootstrapping the ACP coding-agent workers +- [Run a sample scenario](./SAMPLE-SCENARIO.md) — seed criteria and submit + a run +- [`values.yaml`](./values.yaml) — every configurable value +- [Bicep infrastructure README](../azure/README.md) diff --git a/deploy/helm/scope/SAMPLE-SCENARIO.md b/deploy/helm/scope/SAMPLE-SCENARIO.md new file mode 100644 index 000000000..c63964669 --- /dev/null +++ b/deploy/helm/scope/SAMPLE-SCENARIO.md @@ -0,0 +1,52 @@ +# Run a sample scenario + +Once the core app and at least one [worker](./WORKERS.md) are up, try an +end-to-end run. + +## Why you need to seed criteria first + +Scenarios and personas (`config/scenarios/`, `config/personas/`) aren't +server-side data — `scope run submit --scenario --persona ` +reads them straight off disk, so there's nothing to import for those. + +Criteria are different: they're server-side and project-scoped. Several +scenarios (the `version: v2` ones) reference reusable criteria IDs like +`has_react`/`has_typescript` instead of inlining their own, and those IDs +must exist in the target project before the judge can evaluate a run +against them. + +This seed step isn't a Helm hook. Unlike `db-migrate`, it's project-scoped +business data, not infrastructure, and no project exists until the +`db-migrate` Job's "Initial Project" migration has run. Baking a project +assumption into the chart would surprise anyone bringing their own +projects — so run it yourself, once. + +## Steps + +```bash +kubectl -n scope port-forward svc/api 18080:80 & + +# Find the project to seed into: the "Initial Project" created by +# db-migrate on first install, or one of your own. +scope project list --url http://localhost:18080 + +# Import every reusable criterion. This is an upsert, so it's safe to rerun. +scope criteria import config/criteria --project --url http://localhost:18080 + +# Submit the simplest sample scenario. +scope run submit \ + --scenario config/scenarios/hello-world-express-v2.yaml \ + --persona config/personas/vibe-coder.yaml \ + --worker coder-acp-copilot \ + --project \ + --url http://localhost:18080 +``` + +`--worker` must be an agent ID registered in [Workers](./WORKERS.md) — +check with `scope agent list --url http://localhost:18080`. + +`hello-world-express-v2.yaml` only needs `has_azure_doc`/`has_azure_azd` +(plus their `has_iac`/`has_azure`/`has_cloud` ancestors). Importing the +whole `config/criteria` directory is simpler than cherry-picking a +dependency closure, and it covers every sample scenario, including +`react-snake-game-v2.yaml`. diff --git a/deploy/helm/scope/WORKERS.md b/deploy/helm/scope/WORKERS.md new file mode 100644 index 000000000..3294298b9 --- /dev/null +++ b/deploy/helm/scope/WORKERS.md @@ -0,0 +1,132 @@ +# Workers + +The ACP coding-agent workers (`coder-acp-copilot`, `coder-acp-claude-code`) +are disabled by default in this chart. Each gets an MCPJungle MCP-gateway +sidecar (`workers..mcpGateway`, enabled by default — see "MCP-gateway +sidecar" below); the AI Gateway/DevProxy sidecar and Kubedock are not +templated yet (see +[`docs/architecture/kubedock.md`](../../docs/architecture/kubedock.md) for +the intended production pattern). Bring them online with one command: + +```bash +./scripts/bootstrap-workers.sh --acr +``` + +This: + +1. Builds and pushes both worker images with `az acr build`. +2. Runs `helm upgrade` to point each worker's image fields at the image it + just built, and flips `workers..enabled: true`. +3. Waits for both Deployments to roll out. +4. Registers both agent types with the API (`scripts/register-agent.sh`). + +The script does **not** register worker credentials. Add the GitHub PAT +and/or Anthropic API key each worker needs through the Portal's **Secrets** +tab (**Register key**) after the workers are up — see "Registering +credentials" below. + +Run `./scripts/bootstrap-workers.sh --help` for all flags, including +`--skip-build`, `--skip-copilot`, `--skip-claude-code`, +`--skip-agent-registration`, and custom +`--namespace`/`--release`/`--chart`/`--image-tag`. + +> Pass `--values-file ` so the upgrade re-merges your original values +> override file against the chart's current defaults, instead of freezing a +> historical `--reuse-values` snapshot (which silently drops any default key +> added to `values.yaml` after your first `helm install`). + +## Registering credentials + +Workers need a GitHub PAT (`coder-acp-copilot`) and/or an Anthropic API key +(`coder-acp-claude-code`) to drive their respective CLIs/SDKs. Register +these yourself, through either: + +- The Portal's **Secrets** tab (**Register key**), or a direct + `POST /api/v1/keys` call against the API, writes straight to Key Vault + through token-manager's workload identity. No chart changes needed. +- Populating `secrets.keyVaultSecretNames.githubToken`/`.anthropicApiKey` in + Key Vault yourself also works, as a static fallback — the worker only + falls back to it when Token Manager has no credential registered. + +## How the chart templates workers + +Each worker's Deployment is templated by this chart +(`templates/workers/coder-acp-copilot/`, +`templates/workers/coder-acp-claude-code/`), reusing the common env +ConfigMap/Secret the rest of the app uses, but under their own +narrower-identity ServiceAccount (see "Worker identity isolation" below — +no Key Vault CSI mount). The script just flips +`workers..enabled: true` and sets +`workers..image.registry`/`.tag` with `helm upgrade --set`. + +These are minimal background Storage Queue consumers, plus an MCPJungle +MCP-gateway sidecar: + +- No AI Gateway/DevProxy sidecar (HAR capture). +- No Kubedock (in-task `docker build`/`docker run` support — see + [`docs/architecture/kubedock.md`](../../docs/architecture/kubedock.md) + for the intended production pattern). + +Scenarios needing those two features aren't supported by this chart yet. + +## MCP-gateway sidecar + +Each worker Pod runs an [MCPJungle](https://github.com/mcpjungle/mcpjungle) +sidecar (`workers..mcpGateway`, enabled by default) that aggregates +MCP servers for the worker, mirroring docker-compose's +`mcp-gateway-copilot`/`mcp-gateway-claude-code` services: + +- Templated as a **native sidecar** (`initContainers` entry with + `restartPolicy: Always`, GA since Kubernetes 1.29) rather than a regular + container, so kubelet starts it first and holds the Pod at `NotReady` + until its `/health` readiness probe passes — the worker's first task + never races a gateway that isn't listening yet. +- Shares an `emptyDir` volume mounted at `/workspace` with the worker + container (compose's `copilot_workspace`/`claude_workspace` volumes). +- The worker reaches it at `http://localhost:8080` (same Pod — no + `Service` object needed). + +Set `workers..mcpGateway.enabled: false` only if your scenarios never +configure MCP servers. + +## Worker identity isolation + +Both ACP workers run benchmark-controlled agent code, so they get a +separate, narrower `ServiceAccount`/identity than the rest of the app: +`azure.workerWorkloadIdentityClientId`/`azure.workerServiceAccountName` +(Storage Blob/Queue only — no Key Vault access), instead of the shared +identity's Key Vault Secrets Officer role. Their pods also don't mount the +`kv-secrets` CSI volume. They still read `GITHUB_TOKEN`/`ANTHROPIC_API_KEY` +fallback values and `MONGO_CONNECTION_STRING`/`REDIS_PASSWORD` (needed to +claim/lock queue work) via `envFrom.secretRef` on the common K8s Secret — +that secret is synced by the `api` pod's own CSI mount, so the workers don't +need their own mount to receive it. + +`scripts/generate-values-from-bicep.sh` picks up the worker identity's +outputs automatically if your Bicep deployment provisioned one; otherwise +those two values stay blank and are only required once you set +`workers..enabled: true`. + +**Known limitation**: full isolation needs worker code changes this chart +can't make alone. `coder-acp-copilot`/`coder-acp-claude-code` are a single +process today — the same container that claims Storage Queue work via Mongo +and Redis also spawns the benchmark agent subprocess, so the pod can't be +handed queue credentials without also handing them to the agent's +environment/filesystem. True separation (a trusted queue-processor process +with no agent execution, and an executor with zero backend credentials — +`automountServiceAccountToken: false`, no Mongo/Redis/API-key env) is +tracked as follow-up work, not part of this chart. + +## Agent versions + +`workers.coderAcpCopilot.agentVersion`/`.coderAcpClaudeCode.agentVersion` +must match the `agentVersion` in whatever version manifest you register. +The script's defaults point at the `-dev` manifests under each worker's +directory — intended for chart smoke-testing only. For anything beyond +that, pass a real version manifest (and matching +`imageTag`/`queueName`/`gitCommit`/`buildTime`), and keep `values.yaml`'s +`agentVersion` fields in sync. + +Agent registration doesn't run automatically on `helm install`/`upgrade` — +rerun `scripts/bootstrap-workers.sh` (or `scripts/register-agent.sh` +directly) whenever you roll a new worker image or version. diff --git a/deploy/helm/scope/templates/NOTES.txt b/deploy/helm/scope/templates/NOTES.txt new file mode 100644 index 000000000..3a3c4971c --- /dev/null +++ b/deploy/helm/scope/templates/NOTES.txt @@ -0,0 +1,25 @@ +Scope has been deployed as release "{{ .Release.Name }}" in namespace +"{{ .Values.global.namespace }}". + +{{- if .Values.api.ingress.enabled }} +API: https://{{ .Values.api.ingress.host }} +{{- else }} +API: kubectl -n {{ .Values.global.namespace }} port-forward svc/api 3100:{{ .Values.api.port }} +{{- end }} +{{- if .Values.portal.ingress.enabled }} +Portal: https://{{ .Values.portal.ingress.host }} +{{- else }} +Portal: kubectl -n {{ .Values.global.namespace }} port-forward svc/{{ include "scope.componentName" (dict "component" "portal" "context" .) }} 5100:{{ .Values.portal.port }} +{{- end }} + +Before this chart can fully start: + 1. Populate any bring-your-own secrets in Key Vault "{{ .Values.azure.keyVaultName }}" + (github-token, anthropic-api-key, etc. — see values.yaml secrets.keyVaultSecretNames). + 2. Confirm azure.workloadIdentityClientId matches the user-assigned identity + federated by deploy/azure/ to system:serviceaccount:{{ .Values.global.namespace }}:{{ .Values.azure.serviceAccountName }}. + +NOTE: The ACP coding-agent workers (coder-acp-copilot, coder-acp-claude-code) +are NOT deployed by this chart. Build/push their images to your ACR and +bring them online as a separate step — see "Phase 3" in README.md. + +See deploy/helm/scope/README.md for full install instructions. diff --git a/deploy/helm/scope/templates/_helpers.tpl b/deploy/helm/scope/templates/_helpers.tpl new file mode 100644 index 000000000..a3635c7f6 --- /dev/null +++ b/deploy/helm/scope/templates/_helpers.tpl @@ -0,0 +1,99 @@ +{{/* +Chart base name. +*/}} +{{- define "scope.name" -}} +{{- .Chart.Name -}} +{{- end -}} + +{{/* +Fully qualified app name, used as a prefix for resource names. +*/}} +{{- define "scope.fullname" -}} +{{- .Release.Name -}} +{{- end -}} + +{{/* +Common labels applied to every resource. +*/}} +{{- define "scope.labels" -}} +app.kubernetes.io/name: {{ include "scope.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end -}} + +{{/* +Selector labels for a given service (pass `component` in the calling scope's +dict, e.g. (dict "component" "api" "context" $)). +*/}} +{{- define "scope.selectorLabels" -}} +app.kubernetes.io/name: {{ include "scope.name" .context }} +app.kubernetes.io/instance: {{ .context.Release.Name }} +app.kubernetes.io/component: {{ .component }} +{{- end -}} + +{{/* +Resource name for a given service component, e.g. "scope-api". +*/}} +{{- define "scope.componentName" -}} +{{- printf "%s-%s" (include "scope.fullname" .context) .component -}} +{{- end -}} + +{{/* +Fully qualified image reference, combining a registry with a service's +image.repository/tag. Per-image `.image.registry` (used by the workers and +the db-migrate job, which live on the Bicep-provisioned ACR rather than +GHCR) wins; otherwise falls back to `global.imageRegistry` (the default +distribution point for the core app services, conventionally +ghcr.io//scope). Falls back to `global.imageTag` when `.image.tag` +is unset, and to a bare (unregistered) name when no registry applies at all. +*/}} +{{- define "scope.image" -}} +{{- $tag := .image.tag | default .context.Values.global.imageTag -}} +{{- $registry := .image.registry | default .context.Values.global.imageRegistry -}} +{{- if $registry -}} +{{- printf "%s/%s:%s" $registry .image.repository $tag -}} +{{- else -}} +{{- printf "%s:%s" .image.repository $tag -}} +{{- end -}} +{{- end -}} + +{{/* +The db-migrate Job's image tag. GHCR only publishes the api Dockerfile's +"builder" stage as "builder" (latest) and "builder-" — never a bare + or "latest" tag (see .github/workflows/publish-images.yml) — so this +can't reuse "scope.image"'s plain default-to-global.imageTag fallback. +`jobs.dbMigrate.image.tag` (empty by default) wins if explicitly set; +otherwise this derives "builder-" from whatever revision +the rest of the app is pinned to via `global.imageTag`, falling back to the +mutable "builder" tag when `global.imageTag` is still the default "latest" +(matching the one `latest`-equivalent tag GHCR actually publishes). This +keeps migrations and the application on the same revision when +`global.imageTag` is pinned to a commit SHA. +*/}} +{{- define "scope.dbMigrateTag" -}} +{{- if .Values.jobs.dbMigrate.image.tag -}} +{{- .Values.jobs.dbMigrate.image.tag -}} +{{- else if or (not .Values.global.imageTag) (eq .Values.global.imageTag "latest") -}} +{{- "builder" -}} +{{- else -}} +{{- printf "builder-%s" .Values.global.imageTag -}} +{{- end -}} +{{- end -}} + +{{/* +Name of the shared workload-identity ServiceAccount. Must match the Bicep +template's federated subject. +*/}} +{{- define "scope.serviceAccountName" -}} +{{- .Values.azure.serviceAccountName -}} +{{- end -}} + +{{/* +Name of the worker-scoped workload-identity ServiceAccount (ACP workers +only). Narrower identity than scope.serviceAccountName — see WORKERS.md +"Worker identity isolation". +*/}} +{{- define "scope.workerServiceAccountName" -}} +{{- .Values.azure.workerServiceAccountName -}} +{{- end -}} diff --git a/deploy/helm/scope/templates/api/deployment.yaml b/deploy/helm/scope/templates/api/deployment.yaml new file mode 100644 index 000000000..2ab7e43e3 --- /dev/null +++ b/deploy/helm/scope/templates/api/deployment.yaml @@ -0,0 +1,62 @@ +{{- if .Values.api.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "scope.componentName" (dict "component" "api" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: api +spec: + replicas: {{ .Values.api.replicas }} + selector: + matchLabels: + {{- include "scope.selectorLabels" (dict "component" "api" "context" .) | nindent 6 }} + template: + metadata: + labels: + {{- include "scope.selectorLabels" (dict "component" "api" "context" .) | nindent 8 }} + azure.workload.identity/use: "true" + spec: + serviceAccountName: {{ include "scope.serviceAccountName" . }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: api + image: {{ include "scope.image" (dict "image" .Values.api.image "context" .) }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - name: http + containerPort: {{ .Values.api.port }} + envFrom: + - configMapRef: + name: {{ include "scope.componentName" (dict "component" "common-env" "context" .) }} + - secretRef: + name: {{ .Values.secrets.k8sSecretName }} + env: + - name: PORT + value: {{ .Values.api.port | quote }} + volumeMounts: + - name: kv-secrets + mountPath: /mnt/secrets + readOnly: true + readinessProbe: + httpGet: { path: /health, port: http } + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: { path: /health, port: http } + initialDelaySeconds: 15 + periodSeconds: 20 + resources: + {{- toYaml .Values.api.resources | nindent 12 }} + volumes: + - name: kv-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: {{ include "scope.componentName" (dict "component" "azure-kv" "context" .) }} +{{- end -}} diff --git a/deploy/helm/scope/templates/api/ingress.yaml b/deploy/helm/scope/templates/api/ingress.yaml new file mode 100644 index 000000000..c6912e9a5 --- /dev/null +++ b/deploy/helm/scope/templates/api/ingress.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.api.enabled .Values.api.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "scope.componentName" (dict "component" "api" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: api +spec: + {{- with .Values.api.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.api.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + - host: {{ required "api.ingress.host is required when api.ingress.enabled is true" .Values.api.ingress.host | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + # Must match the literal "api" Service name (see api/service.yaml). + name: api + port: + number: {{ .Values.api.port }} +{{- end -}} diff --git a/deploy/helm/scope/templates/api/service.yaml b/deploy/helm/scope/templates/api/service.yaml new file mode 100644 index 000000000..1d930ccb5 --- /dev/null +++ b/deploy/helm/scope/templates/api/service.yaml @@ -0,0 +1,22 @@ +{{- if .Values.api.enabled -}} +apiVersion: v1 +kind: Service +metadata: + # Hardcoded to literal "api" (NOT the usual "-api" componentName + # prefix): the portal image's apps/portal/nginx.conf bakes in + # `proxy_pass http://api:80` at build time with no templating hook, so + # the Service DNS name this chart creates must match it exactly, + # regardless of Helm release name. + name: api + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: api +spec: + selector: + {{- include "scope.selectorLabels" (dict "component" "api" "context" .) | nindent 4 }} + ports: + - name: http + port: {{ .Values.api.port }} + targetPort: http +{{- end -}} diff --git a/deploy/helm/scope/templates/configmap-common-env.yaml b/deploy/helm/scope/templates/configmap-common-env.yaml new file mode 100644 index 000000000..287e983f1 --- /dev/null +++ b/deploy/helm/scope/templates/configmap-common-env.yaml @@ -0,0 +1,28 @@ +{{/* +Non-secret environment shared by most services: internal service hostnames, +storage account name (RBAC-only, no key needed), and queue names — mirrors +the `x-common-env`/`x-worker-env` YAML anchors in docker-compose.yml. +*/}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "scope.componentName" (dict "component" "common-env" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} +data: + AZURE_STORAGE_ACCOUNT_NAME: {{ .Values.azure.storageAccountName | quote }} + # Only token-manager calls the Key Vault SDK directly (DefaultAzureCredential + # via the workload-identity-injected AZURE_CLIENT_ID/AZURE_FEDERATED_TOKEN_FILE + # env vars); harmless no-op for the other services. + AZURE_KEYVAULT_URI: {{ required "azure.keyVaultUri is required (see deploy/azure output keyVaultUri)" .Values.azure.keyVaultUri | quote }} + TOKEN_MANAGER_URL: "http://{{ include "scope.componentName" (dict "component" "token-manager" "context" .) }}:{{ .Values.tokenManager.port }}" + JUDGE_SERVICE_URL: "http://{{ include "scope.componentName" (dict "component" "judge" "context" .) }}:{{ .Values.judge.port }}" + SCOPE_MT_API_URL: "http://api:{{ .Values.api.port }}" + AZURE_STORAGE_QUEUE_REPORT: report-queue + # Discrete Redis connection settings (Azure Cache for Redis: TLS-only on + # 6380). REDIS_PASSWORD (the access key) comes from the KV-synced + # `scope-secrets` Secret, not this ConfigMap. + REDIS_HOST: {{ required "azure.redisHostName is required (see deploy/azure output redisHostName)" .Values.azure.redisHostName | quote }} + REDIS_PORT: {{ .Values.azure.redisPort | quote }} + REDIS_TLS: {{ .Values.azure.redisTls | quote }} diff --git a/deploy/helm/scope/templates/jobs/db-migrate.yaml b/deploy/helm/scope/templates/jobs/db-migrate.yaml new file mode 100644 index 000000000..00b725daa --- /dev/null +++ b/deploy/helm/scope/templates/jobs/db-migrate.yaml @@ -0,0 +1,74 @@ +{{- if .Values.jobs.dbMigrate.enabled -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "scope.componentName" (dict "component" "db-migrate" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: db-migrate + annotations: + # Mirrors docker-compose's `depends_on: db-migrate: condition: + # service_completed_successfully`. Re-run on every `helm upgrade` too, + # since new migrations can ship with any release. + # + # Must be post-install/post-upgrade, NOT pre-install/pre-upgrade: Helm + # creates every pre-install/pre-upgrade hook BEFORE any regular release + # resource exists, so a pre-install db-migrate Job would run before the + # ServiceAccount (workload identity) and SecretProviderClass (Key Vault + # CSI sync) templates below even exist — the pod can't authenticate or + # mount `kv-secrets`, and `envFrom.secretRef: scope-secrets` fails with + # "secret not found". Running post-install/post-upgrade instead means + # those regular resources are already applied, at the cost of api/ + # judge/scheduler/token-manager pods starting up slightly before + # migrations finish (acceptable: these services retry transient DB + # errors — see the `resilience` skill's retry/backoff conventions used + # throughout this codebase). + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + backoffLimit: 3 + template: + metadata: + labels: + {{- include "scope.selectorLabels" (dict "component" "db-migrate" "context" .) | nindent 8 }} + azure.workload.identity/use: "true" + spec: + restartPolicy: Never + serviceAccountName: {{ include "scope.serviceAccountName" . }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + # packages/db-migrations' migrate:up script runs its .ts migrations + # directly via tsx, so this must use the Dockerfile's `builder` + # stage (full TS source + dev deps) rather than the slim production + # scope-api image, which only ships compiled dist/ (no src/, no + # tsx) — see jobs.dbMigrate.image.tag default ("builder") in + # values.yaml. + - name: db-migrate + image: {{ include "scope.image" (dict "image" (merge (dict "tag" (include "scope.dbMigrateTag" .)) .Values.jobs.dbMigrate.image) "context" .) }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + workingDir: /app + command: ["pnpm", "--filter", "db-migrations", "migrate:up"] + envFrom: + - secretRef: + name: {{ .Values.secrets.k8sSecretName }} + volumeMounts: + # Not read directly by migrate:up — mounting the CSI volume is + # what makes the Secrets Store CSI driver sync the Key Vault + # secrets into the `scope-secrets` K8s Secret before this + # container starts, which `envFrom.secretRef` below depends on. + - name: kv-secrets + mountPath: /mnt/secrets + readOnly: true + volumes: + - name: kv-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: {{ include "scope.componentName" (dict "component" "azure-kv" "context" .) }} +{{- end -}} diff --git a/deploy/helm/scope/templates/judge/deployment.yaml b/deploy/helm/scope/templates/judge/deployment.yaml new file mode 100644 index 000000000..5c4038f69 --- /dev/null +++ b/deploy/helm/scope/templates/judge/deployment.yaml @@ -0,0 +1,73 @@ +{{- if .Values.judge.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "scope.componentName" (dict "component" "judge" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: judge +spec: + replicas: {{ .Values.judge.replicas }} + selector: + matchLabels: + {{- include "scope.selectorLabels" (dict "component" "judge" "context" .) | nindent 6 }} + template: + metadata: + labels: + {{- include "scope.selectorLabels" (dict "component" "judge" "context" .) | nindent 8 }} + azure.workload.identity/use: "true" + spec: + serviceAccountName: {{ include "scope.serviceAccountName" . }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: judge + image: {{ include "scope.image" (dict "image" .Values.judge.image "context" .) }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - name: http + containerPort: {{ .Values.judge.port }} + envFrom: + - configMapRef: + name: {{ include "scope.componentName" (dict "component" "common-env" "context" .) }} + - secretRef: + name: {{ .Values.secrets.k8sSecretName }} + env: + - name: PORT + value: {{ .Values.judge.port | quote }} + # Criteria loaded from the API (primary); filesystem fallback + # (CRITERIA_DIR) is unused in prod since this image ships no + # /app/config/criteria — kept only for parity with dev. + - name: CRITERIA_API_URL + value: "http://api:{{ .Values.api.port }}" + - name: JUDGE_MODEL + value: {{ .Values.judge.judgeModel | quote }} + - name: FEEDBACK_MODEL + value: {{ .Values.judge.feedbackModel | quote }} + - name: JUDGE_STRATEGY + value: {{ .Values.judge.judgeStrategy | quote }} + volumeMounts: + - name: kv-secrets + mountPath: /mnt/secrets + readOnly: true + readinessProbe: + httpGet: { path: /health, port: http } + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: { path: /health, port: http } + initialDelaySeconds: 15 + periodSeconds: 20 + resources: + {{- toYaml .Values.judge.resources | nindent 12 }} + volumes: + - name: kv-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: {{ include "scope.componentName" (dict "component" "azure-kv" "context" .) }} +{{- end -}} diff --git a/deploy/helm/scope/templates/judge/service.yaml b/deploy/helm/scope/templates/judge/service.yaml new file mode 100644 index 000000000..801441eea --- /dev/null +++ b/deploy/helm/scope/templates/judge/service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.judge.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "scope.componentName" (dict "component" "judge" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: judge +spec: + selector: + {{- include "scope.selectorLabels" (dict "component" "judge" "context" .) | nindent 4 }} + ports: + - name: http + port: {{ .Values.judge.port }} + targetPort: http +{{- end -}} diff --git a/deploy/helm/scope/templates/portal/deployment.yaml b/deploy/helm/scope/templates/portal/deployment.yaml new file mode 100644 index 000000000..8995aef73 --- /dev/null +++ b/deploy/helm/scope/templates/portal/deployment.yaml @@ -0,0 +1,52 @@ +{{- if .Values.portal.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "scope.componentName" (dict "component" "portal" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: portal +spec: + replicas: {{ .Values.portal.replicas }} + selector: + matchLabels: + {{- include "scope.selectorLabels" (dict "component" "portal" "context" .) | nindent 6 }} + template: + metadata: + labels: + {{- include "scope.selectorLabels" (dict "component" "portal" "context" .) | nindent 8 }} + spec: + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: portal + image: {{ include "scope.image" (dict "image" .Values.portal.image "context" .) }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - name: http + containerPort: {{ .Values.portal.port }} + env: + # Nginx serves purely static, pre-built assets — no Azure + # identity, Key Vault secrets, or backend env vars are needed + # here. Only these two toggles are read at container start by + # docker-entrypoint.sh (see apps/portal/docker-entrypoint.sh). + - name: SCOPE_AUTH_ENABLED + value: {{ .Values.portal.authEnabled | quote }} + {{- if .Values.portal.docsBaseUrl }} + - name: SCOPE_DOCS_BASE_URL + value: {{ .Values.portal.docsBaseUrl | quote }} + {{- end }} + readinessProbe: + httpGet: { path: /health, port: http } + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: { path: /health, port: http } + initialDelaySeconds: 15 + periodSeconds: 20 + resources: + {{- toYaml .Values.portal.resources | nindent 12 }} +{{- end -}} diff --git a/deploy/helm/scope/templates/portal/ingress.yaml b/deploy/helm/scope/templates/portal/ingress.yaml new file mode 100644 index 000000000..aba290408 --- /dev/null +++ b/deploy/helm/scope/templates/portal/ingress.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.portal.enabled .Values.portal.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "scope.componentName" (dict "component" "portal" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: portal +spec: + {{- with .Values.portal.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.portal.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + - host: {{ required "portal.ingress.host is required when portal.ingress.enabled is true" .Values.portal.ingress.host | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "scope.componentName" (dict "component" "portal" "context" .) }} + port: + number: {{ .Values.portal.port }} +{{- end -}} diff --git a/deploy/helm/scope/templates/portal/service.yaml b/deploy/helm/scope/templates/portal/service.yaml new file mode 100644 index 000000000..b661025e2 --- /dev/null +++ b/deploy/helm/scope/templates/portal/service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.portal.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "scope.componentName" (dict "component" "portal" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: portal +spec: + selector: + {{- include "scope.selectorLabels" (dict "component" "portal" "context" .) | nindent 4 }} + ports: + - name: http + port: {{ .Values.portal.port }} + targetPort: http +{{- end -}} diff --git a/deploy/helm/scope/templates/scheduler/deployment.yaml b/deploy/helm/scope/templates/scheduler/deployment.yaml new file mode 100644 index 000000000..b98bf174a --- /dev/null +++ b/deploy/helm/scope/templates/scheduler/deployment.yaml @@ -0,0 +1,68 @@ +{{- if .Values.scheduler.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "scope.componentName" (dict "component" "scheduler" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: scheduler +spec: + replicas: {{ .Values.scheduler.replicas }} + selector: + matchLabels: + {{- include "scope.selectorLabels" (dict "component" "scheduler" "context" .) | nindent 6 }} + template: + metadata: + labels: + {{- include "scope.selectorLabels" (dict "component" "scheduler" "context" .) | nindent 8 }} + azure.workload.identity/use: "true" + spec: + serviceAccountName: {{ include "scope.serviceAccountName" . }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: scheduler + image: {{ include "scope.image" (dict "image" .Values.scheduler.image "context" .) }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - name: http + containerPort: {{ .Values.scheduler.port }} + envFrom: + - configMapRef: + name: {{ include "scope.componentName" (dict "component" "common-env" "context" .) }} + - secretRef: + name: {{ .Values.secrets.k8sSecretName }} + env: + - name: PORT + value: {{ .Values.scheduler.port | quote }} + - name: SCHEDULER_TARGET_QUEUE_DEPTH + value: "2" + - name: SCHEDULER_POLL_INTERVAL_MS + value: "2000" + volumeMounts: + - name: kv-secrets + mountPath: /mnt/secrets + readOnly: true + # The scheduler's health server returns 200 on every path (no + # dedicated /health route) — see apps/scheduler/src/index.ts. + readinessProbe: + httpGet: { path: /, port: http } + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: { path: /, port: http } + initialDelaySeconds: 15 + periodSeconds: 20 + resources: + {{- toYaml .Values.scheduler.resources | nindent 12 }} + volumes: + - name: kv-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: {{ include "scope.componentName" (dict "component" "azure-kv" "context" .) }} +{{- end -}} diff --git a/deploy/helm/scope/templates/scheduler/service.yaml b/deploy/helm/scope/templates/scheduler/service.yaml new file mode 100644 index 000000000..ca0b91178 --- /dev/null +++ b/deploy/helm/scope/templates/scheduler/service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.scheduler.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "scope.componentName" (dict "component" "scheduler" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: scheduler +spec: + selector: + {{- include "scope.selectorLabels" (dict "component" "scheduler" "context" .) | nindent 4 }} + ports: + - name: http + port: {{ .Values.scheduler.port }} + targetPort: http +{{- end -}} diff --git a/deploy/helm/scope/templates/secretproviderclass.yaml b/deploy/helm/scope/templates/secretproviderclass.yaml new file mode 100644 index 000000000..b61396082 --- /dev/null +++ b/deploy/helm/scope/templates/secretproviderclass.yaml @@ -0,0 +1,63 @@ +{{/* +Secrets Store CSI driver SecretProviderClass: syncs the Key Vault secrets +listed in Values.secrets.keyVaultSecretNames into a single K8s Secret +(Values.secrets.k8sSecretName) via secretObjects, so application Deployments +can consume them with a plain `envFrom.secretRef` instead of mounting the CSI +volume individually. At least one Pod must still mount this SecretProviderClass +as a volume for the sync to occur (see the `kv-secrets` volume on the `api` +Deployment) — the Secrets Store CSI driver syncs on volume mount, not +continuously in the background. + +Uses workload identity (clientID) rather than pod identity or the VM's +system-assigned identity, matching the Bicep template's AKS configuration. + +Any `secrets.keyVaultSecretNames.` entry set to an empty string is +skipped entirely (not requested from Key Vault, not synced into the K8s +Secret). This lets optional secrets (e.g. `githubToken`, `anthropicApiKey`, +needed only by the Phase 3 worker images / portal AI features) be omitted +with `--set secrets.keyVaultSecretNames.anthropicApiKey=""` until they're +actually created in the vault — otherwise the Azure provider fails the +*entire* CSI mount (for every secret, not just the missing one) if any +listed object doesn't exist. +*/}} +{{- $logicalToEnvKey := dict + "mongoConnectionString" "MONGO_CONNECTION_STRING" + "redisPassword" "REDIS_PASSWORD" + "githubToken" "GITHUB_TOKEN" + "anthropicApiKey" "ANTHROPIC_API_KEY" +-}} +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: {{ include "scope.componentName" (dict "component" "azure-kv" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} +spec: + provider: azure + secretObjects: + - secretName: {{ .Values.secrets.k8sSecretName }} + type: Opaque + data: + {{- range $logical, $kvObjectName := .Values.secrets.keyVaultSecretNames }} + {{- $envKey := get $logicalToEnvKey $logical }} + {{- if and $envKey $kvObjectName }} + - objectName: {{ $kvObjectName }} + key: {{ $envKey }} + {{- end }} + {{- end }} + parameters: + usePodIdentity: "false" + useVMManagedIdentity: "false" + clientID: {{ required "azure.workloadIdentityClientId is required" .Values.azure.workloadIdentityClientId | quote }} + keyvaultName: {{ required "azure.keyVaultName is required (see deploy/azure output keyVaultName)" .Values.azure.keyVaultName | quote }} + tenantId: {{ required "azure.tenantId is required" .Values.azure.tenantId | quote }} + objects: | + array: + {{- range $logical, $kvObjectName := .Values.secrets.keyVaultSecretNames }} + {{- if $kvObjectName }} + - | + objectName: {{ $kvObjectName }} + objectType: secret + {{- end }} + {{- end }} diff --git a/deploy/helm/scope/templates/serviceaccount.yaml b/deploy/helm/scope/templates/serviceaccount.yaml new file mode 100644 index 000000000..d22d25d70 --- /dev/null +++ b/deploy/helm/scope/templates/serviceaccount.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "scope.serviceAccountName" . }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + azure.workload.identity/use: "true" + annotations: + azure.workload.identity/client-id: {{ required "azure.workloadIdentityClientId is required (see deploy/azure output workloadIdentityClientId)" .Values.azure.workloadIdentityClientId | quote }} +{{- if or .Values.workers.coderAcpCopilot.enabled .Values.workers.coderAcpClaudeCode.enabled }} +--- +# Narrower identity for the ACP workers (Storage Blob/Queue only — no Key +# Vault access), so benchmark-controlled agent code never runs under the +# backend's Key Vault Secrets Officer identity. See WORKERS.md "Worker +# identity isolation". +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "scope.workerServiceAccountName" . }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + azure.workload.identity/use: "true" + annotations: + azure.workload.identity/client-id: {{ required "azure.workerWorkloadIdentityClientId is required when workers are enabled (see deploy/azure output workerWorkloadIdentityClientId)" .Values.azure.workerWorkloadIdentityClientId | quote }} +{{- end }} diff --git a/deploy/helm/scope/templates/token-manager/deployment.yaml b/deploy/helm/scope/templates/token-manager/deployment.yaml new file mode 100644 index 000000000..741f7b525 --- /dev/null +++ b/deploy/helm/scope/templates/token-manager/deployment.yaml @@ -0,0 +1,66 @@ +{{- if .Values.tokenManager.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "scope.componentName" (dict "component" "token-manager" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: token-manager +spec: + replicas: {{ .Values.tokenManager.replicas }} + selector: + matchLabels: + {{- include "scope.selectorLabels" (dict "component" "token-manager" "context" .) | nindent 6 }} + template: + metadata: + labels: + {{- include "scope.selectorLabels" (dict "component" "token-manager" "context" .) | nindent 8 }} + azure.workload.identity/use: "true" + spec: + serviceAccountName: {{ include "scope.serviceAccountName" . }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: token-manager + image: {{ include "scope.image" (dict "image" .Values.tokenManager.image "context" .) }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - name: http + containerPort: {{ .Values.tokenManager.port }} + envFrom: + - configMapRef: + name: {{ include "scope.componentName" (dict "component" "common-env" "context" .) }} + - secretRef: + name: {{ .Values.secrets.k8sSecretName }} + env: + - name: PORT + value: {{ .Values.tokenManager.port | quote }} + # How often token-manager re-validates registered keys against + # their upstream provider. + - name: VALIDATION_INTERVAL_MS + value: "300000" + volumeMounts: + - name: kv-secrets + mountPath: /mnt/secrets + readOnly: true + readinessProbe: + httpGet: { path: /health, port: http } + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: { path: /health, port: http } + initialDelaySeconds: 15 + periodSeconds: 20 + resources: + {{- toYaml .Values.tokenManager.resources | nindent 12 }} + volumes: + - name: kv-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: {{ include "scope.componentName" (dict "component" "azure-kv" "context" .) }} +{{- end -}} diff --git a/deploy/helm/scope/templates/token-manager/service.yaml b/deploy/helm/scope/templates/token-manager/service.yaml new file mode 100644 index 000000000..c0d9a900b --- /dev/null +++ b/deploy/helm/scope/templates/token-manager/service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.tokenManager.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "scope.componentName" (dict "component" "token-manager" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: token-manager +spec: + selector: + {{- include "scope.selectorLabels" (dict "component" "token-manager" "context" .) | nindent 4 }} + ports: + - name: http + port: {{ .Values.tokenManager.port }} + targetPort: http +{{- end -}} diff --git a/deploy/helm/scope/templates/workers/coder-acp-claude-code/deployment.yaml b/deploy/helm/scope/templates/workers/coder-acp-claude-code/deployment.yaml new file mode 100644 index 000000000..60512ee54 --- /dev/null +++ b/deploy/helm/scope/templates/workers/coder-acp-claude-code/deployment.yaml @@ -0,0 +1,120 @@ +{{- if .Values.workers.coderAcpClaudeCode.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "scope.componentName" (dict "component" "coder-acp-claude-code" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: coder-acp-claude-code +spec: + replicas: {{ .Values.workers.coderAcpClaudeCode.replicas }} + selector: + matchLabels: + {{- include "scope.selectorLabels" (dict "component" "coder-acp-claude-code" "context" .) | nindent 6 }} + template: + metadata: + labels: + {{- include "scope.selectorLabels" (dict "component" "coder-acp-claude-code" "context" .) | nindent 8 }} + azure.workload.identity/use: "true" + spec: + # Worker-scoped identity (Storage Blob/Queue only — no Key Vault + # access), not the shared backend ServiceAccount: this pod runs + # benchmark-controlled agent code and must not be able to reach Key + # Vault. See WORKERS.md "Worker identity isolation". + serviceAccountName: {{ include "scope.workerServiceAccountName" . }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.workers.coderAcpClaudeCode.mcpGateway.enabled }} + # Native sidecar (GA since K8s 1.29): kubelet starts this before the + # main container and holds the Pod at NotReady until its readiness + # probe passes, so the worker's first task never races an MCP-gateway + # that isn't listening yet. Mirrors compose's + # mcp-gateway-claude-code `depends_on: service_healthy`. + initContainers: + - name: mcp-gateway + restartPolicy: Always + image: {{ include "scope.image" (dict "image" .Values.workers.coderAcpClaudeCode.mcpGateway.image "context" .) }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + # Suppress MCPJungle's ASCII-art banner (written to stderr). + command: ["/bin/sh", "-c"] + args: ["/mcpjungle start 2>&1 | grep --line-buffered -v '[█═╗╚║╔╝]' >&2"] + env: + - name: SESSION_IDLE_TIMEOUT_SEC + value: "0" + volumeMounts: + - name: workspace + mountPath: /workspace + readinessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 60 + resources: + {{- toYaml .Values.workers.coderAcpClaudeCode.mcpGateway.resources | nindent 12 }} + {{- end }} + containers: + # Pure background Storage Queue consumer — no HTTP server/port, so + # no Service or readiness/liveness HTTP probes (see + # apps/workers/coder-acp-claude-code/src/index.ts). + # + # NOT templated here (see docs/architecture/worker-requirements.md + # and docs/architecture/kubedock.md): the AI Gateway/DevProxy + # sidecar (HAR capture) and Kubedock (in-task `docker + # build`/`docker run` support). The worker runs without them — + # scenarios that need in-task Docker access are not yet supported + # by this chart. Follow up once those sidecars have a templated + # pattern. + - name: coder-acp-claude-code + image: {{ include "scope.image" (dict "image" .Values.workers.coderAcpClaudeCode.image "context" .) }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + envFrom: + - configMapRef: + name: {{ include "scope.componentName" (dict "component" "common-env" "context" .) }} + - secretRef: + # Supplies MONGO_CONNECTION_STRING/REDIS_PASSWORD always, and + # ANTHROPIC_API_KEY as a fallback only if + # secrets.keyVaultSecretNames.anthropicApiKey is populated. + # The worker prefers acquiring its Anthropic credential + # dynamically from Token Manager at runtime (see "Read + # credentials via Token Manager" in + # docs/architecture/worker-requirements.md) — register one + # via the Portal's **Secrets** tab (**Register key**) or + # `POST /api/v1/keys` instead of populating this secret, + # unless you specifically want the static-env-var fallback. + name: {{ .Values.secrets.k8sSecretName }} + env: + - name: WORKER_NAME + value: coder-acp-claude-code + # Must exactly match the agentVersion registered via + # scripts/register-agent.sh (see deploy/helm/scope/README.md + # "Phase 3") — the queue processor refuses to start otherwise. + - name: SCOPE_AGENT_VERSION + value: {{ .Values.workers.coderAcpClaudeCode.agentVersion | quote }} + - name: QUEUE_NAME + value: {{ .Values.workers.coderAcpClaudeCode.queueName | quote }} + {{- if .Values.workers.coderAcpClaudeCode.mcpGateway.enabled }} + # Same Pod, so the sidecar is reachable over loopback — no + # Service object needed. + - name: MCP_GATEWAY_URL + value: http://localhost:8080 + {{- end }} + {{- if .Values.workers.coderAcpClaudeCode.mcpGateway.enabled }} + volumeMounts: + - name: workspace + mountPath: /workspace + {{- end }} + resources: + {{- toYaml .Values.workers.coderAcpClaudeCode.resources | nindent 12 }} + {{- if .Values.workers.coderAcpClaudeCode.mcpGateway.enabled }} + volumes: + # Shared with the mcp-gateway sidecar — mirrors compose's + # claude_workspace named volume. No kv-secrets CSI volume here: + # see the identity note above. + - name: workspace + emptyDir: {} + {{- end }} +{{- end -}} diff --git a/deploy/helm/scope/templates/workers/coder-acp-copilot/deployment.yaml b/deploy/helm/scope/templates/workers/coder-acp-copilot/deployment.yaml new file mode 100644 index 000000000..c99c4d3d7 --- /dev/null +++ b/deploy/helm/scope/templates/workers/coder-acp-copilot/deployment.yaml @@ -0,0 +1,119 @@ +{{- if .Values.workers.coderAcpCopilot.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "scope.componentName" (dict "component" "coder-acp-copilot" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: coder-acp-copilot +spec: + replicas: {{ .Values.workers.coderAcpCopilot.replicas }} + selector: + matchLabels: + {{- include "scope.selectorLabels" (dict "component" "coder-acp-copilot" "context" .) | nindent 6 }} + template: + metadata: + labels: + {{- include "scope.selectorLabels" (dict "component" "coder-acp-copilot" "context" .) | nindent 8 }} + azure.workload.identity/use: "true" + spec: + # Worker-scoped identity (Storage Blob/Queue only — no Key Vault + # access), not the shared backend ServiceAccount: this pod runs + # benchmark-controlled agent code and must not be able to reach Key + # Vault. See WORKERS.md "Worker identity isolation". + serviceAccountName: {{ include "scope.workerServiceAccountName" . }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.workers.coderAcpCopilot.mcpGateway.enabled }} + # Native sidecar (GA since K8s 1.29): kubelet starts this before the + # main container and holds the Pod at NotReady until its readiness + # probe passes, so the worker's first task never races an MCP-gateway + # that isn't listening yet. Mirrors compose's + # mcp-gateway-copilot `depends_on: service_healthy`. + initContainers: + - name: mcp-gateway + restartPolicy: Always + image: {{ include "scope.image" (dict "image" .Values.workers.coderAcpCopilot.mcpGateway.image "context" .) }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + # Suppress MCPJungle's ASCII-art banner (written to stderr). + command: ["/bin/sh", "-c"] + args: ["/mcpjungle start 2>&1 | grep --line-buffered -v '[█═╗╚║╔╝]' >&2"] + env: + - name: SESSION_IDLE_TIMEOUT_SEC + value: "0" + volumeMounts: + - name: workspace + mountPath: /workspace + readinessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 60 + resources: + {{- toYaml .Values.workers.coderAcpCopilot.mcpGateway.resources | nindent 12 }} + {{- end }} + containers: + # Pure background Storage Queue consumer — no HTTP server/port, so + # no Service or readiness/liveness HTTP probes (see + # apps/workers/coder-acp-copilot/src/index.ts). + # + # NOT templated here (see docs/architecture/worker-requirements.md + # and docs/architecture/kubedock.md): the AI Gateway/DevProxy + # sidecar (HAR capture) and Kubedock (in-task `docker + # build`/`docker run` support). The worker runs without them — + # scenarios that need in-task Docker access are not yet supported + # by this chart. Follow up once those sidecars have a templated + # pattern. + - name: coder-acp-copilot + image: {{ include "scope.image" (dict "image" .Values.workers.coderAcpCopilot.image "context" .) }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + envFrom: + - configMapRef: + name: {{ include "scope.componentName" (dict "component" "common-env" "context" .) }} + - secretRef: + # Supplies MONGO_CONNECTION_STRING/REDIS_PASSWORD always, and + # GITHUB_TOKEN as a fallback only if secrets.keyVaultSecretNames.githubToken + # is populated. The worker prefers acquiring its GitHub + # credential dynamically from Token Manager at runtime (see + # "Read credentials via Token Manager" in + # docs/architecture/worker-requirements.md) — register one + # via the Portal's **Secrets** tab (**Register key**) or + # `POST /api/v1/keys` instead of populating this secret, + # unless you specifically want the static-env-var fallback. + name: {{ .Values.secrets.k8sSecretName }} + env: + - name: WORKER_NAME + value: coder-acp-copilot + # Must exactly match the agentVersion registered via + # scripts/register-agent.sh (see deploy/helm/scope/README.md + # "Phase 3") — the queue processor refuses to start otherwise. + - name: SCOPE_AGENT_VERSION + value: {{ .Values.workers.coderAcpCopilot.agentVersion | quote }} + - name: QUEUE_NAME + value: {{ .Values.workers.coderAcpCopilot.queueName | quote }} + {{- if .Values.workers.coderAcpCopilot.mcpGateway.enabled }} + # Same Pod, so the sidecar is reachable over loopback — no + # Service object needed. + - name: MCP_GATEWAY_URL + value: http://localhost:8080 + {{- end }} + {{- if .Values.workers.coderAcpCopilot.mcpGateway.enabled }} + volumeMounts: + - name: workspace + mountPath: /workspace + {{- end }} + resources: + {{- toYaml .Values.workers.coderAcpCopilot.resources | nindent 12 }} + {{- if .Values.workers.coderAcpCopilot.mcpGateway.enabled }} + volumes: + # Shared with the mcp-gateway sidecar — mirrors compose's + # copilot_workspace named volume. No kv-secrets CSI volume here: + # see the identity note above. + - name: workspace + emptyDir: {} + {{- end }} +{{- end -}} diff --git a/deploy/helm/scope/templates/workers/report-generator/deployment.yaml b/deploy/helm/scope/templates/workers/report-generator/deployment.yaml new file mode 100644 index 000000000..deafb1917 --- /dev/null +++ b/deploy/helm/scope/templates/workers/report-generator/deployment.yaml @@ -0,0 +1,54 @@ +{{- if .Values.workers.reportGenerator.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "scope.componentName" (dict "component" "report-generator" "context" .) }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "scope.labels" . | nindent 4 }} + app.kubernetes.io/component: report-generator +spec: + replicas: {{ .Values.workers.reportGenerator.replicas }} + selector: + matchLabels: + {{- include "scope.selectorLabels" (dict "component" "report-generator" "context" .) | nindent 6 }} + template: + metadata: + labels: + {{- include "scope.selectorLabels" (dict "component" "report-generator" "context" .) | nindent 8 }} + azure.workload.identity/use: "true" + spec: + serviceAccountName: {{ include "scope.serviceAccountName" . }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + # Pure background Storage Queue consumer — no HTTP server/port, so + # no Service or readiness/liveness HTTP probes (see + # apps/workers/report-generator/src/index.ts). + - name: report-generator + image: {{ include "scope.image" (dict "image" .Values.workers.reportGenerator.image "context" .) }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + envFrom: + - configMapRef: + name: {{ include "scope.componentName" (dict "component" "common-env" "context" .) }} + - secretRef: + name: {{ .Values.secrets.k8sSecretName }} + env: + - name: REPORT_MODEL + value: {{ .Values.workers.reportGenerator.reportModel | quote }} + volumeMounts: + - name: kv-secrets + mountPath: /mnt/secrets + readOnly: true + resources: + {{- toYaml .Values.workers.reportGenerator.resources | nindent 12 }} + volumes: + - name: kv-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: {{ include "scope.componentName" (dict "component" "azure-kv" "context" .) }} +{{- end -}} diff --git a/deploy/helm/scope/values.yaml b/deploy/helm/scope/values.yaml new file mode 100644 index 000000000..f2d359eaa --- /dev/null +++ b/deploy/helm/scope/values.yaml @@ -0,0 +1,320 @@ +# ----------------------------------------------------------------------------- +# Default values for the Scope umbrella chart. +# +# This chart is designed to be deployed against the AKS cluster provisioned by +# deploy/azure/ (the "Deploy to Azure" Bicep template). Values under `azure:` +# correspond directly to that template's outputs. Run +# scripts/generate-values-from-bicep.sh --resource-group to generate a +# values file from those outputs automatically (see README.md "Deploy the +# core app") instead of transcribing them by hand: +# +# az deployment group show -g -n --query properties.outputs +# +# azure.acrLoginServer <- output acrLoginServer +# azure.keyVaultName <- output keyVaultName +# azure.keyVaultUri <- output keyVaultUri +# azure.workloadIdentityClientId <- output workloadIdentityClientId +# azure.serviceAccountName <- output workloadIdentityServiceAccountName +# azure.tenantId <- your Azure AD tenant ID +# azure.storageAccountName <- output storageAccountName +# azure.redisHostName <- output redisHostName +# azure.redisPort <- output redisPort +# azure.workerWorkloadIdentityClientId <- output workerWorkloadIdentityClientId +# azure.workerServiceAccountName <- output workerWorkloadIdentityServiceAccountName +# +# The last two are only needed if you plan to bring the ACP workers online +# (see WORKERS.md) — they're a separate, narrower identity (Storage Blob/Queue +# only) than the shared one above, so untrusted benchmark agent code running +# in the worker pods never gets the backend's Key Vault access. +# +# This repo ships ONE values.yaml with sane open-source defaults. There are no +# per-environment overlay files in-repo — layer your own `-f my-values.yaml` +# (or `--set`) on top for environment-specific overrides. +# ----------------------------------------------------------------------------- + +global: + # Namespace all resources are deployed into. Must match the namespace the + # Bicep template federated the workload identity to + # (output workloadIdentityServiceAccountNamespace, default "scope"). + namespace: scope + + # Container registry the core app service images (api, portal, judge, + # tokenManager, scheduler, workers.reportGenerator) are pulled from. + # Defaults to this repo's GHCR package namespace — AKS can pull public GHCR + # images with no imagePullSecrets. Override to your own fork's GHCR + # namespace (or any registry you've published these 6 images to, including + # the Bicep-provisioned ACR) with `--set global.imageRegistry=...`. + # + # The two ACP coding-agent workers (workers.coderAcpCopilot/ + # coderAcpClaudeCode) and the db-migrate job are NOT published here — they + # are built/pushed to the Bicep-provisioned ACR as a separate, later step + # (see deploy/helm/scope/README.md "Phase 3"), and use their own + # `image.registry` override below instead of this global default. + imageRegistry: "ghcr.io/microsoft" + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + + imageTag: "latest" + +# ----------------------------------------------------------------------------- +# Azure integration: Key Vault (via Secrets Store CSI driver) + workload +# identity (direct SDK access for Blob/Queue storage and token-manager's +# live Key Vault calls). +# ----------------------------------------------------------------------------- +azure: + tenantId: "" + keyVaultName: "" + # Full Key Vault URI (output keyVaultUri), e.g. https://.vault.azure.net/. + # Used directly by token-manager's Key Vault SDK client (DefaultAzureCredential + # via workload identity) — distinct from keyVaultName, which the Secrets + # Store CSI driver's SecretProviderClass wants instead. + keyVaultUri: "" + # Client ID of the user-assigned managed identity the Bicep template + # created and federated to the Kubernetes ServiceAccount below. + workloadIdentityClientId: "" + # Must exactly match the Bicep's federated subject + # (system:serviceaccount::). + serviceAccountName: scope-workload-identity + storageAccountName: "" + # Hostname of the Azure Managed Redis (redisEnterprise) instance (output + # redisHostName). Non-secret — goes straight into the common-env ConfigMap + # as REDIS_HOST. Always TLS-only, but the port is per-deployment (the + # Bicep template's `output redisPort` varies by SKU/region) — always set + # this from that output rather than relying on the default below, which is + # only a placeholder. + redisHostName: "" + redisPort: 10000 + redisTls: true + # Client ID of the worker-scoped managed identity (Storage Blob/Queue only + # — no Key Vault access), federated to the ServiceAccount below. Only + # needed if you bring the ACP workers online (see WORKERS.md); leave blank + # otherwise. Kept separate from workloadIdentityClientId above so + # benchmark-controlled agent code running in the worker pods never gets + # the backend's Key Vault access. + workerWorkloadIdentityClientId: "" + # Must exactly match the Bicep's federated subject + # (system:serviceaccount::). + workerServiceAccountName: scope-worker-workload-identity + +# Secret object names as they exist in Key Vault, synced into a single K8s +# Secret (named by `secrets.k8sSecretName`) via the Secrets Store CSI +# driver's secretObjects feature. `mongoConnectionString` and +# `redisPassword` are pre-populated by the Bicep template (as +# `cosmos-connection-string` and `redis-primary-key` respectively — Redis is +# an access *key*, not a connection string, so it pairs with +# `azure.redisHostName`/`redisPort`/`redisTls` above to form the discrete +# REDIS_HOST/PORT/PASSWORD/TLS env vars every service actually reads). The +# rest are bring-your-own, and default to "" (skipped — see the comment in +# templates/secretproviderclass.yaml) because they don't exist in Key Vault +# yet on a fresh install. The Azure provider fails the *entire* CSI mount if +# any listed object is missing, which would otherwise block every pod that +# mounts kv-secrets (api, judge, token-manager, scheduler, report-generator, +# and the db-migrate hook) — not just the workers that actually need these. +# Once you've created the Key Vault secrets (see CONFIGURATION.md "Bring +# your own secrets"), set these to the real object names with `helm upgrade` +# to start syncing them. +secrets: + k8sSecretName: scope-secrets + keyVaultSecretNames: + mongoConnectionString: cosmos-connection-string + redisPassword: redis-primary-key + githubToken: "" + anthropicApiKey: "" + +# ----------------------------------------------------------------------------- +# Per-service configuration. Each service supports `enabled`, `image`, +# `replicas`, and `resources`; resource defaults mirror the "mirrors prod +# K8s" limits already documented in docker-compose.yml. +# ----------------------------------------------------------------------------- + +api: + enabled: true + image: + repository: scope-api + tag: "" + port: 80 + replicas: 1 + resources: + requests: { cpu: 250m, memory: 256Mi } + limits: { cpu: "500m", memory: 512Mi } + ingress: + enabled: false + className: "" + host: "" + tls: [] + +portal: + enabled: true + image: + repository: scope-portal + tag: "" + port: 80 + replicas: 1 + resources: + requests: { cpu: 100m, memory: 64Mi } + limits: { cpu: 200m, memory: 128Mi } + # Runtime-only toggles (see apps/portal/docker-entrypoint.sh) — these do + # NOT require rebuilding the portal image. The actual IdP tenant/client + # config (VITE_AUTH_*) is baked into the image at build time instead (see + # docs/architecture/auth-rbac.md §8); rebuild+republish your own portal + # image if you need a different Entra app registration than this chart's + # default GHCR image ships with. + # + # Defaults to false (unlike the portal image's own "secure by default" + # fallback): the publicly-published GHCR portal image has no Entra app + # registration baked in, so setting this true out of the box makes every + # fresh Helm install land on a dead-end "Authentication not configured" + # page (apps/portal/src/components/auth/RequireAuth.tsx) with no way to + # sign in. Set to true (and rebuild the portal image with your own + # VITE_AUTH_* values) once you've wired up a real IdP. + authEnabled: false + docsBaseUrl: "" + ingress: + enabled: false + className: "" + host: "" + tls: [] + +judge: + enabled: true + image: + repository: scope-judge + tag: "" + port: 80 + replicas: 1 + resources: + requests: { cpu: 500m, memory: 512Mi } + limits: { cpu: "1", memory: 1Gi } + judgeModel: gpt-5.4-mini + feedbackModel: gpt-5.4-mini + judgeStrategy: bundled + +tokenManager: + enabled: true + image: + repository: scope-token-manager + tag: "" + port: 80 + replicas: 1 + resources: + requests: { cpu: 100m, memory: 128Mi } + limits: { cpu: 200m, memory: 256Mi } + +scheduler: + enabled: true + image: + repository: scope-scheduler + tag: "" + port: 8080 + replicas: 1 + resources: + requests: { cpu: 100m, memory: 128Mi } + limits: { cpu: 200m, memory: 256Mi } + +workers: + reportGenerator: + enabled: true + image: + repository: scope-report-generator + tag: "" + replicas: 1 + reportModel: gpt-5.4-mini + resources: + requests: { cpu: 250m, memory: 256Mi } + limits: { cpu: "500m", memory: 512Mi } + # ACP coding-agent workers are intentionally disabled by default and kept + # out of the main `helm install`. They are not published to GHCR (their + # images need the Kubedock sidecar story in docs/architecture/kubedock.md, + # which this chart does not yet template) and are built/pushed to the + # Bicep-provisioned ACR as "Phase 3" — see deploy/helm/scope/README.md. + # Set `enabled: true` and `image.registry` to your ACR login server + # (`az acr build` pushes there) only once you're ready to deploy them. + coderAcpCopilot: + enabled: false + image: + # Bicep-provisioned ACR login server, e.g. myacr.azurecr.io. Not GHCR — + # this image is never published there. + registry: "" + repository: scope-coder-acp-copilot + tag: "" + queueName: queue-coder-acp-copilot + # Must match the `agentVersion` you register via + # scripts/register-agent.sh (see deploy/helm/scope/README.md "Phase 3"). + # apps/workers/coder-acp-copilot/agent-version.dev.yaml's "copilot-dev" + # is a dev-only placeholder — set this to a real version string + # (and register a matching, non-dev agent-version manifest) for + # anything beyond chart smoke-testing. + agentVersion: copilot-dev + replicas: 1 + resources: + requests: { cpu: 500m, memory: 512Mi } + limits: { cpu: "1", memory: 1Gi } + # MCPJungle sidecar (MCP-server aggregation) — mirrors docker-compose's + # mcp-gateway-copilot service. Templated as a native sidecar + # (initContainer with restartPolicy: Always, GA since K8s 1.29) so + # kubelet holds the Pod at NotReady until the gateway's health check + # passes, instead of racing the worker's first task. Disable only if + # your scenarios never configure MCP servers. + mcpGateway: + enabled: true + image: + registry: ghcr.io + repository: mcpjungle/mcpjungle + tag: latest-stdio + resources: + requests: { cpu: 250m, memory: 256Mi } + limits: { cpu: "500m", memory: 512Mi } + coderAcpClaudeCode: + enabled: false + image: + registry: "" + repository: scope-coder-acp-claude-code + tag: "" + queueName: queue-coder-acp-claude-code + # See workers.coderAcpCopilot.agentVersion above for the registration + # contract this must satisfy. + agentVersion: claude-agent-acp-dev-sdk-dev + replicas: 1 + resources: + requests: { cpu: 500m, memory: 512Mi } + limits: { cpu: "1", memory: 1Gi } + # See workers.coderAcpCopilot.mcpGateway above. + mcpGateway: + enabled: true + image: + registry: ghcr.io + repository: mcpjungle/mcpjungle + tag: latest-stdio + resources: + requests: { cpu: 250m, memory: 256Mi } + limits: { cpu: "500m", memory: 512Mi } + +jobs: + dbMigrate: + enabled: true + image: + repository: scope-api + # packages/db-migrations' migrate:up script runs its .ts migrations + # directly via tsx; the production `scope-api:latest` image only ships + # compiled dist/ for db-migrations (no src/, no tsx), so this job must + # run against the Dockerfile's `builder` stage instead. The GHCR + # publish workflow pushes that stage as "builder" (mutable) and + # "builder-" (immutable) variants of the same scope-api image. + # Left empty so `scope.dbMigrateTag` (templates/_helpers.tpl) derives + # "builder-" automatically when you pin + # `global.imageTag` to a commit SHA, keeping migrations on the same + # revision as the rest of the app — set this explicitly only to + # override that derivation. + tag: "" + # NOT YET IMPLEMENTED as a chart template. Registering the ACP + # coding-agent types (scripts/register-agent.sh + each worker's + # agent.yaml) only matters once those workers are actually deployed — + # which, per this chart's split, happens in Phase 3 (see + # deploy/helm/scope/README.md), alongside building/pushing their images + # to ACR. Run scripts/register-agent.sh manually (or curl the API) at + # that point instead of via `helm install`. + registerAgents: + enabled: false + image: + repository: alpine + tag: "3.21" diff --git a/packages/shared/src/storage/blob-storage.test.ts b/packages/shared/src/storage/blob-storage.test.ts index 651c34fe8..a72865e1f 100644 --- a/packages/shared/src/storage/blob-storage.test.ts +++ b/packages/shared/src/storage/blob-storage.test.ts @@ -17,8 +17,15 @@ const mockLogsContainerClient = { getAppendBlobClient: vi.fn(() => mockAppendBlobClient), }; +const mockBlockBlobClient = { + upload: vi.fn().mockResolvedValue(undefined), + uploadData: vi.fn().mockResolvedValue(undefined), + url: "https://test.blob.core.windows.net/snapshots/mock-blob", +}; + const mockSnapshotsContainerClient = { createIfNotExists: vi.fn().mockResolvedValue(undefined), + getBlockBlobClient: vi.fn(() => mockBlockBlobClient), }; const mockBlobServiceClient = { @@ -264,3 +271,63 @@ describe("BlobStorage — log helpers", () => { }); }); }); + +describe("BlobStorage — container-creation authorization retry", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + function makeAuthzError(): Error & { statusCode: number; code: string } { + return Object.assign( + new Error( + "This request is not authorized to perform this operation using this permission.", + ), + { statusCode: 403, code: "AuthorizationPermissionMismatch" }, + ); + } + + it("retries uploadJson on a transient AuthorizationPermissionMismatch and succeeds", async () => { + mockBlockBlobClient.uploadData + .mockRejectedValueOnce(makeAuthzError()) + .mockResolvedValueOnce(undefined); + + const storage = makeStorage(); + const url = await storage.uploadJson("some/blob.json", { ok: true }); + + expect(mockBlockBlobClient.uploadData).toHaveBeenCalledTimes(2); + expect(url).toBe(mockBlockBlobClient.url); + }); + + it("does not retry uploadJson on an unrelated error", async () => { + const otherError = Object.assign(new Error("Boom"), { statusCode: 500 }); + mockBlockBlobClient.uploadData.mockRejectedValueOnce(otherError); + + const storage = makeStorage(); + await expect(storage.uploadJson("some/blob.json", { ok: true })).rejects.toThrow("Boom"); + + expect(mockBlockBlobClient.uploadData).toHaveBeenCalledTimes(1); + }); + + it("gives up after repeated AuthorizationPermissionMismatch failures", async () => { + vi.useFakeTimers(); + try { + mockBlockBlobClient.uploadData.mockRejectedValue(makeAuthzError()); + + const storage = makeStorage(); + const pending = storage.uploadText("some/blob.txt", "hello"); + // Attach the rejection assertion before advancing timers so the + // rejection handler is registered before fake timers resolve it. + const assertion = expect(pending).rejects.toThrow("not authorized to perform this operation"); + // Lets the retry policy's backoff timers (up to 4 retries) fire without + // waiting for real wall-clock time. + await vi.runAllTimersAsync(); + await assertion; + + // maxRetries: 4 => up to 5 total attempts + expect(mockBlockBlobClient.uploadData.mock.calls.length).toBeLessThanOrEqual(5); + expect(mockBlockBlobClient.uploadData.mock.calls.length).toBeGreaterThan(1); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/packages/shared/src/storage/blob-storage.ts b/packages/shared/src/storage/blob-storage.ts index c9b50038b..cb97557dc 100644 --- a/packages/shared/src/storage/blob-storage.ts +++ b/packages/shared/src/storage/blob-storage.ts @@ -16,6 +16,7 @@ import { createGunzip } from "zlib"; import { extract } from "tar"; import type { LogEvent } from "../types/types.js"; import type { ToolCall } from "../har/types.js"; +import { withRetry } from "../utils/retry.js"; const SNAPSHOTS_CONTAINER = "snapshots"; const LOGS_CONTAINER = "logs"; @@ -40,6 +41,22 @@ export interface BlobStorageConfig { storageConnectionString?: string; // For local Azurite } +/** + * True for Azure Storage's `AuthorizationPermissionMismatch` error. In an + * RBAC-only (workload identity) setup this shows up as a transient failure — + * most commonly in the few seconds/minutes right after `createIfNotExists` + * auto-creates a container for the first time, where the data-plane + * authorization check for the new container hasn't fully propagated yet even + * though the account-level role assignment has existed for a long time. + * Retrying with backoff resolves it without any code or infra change. + */ +function isAuthorizationPropagationError(error: unknown): boolean { + const err = error as { statusCode?: number; code?: string; message?: string } | undefined; + if (!err) return false; + if (err.code === "AuthorizationPermissionMismatch") return true; + return err.statusCode === 403 && /not authorized to perform this operation/i.test(err.message ?? ""); +} + export class BlobStorage { private containerClient: ContainerClient; private logsContainerClient: ContainerClient; @@ -73,6 +90,22 @@ export class BlobStorage { await this.containerClient.createIfNotExists(); } + /** + * Wraps a snapshots-container write with a short retry for the + * `AuthorizationPermissionMismatch` container-creation propagation race + * described on {@link isAuthorizationPropagationError}. Non-idempotent + * writes here (blob uploads) are safe to retry: each overwrites the same + * blob name, so a retried attempt simply re-sends identical content. + */ + private async withContainerRetry(fn: () => Promise): Promise { + return withRetry(fn, { + maxRetries: 4, + baseDelayMs: 1000, + maxDelayMs: 8000, + isRetryable: isAuthorizationPropagationError, + }); + } + /** * Ensures the logs container exists. Lazily initialized — the HTTP call is made * at most once per BlobStorage instance, regardless of concurrent callers. @@ -219,9 +252,11 @@ export class BlobStorage { const blobName = `${requestId}/runs/${runId}/iteration-${iteration}/tool-calls.jsonl`; const blockBlobClient = this.containerClient.getBlockBlobClient(blobName); const body = toolCalls.map((tc) => JSON.stringify(tc)).join("\n") + "\n"; - await blockBlobClient.upload(body, Buffer.byteLength(body), { - blobHTTPHeaders: { blobContentType: "application/x-ndjson" }, - }); + await this.withContainerRetry(() => + blockBlobClient.upload(body, Buffer.byteLength(body), { + blobHTTPHeaders: { blobContentType: "application/x-ndjson" }, + }), + ); } /** @@ -328,16 +363,18 @@ export class BlobStorage { ); // Upload to blob storage - await blockBlobClient.uploadFile(archivePath, { - blobHTTPHeaders: { - blobContentType: "application/gzip", - }, - tags: { - requestId, - runId, - iteration: String(iteration), - }, - }); + await this.withContainerRetry(() => + blockBlobClient.uploadFile(archivePath, { + blobHTTPHeaders: { + blobContentType: "application/gzip", + }, + tags: { + requestId, + runId, + iteration: String(iteration), + }, + }), + ); return blockBlobClient.url; } finally { @@ -377,15 +414,17 @@ export class BlobStorage { ); // Upload to blob storage - await blockBlobClient.uploadFile(archivePath, { - blobHTTPHeaders: { - blobContentType: "application/gzip", - }, - tags: { - requestId, - iteration: String(iteration), - }, - }); + await this.withContainerRetry(() => + blockBlobClient.uploadFile(archivePath, { + blobHTTPHeaders: { + blobContentType: "application/gzip", + }, + tags: { + requestId, + iteration: String(iteration), + }, + }), + ); return blockBlobClient.url; } finally { @@ -406,11 +445,13 @@ export class BlobStorage { await this.ensureContainer(); const blockBlobClient = this.containerClient.getBlockBlobClient(blobName); - await blockBlobClient.uploadFile(filePath, { - blobHTTPHeaders: { - blobContentType: contentType, - }, - }); + await this.withContainerRetry(() => + blockBlobClient.uploadFile(filePath, { + blobHTTPHeaders: { + blobContentType: contentType, + }, + }), + ); return blockBlobClient.url; } @@ -432,9 +473,11 @@ export class BlobStorage { const body = typeof data === "string" ? data : JSON.stringify(data); const buf = Buffer.from(body, "utf-8"); const blockBlobClient = this.containerClient.getBlockBlobClient(blobName); - await blockBlobClient.uploadData(buf, { - blobHTTPHeaders: { blobContentType: "application/json" }, - }); + await this.withContainerRetry(() => + blockBlobClient.uploadData(buf, { + blobHTTPHeaders: { blobContentType: "application/json" }, + }), + ); return blockBlobClient.url; } @@ -454,9 +497,11 @@ export class BlobStorage { const buf = Buffer.from(text, "utf-8"); const blockBlobClient = this.containerClient.getBlockBlobClient(blobName); - await blockBlobClient.uploadData(buf, { - blobHTTPHeaders: { blobContentType: contentType }, - }); + await this.withContainerRetry(() => + blockBlobClient.uploadData(buf, { + blobHTTPHeaders: { blobContentType: contentType }, + }), + ); return blockBlobClient.url; } diff --git a/scripts/bootstrap-workers.sh b/scripts/bootstrap-workers.sh new file mode 100755 index 000000000..77df926e4 --- /dev/null +++ b/scripts/bootstrap-workers.sh @@ -0,0 +1,295 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# One-shot setup for the two ACP coding-agent workers (coder-acp-copilot, +# coder-acp-claude-code) on top of an already-`helm install`ed Scope +# release (see deploy/helm/scope/README.md "Phase 1/2"). This is the +# single entry point for "Phase 3": it builds + pushes both worker images +# to your ACR, flips them on via `helm upgrade`, and registers each agent +# with the API. You should not need to run any `az acr build`/`helm +# upgrade --set workers...`/`register-agent.sh` commands by hand — this +# script wraps all of them. +# +# This script does NOT register worker credentials (GitHub token, +# Anthropic API key). Add those yourself via the Portal's "Create Token" +# page (or POST /api/v1/keys directly) after the workers are up — see +# deploy/helm/scope/WORKERS.md. +# +# Usage: +# scripts/bootstrap-workers.sh --acr [options] +# +# Options: +# --acr ACR name or login server (required; e.g. myacr +# or myacr.azurecr.io) +# --namespace Kubernetes namespace (default: scope) +# --release Helm release name (default: scope) +# --chart Helm chart path (default: deploy/helm/scope, +# relative to repo root) +# --image-tag Tag to build/push/deploy (default: a unique tag +# derived from the current git commit, e.g. +# a1b2c3d or a1b2c3d-dirty). Rerunning this script +# with the mutable "latest" tag would push new +# image content without changing the rendered +# Deployment PodTemplate, so Helm would leave the +# old pods running — see deploy/helm/scope/ +# README.md "Phase 3" for details. +# --values-file Same values override file passed to the +# original `helm install ... -f ` (see +# README.md "Install (Phase 2)"). Strongly +# recommended: without it, this script falls back +# to `helm upgrade --reuse-values`, which freezes +# the release's previously computed values and +# will NOT pick up new default keys added to the +# chart's values.yaml since the initial install +# (this silently broke worker agentVersion/ +# queueName in testing — see README "Phase 3"). +# --skip-copilot Skip the coder-acp-copilot worker entirely +# --skip-claude-code Skip the coder-acp-claude-code worker entirely +# --skip-build Reuse already-pushed images; skip `az acr build` +# --skip-agent-registration +# Skip the scripts/register-agent.sh step +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +NAMESPACE="scope" +RELEASE_NAME="scope" +CHART_PATH="deploy/helm/scope" +# Default to a unique tag derived from the current commit so each run +# produces a distinct tag and the rendered Deployment PodTemplate actually +# changes (a mutable tag like "latest" lets Helm think nothing changed and +# leave old pods running — see --image-tag help above). Falls back to a +# timestamp if this isn't a git checkout. +if git -C "$REPO_ROOT" rev-parse --short HEAD >/dev/null 2>&1; then + IMAGE_TAG="$(git -C "$REPO_ROOT" rev-parse --short HEAD)" + if ! git -C "$REPO_ROOT" diff --quiet HEAD -- 2>/dev/null; then + IMAGE_TAG="${IMAGE_TAG}-dirty" + fi +else + IMAGE_TAG="$(date +%Y%m%d%H%M%S)" +fi +VALUES_FILE="" +ACR_NAME="" +SKIP_COPILOT=false +SKIP_CLAUDE_CODE=false +SKIP_BUILD=false +SKIP_AGENT_REGISTRATION=false + +log() { printf '\n==> %s\n' "$1"; } +die() { + printf 'error: %s\n' "$1" >&2 + exit 1 +} + +usage() { + sed -n '2,51p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --acr) + ACR_NAME="$2" + shift 2 + ;; + --namespace) + NAMESPACE="$2" + shift 2 + ;; + --release) + RELEASE_NAME="$2" + shift 2 + ;; + --chart) + CHART_PATH="$2" + shift 2 + ;; + --image-tag) + IMAGE_TAG="$2" + shift 2 + ;; + --values-file) + VALUES_FILE="$2" + shift 2 + ;; + --skip-copilot) + SKIP_COPILOT=true + shift + ;; + --skip-claude-code) + SKIP_CLAUDE_CODE=true + shift + ;; + --skip-build) + SKIP_BUILD=true + shift + ;; + --skip-agent-registration) + SKIP_AGENT_REGISTRATION=true + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + die "unknown argument: $1 (see --help)" + ;; + esac +done + +[ -n "$ACR_NAME" ] || die "--acr is required (ACR name or login server)" +ACR_LOGIN_SERVER="$ACR_NAME" +case "$ACR_LOGIN_SERVER" in + *.azurecr.io) ;; + *) ACR_LOGIN_SERVER="${ACR_NAME}.azurecr.io" ;; +esac +ACR_SHORT_NAME="${ACR_LOGIN_SERVER%%.*}" + +if [ -n "$VALUES_FILE" ] && [ ! -f "$VALUES_FILE" ]; then + die "--values-file not found: $VALUES_FILE" +fi + +for bin in az helm kubectl curl; do + command -v "$bin" >/dev/null 2>&1 || die "$bin is required on PATH" +done +if [ "$SKIP_AGENT_REGISTRATION" = false ] && ! command -v yq >/dev/null 2>&1; then + die "yq is required for agent registration (brew install yq), or pass --skip-agent-registration" +fi + +cd "$REPO_ROOT" + +# ----------------------------------------------------------------------------- +# 1. Build + push worker images to ACR (az acr build — no local Docker +# needed). +# ----------------------------------------------------------------------------- +if [ "$SKIP_BUILD" = false ]; then + if [ "$SKIP_COPILOT" = false ]; then + log "Building coder-acp-copilot:$IMAGE_TAG on $ACR_SHORT_NAME" + COPILOT_CLI_VERSION=$(grep '^COPILOT_CLI_VERSION=' apps/workers/coder-acp-copilot/versions.env | cut -d= -f2) + az acr build --registry "$ACR_SHORT_NAME" --image "scope-coder-acp-copilot:$IMAGE_TAG" \ + -f apps/workers/coder-acp-copilot/Dockerfile \ + --build-arg "COPILOT_CLI_VERSION=$COPILOT_CLI_VERSION" . + fi + if [ "$SKIP_CLAUDE_CODE" = false ]; then + log "Building coder-acp-claude-code:$IMAGE_TAG on $ACR_SHORT_NAME" + CLAUDE_CODE_ACP_VERSION=$(grep '^CLAUDE_CODE_ACP_VERSION=' apps/workers/coder-acp-claude-code/versions.env | cut -d= -f2) + CLAUDE_AGENT_SDK_VERSION=$(grep '^CLAUDE_AGENT_SDK_VERSION=' apps/workers/coder-acp-claude-code/versions.env | cut -d= -f2) + az acr build --registry "$ACR_SHORT_NAME" --image "scope-coder-acp-claude-code:$IMAGE_TAG" \ + -f apps/workers/coder-acp-claude-code/Dockerfile \ + --build-arg "CLAUDE_CODE_ACP_VERSION=$CLAUDE_CODE_ACP_VERSION" \ + --build-arg "CLAUDE_AGENT_SDK_VERSION=$CLAUDE_AGENT_SDK_VERSION" . + fi +else + log "Skipping image build (--skip-build); assuming $ACR_LOGIN_SERVER already has the images tagged $IMAGE_TAG" +fi + +# ----------------------------------------------------------------------------- +# 2. Temporary port-forward to the API, used for agent registration +# below. +# ----------------------------------------------------------------------------- +API_LOCAL_PORT=18080 +API_URL="http://127.0.0.1:$API_LOCAL_PORT" +PORT_FORWARD_PID="" + +start_port_forward() { + [ -n "$PORT_FORWARD_PID" ] && return 0 + log "Port-forwarding svc/api -n $NAMESPACE -> localhost:$API_LOCAL_PORT" + kubectl -n "$NAMESPACE" port-forward svc/api "$API_LOCAL_PORT:80" >/tmp/scope-bootstrap-workers-pf.log 2>&1 & + PORT_FORWARD_PID=$! + for _ in $(seq 1 30); do + if curl -sf -o /dev/null "$API_URL/health"; then + return 0 + fi + sleep 1 + done + die "api did not become reachable through the port-forward (see /tmp/scope-bootstrap-workers-pf.log)" +} + +stop_port_forward() { + if [ -n "$PORT_FORWARD_PID" ]; then + kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true + wait "$PORT_FORWARD_PID" 2>/dev/null || true + PORT_FORWARD_PID="" + fi +} +trap stop_port_forward EXIT + +# ----------------------------------------------------------------------------- +# 3. Enable the worker Deployments via `helm upgrade` (chart already has +# them templated, gated on workers..enabled — see +# deploy/helm/scope/templates/workers/). +# ----------------------------------------------------------------------------- +HELM_SET_ARGS=() +if [ "$SKIP_COPILOT" = false ]; then + HELM_SET_ARGS+=( + --set "workers.coderAcpCopilot.enabled=true" + --set "workers.coderAcpCopilot.image.registry=$ACR_LOGIN_SERVER" + --set "workers.coderAcpCopilot.image.tag=$IMAGE_TAG" + ) +fi +if [ "$SKIP_CLAUDE_CODE" = false ]; then + HELM_SET_ARGS+=( + --set "workers.coderAcpClaudeCode.enabled=true" + --set "workers.coderAcpClaudeCode.image.registry=$ACR_LOGIN_SERVER" + --set "workers.coderAcpClaudeCode.image.tag=$IMAGE_TAG" + ) +fi + +if [ "${#HELM_SET_ARGS[@]}" -gt 0 ]; then + log "helm upgrade $RELEASE_NAME (enabling workers)" + if [ -n "$VALUES_FILE" ]; then + # Re-merge the chart's current values.yaml with the same override file + # used at install time. This is deliberately NOT `--reuse-values`: + # that flag freezes the release's previously *computed* values and + # silently drops any new default key added to values.yaml since install + # (e.g. workers..agentVersion/queueName), because the frozen + # object simply never had that key to begin with. + helm upgrade "$RELEASE_NAME" "$CHART_PATH" -n "$NAMESPACE" -f "$VALUES_FILE" "${HELM_SET_ARGS[@]}" --wait --timeout 5m + else + echo "warning: --values-file not provided; falling back to --reuse-values." >&2 + echo " This reuses this release's frozen computed values and will NOT" >&2 + echo " pick up new chart defaults (e.g. agentVersion/queueName) added" >&2 + echo " to values.yaml since the original install. Re-run with" >&2 + echo " --values-file to avoid this." >&2 + helm upgrade "$RELEASE_NAME" "$CHART_PATH" -n "$NAMESPACE" --reuse-values "${HELM_SET_ARGS[@]}" --wait --timeout 5m + fi + + log "Waiting for worker Deployments to roll out" + [ "$SKIP_COPILOT" = false ] && kubectl -n "$NAMESPACE" rollout status deployment/"$RELEASE_NAME"-coder-acp-copilot --timeout=5m + [ "$SKIP_CLAUDE_CODE" = false ] && kubectl -n "$NAMESPACE" rollout status deployment/"$RELEASE_NAME"-coder-acp-claude-code --timeout=5m +fi + +# ----------------------------------------------------------------------------- +# 4. Register each enabled agent with the API. Uses the *-dev.yaml version +# manifests shipped in-repo — fine for chart smoke-testing; for a real +# deployment, register a version manifest whose imageTag/queueName match +# what you actually built above, and keep +# workers..agentVersion (values.yaml) in sync with its agentVersion. +# ----------------------------------------------------------------------------- +if [ "$SKIP_AGENT_REGISTRATION" = false ]; then + start_port_forward + if [ "$SKIP_COPILOT" = false ]; then + log "Registering coder-acp-copilot agent" + "$REPO_ROOT/scripts/register-agent.sh" "$API_URL" \ + apps/workers/coder-acp-copilot/agent.yaml \ + --available true \ + apps/workers/coder-acp-copilot/agent-version.dev.yaml + fi + if [ "$SKIP_CLAUDE_CODE" = false ]; then + log "Registering coder-acp-claude-code agent" + "$REPO_ROOT/scripts/register-agent.sh" "$API_URL" \ + apps/workers/coder-acp-claude-code/agent.yaml \ + --available true \ + apps/workers/coder-acp-claude-code/agent-version.dev.yaml + fi +fi + +stop_port_forward +trap - EXIT + +log "Done." +[ "$SKIP_COPILOT" = false ] && echo " coder-acp-copilot: $ACR_LOGIN_SERVER/scope-coder-acp-copilot:$IMAGE_TAG" +[ "$SKIP_CLAUDE_CODE" = false ] && echo " coder-acp-claude-code: $ACR_LOGIN_SERVER/scope-coder-acp-claude-code:$IMAGE_TAG" +echo "Check pod status with: kubectl -n $NAMESPACE get pods -l 'app.kubernetes.io/component in (coder-acp-copilot,coder-acp-claude-code)'" diff --git a/scripts/generate-values-from-bicep.sh b/scripts/generate-values-from-bicep.sh new file mode 100755 index 000000000..120284fca --- /dev/null +++ b/scripts/generate-values-from-bicep.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# Reads the outputs of an already-provisioned `deploy/azure/` Bicep +# deployment and generates a Helm values file from them. Closes the gap +# between Bicep's outputs and a hand-authored values.yaml, which is where +# transcription mistakes (wrong Redis port, stale Key Vault name, etc.) have +# historically crept in. Does not touch Helm at all — once this writes your +# values file, install the chart yourself (see deploy/helm/scope/README.md). +# +# Usage: +# scripts/generate-values-from-bicep.sh --resource-group [options] +# +# Options: +# --resource-group Resource group the Bicep template deployed into +# (required). +# --deployment-name Deployment within that resource group to read +# outputs from. Optional: if omitted, the most +# recent deployment whose outputs match this +# template (keyVaultName, redisHostName, +# workloadIdentityClientId all present) is used. +# This combination excludes nested module +# deployments (e.g. a `keyvault` module), which +# can re-expose a subset of the same output names. +# Pass this explicitly to pick a different one. +# --values-file Where to write the generated values file +# (default: my-values.yaml). If this file already +# exists, the script refuses to overwrite it +# unless --force is also given. +# --image-registry Baked into the generated file as +# global.imageRegistry (e.g. your own fork's GHCR +# namespace, or the Bicep-provisioned ACR). Omit +# to use the chart's ghcr.io/microsoft default. +# --force Overwrite --values-file if it already exists. +# -h, --help Show this help. +# +# If the deployment also has worker-identity outputs +# (workerWorkloadIdentityClientId/workerWorkloadIdentityServiceAccountName — +# only present if you provisioned the ACP worker identity), those are +# written too. Missing is not an error: older deployments, or ones that +# never provisioned worker identity, just won't get the worker.* keys, and +# the chart falls back to its defaults (empty client ID, required only once +# you enable a worker — see WORKERS.md). +# +# Requires: az (logged in, correct subscription active), jq. +set -euo pipefail + +RESOURCE_GROUP="" +DEPLOYMENT_NAME="" +VALUES_FILE="my-values.yaml" +IMAGE_REGISTRY="" +FORCE=false + +usage() { + sed -n '2,38p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --resource-group) RESOURCE_GROUP="$2"; shift 2 ;; + --deployment-name) DEPLOYMENT_NAME="$2"; shift 2 ;; + --values-file) VALUES_FILE="$2"; shift 2 ;; + --image-registry) IMAGE_REGISTRY="$2"; shift 2 ;; + --force) FORCE=true; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +for bin in az jq; do + command -v "$bin" >/dev/null 2>&1 || { echo "ERROR: $bin not found on PATH" >&2; exit 1; } +done + +[[ -n "$RESOURCE_GROUP" ]] || { echo "ERROR: --resource-group is required" >&2; usage; exit 1; } + +if [[ -f "$VALUES_FILE" && "$FORCE" != true ]]; then + echo "ERROR: $VALUES_FILE already exists. Pass --force to overwrite it or --values-file to write elsewhere." >&2 + exit 1 +fi + +if [[ -z "$DEPLOYMENT_NAME" ]]; then + echo "==> --deployment-name not given; finding the most recent matching deployment in $RESOURCE_GROUP" + DEPLOYMENT_NAME="$( + az deployment group list --resource-group "$RESOURCE_GROUP" -o json \ + | jq -r ' + [.[] | select( + .properties.outputs.keyVaultName != null + and .properties.outputs.redisHostName != null + and .properties.outputs.workloadIdentityClientId != null + )] + | sort_by(.properties.timestamp) + | reverse + | .[0].name // empty + ' + )" + [[ -n "$DEPLOYMENT_NAME" ]] || { + echo "ERROR: no deployment in '$RESOURCE_GROUP' has this template's outputs. Pass --deployment-name explicitly." >&2 + exit 1 + } + echo " Using deployment: $DEPLOYMENT_NAME" +fi + +echo "==> Reading Bicep outputs from $RESOURCE_GROUP/$DEPLOYMENT_NAME" +OUTPUTS_JSON="$(az deployment group show -g "$RESOURCE_GROUP" -n "$DEPLOYMENT_NAME" --query properties.outputs -o json)" + +get_output() { + jq -r --arg k "$1" '.[$k].value // empty' <<<"$OUTPUTS_JSON" +} + +TENANT_ID="$(az account show --query tenantId -o tsv)" +KEY_VAULT_NAME="$(get_output keyVaultName)" +KEY_VAULT_URI="$(get_output keyVaultUri)" +WORKLOAD_IDENTITY_CLIENT_ID="$(get_output workloadIdentityClientId)" +SERVICE_ACCOUNT_NAME="$(get_output workloadIdentityServiceAccountName)" +STORAGE_ACCOUNT_NAME="$(get_output storageAccountName)" +REDIS_HOST_NAME="$(get_output redisHostName)" +REDIS_PORT="$(get_output redisPort)" +# Optional: only present if the Bicep deployment provisioned the ACP worker +# identity. Not required below — see WORKERS.md. +WORKER_WORKLOAD_IDENTITY_CLIENT_ID="$(get_output workerWorkloadIdentityClientId)" +WORKER_SERVICE_ACCOUNT_NAME="$(get_output workerWorkloadIdentityServiceAccountName)" + +for pair in \ + "tenantId:$TENANT_ID" \ + "keyVaultName:$KEY_VAULT_NAME" \ + "keyVaultUri:$KEY_VAULT_URI" \ + "workloadIdentityClientId:$WORKLOAD_IDENTITY_CLIENT_ID" \ + "workloadIdentityServiceAccountName:$SERVICE_ACCOUNT_NAME" \ + "storageAccountName:$STORAGE_ACCOUNT_NAME" \ + "redisHostName:$REDIS_HOST_NAME" \ + "redisPort:$REDIS_PORT" +do + key="${pair%%:*}" + val="${pair#*:}" + if [[ -z "$val" ]]; then + echo "ERROR: value for '$key' was empty. Check --resource-group/--deployment-name point at the right Bicep deployment." >&2 + exit 1 + fi +done + +if [[ -z "$WORKER_WORKLOAD_IDENTITY_CLIENT_ID" || -z "$WORKER_SERVICE_ACCOUNT_NAME" ]]; then + echo " Note: worker-identity outputs not found on this deployment — skipping worker.* values. Bring the ACP workers online only after re-provisioning with that identity (see WORKERS.md)." +fi + +echo "==> Writing $VALUES_FILE" +{ + if [[ -n "$IMAGE_REGISTRY" ]]; then + cat < "$VALUES_FILE" + +echo " Done. Review/edit $VALUES_FILE (e.g. to add ingress hosts), then install" +echo " the chart — see deploy/helm/scope/README.md." diff --git a/scripts/get-copilot-token.ts b/scripts/get-copilot-token.ts new file mode 100644 index 000000000..8cb1cf04d --- /dev/null +++ b/scripts/get-copilot-token.ts @@ -0,0 +1,87 @@ +#!/usr/bin/env npx tsx +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Get a GitHub OAuth token for Copilot via device-code flow. + * + * Uses VS Code's OAuth app (client_id: 01ab8ac9400c4e429b23) with NO scopes. + * The resulting scopeless token can be exchanged for a VSCODE_COPILOT_CHAT_TOKEN. + * + * Usage: npx tsx scripts/get-copilot-token.ts + * Output: prints the token to stdout + * + * Example: export COPILOT_GITHUB_TOKEN=$(pnpm -s get-copilot-token) + * + * Based on: vscode-copilot-evaluation/src/proxy/ghOAuthTokenFetcher.ts + */ +import { execSync } from "child_process"; + +const CLIENT_ID = "01ab8ac9400c4e429b23"; +const DEVICE_CODE_URL = "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/login/device/code"; +const ACCESS_TOKEN_URL = "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/login/oauth/access_token"; + +async function main() { + // Step 1: Request device code + const codeResponse = await fetch(DEVICE_CODE_URL, { + method: "POST", + headers: { Accept: "application/json", "Content-Type": "application/json" }, + body: JSON.stringify({ client_id: CLIENT_ID }), + }); + const codeData = await codeResponse.json(); + + console.error("=".repeat(80)); + console.error("ATTENTION: To authenticate with GitHub for Copilot:"); + console.error(); + console.error(` 1. Copy this code: ${codeData.user_code}`); + console.error(` 2. Open: ${codeData.verification_uri}`); + console.error(` 3. Paste the code and approve access.`); + console.error(); + + // Try to open the browser + try { + const openCmd = process.platform === "darwin" ? "open" : process.platform === "win32" ? "start" : "xdg-open"; + execSync(`${openCmd} ${codeData.verification_uri}`, { stdio: "ignore" }); + console.error("(Browser opened automatically)"); + } catch { + console.error("(Could not open browser — please navigate manually)"); + } + console.error(); + console.error(`Polling for approval (up to ${codeData.expires_in}s)...`); + + // Step 2: Poll for access token + let expiresIn = codeData.expires_in; + let accessToken; + while (expiresIn > 0) { + await new Promise((r) => setTimeout(r, codeData.interval * 1000)); + expiresIn -= codeData.interval; + + const tokenResponse = await fetch(ACCESS_TOKEN_URL, { + method: "POST", + headers: { Accept: "application/json", "Content-Type": "application/json" }, + body: JSON.stringify({ + client_id: CLIENT_ID, + device_code: codeData.device_code, + grant_type: "urn:ietf:params:oauth:grant-type:device_code", + }), + }); + const tokenData = await tokenResponse.json(); + + if (tokenData.access_token) { + accessToken = tokenData.access_token; + break; + } + } + + if (!accessToken) { + console.error("❌ Failed to obtain access token within the time limit."); + process.exit(1); + } + + console.log(accessToken); +} + +main().catch((err) => { + console.error("❌", err.message); + process.exit(1); +});