diff --git a/.github/compose/compose.yaml b/.github/compose/compose.yaml index 95f2d81c..df2d9d73 100644 --- a/.github/compose/compose.yaml +++ b/.github/compose/compose.yaml @@ -10,6 +10,14 @@ services: KUBECONFIG: /home/coder/.kube/config volumes: - ${CODER_KUBECONFIG}:/home/coder/.kube/config:ro + # provisionerd resolves its provisioning binary by LookPath("terraform") - there is no + # OpenTofu-specific config surface at all (see DESIGN.md). This bind-mounts an OpenTofu + # binary directly over that lookup path so provisionerd applies the template with tofu + # while still finding a binary literally named "terraform". The image's real terraform + # binary at that path is root-owned (0755, root:root) and the container runs as uid 1000, + # so overwriting it from inside the container is not an option - the bind mount replaces + # the mount point at the kernel level regardless of that ownership. + - ${CODER_TOFU_BINARY}:/usr/local/bin/terraform:ro networks: default: kind: diff --git a/.github/renovate.json b/.github/renovate.json index ed28b755..fd4b6397 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -25,7 +25,7 @@ "github>ppat/renovate-presets#v0.3.0", "github>ppat/renovate-presets:dev-tools#v0.3.0", "github>ppat/renovate-presets:github-actions#v0.3.0", - "github>ppat/coder//.github/renovate/template-terraform-provider", + "github>ppat/coder//.github/renovate/template-opentofu-provider", "github>ppat/coder//.github/renovate/image-cli-tools", "github>ppat/coder//.github/renovate/exceptions" ], diff --git a/.github/renovate/exceptions.json b/.github/renovate/exceptions.json index bc7fda8b..2e981a3c 100644 --- a/.github/renovate/exceptions.json +++ b/.github/renovate/exceptions.json @@ -2,18 +2,16 @@ "$schema": "https://docs.renovatebot.com/renovate-schema.json", "packageRules": [ { - "commitMessageTopic": "terraform", - "description": "group terraform upgrades together", - "groupName": "terraform-{{newVersion}}", + "commitMessageTopic": "opentofu", + "description": "group opentofu upgrades together", + "groupName": "opentofu-{{newVersion}}", "matchManagers": [ - "mise", - "terraform-version" + "mise" ], "matchPackageNames": [ - "hashicorp/terraform", - "terraform" + "opentofu/opentofu" ], - "semanticCommitScope": "terraform-version" + "semanticCommitScope": "opentofu-version" }, { "automerge": false, diff --git a/.github/renovate/template-terraform-provider.json b/.github/renovate/template-opentofu-provider.json similarity index 94% rename from .github/renovate/template-terraform-provider.json rename to .github/renovate/template-opentofu-provider.json index d3805ff2..318ceeee 100644 --- a/.github/renovate/template-terraform-provider.json +++ b/.github/renovate/template-opentofu-provider.json @@ -3,7 +3,7 @@ "packageRules": [ { "addLabels": [ - "template:terraform-provider" + "template:opentofu-provider" ], "commitMessageTopic": "{{packageName}}", "groupName": "{{datasource}}-{{packageName}}", @@ -13,7 +13,7 @@ "matchFileNames": [ "templates/**" ], - "semanticCommitScope": "terraform-provider", + "semanticCommitScope": "opentofu-provider", "separateMajorMinor": true, "separateMinorPatch": false, "separateMultipleMajor": true, diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 34ba4451..038c5fc3 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -32,7 +32,7 @@ jobs: - .github/renovate/** shellscripts: - '**.sh' - terraform: + opentofu: - '**.tf' - '**/.terraform-version' - '**/.terraform.lock.hcl' @@ -41,12 +41,12 @@ jobs: - '**.yaml' git_ref: ${{ github.head_ref || github.ref }} - terraform-dirs: + opentofu-dirs: needs: [detect-changes] runs-on: ubuntu-24.04 timeout-minutes: 2 outputs: - terraform_dirs: ${{ steps.determine-tf-dirs.outputs.dirs }} + opentofu_dirs: ${{ steps.determine-tf-dirs.outputs.dirs }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -54,22 +54,22 @@ jobs: fetch-depth: 1 persist-credentials: false - - name: Determine Terraform directories to validate + - name: Determine OpenTofu directories to validate id: determine-tf-dirs # yamllint disable-line rule:indentation run: | if [[ "${{ github.event_name }}" != "pull_request" ]]; then - # For workflow_dispatch, validate all terraform directories + # For workflow_dispatch, validate all opentofu directories TF_DIRS=$(find templates/ -maxdepth 2 -mindepth 2 -type d -not -name '.*' 2>/dev/null | sort | jq -R -s -c 'split("\n") | map(select(length > 0))') - elif [[ "${{ fromJSON(needs.detect-changes.outputs.results).terraform_any_changed }}" == "true" ]]; then + elif [[ "${{ fromJSON(needs.detect-changes.outputs.results).opentofu_any_changed }}" == "true" ]]; then # For pull requests, check which files changed - CHANGED_FILES="${{ fromJSON(needs.detect-changes.outputs.results).terraform_all_changed_files }}" + CHANGED_FILES="${{ fromJSON(needs.detect-changes.outputs.results).opentofu_all_changed_files }}" TEMPLATE_CHANGES=$(echo "$CHANGED_FILES" | tr ' ' '\n' | grep '^templates/' || true) # Otherwise only validate affected templates TF_DIRS=$(echo "$TEMPLATE_CHANGES" | grep -o 'templates/[^/]*' | sort | uniq | tr -d ' ' | jq -R -s -c 'split("\n") | map(select(length > 0))') else TF_DIRS="[]" - echo "No Terraform changes detected" + echo "No OpenTofu changes detected" fi echo "dirs=$TF_DIRS" >> "$GITHUB_OUTPUT" @@ -128,13 +128,13 @@ jobs: git_ref: ${{ github.head_ref || github.ref }} files: ${{ github.event_name != 'pull_request' && 'ALL' || fromJSON(needs.detect-changes.outputs.results).shellscripts_all_changed_files }} - terraform: - needs: [terraform-dirs] - if: ${{ github.event_name != 'pull_request' || needs.terraform-dirs.outputs.terraform_dirs != '[]' }} - uses: ppat/github-workflows/.github/workflows/lint-terraform.yaml@5a96ced8ceefd58062f6b91ee9d6f3a31cd06e1c # v6.0.0 + opentofu: + needs: [opentofu-dirs] + if: ${{ github.event_name != 'pull_request' || needs.opentofu-dirs.outputs.opentofu_dirs != '[]' }} + uses: ppat/github-workflows/.github/workflows/lint-opentofu.yaml@54ce2e4ac00927d6164d032b321f0142aab01c69 # v6.1.0 with: git_ref: ${{ github.head_ref || github.ref }} - tf_dirs: ${{ needs.terraform-dirs.outputs.terraform_dirs }} + tofu_dirs: ${{ needs.opentofu-dirs.outputs.opentofu_dirs }} yaml: needs: [detect-changes] diff --git a/.github/workflows/test-template.yaml b/.github/workflows/test-template.yaml index f5bff1b9..88f845d2 100644 --- a/.github/workflows/test-template.yaml +++ b/.github/workflows/test-template.yaml @@ -18,7 +18,7 @@ permissions: env: CODER_URL: http://localhost:7080 - COMPOSE_FILE: .github/compose/compose.yaml + COMPOSE_FILE: current/.github/compose/compose.yaml KIND_CLUSTER: coder-template-test TEMPLATE_NAME: homelab-workspace-test WORKSPACE_NAME: template-test @@ -28,11 +28,16 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup repository and tools + # Checks the repo out into current/ (hence the current/ prefix on every path + # reference below) and, with no mise_toml override, installs straight from + # current/mise.toml - the same `opentofu` pin the authoring toolchain uses + # locally, so there's no second version string to drift from it. + uses: ppat/homelab-ops-actions/actions/setup-repository-tools@f66c9acb40309305e0e1fd3179406b85647aa54c # v2.3.2 with: - fetch-depth: 1 - persist-credentials: false + current_git_ref: ${{ github.head_ref || github.ref }} + current_repository: ${{ github.repository }} + token: ${{ github.token }} - name: Create Kubernetes test cluster run: kind create cluster --name "${KIND_CLUSTER}" @@ -55,6 +60,20 @@ jobs: sed -Ei "s#https://127\\.0\\.0\\.1:[0-9]+#https://${KIND_CLUSTER}-control-plane:6443#" "${CODER_KUBECONFIG}" chmod 644 "${CODER_KUBECONFIG}" echo "CODER_KUBECONFIG=${CODER_KUBECONFIG}" >> "${GITHUB_ENV}" + # $GITHUB_ENV only takes effect for steps that run AFTER this one - it does + # nothing for the `docker compose up` a few lines below, in this same step, + # which is why CODER_KUBECONFIG above works (it's this step's own `env:`, + # set from the start) while a same-step echo-then-use of CODER_TOFU_BINARY + # silently resolved empty and produced docker compose's opaque "invalid + # spec: empty section between colons". Exporting it directly makes it visible + # to this step's own `docker compose up`; the $GITHUB_ENV write is still needed + # so later steps' `docker compose exec`/`logs` calls see it too, since compose + # re-resolves the whole file - volumes included - on every subcommand. + # cd'd into current/ for this one command only - that's where mise.toml + # actually is, unlike this step's own default working directory. + CODER_TOFU_BINARY="$(cd current && mise which tofu)" + export CODER_TOFU_BINARY + echo "CODER_TOFU_BINARY=${CODER_TOFU_BINARY}" >> "${GITHUB_ENV}" docker compose -f "${COMPOSE_FILE}" up --detach - name: Wait for local Coder @@ -72,7 +91,7 @@ jobs: coder server create-admin-user --username ci --email ci@example.invalid --password ci-password - name: Install and log in to Coder CLI - uses: ./actions/coder-cli-login + uses: ./current/actions/coder-cli-login with: coder_url: ${{ env.CODER_URL }} coder_email: ci@example.invalid @@ -88,9 +107,9 @@ jobs: echo "workspace_image=ghcr.io/ppat/coder-workspace:${RELEASE_TAG}" >> "${GITHUB_OUTPUT}" - name: Publish template - uses: ./actions/coder-template-push + uses: ./current/actions/coder-template-push with: - template_dir: templates/kubernetes/homelab-workspace + template_dir: current/templates/kubernetes/homelab-workspace template_name: ${{ env.TEMPLATE_NAME }} template_version: ${{ github.sha }} workspace_image: ${{ steps.image.outputs.workspace_image }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1bc8453f..22470b29 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,8 +52,8 @@ repos: - repo: https://github.com/gruntwork-io/pre-commit rev: v0.1.30 hooks: - - id: terraform-fmt - - id: terraform-validate + - id: tofu-fmt + - id: tofu-validate - id: tflint args: - "--config=__GIT_ROOT__/.tflint.hcl" diff --git a/CLAUDE.md b/CLAUDE.md index 39f65f3f..6e124562 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,9 +4,9 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Repository purpose -This repo holds [Coder](https://coder.com/) workspace **templates** (Terraform) and the **container images** those templates provision — it is not an application codebase. There is currently one template/image pair, both named `homelab-workspace`: +This repo holds [Coder](https://coder.com/) workspace **templates** (OpenTofu) and the **container images** those templates provision — it is not an application codebase. There is currently one template/image pair, both named `homelab-workspace`: -- `templates/kubernetes/homelab-workspace/` — Terraform template (`coder/coder` + `hashicorp/kubernetes` providers), deployed to a personal Kubernetes cluster. +- `templates/kubernetes/homelab-workspace/` — OpenTofu template (`coder/coder` + `hashicorp/kubernetes` providers), deployed to a personal Kubernetes cluster. - `images/homelab-workspace/Dockerfile` — the Ubuntu-based image the template's pod runs. Almost all day-to-day change here is dependency bumps (Renovate) or edits to the template/image; there's no application logic, unit tests, or build step in the traditional sense. @@ -15,18 +15,18 @@ Almost all day-to-day change here is dependency bumps (Renovate) or edits to the ## Commands -No package-manager project lives here (`mise.toml` just pins `bun`/`node`/`terraform`/`tflint` tool versions). Local validation is `pre-commit`: +No package-manager project lives here (`mise.toml` just pins `bun`/`node`/`opentofu`/`tflint` tool versions). Local validation is `pre-commit`: ```bash -pre-commit run --all-files # yamllint, markdownlint, shellcheck, hadolint, commitlint, terraform fmt/validate/tflint +pre-commit run --all-files # yamllint, markdownlint, shellcheck, hadolint, commitlint, tofu fmt/validate/tflint ``` -Terraform checks scoped to the one template directory: +OpenTofu checks scoped to the one template directory: ```bash cd templates/kubernetes/homelab-workspace -terraform fmt -check -terraform validate +tofu fmt -check +tofu validate tflint --config=../../../.tflint.hcl ``` @@ -44,7 +44,7 @@ There is no local way to build/publish the image against the private registry ca Commitlint (`commitlint.config.js`) enforces Conventional Commits. -- Allowed scopes only: `cli-tools`, `dev-tools`, `deps`, `github-actions`, `release`, `renovate`, `terraform-provider`, `terraform-version`, or no scope. An unlisted scope fails commit-msg validation. +- Allowed scopes only: `cli-tools`, `dev-tools`, `deps`, `github-actions`, `opentofu-provider`, `opentofu-version`, `release`, `renovate`, or no scope. An unlisted scope fails commit-msg validation. - Body lines ≤120 chars, except `chore(deps)` commits (Renovate generates these verbatim, so that rule is relaxed for them). ## Release flow @@ -53,7 +53,7 @@ Commitlint (`commitlint.config.js`) enforces Conventional Commits. 1. It creates or updates the release PR from Conventional Commit history. Merging that PR creates the GitHub release/tag and the next release PR. 2. The published GitHub release triggers `.github/workflows/publish.yaml`, which builds the workspace image for `linux/amd64,linux/arm64` and pushes it to the private registry. -3. The same publish workflow pushes the Terraform template to the live Coder deployment, tagged with the released version. +3. The same publish workflow pushes the template to the live Coder deployment, tagged with the released version. `coder template push` is a local HCL parse — it never execs terraform/tofu itself (see the gotcha below) — the binary that actually applies it belongs to whichever provisioner runs it, and that provisioner lives in `homelab-ops-kubernetes-apps`, out of this repo's scope (see `homelab-ops-kubernetes-apps#3900` for that repo's own OpenTofu swap). PR validation is deliberately narrower than publishing: `test-image.yaml` builds image changes against the private registry cache, while `test-template.yaml` publishes template changes to a local Coder/Postgres test control plane @@ -84,7 +84,7 @@ Quick orientation map — for what each piece is *for* and the decisions behind **Image** (`images/homelab-workspace/Dockerfile`): three build stages — `base` (minimal bootstrap deps) → `system-base` (`unminimize` + full interactive toolset) → final stage (env vars into `/etc/environment`, fixed-UID/GID `coder` user, `USER coder`). All `apt`-touching `RUN` steps use BuildKit cache mounts — match that pattern when adding packages. -**Renovate** (`.github/renovate.json` + `.github/renovate/*.json`): extends shared `ppat/renovate-presets` plus repo-local rules in `exceptions.json`, `image-cli-tools.json`, `template-terraform-provider.json` that set different automerge delays per dependency class. +**Renovate** (`.github/renovate.json` + `.github/renovate/*.json`): extends shared `ppat/renovate-presets` plus repo-local rules in `exceptions.json`, `image-cli-tools.json`, `template-opentofu-provider.json` that set different automerge delays per dependency class. **Local test control plane** (`.github/compose/compose.yaml`): brings up a disposable Coder/Postgres pair for `test-template.yaml`, versioned to match the live deployment in `homelab-ops-kubernetes-apps` (see [TESTING.md](TESTING.md)). Named `compose.yaml`, not e.g. `coder-template-test.yaml`, for two reasons: it runs with no `-f` flag from inside the directory, and it's what makes Renovate's built-in `docker-compose` manager match the file at all — that manager is enabled by default and needs no config here, but only extracts a version it can see as a literal in an `image:` line, so the versions are pinned there directly rather than behind a `${VAR}`/`.env` indirection it can't see through. @@ -97,6 +97,7 @@ Things that look arbitrary in the code but are load-bearing (full reasoning in [ - `deployment.tf`'s `system` volume is an `empty_dir`, rebuilt from the image on every pod start — a fix to anything under `/usr`, `/etc`, `/var` must go in the image or the init script, not be treated as a one-time patch. - The Dockerfile writes shared env vars to `/etc/environment` rather than using `ENV`, because `PATH` needs to be extended by a script running after the image is built, not fixed at build time. - `parameters.tf`'s `local.validated_*` allowlist is the only thing stopping `system_packages`/`preferred_nodes` from injecting shell metacharacters into the init container — any new parameter whose value reaches a shell must go through the same validate-then-use step. +- **This repo's disposable test control plane (`.github/compose/compose.yaml`, `test-template.yaml`) proves the template applies under OpenTofu before that mechanism ever reaches a real deployment.** `coder template push` never execs terraform/tofu itself (it's a local, pure-Go HCL parse); the binary that applies the template is whatever `provisionerd` finds via `LookPath("terraform")`, and Coder's provisioner has zero vendor awareness — only a numeric `terraform version -json` range check (currently 1.1.0–1.15.9), which `tofu version -json` satisfies because OpenTofu deliberately keeps the same JSON key. The official `ghcr.io/coder/coder` image ships its real Terraform binary at `/usr/local/bin/terraform`, root-owned 0755, while the container runs as uid 1000 — so the coder user cannot overwrite it in place, and a bind mount (`${CODER_TOFU_BINARY}:/usr/local/bin/terraform:ro` in `compose.yaml`) is what actually swaps it, not an in-container step. `test-template.yaml` sources that binary via `ppat/homelab-ops-actions/actions/setup-repository-tools` — the same checkout+mise action every reusable lint workflow in `ppat/github-workflows` already uses — reading `mise.toml`'s own `opentofu` pin directly rather than a second hardcoded version string. `opentofu/setup-opentofu` is rejected outright by this repo's locked-down Actions allowlist ("selected" patterns) at workflow-intake time — a real failure mode invisible to `actionlint`, `yamllint`, or any local run, surfacing only as a bare "workflow file issue" with zero jobs created. `setup-repository-tools` does its own checkout into `current/`, so every path this job touches after that step is prefixed accordingly. The local recipe in [TESTING.md](TESTING.md) sources the same mise-installed `tofu` already on `PATH`. Verified directly, not inferred: `docker compose exec coder terraform version` reports an `OpenTofu vX.Y.Z` banner, and a real template push's provisioner log reads `OpenTofu 1.12.6` / `OpenTofu has been successfully initialized!` before `Apply complete!`. **The live Coder deployment's provisioner is a separate repo's decision** — `homelab-ops-kubernetes-apps#3900` makes the equivalent swap there (a Kubernetes init-container-plus-`emptyDir`, since that cluster's containerd can't do single-file image-volume subPath mounts), and the HCL this template emits is fully backward compatible with real Terraform regardless, so this repo's own correctness never depended on that PR landing. - `script-memory-watchdog.sh` computes headroom as `memory.max − U`, where `U` sums only the *unreclaimable* fields of `memory.stat` (`anon`, `shmem`, `unevictable`, `slab_unreclaimable`, `kernel_stack`, `pagetables`, `sec_pagetables`, `percpu`, `sock`). Do not "simplify" it to `memory.current` or to `memory.stat`'s `kernel` roll-up: on the live pod those read 96% and 42% of the limit while true `U` is 28%. Nothing acts on this number any more — it is pod-level context for the per-process rows and the honest figure published in the workspace UI. - `calibration.csv`'s `du_bytes_per_s` column is a per-sample instantaneous rate (`(M_U - PREV_U) / SAMPLE_INTERVAL`, 10s by default), not a drift rate, and naively averaging it overstates drift roughly fourfold: ~19.8 kB/s (~68 MB/hour) naive mean on the live pod versus ~18.4 MB/hour computed from `u_mb` across two `event=census` rows 26.1h apart. The naive mean is dominated by a handful of 10s allocation/GC spikes (+102 to −58 MB/s) that an hourly comparison washes out. The column and its caveat are documented together at its declaration in the script; nothing currently reads this column programmatically, so the trap is for a human, not a bug. - **The watchdog is a drift policer, not an OOM preventer, and the difference is measured.** The graded L1–L4 shedding ladder that used to be here was removed, not tuned: the recorded kills are 70–220 MB/s spikes that go from idle to dead inside a minute, a live reproduction climbed the ladder correctly and logged `no-candidates` because the runaway was not in the tree it managed, and the entire editor tree it could shed is ~0.7 GiB — five seconds of that growth. Before re-adding anything reactive, establish that a poll loop can see the event at all. What the loop *is* good at is MB-per-minute growth in the standing population, which is what it now does. @@ -136,5 +137,5 @@ This repo is only the image+template layer. When a task's real cause is above or ## Working on a template or image change -1. Make the change, run `pre-commit run --all-files` and the scoped `terraform validate`/`tflint` commands above. +1. Make the change, run `pre-commit run --all-files` and the scoped `tofu validate`/`tflint` commands above. 2. Follow [TESTING.md](TESTING.md) to exercise it via `test_mode` before merging — merging to `main` publishes to the real template/image with no separate promotion step. diff --git a/DESIGN.md b/DESIGN.md index e679cc6d..acf283ec 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -14,7 +14,7 @@ flowchart LR Renovate --> Template Image[Container image build] --> Push[coder template push] - Template[Terraform template] --> Push + Template[OpenTofu template] --> Push Push --> Live[Live Coder deployment
provisions workspace pods] ``` @@ -22,7 +22,7 @@ The image and template are two halves of one release, not independent artifacts: The "live Coder deployment" this pushes to is itself provisioned separately, and what an operator actually works in is assembled from more layers than these two — see [Where the workspace environment comes from](#where-the-workspace-environment-comes-from) below. -Renovate feeds both halves continuously (Terraform provider versions, image package/tool versions, GitHub Actions), so the day-to-day work in this repo is mostly reviewing and merging those bumps rather than writing new template/image logic. +Renovate feeds both halves continuously (OpenTofu provider versions, image package/tool versions, GitHub Actions), so the day-to-day work in this repo is mostly reviewing and merging those bumps rather than writing new template/image logic. ## Where the workspace environment comes from @@ -61,6 +61,8 @@ The rule that ties the layers together: a package or tool belongs in the *lowest **No staging environment, so the release pipeline carries its own rehearsal path.** There's exactly one live template and one live cluster — no separate staging Coder deployment to try changes against first. Rather than accept "every merge to main is a live-fire test," the release pipeline itself can run in a mode that exercises a real build and a real (but disposable, clearly-named) template push without touching the production template or its persistent state. That path is what makes it safe to iterate on template/image changes at the same pace as everything else in the repo. See [TESTING.md](TESTING.md) for how to use it. +**The template is authored in OpenTofu; the live deployment's provisioner is a separate repo's decision, not this one's.** Coder has no concept of a provisioning tool beyond "whatever binary is named `terraform` on `PATH`, in a supported numeric version range" — there is no vendor check, so a `tofu` binary standing in for that name applies the template identically. This repo can only control the provisioner in its own disposable test control plane (`.github/compose/compose.yaml`), which bind-mounts an OpenTofu binary over the official Coder image's own `terraform` path — both to dogfood the toolchain this repo now authors in, and because that swap is what surfaces a real Terraform/OpenTofu behavioral divergence, if this template's HCL ever grows one, before it reaches a real deployment. The live deployment's provisioner is owned by `homelab-ops-kubernetes-apps` — `homelab-ops-kubernetes-apps#3900` makes the same swap there, via a Kubernetes-native init-container-plus-`emptyDir` mechanism instead of a Docker bind mount — that cluster's containerd doesn't support single-file subPath mounts on image volumes, only whole-directory. + **`/tmp` is node-local scratch space, deliberately not the shared home volume.** Every workspace's `/tmp` used to be whatever the container's writable overlay layer gave it for free - fast, but unbounded, and on the node's root filesystem. On the operator's own long-lived workspace that grew to several GiB (dominated by Claude Code's own scratch directory, `$TMPDIR/claude-/...`, which agent sessions use for downloads and experiments) and pushed the node toward the kubelet's disk-pressure eviction threshold - a risk to every other pod on that node, not just the workspace that caused it. The home PVC has ample free space, but is NFS-backed, which is a bad fit for what actually lives in `/tmp`: build caches and compiler intermediates are exactly the write-heavy, latency-sensitive workload NFS handles worst. A plain `empty_dir` would keep `/tmp` fast but doesn't fix anything, because `empty_dir` lives on the same constrained node root filesystem the container overlay already did. The fix is a Kubernetes "generic ephemeral volume" on a Longhorn storage class that is both node-local (so still fast) and backed by a separate, much larger partition on the same node than the root filesystem is - see the comment on the `tmp` volume in `deployment.tf` for the specific class and why it's the non-replicated one (scratch data costs nothing to lose) rather than the default replicated class other PVCs in this cluster use. Its size is a fixed ceiling rather than left unbounded, so a runaway consumer now fails predictably inside its own volume instead of eventually pressuring the node. Because this volume's lifecycle is tied to the Pod rather than the container, something has to wipe it on every container start, so a container restart within a live Pod doesn't just inherit whatever the previous container left behind. **That wipe runs in the container entrypoint because `/tmp` is not only scratch space - the Coder agent lives there too.** The first attempt put the wipe in `script-agent-startup.sh`, which is the wrong side of the ordering and shipped a broken template: Coder's generated bootstrap downloads the agent CLI into a per-boot `mktemp -d -t coder.XXXXXX` directory under `/tmp`, makes it the agent's working directory, appends it to the PATH of every session and script the agent runs, and *then* runs the startup script - which deleted it, leaving every `coder stat` metadata panel reporting `coder: command not found` while the agent process itself carried on from an unlinked binary. The tempting repair is to exclude the agent's own paths from the wipe, and it is the wrong one: that set is an implementation detail of whatever agent version the control plane happens to serve, and it is not even uniformly named - v2.35.3 owns a random-suffixed `coder.XXXXXX/`, `coder-agent.sock` (a hardcoded absolute path, not a `TMPDIR`-relative one), rotated `coder-agent*.log` files, `coder-script-data/`, `coder-screen/`, and `boundary-audit.sock`, which carries no `coder` prefix at all. An allowlist would stop matching on some future upgrade and fail exactly as invisibly as the original bug. The workspace container's `command` is therefore a small entrypoint script that wipes `/tmp` and `exec`s Coder's bootstrap: it is the only hook that runs on a container-only restart within a live Pod (init containers run once per Pod), it runs before the agent exists so there is nothing to exclude, and `exec` keeps the agent as PID 1 for orphan reaping and the liveness probe. The wipe there is best-effort rather than fatal, because that entrypoint is the only path to a running agent and a hard failure would be a `CrashLoopBackOff` nobody can shell into; it records its outcome instead, and `script-agent-startup.sh` asserts both that outcome and that the agent CLI still resolves and runs - the check the original change lacked, which turns a recurrence into a failed startup script in the workspace UI instead of eight quietly broken metadata panels. diff --git a/README.md b/README.md index 4ff743b7..94ca0734 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # coder -Terraform templates and container images for [Coder](https://coder.com/) workspaces, running on a personal Kubernetes homelab cluster. +OpenTofu templates and container images for [Coder](https://coder.com/) workspaces, running on a personal Kubernetes homelab cluster. ## Contents -- **`templates/kubernetes/homelab-workspace/`** — the Coder/Kubernetes Terraform template that provisions a workspace pod. +- **`templates/kubernetes/homelab-workspace/`** — the Coder/Kubernetes OpenTofu template that provisions a workspace pod. - **`images/homelab-workspace/`** — the Ubuntu-based container image that pod runs. -Template and image are versioned and released together; see [DESIGN.md](DESIGN.md) for why. Dependency versions (Terraform providers, image packages, GitHub Actions) are kept current mostly by Renovate. +Template and image are versioned and released together; see [DESIGN.md](DESIGN.md) for why. Dependency versions (OpenTofu providers, image packages, GitHub Actions) are kept current mostly by Renovate. This repo is the middle of a larger stack: the Coder control plane is deployed separately (from the `homelab-ops-kubernetes-*` repos), and an operator's day-to-day tooling comes from their [dotfiles](../dotfiles) at provision time. See [DESIGN.md](DESIGN.md#where-the-workspace-environment-comes-from) for how the image, template, and those neighbours layer into one workspace environment. diff --git a/TESTING.md b/TESTING.md index 768e287b..22827a73 100644 --- a/TESTING.md +++ b/TESTING.md @@ -19,8 +19,15 @@ PR workflows are scoped directly by changed paths: - `test-image.yaml` runs for image changes. It delegates the multi-architecture build and private-registry cache to the shared image-build workflow, which connects to Tailscale for that registry. - `test-template.yaml` runs for template changes. It creates a local Coder/Postgres compose deployment, gives Coder - the test-cluster kubeconfig, publishes the template with the latest released GHCR workspace image, creates a - workspace, pings its agent three times with a timeout, and verifies SSH by running `env`. + the test-cluster kubeconfig and an OpenTofu binary bind-mounted over its bundled `terraform`, publishes the + template with the latest released GHCR workspace image, creates a workspace, pings its agent three times with a + timeout, and verifies SSH by running `env`. + +This test control plane is where this repo proves OpenTofu is the thing that actually *applies* the template — +the live deployment's provisioner is a separate repo's decision, out of this repo's control (see the +`CODER_TOFU_BINARY` gotcha in [CLAUDE.md](CLAUDE.md), and `homelab-ops-kubernetes-apps#3900` for that repo's own +swap). The template is OpenTofu-authored HCL that either binary applies identically, so this pins down what this +repo verified rather than carrying an untested assumption that OpenTofu behaves identically end to end. The temporary Coder deployment has no `CODER_ACCESS_URL`, so Coder creates its development tunnel. This is the workspace agent's route back from Kind; the runner continues to call Coder on localhost and template testing needs @@ -36,7 +43,8 @@ pinning the Dockerfile's `FROM` line gets. ## Running the template test locally `test-template.yaml` needs nothing CI has that a laptop doesn't — no secrets, no Tailscale, no private registry. With -`kind`, `docker compose`, `kubectl`, and the `coder` CLI on `PATH`, from the repo root: +`kind`, `docker compose`, `kubectl`, the `coder` CLI, and `tofu` (`mise install`, pinned in `mise.toml`) on `PATH`, +from the repo root: ```bash kind create cluster --name coder-template-test @@ -47,12 +55,16 @@ cp /tmp/kind-kubeconfig /tmp/coder-kubeconfig sed -Ei 's#https://127\.0\.0\.1:[0-9]+#https://coder-template-test-control-plane:6443#' /tmp/coder-kubeconfig chmod 644 /tmp/coder-kubeconfig -CODER_KUBECONFIG=/tmp/coder-kubeconfig docker compose -f .github/compose/compose.yaml up --detach +CODER_KUBECONFIG=/tmp/coder-kubeconfig CODER_TOFU_BINARY="$(mise which tofu)" \ + docker compose -f .github/compose/compose.yaml up --detach # once http://localhost:7080/api/v2/buildinfo responds: docker compose -f .github/compose/compose.yaml exec coder \ coder server create-admin-user --username ci --email ci@example.invalid --password ci-password -coder login http://localhost:7080 --username ci --password ci-password +CODER_SESSION_TOKEN="$(curl --fail --silent --show-error -X POST http://localhost:7080/api/v2/users/login \ + -H 'Content-Type: application/json' -H 'Accept: application/json' \ + --data '{"email":"ci@example.invalid","password":"ci-password"}' | jq --raw-output .session_token)" +CODER_SESSION_TOKEN="${CODER_SESSION_TOKEN}" coder login --use-token-as-session http://localhost:7080 coder template push --directory templates/kubernetes/homelab-workspace \ --var workspace_image=ghcr.io/ppat/coder-workspace: --var test_mode=true \ @@ -63,6 +75,11 @@ coder ping --num 3 --timeout 30s local-test coder ssh local-test -- env ``` +`docker compose exec coder terraform version` should report an `OpenTofu vX.Y.Z` banner, not a Terraform one — see +the `CODER_TOFU_BINARY` gotcha in [CLAUDE.md](CLAUDE.md) for why a binary literally named `terraform` is OpenTofu +here. (`coder login`'s `--username`/`--password` flags apply only to first-user bootstrap, not to logging in as an +already-created user — the token dance above is what `actions/coder-cli-login` does for the same reason.) + Tear down with `docker compose -f .github/compose/compose.yaml down --volumes` and `kind delete cluster --name coder-template-test`. `compose.yaml` uses Compose's default project-directory-from-file resolution, so the same commands also work as `docker compose up --detach` etc. from inside `.github/compose/` with diff --git a/commitlint.config.js b/commitlint.config.js index 8cf97073..16c267e5 100644 --- a/commitlint.config.js +++ b/commitlint.config.js @@ -38,10 +38,10 @@ module.exports = { 'dev-tools', 'deps', 'github-actions', + 'opentofu-provider', + 'opentofu-version', 'release', - 'renovate', - 'terraform-provider', - 'terraform-version' + 'renovate' ] ], // 'scope-empty': [2, 'always'], diff --git a/mise.toml b/mise.toml index 41f74942..163114b0 100644 --- a/mise.toml +++ b/mise.toml @@ -1,7 +1,7 @@ [tools] bun = "1.4.0" node = "24.19.0" -terraform = "1.15.9" +opentofu = "1.12.6" tflint = "0.64.0" [settings.npm] diff --git a/templates/kubernetes/homelab-workspace/.terraform.lock.hcl b/templates/kubernetes/homelab-workspace/.terraform.lock.hcl index 72d6bd3a..ba4473bd 100644 --- a/templates/kubernetes/homelab-workspace/.terraform.lock.hcl +++ b/templates/kubernetes/homelab-workspace/.terraform.lock.hcl @@ -1,7 +1,7 @@ -# This file is maintained automatically by "terraform init". +# This file is maintained automatically by "tofu init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/coder/coder" { +provider "registry.opentofu.org/coder/coder" { version = "2.18.0" constraints = "2.18.0" hashes = [ @@ -35,32 +35,39 @@ provider "registry.terraform.io/coder/coder" { ] } -provider "registry.terraform.io/hashicorp/kubernetes" { +provider "registry.opentofu.org/hashicorp/kubernetes" { version = "3.2.1" constraints = "3.2.1" hashes = [ - "h1:+7JyEgDwBopivx6mO959NaoetqLKTdwi+TiTiBfDY/0=", - "h1:+FgzP28F1P2LvmTWxtb63edwDyI2V9EBjPKBTM1L4C4=", - "h1:7o13eohIV7A59BWK02ydJfYfwDUp4oQgnYf5Lfn+dc4=", - "h1:Fr+b5Yyuvz6kfg55XMrRm6tbfqXGPrLgiUIupML+rNw=", - "h1:QxEYX+4ndd8XCcyiq8044Ofhkm6WVcNP5cG3PabjC9M=", - "h1:XccFuTe/eJ94vkoZNYUL3wsdXJwlxHguIY36jildRQ8=", - "h1:Y5SlUIoL8IKF44uAtoyqJHXx7fJqSWvuCzcGQQ0Nz2g=", - "h1:isSCfF3UQzPtjCdgLK32KJz9FN/YBjAAb+qDzmrFaZM=", - "h1:mcG69DdvaQvDNQzIo+SLVekECRLiNKavq5jbp/yieOU=", - "h1:mleH9HyRdgUhjURauw2LezYc6c7IxJKvmgQGy4fXn8E=", - "h1:zkgz+1VNLAjLM8WumAx5Y0AMK9tLSa1GvRT7XjpFmp4=", - "zh:067fe16a852d42e0f571712e36cb3e71855f917ea2041415e155f56ebc480d7f", - "zh:2815e174f8f0f032ea3a64f2196740ad000a39f88ae5646e7061bf15ed589f62", - "zh:2f94f6b689c59c43e596e724228f2861095d02c2a2ac2a257a4619667135ac75", - "zh:3e807310c84f11561b9ba06b978f03c46cfeca2e84ad0803d34d5d30a8a637cb", - "zh:5cba6f92202c60cac6898141356420709f5341b80ae4c360725cc647f86188ff", - "zh:72b841b6f0820d8f87c3d7c5a3611c35121ab9a4c1db4ea7a98b0319f209e474", - "zh:74770b892ee9b04829d92318d9e8ca96f8143b0c6c766e4141901908173fd01d", - "zh:7a723c8ebf9e218d0f7a0cfe6c0437f2b5eeb7ae015a14fad16e0f7fd9ef79ab", - "zh:a0f5073b2636a3894d4e9dd1b6853d5f324dd78728313bff79b842e5e9eca96f", - "zh:c13241cba993ef63a537beb6a1caf00e233bb045b50d197349530de0ee3276d5", - "zh:d52826f4b0227b7db99ea4a1d48f49a0bfb440563c92ebd2f8faec273c856c2d", - "zh:dc1cf5505a39a264a650b0830f74150ad02368787e5ead89e4007034f8f47831", + "h1:+VM77HEd3CgzxwQ0PtfOc5tQrRQ2FZ9YJj1ShiMTsoI=", + "h1:32YYVAK/LUIheXfaxxnwL4qzf6QgWaMVkByneQH8EpA=", + "h1:8WnHwjdwYmtpTJg5reAs9VGtKq67E3dQdIrcjDW1Xao=", + "h1:8pNFJAX4IXSaPr+f02FsD6bUlZrMT5rkPPnA5sm2szI=", + "h1:Ad9kERqM2KFLkW/beJ6VsV3QxD05o98ZjC/epjpDcnI=", + "h1:EtS/ztrc03Vy+u9Q6eflu+HYVMMhQIufMaHS+vlbprY=", + "h1:GROcvqLewzybvZa6LjWjU7EWE7aOttp9ovBkT1b8lhc=", + "h1:OR1GGSnvGgFNKA8ilnttn1cy5wT8N0qzGpbqWOZ+yG8=", + "h1:PJBBJz/9kOjaGjGXCQpVslstEbeTxQBx8zu/ZAvRHak=", + "h1:QzzyadpGU0iHlOKsXX/ge6/apcwRXmzVlJxSzVclM/I=", + "h1:Un9WHqrP13fUt64c7VcO60VfvJty0tnrX88b8wy/4SU=", + "h1:VDyHxveT5+lt75fY3oOpa1D/08i1+AC6Af5XoQXEK7E=", + "h1:glcBsCYHGLC1os31lZ22X0se9EyTjQO/1kVMeHj5uJ8=", + "h1:qGon41gIBlkXIbSpsovc5UJr4okWcjaw7KkgPFxhNlU=", + "h1:tsq7mekr30obQVOK83ZNAiRb7rzRSZnWxQLdjD1Rvo0=", + "zh:018e382285ab0ca5a68e4e4116e1cfc3a20c57194f4e678bd87654f5aecf5b14", + "zh:0e8d233199b6896a6b9379b2f41bfdcdbb42b80f5f1c4ed7d87825ce53bae639", + "zh:112221d98fae795d89e6a0adc526fbe2a115316bd052fe10433d93061183b5e2", + "zh:1487c229ad673ddbd6c46ac511985d9a7ce86120ccf1c929acacb3761f3f1f9c", + "zh:2b73e4087cfbf20049240914a02f1344e43eb96a3c7d734454e4c49875aba1df", + "zh:3de5de4e50e58228f612fa158577f623c3a679b18918f62e6b0a925b23e4ec04", + "zh:6501c980ecb830728929798848105b27ce1cba1b604745fc609ecc54a80e0bd4", + "zh:67db18b88e93520eb052c5fae1f0dbedefe8433692b98ebdd3e29c3824ef08eb", + "zh:6aa9bada1a35dd1eb1a37e0233a031d8ecc1ae76ead3e851419bc49ce0dd67ba", + "zh:80e499f1971e2f5dfa7bbb4c40fa2bb303d6c0752b3d04f9a6869b7051071913", + "zh:93d63aee4fbacd28eef5330d517f37da2ead51dbd9c1d89e3e2a19a158c15a5d", + "zh:97c9d6971915400300f78300b807470c00793068b2b90506463da96cf5067092", + "zh:9a626f4c13f32317e8b891950bfd34eb69970599b9e8f62a5faa565e8c60f962", + "zh:d233fc2f7173db3ce5de21dc17e37b7757956b48783be3a81b8d92a6893a4fbc", + "zh:dfc6282c33ab4747d899915027de314687c6f8b49b7c95cafa8544bc45a0d15a", ] } diff --git a/templates/kubernetes/homelab-workspace/terraform.tf b/templates/kubernetes/homelab-workspace/terraform.tf index 8cb9541c..145b12b1 100644 --- a/templates/kubernetes/homelab-workspace/terraform.tf +++ b/templates/kubernetes/homelab-workspace/terraform.tf @@ -3,11 +3,11 @@ terraform { required_providers { coder = { - source = "coder/coder" + source = "registry.opentofu.org/coder/coder" version = "2.18.0" } kubernetes = { - source = "hashicorp/kubernetes" + source = "registry.opentofu.org/hashicorp/kubernetes" version = "3.2.1" } }