diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1d65e36..3a04c76 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -31,7 +31,7 @@ jobs: fail-fast: false matrix: system: [debian-12, debian-13, ubuntu-22.04, ubuntu-24.04, ubuntu-26.04, rocky-9, almalinux-10] - group: [base, security, scanners, web, runtime, database, database-alt, container, backup, backup-alt, monitoring, network, mail] + group: [base, security, scanners, web, runtime, database, database-alt, container, backup, backup-alt, monitoring, network, mail, apps] arch: [x64, arm64] include: - group: base @@ -51,7 +51,8 @@ jobs: - group: database-alt modules: mysql valkey mongodb # Podman runs inside a test container (its storage on a volume, not on overlayfs). Docker, Portainer and - # k3s need a kernel of their own (bridges, iptables, cgroups) and run in the VM jobs below. + # k3s run in the VM jobs below with a kernel of their own; on arm64 (no KVM) the app jobs run Docker inside + # these containers. - group: container modules: podman # restic alone (local repository), then the database dumps of every engine; the conflicting engines in a job @@ -71,6 +72,10 @@ jobs: # filter without public DNS. - group: mail modules: mail-relay guard,mail-relay mailserver nginx,php,mailserver,webmail + # The apps that run without Docker: Self-Updater deploys a Git repository (its own, by tag) and MinIO (Silo) + # runs from its package behind nginx. The Docker stacks of the other apps run in the apps VM jobs below. + - group: apps + modules: node,self-updater nginx,minio steps: - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: @@ -139,12 +144,14 @@ jobs: # Debian 13 for every change, on every distribution in the nightly and the manual runs. Docker with Portainer, # then k3s (after Docker is removed; they conflict) on every supported system, in a clean VM because the runner # already has Docker; minimal-secure on the newest Ubuntu release; the Kubernetes node profile with a Lynis audit; - # Docker with the monitoring stack, as on a DevOps server. + # Docker with the monitoring stack and Forgejo, as on a DevOps server; mail, Nextcloud with Collabora Online and + # Vaultwarden, as on an office server, also on Rocky Linux 9 (the SELinux policy of the RHEL family 9 and 10). + # With the apps, profile all needs most of the memory and processors of the runner. include: >- - ${{ fromJSON(format('[{0},{1}]', '{"system":"debian-12","run":"containers"},{"system":"debian-13","run":"containers"},{"system":"ubuntu-22.04","run":"containers"},{"system":"ubuntu-24.04","run":"containers"},{"system":"ubuntu-26.04","run":"containers"},{"system":"rocky-9","run":"containers"},{"system":"almalinux-10","run":"containers"},{"system":"ubuntu-26.04","run":"profile minimal-secure"},{"system":"debian-13","run":"profile k8s-node"},{"system":"almalinux-10","run":"profile k8s-node"},{"system":"debian-13","run":"profile devops"},{"system":"almalinux-10","run":"profile devops"}', + ${{ fromJSON(format('[{0},{1}]', '{"system":"debian-12","run":"containers"},{"system":"debian-13","run":"containers"},{"system":"ubuntu-22.04","run":"containers"},{"system":"ubuntu-24.04","run":"containers"},{"system":"ubuntu-26.04","run":"containers"},{"system":"rocky-9","run":"containers"},{"system":"almalinux-10","run":"containers"},{"system":"ubuntu-26.04","run":"profile minimal-secure"},{"system":"debian-13","run":"profile k8s-node"},{"system":"almalinux-10","run":"profile k8s-node"},{"system":"debian-13","run":"profile devops"},{"system":"almalinux-10","run":"profile devops"},{"system":"debian-13","run":"profile office","memory":8192},{"system":"rocky-9","run":"profile office","memory":8192},{"system":"almalinux-10","run":"profile office","memory":8192}', (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') - && '{"system":"debian-12","run":"profile all"},{"system":"debian-13","run":"profile all"},{"system":"rocky-9","run":"profile all"},{"system":"almalinux-10","run":"profile all"},{"system":"ubuntu-22.04","run":"profile all"},{"system":"ubuntu-24.04","run":"profile all"},{"system":"ubuntu-26.04","run":"profile all"}' - || '{"system":"debian-13","run":"profile all"}')) }} + && '{"system":"debian-12","run":"profile all","memory":12288,"cpus":4,"disk":"40G"},{"system":"debian-13","run":"profile all","memory":12288,"cpus":4,"disk":"40G"},{"system":"rocky-9","run":"profile all","memory":12288,"cpus":4,"disk":"40G"},{"system":"almalinux-10","run":"profile all","memory":12288,"cpus":4,"disk":"40G"},{"system":"ubuntu-22.04","run":"profile all","memory":12288,"cpus":4,"disk":"40G"},{"system":"ubuntu-24.04","run":"profile all","memory":12288,"cpus":4,"disk":"40G"},{"system":"ubuntu-26.04","run":"profile all","memory":12288,"cpus":4,"disk":"40G"}' + || '{"system":"debian-13","run":"profile all","memory":12288,"cpus":4,"disk":"40G"}')) }} steps: - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: @@ -169,6 +176,11 @@ jobs: env: SYSTEM: vm-${{ matrix.system }} RUN: ${{ matrix.run }} + LSS_VM_MEMORY: ${{ matrix.memory || 4096 }} + LSS_VM_CPUS: ${{ matrix.cpus || 2 }} + LSS_VM_DISK: ${{ matrix.disk || '20G' }} + # Every module at once (profile all) takes close to 40 minutes; the limit of one run is 70 minutes there. + LSS_TEST_TIMEOUT: ${{ matrix.run == 'profile all' && 4200 || 2400 }} run: | if [[ $RUN == kernel ]]; then tests/integration/run.sh "$SYSTEM" time swap firewall kernel-hardening mac auditd usbguard wireguard @@ -186,3 +198,104 @@ jobs: esac tests/integration/run.sh "$SYSTEM" --profile "${RUN#profile }" "$minimum" fi + + # The apps as Docker Compose stacks behind each web server, on every supported system (x86_64 VMs: Docker needs a + # kernel of its own). Docker and the web server are the base; each app is installed, used (mod_verify), installed + # again and removed on its own. Nextcloud alone, then with Collabora Online (Nextcloud Office). + apps: + name: ${{ matrix.system }} VM / apps ${{ matrix.set }} + runs-on: ubuntu-24.04 + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: + system: [debian-12, debian-13, ubuntu-22.04, ubuntu-24.04, ubuntu-26.04, rocky-9, almalinux-10] + set: [office, dev, web] + include: + - set: office + base: docker,nginx + modules: nextcloud collabora,nextcloud vaultwarden + - set: dev + base: docker,apache + modules: forgejo uptime-kuma n8n + - set: web + base: docker,caddy,tls + modules: keycloak matomo wordpress minio + settings: LSS_MINIO_MODE=docker + steps: + - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install QEMU and allow access to KVM + run: | + sudo apt-get -qq update + sudo NEEDRESTART_SUSPEND=1 apt-get -qq install -y --no-install-recommends qemu-system-x86 qemu-utils cloud-image-utils >/dev/null + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Install, verify, reinstall, remove + env: + SYSTEM: vm-${{ matrix.system }} + BASE: ${{ matrix.base }} + MODULES: ${{ matrix.modules }} + LSS_TEST_SETTINGS: ${{ matrix.settings }} + LSS_VM_MEMORY: 8192 + LSS_VM_CPUS: 4 + LSS_VM_DISK: 40G + run: | + read -ra modules <<<"$MODULES" + tests/integration/run.sh "$SYSTEM" --base "$BASE" "${modules[@]}" + + # The app stacks on arm64. The arm64 runners have no KVM, so Ubuntu 24.04 runs them on the runner itself (host); the + # runner's own Docker goes first, so the docker module sets Docker up as on a fresh server. The other systems run in + # the privileged systemd containers of the container jobs, with Docker inside (its storage on a volume) and the + # runner's kernel: its netfilter modules load on the runner, as a container cannot load them. + apps-arm64: + name: ${{ matrix.system == 'host' && 'ubuntu-24.04 arm64' || format('{0} arm64 container', matrix.system) }} / apps ${{ matrix.set }} + runs-on: ubuntu-24.04-arm + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: + system: [host, debian-12, debian-13, ubuntu-22.04, ubuntu-26.04, rocky-9, almalinux-10] + set: [office, dev, web] + include: + - set: office + base: docker,nginx + modules: nextcloud collabora,nextcloud vaultwarden + - set: dev + base: docker,apache + modules: forgejo uptime-kuma n8n + - set: web + base: docker,caddy,tls + modules: keycloak matomo wordpress minio + settings: LSS_MINIO_MODE=docker + steps: + - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Remove the runner's Docker + if: matrix.system == 'host' + run: | + sudo systemctl stop docker.socket docker.service containerd.service 2>/dev/null || true + mapfile -t packages < <(dpkg-query -W -f "\${Package}\n" 'moby-*' 'docker-*' containerd 'containerd.io' 2>/dev/null) + [ "${#packages[@]}" -eq 0 ] || sudo NEEDRESTART_SUSPEND=1 apt-get -qq purge -y "${packages[@]}" >/dev/null + sudo rm -rf /var/lib/docker /var/lib/containerd /etc/docker + - name: Load the netfilter modules of Docker + if: matrix.system != 'host' + run: sudo modprobe -a br_netfilter xt_addrtype + - name: Install, verify, reinstall, remove + env: + SYSTEM: ${{ matrix.system }} + BASE: ${{ matrix.base }} + MODULES: ${{ matrix.modules }} + LSS_TEST_SETTINGS: ${{ matrix.settings }} + run: | + read -ra modules <<<"$MODULES" + tests/integration/run.sh "$SYSTEM" --base "$BASE" "${modules[@]}" diff --git a/README.md b/README.md index 9c3ca81..5ffdbb8 100644 --- a/README.md +++ b/README.md @@ -31,12 +31,19 @@ every module on each tested system. | Profile `minimal-secure` | 1 GB; 2 GB on the RHEL family | 4 GB | 3 to 5 min plus AIDE | | `minimal-secure` with `crowdsec` | 1.5 GB; 2 GB on the RHEL family | 4.5 GB | about 30 s more | | `minimal-secure` with `clamav` | 2.5 GB | 4.5 GB | about 1 min more | +| Profile `office` | 1.5 GB; 2 GB on the RHEL family | 10 GB | 9 to 16 min | +| Profile `devops` | 1 GB; 1.5 GB on the RHEL family | 7 GB | 6 to 11 min | One CPU core is enough; more cores mainly shorten the first AIDE database build. The disk figure includes the swap file (`LSS_SWAP_SIZE`, 1 GB in the measurement), the journal (up to `LSS_LOGGING_MAX_USE`, 1 GB) and the audit logs (up to 10 × 32 MiB). The RHEL family needs more memory because dnf loads its repository metadata during every install, up to 0.7 GB. +The `office` and `devops` profiles are measured in the CI VMs of Debian 13 and AlmaLinux 10. Their disk figure includes +the container images of the apps. After the install, `office` keeps about 0.9 GiB resident (Nextcloud, Collabora +Online, Vaultwarden, the mail server and the web server) and `devops` about 0.8 GiB; plan more for real use, since +Collabora needs memory for every open document. + The first AIDE run reads every file under the directories it watches, so its time depends on the server: 12 to 90 seconds on a fresh system, 16 minutes on a CI machine full of preinstalled toolchains. It runs with the lowest CPU and I/O priority. @@ -68,7 +75,7 @@ up). Peak RAM covers install and `lss verify`; "while running" is what stays res | `lynis` | 25 s | 80 MiB | 5 MiB | weekly audit | | `usbguard` | 10 s | 90 MiB | 10 MiB | usbguard-daemon | | `guard` | 5 s | 90 MiB | – | daily check | -| `docker` | not measured yet | not measured yet | not measured yet | dockerd and containerd | +| `docker` | 105 s with the web server | 660 MiB with the web server | 940 MiB with the web server | dockerd and containerd, about 260 MiB with the web server | | `podman` | not measured yet | not measured yet | not measured yet | – (no daemon) | | `portainer` | not measured yet | not measured yet | not measured yet | Portainer container, up to 512 MiB | | `k3s` | not measured yet | not measured yet | not measured yet | k3s server, about 1 GiB with Traefik | @@ -88,9 +95,24 @@ up). Peak RAM covers install and `lss verify`; "while running" is what stays res | `mail-relay` | 35 s | 250 MiB | 180 MiB (dnf), else 135 MiB | Postfix (master, queue manager, pickup) | | `mailserver` | 90 s | 800 MiB (dnf), else 225 MiB | 235 MiB (dnf), else 120 MiB | Postfix, Dovecot and Rspamd, up to 180 MiB | | `webmail` | 135 s with nginx, php and mailserver | 390 MiB together | 265 MiB together | Roundcube in PHP-FPM; with nginx and the mail server up to 290 MiB | +| `nextcloud` | 295 s | 810 MiB (dnf), else 745 MiB | 2.8 GiB | Nextcloud (Apache) and PostgreSQL, about 420 MiB (limit 1.5 GiB) | +| `collabora` | 90 s | 830 MiB | 1.4 GiB | Collabora Online, about 640 MiB (limit 2 GiB) | +| `forgejo` | 55 s | 370 MiB | 195 MiB | Forgejo, about 300 MiB (limit 1 GiB) | +| `uptime-kuma` | 60 s | 295 MiB | 665 MiB | Uptime Kuma, about 170 MiB (limit 512 MiB) | +| `vaultwarden` | 45 s | 750 MiB (dnf), else 260 MiB | 380 MiB | Vaultwarden, about 125 MiB (limit 512 MiB) | +| `keycloak` | 85 s | 960 MiB | 1.0 GiB | Keycloak and PostgreSQL, about 900 MiB (limit 2 GiB) | +| `matomo` | 75 s | 540 MiB | 1.3 GiB | Matomo (Apache) and MariaDB, about 280 MiB (limit 1.5 GiB) | +| `wordpress` | 85 s | 360 MiB | 580 MiB | WordPress (Apache) and MariaDB, about 260 MiB (limit 1.3 GiB) | +| `n8n` | 205 s | 1.2 GiB (dnf), else 935 MiB | 1.8 GiB | n8n and PostgreSQL, about 680 MiB (limit 1.5 GiB) | +| `minio` | 30 s | 250 MiB | 160 MiB | Silo, about 210 MiB (limit 1 GiB in a container) | +| `self-updater` | 65 s with node | 475 MiB with node | 440 MiB with node | Self-Updater (Node.js) | Modules the tests install together are measured together. The Rocky 9 VM in CI is slow throughout (eight minutes even for `wireguard`); those outliers are left out, like one Rocky 9 container run of `mail-relay` that took six minutes. +The apps run as Docker stacks in the x86_64 VMs of all seven systems and on an Ubuntu 24.04 arm64 runner; their disk +figure includes the container images. `collabora` is measured with `nextcloud` already installed. `minio` in native +mode and `self-updater` also run in the containers of all seven systems on both architectures. For an app, "while +running" is what its stack holds after `lss verify`, and the limit is the memory limit lss sets for its containers. Web, runtime and database modules, measured the same way. Modules that only work together are tested together; their row is the cost of the whole set, including the web server or PHP it needs. @@ -230,6 +252,11 @@ See [WireGuard, DNS and NTP](#wireguard-dns-and-ntp). **Mail:** `mail-relay` (the server sends its own mail, through a smarthost or directly) or `mailserver` (Postfix, Dovecot and Rspamd for your domains), `webmail` (Roundcube on the web server). See [Mail](#mail). +**Apps:** `nextcloud`, `collabora` (Collabora Online for Nextcloud Office), `forgejo` (Git hosting), `uptime-kuma`, +`vaultwarden` (password manager), `keycloak` (single sign-on), `matomo` (web analytics), `wordpress`, `n8n` (workflow +automation), `minio` (S3 object storage) and `self-updater` (deploys a Git repository when it changes). See +[Apps](#apps). + ### Web sites One web server owns ports 80 and 443. It answers only for the sites you set up; requests for any other host name get @@ -280,6 +307,54 @@ sudo lss install --profile k8s-node --set LSS_ADMIN_USER_NAME=alice --set LSS_AD With `docker` or `podman` installed, the guard scans every image daily with Trivy and reports HIGH and CRITICAL vulnerabilities per image. +### Apps + +The apps run as Docker Compose stacks behind the web server (nginx, Apache or Caddy), each on host names of its own: +`LSS__DOMAINS`, by default the module name in front of the server's name (`nextcloud.example.com`). Those names +must point to the server; with the `tls` module every app gets its own certificate. `minio` and `self-updater` also +run without Docker: Silo from its package, Self-Updater from its npm release. + +Every stack follows the container rules above: images pinned by digest, no capabilities, no new privileges, a +read-only root file system where the image allows it (Collabora Online and Keycloak write below their program +directory and are marked as exceptions), memory and process limits. The containers run as unprivileged users, mapped +to unused IDs on the host by Docker's user namespace. Ports bind to 127.0.0.1 only (`LSS__PORT`, 8081 to 8093); +the web server publishes them with TLS and the security headers of every lss site. Databases run in the same stack on +an internal network without a route out. + +Passwords never appear in a Compose file, an environment variable or a command line. lss creates them in +`/etc/lss/secrets` and hands each container a read-only copy as a file under `/run/secrets`. The first administrator +gets a generated password that lss does not print: `lss secrets show nextcloud-admin-password` and so on (for +Vaultwarden, `vaultwarden-admin-token` opens `/admin`, where you invite users; open registration is off everywhere). +Matomo stores its database password in its own `config.ini.php`, which lss cannot avoid. + +`lss verify` uses every app, not just its start page: it stores and reads a file in Nextcloud over WebDAV, converts a +document with Collabora Online, creates and deletes a repository in Forgejo, a monitor in Uptime Kuma, a user in +Keycloak and Vaultwarden and a workflow in n8n, tracks a page view in Matomo and reads it back, logs in to WordPress +and writes a bucket and an object to MinIO. Nextcloud and Collabora Online link themselves when both are installed, +in either order. + +Data lives in Docker volumes named after the stack (`lss-nextcloud_html`, `lss-nextcloud_db`, ...). `lss remove` +keeps them; `--purge` deletes them with the secrets. Before every restic backup, `/usr/local/lib/lss/app-backup` saves +what each app registered in `/etc/lss/app-backup`. It dumps the PostgreSQL and MariaDB databases (Nextcloud, Keycloak, +n8n, Matomo, WordPress) and copies the SQLite apps (Forgejo, Vaultwarden, Uptime Kuma) while the app is paused for a +moment, both to `/var/backups/lss-apps`. Restic backs up that directory together with the file volumes, which stay where +they are (Nextcloud's files, MinIO's objects). `lss verify` runs this step for each app, so a broken dump shows up before +the night's backup. + +Nextcloud and Collabora Online call each other through the web server, and both check the other's certificate. Without +the `tls` module, every site gets a self-signed certificate for its own names, and each app trusts the other's site. + +`minio` is the MinIO server code, maintained as [PGSTY Silo](https://github.com/pgsty/silo) since MinIO archived its +community edition in 2026. The S3 API answers on `LSS_MINIO_DOMAINS`, the web console on +`LSS_MINIO_CONSOLE_DOMAINS`. `LSS_MINIO_MODE=docker` runs it as a container instead of the package. + +`self-updater` installs [Self-Updater](https://github.com/VectoDE/Self-Updater), which keeps a checkout of +`LSS_SELF_UPDATER_REPO` in `LSS_SELF_UPDATER_PATH` up to date (every new commit of a branch, or every release tag with +`LSS_SELF_UPDATER_TRACK=tag`), restarts `LSS_SELF_UPDATER_SERVICE` and rolls back when the health check fails. + +The profile `office` installs `minimal-secure`, mail with webmail, Nextcloud with Collabora Online, Vaultwarden and +restic; `devops` adds Forgejo to Docker and the monitoring stack. + ### Backup `restic` backs up `LSS_RESTIC_PATHS` (configuration, home directories, web roots, the database dumps) every night at @@ -325,7 +400,8 @@ node exporter from the distribution, the RHEL family the checksum-verified upstr come from Grafana's repository and Netdata from Netdata's, both with pinned keys. `smartd` watches the drives, also in virtual machines, where it usually finds none. -The profile `devops` installs `minimal-secure`, `docker` and the monitoring stack without Netdata: +The profile `devops` installs `minimal-secure`, `docker`, nginx with Forgejo and the monitoring stack without +Netdata: ```bash sudo lss install --profile devops --set LSS_ADMIN_USER_NAME=alice --set LSS_ADMIN_USER_GITHUB_USER=alice-on-github @@ -392,7 +468,8 @@ Many providers block outgoing port 25; ask for it to be opened or relay through `webmail` runs Roundcube as a site of the web server (`LSS_WEBMAIL_DOMAINS`, by default the mail host), with the certificate of the `tls` module when it is installed. It talks to Dovecot and Postfix on 127.0.0.1 only. The profile -`office` installs `minimal-secure`, nginx with `tls` and `php`, `mailserver`, `webmail` and `restic`: +`office` installs `minimal-secure`, nginx with `tls` and `php`, `mailserver`, `webmail`, Docker with `nextcloud`, +`collabora` and `vaultwarden` (see [Apps](#apps)) and `restic`: ```bash sudo lss install --profile office --set LSS_ADMIN_USER_NAME=alice --set LSS_ADMIN_USER_GITHUB_USER=alice-on-github --set LSS_MAILSERVER_DOMAINS=example.com --set LSS_MAILSERVER_USERS=alice --set LSS_TLS_DOMAINS=mail.example.com diff --git a/data/checksums.txt b/data/checksums.txt index 5c5f5be..42a2a90 100644 --- a/data/checksums.txt +++ b/data/checksums.txt @@ -106,3 +106,14 @@ alertmanager-0.34.1 x86_64 265b9d1e55ef0d5306a436018af6d2b686c2ce051f03d968f7464 alertmanager-0.34.1 aarch64 d98d6cbaf52151c7e76e24355fec88b11cebcb9875d4cdd8b76ddce7a7e5535c https://github.com/prometheus/alertmanager/releases/download/v0.34.1/alertmanager-0.34.1.linux-arm64.tar.gz node_exporter-1.12.1 x86_64 b51d8a76aa2a9156a55d501aca6276fae09e262259a5e4e831d2c2222f084e63 https://github.com/prometheus/node_exporter/releases/download/v1.12.1/node_exporter-1.12.1.linux-amd64.tar.gz node_exporter-1.12.1 aarch64 ad35b605f9954b9f1ffddf5ba054bdc5a98d790b9eae5291e1eeb83f1ecbd0e7 https://github.com/prometheus/node_exporter/releases/download/v1.12.1/node_exporter-1.12.1.linux-arm64.tar.gz + +# Self-Updater 4.2.1 (VectoDE, npm package of the release): SHA-256 digest of the release asset on GitHub (gh api +# repos/VectoDE/Self-Updater/releases/tags/v4.2.1, checked 2026-10-15; the download matched it). +self-updater-4.2.1 any 2a864c6fcdc6325b7ef87916ee277202ff675937223fdd6d585b7d4fa313e74f https://github.com/VectoDE/Self-Updater/releases/download/v4.2.1/vectode-self-updater-4.2.1.tgz +# PGSTY Silo RELEASE.2026-09-16T00-00-00Z (the maintained fork of the MinIO server; module minio, native mode): the +# packages of the GitHub release, SHA-256 digests of the release assets (gh api repos/pgsty/silo/releases, checked +# 2026-10-15). +silo-deb x86_64 176aad3944d9e9f949baf47bcea86e62a3271094be075c6547c056f652ae7e5f https://github.com/pgsty/silo/releases/download/RELEASE.2026-09-16T00-00-00Z/silo_20260916000000.0.0-1PGSTY_amd64.deb +silo-deb aarch64 32f7fab06f38a90920bf7295017b031da0eb03fcb1e54fd948f109fd30e3a711 https://github.com/pgsty/silo/releases/download/RELEASE.2026-09-16T00-00-00Z/silo_20260916000000.0.0-1PGSTY_arm64.deb +silo-rpm x86_64 cb4d6690d386e43dca0b9c6ae4e3abb1fad49c75f9df7d36502c5f8742ef3a77 https://github.com/pgsty/silo/releases/download/RELEASE.2026-09-16T00-00-00Z/silo-20260916000000.0.0-1PGSTY.x86_64.rpm +silo-rpm aarch64 e7fa4c2fcf49a9d7632e91b4aceef3225c5330571cb19116e5ba59c8d2dc031c https://github.com/pgsty/silo/releases/download/RELEASE.2026-09-16T00-00-00Z/silo-20260916000000.0.0-1PGSTY.aarch64.rpm diff --git a/lib/apps.sh b/lib/apps.sh new file mode 100644 index 0000000..7df0868 --- /dev/null +++ b/lib/apps.sh @@ -0,0 +1,687 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# Apps: the modules of the group apps run as Docker Compose stacks behind the installed web server. Each stack is the +# project lss-NAME with its files in CTR_CONFIG_DIR/lss-NAME (the one directory of /etc its containers may mount): +# compose.yaml the stack, checked against the container rules (lib/container.sh) before every start +# secrets/ copies of the secrets in /etc/lss/secrets that a container reads, mounted read-only as +# /run/secrets/NAME and owned by the user the container runs as (seen from the host) +# config/ other files a container reads +# Nothing secret is written into compose.yaml, an environment variable or a command line: the images read +# /run/secrets/NAME themselves (*_FILE variables) or a small wrapper reads it before the program starts. +# +# Every published port binds to 127.0.0.1; the web server publishes the app as an HTTPS site with the security headers +# of lib/web.sh. Data lives in named volumes (lss-NAME_*), which stay when the module is removed, unless --purge. + +APP_START_TIMEOUT=${APP_START_TIMEOUT:-600} + +# app_project: the Compose project of the module running this step. +app_project() { + printf 'lss-%s' "${LSS_MODULE}" + return +} + +# app_dir: the directory of the stack. +app_dir() { + printf '%s/%s' "${CTR_CONFIG_DIR}" "$(app_project)" + return +} + +app_compose_file() { + printf '%s/compose.yaml' "$(app_dir)" + return +} + +# app_valid_domain NAME: a host name like cloud.example.com (lowercase, at least two labels). +app_valid_domain() { + local name=$1 + [[ ${name} =~ ^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z][a-z0-9-]{0,61}[a-z0-9]$ ]] + return +} + +# app_check_domains: checks the option domains of the module (host names separated by spaces). +app_check_domains() { + local domain + for domain in $(opt domains); do + app_valid_domain "${domain}" || + die "$(config_key "${LSS_MODULE}" domains): '${domain}' is not a host name like ${LSS_MODULE}.example.com." + done + return +} + +# app_check_port OPTION: the option is a port from 1024 to 65535 other than the admin port of the web server. +app_check_port() { + local option=$1 port + port=$(opt_int "${option}") + ((port >= 1024 && port <= 65535)) || die "$(config_key "${LSS_MODULE}" "${option}") must be a port from 1024 to 65535." + [[ ${port} != "${WEB_ADMIN_PORT}" ]] || + die "$(config_key "${LSS_MODULE}" "${option}") must differ from the admin port of the web server (${WEB_ADMIN_PORT})." + return +} + +# app_domains: the host names of the app: the option domains, or NAME.. +app_domains() { + local domains host + domains=$(opt domains) + if [[ -z ${domains} ]]; then + host=$(hostname -f 2>/dev/null || true) + [[ -n ${host} ]] || host=$(hostname) + domains=${LSS_MODULE}.${host,,} + fi + printf '%s' "${domains}" + return +} + +# app_domain: the first host name of the app, the one its links use. +app_domain() { + local -a domains=() + read -ra domains <<<"$(app_domains)" + printf '%s' "${domains[0]}" + return +} + +# app_compose_ready: docker compose is there (the Compose plugin of Docker CE). +app_compose_ready() { + docker compose version >/dev/null 2>&1 || + die "${LSS_MODULE}: docker compose is missing; the apps need the Compose plugin (LSS_DOCKER_SOURCE=auto installs it)." + return +} + +# _app_userns_base FILE: the first ID of the range Docker maps containers into (/etc/subuid or /etc/subgid), or 0 +# when the daemon runs without user namespace remapping. +_app_userns_base() { + local file=$1 user base + # The daemon.json of the docker module has the setting on a line of its own; read without jq. + user=$(sed -n 's/.*"userns-remap"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "${APP_DOCKER_CONFIG:-/etc/docker/daemon.json}" 2>/dev/null | + head -n 1) || user='' + if [[ -z ${user} ]]; then + printf '0' + return 0 + fi + [[ ${user} != default ]] || user=dockremap + user=${user%%:*} + base=$(awk -F ':' -v user="${user}" '$1 == user { print $2; exit }' "${file}" 2>/dev/null) || base='' + [[ ${base} =~ ^[0-9]+$ ]] || die "${LSS_MODULE}: ${file} has no ID range for ${user}, which Docker maps containers into." + printf '%s' "${base}" + return +} + +# app_host_owner UID[:GID]: the owner, as the host sees it, of a file the container user UID:GID must read. +app_host_owner() { + local ids=$1 uid gid + uid=${ids%%:*} + gid=${ids#*:} + [[ ${ids} == *:* ]] || gid=${uid} + [[ ${uid} =~ ^[0-9]+$ && ${gid} =~ ^[0-9]+$ ]] || die "${LSS_MODULE}: '${ids}' is not UID or UID:GID." + printf '%s:%s' "$((uid + $(_app_userns_base "${APP_SUBUID:-/etc/subuid}")))" \ + "$((gid + $(_app_userns_base "${APP_SUBGID:-/etc/subgid}")))" + return +} + +# APP_CHANGED: a file the containers mount changed in this step, so they must be created again (a bind-mounted file +# keeps showing the old copy once file_write replaced it). APP_COMPOSE_CHANGED: compose.yaml changed. +APP_CHANGED=0 +APP_COMPOSE_CHANGED=0 + +# app_file PATH UID[:GID] MODE: writes stdin to $(app_dir)/PATH for the container user UID:GID. +app_file() { + local path=$1 ids=$2 mode=$3 + [[ ${path} =~ ^[a-z0-9][a-z0-9._/-]*$ && /${path}/ != */../* ]] || die "${LSS_MODULE}: '${path}' is not a valid file name." + install -d -m 0700 "$(app_dir)" + install -d -m 0700 "$(dirname "$(app_dir)/${path}")" + file_write "${mode}" "$(app_host_owner "${ids}")" "$(app_dir)/${path}" + [[ ${LSS_FILE_CHANGED} == 0 ]] || APP_CHANGED=1 + return +} + +# app_secret SECRET UID[:GID] [LENGTH] [FILE]: the secret from /etc/lss/secrets (created on first use), copied to +# secrets/FILE (default: SECRET) for the container user UID:GID, mode 0400, without a final newline. The container +# mounts it with app_mount secrets/FILE /run/secrets/NAME. +app_secret() { + local secret=$1 ids=$2 length=${3:-32} file=${4:-$1} value + secret_load "${secret}" value "${length}" + printf '%s' "${value}" | app_file "secrets/${file}" "${ids}" 0400 + return +} + +# app_mount PATH TARGET: the volume line of a Compose service that mounts $(app_dir)/PATH read-only at TARGET (with +# the shared SELinux label on the RHEL family, so every container of the stack may read it). +app_mount() { + local path=$1 target=$2 + printf ' - %s/%s:%s:ro,z\n' "$(app_dir)" "${path}" "${target}" + return +} + +# app_hardening MEMORY PIDS [writable]: the lines every Compose service of lss gets (indented for a service): no +# capabilities, no new privileges, memory and process limits, a read-only root file system unless "writable", restart +# after a crash or reboot. Services add the capabilities they need with cap_add. +app_hardening() { + local memory=$1 pids=$2 writable=${3:-} + printf ' restart: unless-stopped\n cap_drop: [ALL]\n security_opt: ["no-new-privileges:true"]\n' + printf ' mem_limit: %s\n memswap_limit: %s\n pids_limit: %s\n' "${memory}" "${memory}" "${pids}" + if [[ ${writable} == writable ]]; then + printf ' labels:\n lss.managed: "true"\n lss.writable: "true"\n' + else + printf ' read_only: true\n labels:\n lss.managed: "true"\n' + fi + return +} + +# app_compose: writes the stack from stdin to compose.yaml (root only: it names the files with the secrets). +app_compose() { + install -d -m 0700 "$(app_dir)" + file_write 0600 root:root "$(app_compose_file)" + APP_COMPOSE_CHANGED=${LSS_FILE_CHANGED} + return +} + +# _app_compose COMMAND...: docker compose for the stack of the module. +_app_compose() { + docker compose --project-name "$(app_project)" -f "$(app_compose_file)" "$@" + return +} + +# app_running: every service of the stack has a running container (except "init", which runs once per start). +app_running() { + local services running + services=$(_app_compose config --services 2>/dev/null | grep -vx init | sort) || return 1 + running=$(_app_compose ps --status running --services 2>/dev/null | sort) || return 1 + [[ -n ${services} && ${services} == "${running}" ]] + return +} + +# app_up: starts the stack after compose.yaml or a mounted file changed, or when a service is not running. A changed +# mounted file creates every container again, because a bind mount keeps the replaced file. +app_up() { + local -a options=() + app_compose_ready + if [[ ${APP_CHANGED} == 1 ]]; then + options=(--force-recreate) + elif [[ ${APP_COMPOSE_CHANGED} == 0 ]] && app_running; then + return 0 + fi + log_info "Starting $(app_project) (images are pulled on the first start)" + ctr_compose_up "$(app_project)" "$(app_compose_file)" "${options[@]}" + files_note_change "started $(app_project)" + APP_CHANGED=0 + APP_COMPOSE_CHANGED=0 + return +} + +# app_exec [OPTION...] SERVICE COMMAND...: runs COMMAND in the running container of SERVICE (no terminal, stdin passed +# on); OPTIONs of "docker compose exec" such as --user or --env come first. +app_exec() { + _app_compose exec -T "$@" + return +} + +# Nextcloud Office: the collabora module and the nextcloud module link each other, whichever comes second. +APP_NEXTCLOUD_PROJECT=lss-nextcloud +APP_COLLABORA_PROJECT=lss-collabora +# The certificates Collabora checks Nextcloud's against (mounted into the container; see app_collabora_trust). +APP_COLLABORA_TRUST=trust/ca.pem + +# app_domains_of MODULE: the host names of another app module (its option domains, or MODULE.). +app_domains_of() { + local module=$1 domains host + domains=$(config_option "${module}" domains '') + if [[ -z ${domains} ]]; then + host=$(hostname -f 2>/dev/null || true) + [[ -n ${host} ]] || host=$(hostname) + domains=${module}.${host,,} + fi + printf '%s' "${domains}" + return +} + +# _app_nc_exec COMMAND...: a command in the Nextcloud container (stdin passed on). +_app_nc_exec() { + docker compose --project-name "${APP_NEXTCLOUD_PROJECT}" -f "${CTR_CONFIG_DIR}/${APP_NEXTCLOUD_PROJECT}/compose.yaml" exec -T app "$@" + return +} + +# _app_nc_occ ARG...: occ in the Nextcloud container. +_app_nc_occ() { + _app_nc_exec php occ "$@" + return +} + +# _app_route_trust ROUTE: the certificate an app must trust to reach the route ROUTE of this server through the web +# server: the CA of a private ACME server (LSS_TLS_CA_BUNDLE) or, without the tls module, the self-signed certificate +# of the route; nothing for Let's Encrypt. Prints the file, or nothing. +_app_route_trust() { + local route=$1 bundle + if state_installed tls; then + bundle=$(config_option tls ca_bundle '') + [[ -z ${bundle} || ! -s ${bundle} ]] || printf '%s' "${bundle}" + elif [[ -s ${WEB_TLS_DIR}/${route}/fullchain.pem ]]; then + printf '%s/%s/fullchain.pem' "${WEB_TLS_DIR}" "${route}" + fi + return +} + +# app_collabora_trust: the certificates Collabora checks the one of Nextcloud against when it fetches and saves +# documents (WOPI): the CAs of the host and the trust anchor of the Nextcloud route (_app_route_trust), in +# APP_COLLABORA_TRUST of the collabora stack. LSS_FILE_CHANGED tells whether it changed. +app_collabora_trust() { + local dir="${CTR_CONFIG_DIR}/${APP_COLLABORA_PROJECT}" bundle trust content='' + for bundle in /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt; do + if [[ -s ${bundle} ]]; then + content=$(<"${bundle}") + break + fi + done + trust=$(_app_route_trust nextcloud) + [[ -z ${trust} ]] || content+=${content:+$'\n'}$(<"${trust}") + # Collabora refuses an empty file; the default certificate of the web server always exists. + [[ -n ${content} ]] || content=$(<"${WEB_TLS_DIR}/default/fullchain.pem") + install -d -m 0700 "${dir}" "$(dirname "${dir}/${APP_COLLABORA_TRUST}")" + printf '%s\n' "${content}" | file_write 0444 root:root "${dir}/${APP_COLLABORA_TRUST}" + return +} + +# app_collabora_check: Collabora reaches Nextcloud through the web server and accepts its certificate (the WOPI +# access check of Collabora, the same connection it opens for every document). +app_collabora_check() { + local nextcloud collabora page + read -r nextcloud _ <<<"$(app_domains_of nextcloud)" + read -r collabora _ <<<"$(app_domains_of collabora)" + page=$(APP_GET_DOMAIN=${collabora} app_get hosting/wopiAccessCheck -X POST -H 'Content-Type: application/json' \ + --data "{\"callbackUrl\":\"https://${nextcloud}/status.php\"}") || true + [[ $(app_status "${page}") == 200 && ${page%$'\n'*} =~ \"status\":\ *\"Ok\" ]] || + die "${LSS_MODULE}: Collabora Online cannot reach Nextcloud at https://${nextcloud} with a valid certificate (${page%$'\n'*})." + return +} + +# app_nextcloud_office: with both modules installed (or the other one being installed), Nextcloud Office +# (richdocuments) points at the Collabora site and checks that it reaches it. +app_nextcloud_office() { + local url trust sum module first + for module in nextcloud collabora; do + state_installed "${module}" || [[ ${LSS_MODULE} == "${module}" ]] || return 0 + done + read -r first _ <<<"$(app_domains_of collabora)" + url=https://${first} + if ! _app_nc_occ app:list --output=json 2>/dev/null | jq -e '.enabled.richdocuments' >/dev/null 2>&1; then + log_command _app_nc_occ app:install richdocuments || log_command _app_nc_occ app:enable richdocuments || + die "${LSS_MODULE}: the Nextcloud app Nextcloud Office (richdocuments) cannot be installed." + files_note_change "installed Nextcloud Office" + fi + if [[ $(_app_nc_occ config:app:get richdocuments wopi_url 2>/dev/null) != "${url}" ]]; then + log_command _app_nc_occ config:app:set richdocuments wopi_url --value="${url}" || + die "${LSS_MODULE}: occ cannot set the address of Collabora Online." + files_note_change "linked Nextcloud Office to ${url}" + fi + trust=$(_app_route_trust collabora) + if [[ -n ${trust} ]]; then + sum=$(sha256sum "${trust}" | cut -d ' ' -f 1) + if [[ $(state_get nextcloud trusted_cert) != "${sum}" ]]; then + _app_nc_exec sh -c 'cat >/tmp/lss-web.pem' <"${trust}" || + die "${LSS_MODULE}: cannot copy the certificate of the web server into the Nextcloud container." + log_command _app_nc_occ security:certificates:import /tmp/lss-web.pem || + die "${LSS_MODULE}: occ cannot add the certificate of the web server to the trusted ones of Nextcloud." + state_set nextcloud trusted_cert "${sum}" + files_note_change "Nextcloud trusts the certificate of the web server" + fi + fi + log_command _app_nc_occ richdocuments:activate-config || + die "${LSS_MODULE}: Nextcloud Office cannot reach Collabora Online at ${url}; see the log." + # Collabora installed first knew no certificate of Nextcloud yet: it gets it now and starts again. + if [[ ${LSS_MODULE} == nextcloud ]]; then + app_collabora_trust + if [[ ${LSS_FILE_CHANGED} == 1 ]]; then + log_info "Starting ${APP_COLLABORA_PROJECT} again, so it trusts the certificate of Nextcloud" + ctr_compose_up "${APP_COLLABORA_PROJECT}" "${CTR_CONFIG_DIR}/${APP_COLLABORA_PROJECT}/compose.yaml" --force-recreate + APP_GET_DOMAIN=${first} app_wait '' 200 >/dev/null || die "${LSS_MODULE}: Collabora Online does not answer at ${url}." + fi + fi + app_collabora_check + return +} + +# app_timer NAME INTERVAL DESCRIPTION SERVICE COMMAND...: the systemd timer lss-NAME.timer, which runs COMMAND in the +# container of SERVICE every INTERVAL (a systemd time span such as 5min), first INTERVAL after boot. +app_timer() { + local name=$1 interval=$2 description=$3 service=$4 docker + shift 4 + # The real path: with /bin a link to /usr/bin, command -v answers either, depending on PATH, and the unit would change. + docker=$(command -v docker) || die "${LSS_MODULE}: docker is missing." + [[ ${docker} != /* ]] || docker=$(readlink -f "${docker}") + svc_unit "lss-${name}.service" <&2 2>/dev/null || true + _app_compose logs --tail 30 >&2 2>/dev/null || true + return 1 +} + +# app_verify_stack: the stack runs, its compose.yaml still passes the container rules and the site serves security +# headers. +app_verify_stack() { + local page + [[ -s $(app_compose_file) ]] || die "${LSS_MODULE}: $(app_compose_file) is missing." + [[ $(stat -c '%a %U' "$(app_compose_file)") == '600 root' ]] || die "${LSS_MODULE}: $(app_compose_file) must be mode 600, owned by root." + ctr_compose_check "$(app_compose_file)" "$(app_project)" || die "${LSS_MODULE}: the stack breaks the container rules; see above." + app_running || die "${LSS_MODULE}: a container of $(app_project) is not running (docker compose -p $(app_project) ps)." + page=$(app_get '' --head) || true + grep -qi '^strict-transport-security:' <<<"${page}" || + die "${LSS_MODULE}: https://$(app_domain)/ sends no Strict-Transport-Security header; the web server site is missing." + return +} + +# The data of the apps for the restic backup: each app registers what lss-app-backup (libexec) saves of it before every +# backup, whether or not the restic module is installed yet. +APP_BACKUP_SCRIPT=/usr/local/lib/lss/app-backup +APP_BACKUP_REGISTRY=/etc/lss/app-backup +APP_BACKUP_DIR=/var/backups/lss-apps + +# app_backup KEY=VALUE...: registers the data of the app with lss-app-backup: DB=postgres|mariadb (the service "db", +# dumped) with DB_USER, COPY=VOLUMES (copied while the app is paused: SQLite), LIVE=VOLUMES (backed up in place), +# PATHS=DIRECTORIES (native installations); volumes and directories separated by spaces. +app_backup() { + local pair key + local -A values=([DB]=none [DB_USER]='' [COPY]='' [LIVE]='' [PATHS]='') + for pair in "$@"; do + key=${pair%%=*} + [[ ${pair} == *=* && ${key} =~ ^[A-Z_]+$ && -v values[${key}] && ${pair#*=} =~ ^[A-Za-z0-9_./\ -]*$ ]] || + die "${LSS_MODULE}: app_backup does not know '${pair}'." + values[${key}]=${pair#*=} + done + file_write 0755 root:root "${APP_BACKUP_SCRIPT}" <"${LSS_ROOT}/libexec/lss-app-backup" + install -d -m 0700 "${APP_BACKUP_REGISTRY}" + file_write 0600 root:root "${APP_BACKUP_REGISTRY}/${LSS_MODULE}.conf" <-db-password, mounted as /run/secrets/db-password. Call app_secret for it with APP_POSTGRES_USER and the file +# name db-password.db first. +app_postgres() { + local name=$1 + cat <-db-root-password (db-root-password.db). +app_mariadb() { + local name=$1 + cat <"${file}" + printf '%s' "${file}" + return +} + +# app_site_remove NAME: removes the route NAME of app_site. +app_site_remove() { + local name=$1 + web_proxy_route_remove "${name}" + return +} + +# app_remove: stops the stack and removes its site and files; the named volumes (the data) and the secrets stay unless +# --purge. +app_remove() { + local name=${LSS_MODULE} secret + app_site_remove "${name}" + app_backup_remove + if [[ -s $(app_compose_file) ]] && command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + local -a options=(--remove-orphans) + [[ ${LSS_PURGE:-0} != 1 ]] || options+=(--volumes) + log_command _app_compose down "${options[@]}" || die "${name}: docker compose cannot stop $(app_project)." + files_note_change "stopped $(app_project)" + fi + file_remove "$(app_dir)" + rmdir "${CTR_CONFIG_DIR}" 2>/dev/null || true + if [[ ${LSS_PURGE:-0} == 1 ]]; then + for secret in ${MOD_SECRETS:-}; do + secret_remove "${secret}" + done + fi + return +} diff --git a/lib/container.sh b/lib/container.sh index b6cc622..2554266 100644 --- a/lib/container.sh +++ b/lib/container.sh @@ -424,11 +424,12 @@ CTR_COMPOSE_RULES=' # shellcheck disable=SC2016 # jq variables, not shell variables CTR_COMPOSE_BINDS='.services // {} | to_entries[] | .key as $name | .value.volumes[]? | select(.type == "bind") | "\($name)\t\(.source // "")"' -# ctr_compose_check FILE: checks every service of a Compose file against the container rules. docker compose resolves -# the file first (extends, anchors, defaults), so the checks see what Compose would run. Prints each violation and fails -# when there is one; a file it cannot read or check fails as well. +# ctr_compose_check FILE [PROJECT]: checks every service of a Compose file against the container rules. docker +# compose resolves the file first (extends, anchors, defaults), so the checks see what Compose would run. Prints each +# violation and fails when there is one; a file it cannot read or check fails as well. Bind mounts from /etc may come +# from CTR_CONFIG_DIR/PROJECT (the stacks of the app modules), without PROJECT from CTR_CONFIG_DIR/compose-SERVICE. ctr_compose_check() { - local file=$1 json problems binds service source problem line + local file=$1 project=${2:-} json problems binds service source problem line owner [[ -r ${file} ]] || die "container: cannot read the Compose file ${file}." json=$(docker compose -f "${file}" config --format json 2>/dev/null) || die "container: docker compose cannot read ${file} (docker compose -f ${file} config shows why)." @@ -437,7 +438,8 @@ ctr_compose_check() { binds=$(jq -r "${CTR_COMPOSE_BINDS}" <<<"${json}") || die "container: jq cannot read the mounts of ${file}." while IFS=$'\t' read -r service source; do [[ -n ${service} ]] || continue - problem=$(_ctr_host_source_problem "compose-${service}" "${source}") || + owner=${project:-compose-${service}} + problem=$(_ctr_host_source_problem "${owner}" "${source}") || die "container: cannot resolve the host path ${source} of ${service} in ${file}." [[ -z ${problem} ]] || problems+=$'\n'"${service}: ${problem}" done <<<"${binds}" @@ -449,13 +451,27 @@ ctr_compose_check() { return 1 } -# ctr_compose_up PROJECT FILE: starts a Compose project after ctr_compose_check passed (Docker only). +# Attempts and the pause in seconds between them for pulling the images of a Compose project. Registries answer +# 500 or 429 now and then; one such answer should not undo an installation. +CTR_PULL_TRIES=3 +CTR_PULL_DELAY=15 + +# ctr_compose_up PROJECT FILE [OPTION...]: starts a Compose project after ctr_compose_check passed (Docker only). The +# images that are missing are pulled first, with retries. The OPTIONs go to "docker compose up" (--force-recreate +# after a mounted file changed). ctr_compose_up() { - local project=$1 file=$2 + local project=$1 file=$2 try=1 + shift 2 [[ $(ctr_engine) == docker ]] || die "container: Compose projects need the docker module." [[ ${project} =~ ^[a-z0-9][a-z0-9_-]*$ ]] || die "container: '${project}' is not a valid Compose project name." - ctr_compose_check "${file}" || die "container: ${file} breaks the container rules; see above." - log_command docker compose --project-name "${project}" -f "${file}" up --detach --remove-orphans --quiet-pull || + ctr_compose_check "${file}" "${project}" || die "container: ${file} breaks the container rules; see above." + until log_command docker compose --project-name "${project}" -f "${file}" pull --quiet --policy missing; do + ((try < CTR_PULL_TRIES)) || die "container: the images of ${project} cannot be pulled; see the log." + log_warn "container: pulling the images of ${project} failed (attempt ${try} of ${CTR_PULL_TRIES}); retrying in ${CTR_PULL_DELAY} s." + sleep "${CTR_PULL_DELAY}" + try=$((try + 1)) + done + log_command docker compose --project-name "${project}" -f "${file}" up --detach --remove-orphans --quiet-pull "$@" || die "container: docker compose cannot start ${project}; see the log." return } diff --git a/lib/files.sh b/lib/files.sh index 1995adc..0271b46 100644 --- a/lib/files.sh +++ b/lib/files.sh @@ -94,16 +94,18 @@ _files_record_checksum() { return } -# file_write MODE OWNER:GROUP DEST, content on stdin. Sets LSS_FILE_CHANGED to 1 or 0. +# file_write MODE OWNER:GROUP DEST, content on stdin. Sets LSS_FILE_CHANGED to 1 or 0. OWNER:GROUP are names, or +# numbers (UID:GID) for IDs without an account, such as the users of a container in a user namespace. file_write() { - local mode=$1 owner=$2 dest=$3 tmp current_mode current_owner + local mode=$1 owner=$2 dest=$3 tmp current_mode current_owner format='%U:%G' LSS_FILE_CHANGED=0 + [[ ! ${owner} =~ ^[0-9]+:[0-9]+$ ]] || format='%u:%g' mkdir -p "$(dirname "${dest}")" tmp=$(mktemp "${dest}.lss.XXXXXX") cat >"${tmp}" if [[ -f ${dest} ]] && cmp -s "${tmp}" "${dest}"; then current_mode=$(stat -c '%a' "${dest}") - current_owner=$(stat -c '%U:%G' "${dest}") + current_owner=$(stat -c "${format}" "${dest}") if [[ ${current_mode} == "${mode#0}" && ${current_owner} == "${owner}" ]]; then rm -f -- "${tmp}" _files_record_checksum "${dest}" diff --git a/lib/firewall.sh b/lib/firewall.sh index 37d9cd0..e15584a 100644 --- a/lib/firewall.sh +++ b/lib/firewall.sh @@ -20,6 +20,31 @@ fw_backend() { return } +# Attempts and the pause in seconds for a ufw command that loses a race in iptables. +FW_UFW_TRIES=3 +FW_UFW_DELAY=2 + +# ufw ARG...: ufw, again after a short pause when it fails with "initcaps". Before every change ufw tests the +# capabilities of iptables with a chain of its own; when k3s (or Docker) rewrites its chains at the same moment, that +# test fails ("CHAIN_DEL failed") although nothing is wrong. Other failures and the output pass through unchanged. +# The function hides the command from command -v; "type -P ufw" tells whether ufw is installed. +ufw() { + local try=1 code err + err=$(mktemp) + while :; do + code=0 + command ufw "$@" 2>"${err}" || code=$? + if ((code == 0 || try >= FW_UFW_TRIES)) || ! grep -q initcaps "${err}"; then + break + fi + try=$((try + 1)) + sleep "${FW_UFW_DELAY}" + done + cat "${err}" >&2 + rm -f -- "${err}" + return "${code}" +} + fw_vpn_network() { config_get LSS_WIREGUARD_NETWORK 10.66.0.0/24 return @@ -42,7 +67,7 @@ fw_vpn_address() { fw_is_enabled() { if [[ $(fw_backend) == ufw ]]; then - command -v ufw >/dev/null 2>&1 && ufw status 2>/dev/null | grep -q '^Status: active' + type -P ufw >/dev/null && ufw status 2>/dev/null | grep -q '^Status: active' else svc_is_active firewalld fi @@ -120,7 +145,7 @@ _fw_apply() { # The backend tool is installed (the firewall module installs it). fw_available() { if [[ $(fw_backend) == ufw ]]; then - command -v ufw >/dev/null 2>&1 + type -P ufw >/dev/null else command -v firewall-offline-cmd >/dev/null 2>&1 fi diff --git a/lib/mail.sh b/lib/mail.sh index 5b84cec..7622c62 100644 --- a/lib/mail.sh +++ b/lib/mail.sh @@ -116,9 +116,21 @@ EOF } # mail_postfix_sandbox [DIRECTIVE...]: the systemd sandbox of the Postfix master and its daemons. They write the queue, -# their state and the local mailboxes only. Postfix needs /dev/log of the host for syslog, so /dev stays. +# their state and the local mailboxes only. Postfix needs /dev/log of the host for syslog, so /dev stays. Under SELinux +# the master starts its daemons in their own domains with NoNewPrivileges (svc_selinux_spawner), and so does systemd +# for the helpers of the unit: on the RHEL family postfix.service relabels its directories (restorecon) and rebuilds +# the aliases (newaliases, that is sendmail) before the start. mail_postfix_sandbox() { - svc_sandbox "$(mail_postfix_template)" PrivateDevices=no ReadWritePaths=/var/spool/postfix ReadWritePaths=/var/lib/postfix \ + local daemons sendmail unit + local -a programs + unit=$(mail_postfix_template) + daemons=$(postconf -h daemon_directory 2>/dev/null) || true + daemons=${daemons:-/usr/libexec/postfix} + sendmail=$(postconf -h sendmail_path 2>/dev/null) || true + svc_selinux_spawner postfix "${daemons}/master" "${daemons}" + mapfile -t programs < <(svc_exec_programs "${unit}") + svc_selinux_spawner postfix_start init_t "${programs[@]}" "${sendmail:-/usr/sbin/sendmail}" + svc_sandbox "${unit}" PrivateDevices=no ReadWritePaths=/var/spool/postfix ReadWritePaths=/var/lib/postfix \ "$@" return } diff --git a/lib/pkg.sh b/lib/pkg.sh index b399070..c70875a 100644 --- a/lib/pkg.sh +++ b/lib/pkg.sh @@ -19,6 +19,28 @@ _pkg_apt() { return "${PIPESTATUS[0]}" } +# Third-party archives (Launchpad PPAs, vendor repositories) answer 503 for minutes at a time. apt-get update only +# warns when an index cannot be fetched (W: Failed to fetch), so a package of that source looks missing; apt-get +# install fails on the first archive it cannot download. Both are tried again; apt changes nothing when a download +# fails. A warning that stays after the last attempt is left to the caller, as apt-get itself does. +PKG_APT_DELAY=30 +_pkg_apt_retry() { + local attempt output status + for attempt in 1 2 3; do + status=0 + output=$(_pkg_apt "$@") || status=$? + [[ -z ${output} ]] || printf '%s\n' "${output}" + if ((status == 0 && attempt < 3)) && grep -qE '^[WE]: Failed to fetch' <<<"${output}"; then + status=1 + fi + ((status != 0)) || return 0 + ((attempt < 3)) || break + printf 'apt-get %s failed (attempt %s of 3); trying again in %s seconds.\n' "$1" "${attempt}" "${PKG_APT_DELAY}" + sleep "${PKG_APT_DELAY}" + done + return "${status}" +} + # Mirrors of the RHEL family drop out for minutes at a time; dnf gives up on the first broken mirror list, so a # failed transaction is tried again with fresh metadata. dnf changes nothing when a transaction fails. _pkg_dnf() { @@ -35,7 +57,7 @@ _pkg_dnf() { pkg_update() { if os_is_debian; then - log_command _pkg_apt update + log_command _pkg_apt_retry update else log_command _pkg_dnf makecache fi @@ -79,7 +101,7 @@ pkg_install() { log_info "Installing ${missing[*]}" if os_is_debian; then pkg_update_once - log_command _pkg_apt install --no-install-recommends "${missing[@]}" + log_command _pkg_apt_retry install --no-install-recommends "${missing[@]}" else log_command _pkg_dnf install "${missing[@]}" fi @@ -131,7 +153,7 @@ pkg_remove_family() { pkg_upgrade() { if os_is_debian; then pkg_update - log_command _pkg_apt full-upgrade + log_command _pkg_apt_retry full-upgrade else log_command _pkg_dnf upgrade --refresh fi diff --git a/lib/service.sh b/lib/service.sh index 2302e93..0ebe94b 100644 --- a/lib/service.sh +++ b/lib/service.sh @@ -7,6 +7,8 @@ # Seconds lss waits for a unit to start. Some units (MySQL) wait for their service without a limit; a service that # never reports ready would stop the run for good. SVC_START_TIMEOUT=${SVC_START_TIMEOUT:-300} +# Sources of the SELinux policy modules that let services start their programs in their domains (svc_selinux_spawner). +SVC_SELINUX_DIR=${SVC_SELINUX_DIR:-/etc/lss/selinux} svc_exists() { systemctl list-unit-files --no-legend "$1" "$1.service" 2>/dev/null | grep -q . @@ -130,9 +132,7 @@ _svc_selinux_nnp() { [[ ${type} == ?*_exec_t ]] || return 0 domain=${type%_exec_t}_t module=lss_nnp_${type%_exec_t} - if semodule -l 2>/dev/null | awk '{ print $1 }' | grep -qxF "${module}"; then - return 0 - fi + ! _svc_selinux_loaded "${module}" || return 0 dir=$(mktemp -d) printf '(allow init_t %s (process2 (nnp_transition nosuid_transition)))\n' "${domain}" >"${dir}/${module}.cil" if ! log_command semodule -i "${dir}/${module}.cil"; then @@ -144,6 +144,79 @@ _svc_selinux_nnp() { return } +# _svc_selinux_loaded MODULE: the SELinux policy module MODULE is installed. +_svc_selinux_loaded() { + semodule -l 2>/dev/null | awk '{ print $1 }' | grep -qxF "$1" + return +} + +# svc_exec_programs UNIT: the programs systemd runs for UNIT (ExecStartPre, ExecStart, ExecStartPost), one per line. +svc_exec_programs() { + local property + for property in ExecStartPre ExecStart ExecStartPost; do + systemctl show -P "${property}" "$1" 2>/dev/null | grep -o 'path=[^ ;]*' | cut -d = -f 2 + done + return 0 +} + +# svc_selinux_spawner NAME SOURCE PATH...: lets a process start programs in their own SELinux domains while its +# sandbox keeps NoNewPrivileges: the Postfix master its daemons (postfix_qmgr_t, postfix_smtpd_t), Dovecot its auth +# process (dovecot_auth_t), systemd the helpers of postfix.service (restorecon, newaliases). NoNewPrivileges forbids +# each such change unless the policy allows it (avc: nnp_transition), and the program does not start. SOURCE is the +# domain of the process (init_t for systemd) or the program systemd starts it from; the PATHs are programs or +# directories of programs. The policy itself names the domain of every program (selinuxexeccon), and the policy module +# lss_spawn_NAME allows exactly these changes and nothing else. Its source stays in SVC_SELINUX_DIR, so a second run +# loads it only when an update brought other programs. Without SELinux it does nothing. +svc_selinux_spawner() { + local name=$1 source=$2 module=lss_spawn_$1 path file domain + shift 2 + _svc_selinux_on || return 0 + if [[ ${source} == /* ]]; then + domain=$(selinuxexeccon "${source}" system_u:system_r:init_t:s0 2>/dev/null | cut -d : -f 3) || true + [[ -n ${domain} ]] || die "${LSS_MODULE:-lss}: SELinux names no domain for ${source}." + source=${domain} + fi + local -a rules=() files=() + for path in "$@"; do + if [[ -d ${path} ]]; then + files=("${path}"/*) + else + files=("${path}") + fi + for file in "${files[@]}"; do + [[ -f ${file} && -x ${file} ]] || continue + domain=$(selinuxexeccon "${file}" "system_u:system_r:${source}:s0" 2>/dev/null | cut -d : -f 3) || true + [[ -z ${domain} || ${domain} == "${source}" ]] || + rules+=("(allow ${source} ${domain} (process2 (nnp_transition nosuid_transition)))") + done + done + if [[ ${#rules[@]} -eq 0 ]]; then + svc_selinux_spawner_remove "${name}" + return 0 + fi + install -d -m 0755 "${SVC_SELINUX_DIR}" + file_write 0644 root:root "${SVC_SELINUX_DIR}/${module}.cil" < <(printf '%s\n' "${rules[@]}" | LC_ALL=C sort -u) + if [[ ${LSS_FILE_CHANGED} == 0 ]] && _svc_selinux_loaded "${module}"; then + return 0 + fi + log_command semodule -i "${SVC_SELINUX_DIR}/${module}.cil" || + die "${LSS_MODULE:-lss}: SELinux does not take the policy module ${module} for ${source}; see the log." + files_note_change "SELinux module ${module}: ${source} starts its programs in their domains with NoNewPrivileges" + return +} + +# svc_selinux_spawner_remove NAME: unloads the policy module of svc_selinux_spawner and deletes its source. +svc_selinux_spawner_remove() { + local module=lss_spawn_$1 + if command -v semodule >/dev/null 2>&1 && _svc_selinux_loaded "${module}"; then + log_command semodule -r "${module}" || + die "${LSS_MODULE:-lss}: SELinux keeps the policy module ${module}; see the log." + files_note_change "removed SELinux module ${module}" + fi + file_remove "${SVC_SELINUX_DIR}/${module}.cil" + return +} + # svc_scanner_sandbox full|strict [PATH...]: [Service] lines that confine lss's own scanner units. They read the whole # system but write only to the PATHs (with strict) or below /var (with full). The real /tmp, /proc/kallsyms, the # kernel modules and the kernel log stay visible: a private /tmp would hide it from ClamAV and rkhunter, and rkhunter, diff --git a/lib/step.sh b/lib/step.sh index 757da7a..be9f141 100755 --- a/lib/step.sh +++ b/lib/step.sh @@ -15,7 +15,7 @@ set -Eeuo pipefail # change notes it sets reach the caller. shopt -s inherit_errexit lastpipe -for library in log os config files template secrets state pkg repo fetch service firewall container web mail monitoring module; do +for library in log os config files template secrets state pkg repo fetch service firewall container web mail monitoring apps module; do # shellcheck source=/dev/null source "${LSS_ROOT}/lib/${library}.sh" done diff --git a/lib/web.sh b/lib/web.sh index 50c6476..fc38a9c 100644 --- a/lib/web.sh +++ b/lib/web.sh @@ -33,6 +33,8 @@ WEB_ADMIN_CADDY_AUTH=/etc/lss/web/admin-auth.caddy WEB_CERT_HOOK=/etc/letsencrypt/renewal-hooks/deploy/lss-deploy # CA certificate of a private ACME server (LSS_TLS_CA_BUNDLE, for example Pebble or step-ca); certbot trusts it. WEB_ACME_CA=/etc/letsencrypt/lss-acme-ca.pem +# Rule exclusions of the routes for ModSecurity (web_waf_exclude), one file per route. +WEB_WAF_DIR=/etc/lss/waf # certbot's certificate lineages (one directory per certificate name). WEB_CERT_LIVE=/etc/letsencrypt/live # The fail2ban jails and the CrowdSec log acquisition for the installed web server. @@ -161,26 +163,55 @@ web_default_cert() { return 0 fi name=$(hostname -f 2>/dev/null || hostname) + _web_self_signed "${dir}" "${name}" + files_note_change "self-signed default certificate" + return +} + +# _web_self_signed DIR DOMAIN...: a self-signed certificate for the names DOMAIN (the first one is the subject) in +# DIR/fullchain.pem and DIR/privkey.pem, valid for 825 days; the key is readable by WEB_GROUP. +_web_self_signed() { + local dir=$1 subject=$2 tmp san domain + shift + san='' + for domain in "$@"; do + san+="${san:+,}DNS:${domain}" + done tmp=$(mktemp -d) - openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 825 -subj "/CN=${name}" \ - -addext "subjectAltName=DNS:${name}" -keyout "${tmp}/privkey.pem" -out "${tmp}/fullchain.pem" >/dev/null 2>&1 || - die "web: could not create the self-signed default certificate." - install -d -m 0755 "${WEB_TLS_DIR}" + if ! openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 825 -subj "/CN=${subject}" \ + -addext "subjectAltName=${san}" -keyout "${tmp}/privkey.pem" -out "${tmp}/fullchain.pem" >/dev/null 2>&1; then + rm -rf -- "${tmp}" + die "web: could not create a self-signed certificate for ${subject}." + fi # Certificates are state, not configuration: the tls module's deploy hook replaces them, so they are not # recorded as managed files (lss verify would report every renewal as a change made outside lss). - install -d -m 0755 "${dir}" + install -d -m 0755 "${WEB_TLS_DIR}" "${dir}" install -m 0640 -o root -g "${WEB_GROUP}" "${tmp}/privkey.pem" "${dir}/privkey.pem" install -m 0644 -o root -g root "${tmp}/fullchain.pem" "${dir}/fullchain.pem" - files_note_change "self-signed default certificate" rm -rf -- "${tmp}" return } -# web_cert_placeholder NAME: until NAME has a certificate of its own, its directory holds a copy of the default -# certificate, so a site can point at WEB_TLS_DIR/NAME from the start and keeps that path after the real one arrives. +# web_cert_placeholder NAME [DOMAIN...]: until NAME has a certificate of its own, its directory holds a copy of the +# default certificate, so a site can point at WEB_TLS_DIR/NAME from the start and keeps that path after the real one +# arrives. With DOMAIN, the placeholder is a self-signed certificate for these names instead, so clients that trust +# it (a Nextcloud that calls Collabora through the web server) can check the host name; it follows the names and is +# renewed within 30 days of expiry, and a certificate of the tls module (lineage NAME) is never replaced. web_cert_placeholder() { local name=$1 dir + shift dir="${WEB_TLS_DIR}/${name}" + if [[ $# -gt 0 ]]; then + [[ ! -s ${WEB_CERT_LIVE}/${name}/fullchain.pem ]] || return 0 + if [[ -s ${dir}/fullchain.pem && -s ${dir}/privkey.pem ]] && + [[ $(web_cert_domains "${dir}/fullchain.pem") == "$(printf '%s\n' "$@" | sort -u)" ]] && + openssl x509 -checkend $((30 * 86400)) -noout -in "${dir}/fullchain.pem" >/dev/null 2>&1; then + return 0 + fi + _web_self_signed "${dir}" "$@" + files_note_change "self-signed certificate for ${name}" + return + fi [[ -s ${dir}/fullchain.pem && -s ${dir}/privkey.pem ]] && return 0 install -d -m 0755 "${dir}" install -m 0644 -o root -g root "${WEB_TLS_DIR}/default/fullchain.pem" "${dir}/fullchain.pem" @@ -648,6 +679,105 @@ web_site_remove() { return } +# web_proxy_route NAME UPSTREAM DOMAIN...: publishes a local service (UPSTREAM, such as http://127.0.0.1:8081) as the +# HTTPS route NAME for the host names DOMAIN, with a certificate from the tls module when it is installed. The one +# place where an app becomes reachable from outside: today it writes a reverse-proxy site of the web server, and an +# edge proxy in front of the web server can take its routes over here. Reloads the web server when the site changed. +web_proxy_route() { + local name=$1 upstream=$2 + shift 2 + [[ $# -gt 0 ]] || die "web: the route ${name} has no host names." + web_cert_placeholder "${name}" "$@" + web_site_write "${name}" proxy "${upstream}" "$@" + [[ ${LSS_FILE_CHANGED} == 0 ]] || web_reload + if state_installed tls; then + web_cert_issue "${name}" "$@" + fi + return +} + +# web_proxy_route_remove NAME: removes the route of web_proxy_route with its certificates and ModSecurity exclusions. +web_proxy_route_remove() { + local name=$1 + web_waf_exclude_remove "${name}" + if [[ -n $(web_server) ]]; then + web_site_remove "${name}" + web_reload + fi + file_remove "${WEB_TLS_DIR}/${name}" + if [[ -d ${WEB_CERT_LIVE}/${name} ]] && command -v certbot >/dev/null 2>&1; then + log_command certbot delete --non-interactive --cert-name "${name}" || + log_warn "${name}: certbot could not delete the certificate ${name}." + fi + return +} + +# web_waf_exclude NAME "RULE_ID..." DOMAIN...: the Core Rule Set rules ModSecurity skips for the host names of the +# route NAME, for an application whose normal pages a rule takes for an attack (the system check of the Matomo +# installer shows PHP code, which rule 953110 reads as a leak). The exclusions are data in WEB_WAF_DIR, so they apply +# whenever the modsecurity module is installed, before or after the route; web_waf_rules turns them into rules. +web_waf_exclude() { + local name=$1 rules=$2 domain + shift 2 + _web_name_check "${name}" + [[ ${rules} =~ ^[0-9]{6}( [0-9]{6})*$ ]] || die "web: '${rules}' are not rule IDs of the Core Rule Set." + [[ $# -gt 0 ]] || die "web: the ModSecurity exclusions of ${name} have no host names." + for domain in "$@"; do + [[ ${domain} =~ ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ ]] || die "web: '${domain}' is not a host name." + done + install -d -m 0755 "${WEB_WAF_DIR}" + printf '# Managed by lss (route %s). ModSecurity skips these rules for these host names.\nRULES=%s\nDOMAINS=%s\n' \ + "${name}" "${rules}" "$*" | file_write 0644 root:root "${WEB_WAF_DIR}/${name}.conf" + [[ ${LSS_FILE_CHANGED} == 0 ]] || web_waf_apply + return +} + +# web_waf_exclude_remove NAME: drops the exclusions of the route NAME. +web_waf_exclude_remove() { + local name=$1 + [[ -e ${WEB_WAF_DIR}/${name}.conf ]] || return 0 + file_remove "${WEB_WAF_DIR}/${name}.conf" + rmdir "${WEB_WAF_DIR}" 2>/dev/null || true + web_waf_apply + return +} + +# web_waf_rules: the exclusions of every route as ModSecurity rules (phase 1, before the rules they switch off), with +# IDs from 9500001 on in the order of the route names. +web_waf_rules() { + local file id=9500000 rules domains pattern ctl rule + printf '# Managed by lss. Rule exclusions of the routes in %s (web_waf_exclude).\n' "${WEB_WAF_DIR}" + for file in "${WEB_WAF_DIR}"/*.conf; do + [[ -f ${file} ]] || continue + rules=$(sed -n 's/^RULES=//p' "${file}") + domains=$(sed -n 's/^DOMAINS=//p' "${file}") + [[ ${rules} =~ ^[0-9]{6}( [0-9]{6})*$ && ${domains} =~ ^[a-z0-9.\ -]+$ ]] || continue + pattern=$(sed 's/\./\\./g; s/ /|/g' <<<"${domains}") + ctl='' + for rule in ${rules}; do + ctl+=",ctl:ruleRemoveById=${rule}" + done + id=$((id + 1)) + printf 'SecRule REQUEST_HEADERS:Host "@rx ^(?:%s)(?::[0-9]+)?$" "id:%s,phase:1,pass,nolog,t:none,t:lowercase%s"\n' \ + "${pattern}" "${id}" "${ctl}" + done + return +} + +# web_waf_apply: writes the exclusions (web_waf_rules) into the ModSecurity configuration of the modsecurity module +# and reloads the web server when they changed; without ModSecurity there is nothing to do. +web_waf_apply() { + local server dir + state_installed modsecurity || return 0 + server=$(web_server) + [[ ${server} == nginx || ${server} == apache ]] || return 0 + dir=$(web_conf_dir "${server}")/modsecurity + [[ -d ${dir} ]] || return 0 + web_waf_rules | file_write 0644 root:root "${dir}/sites.conf" + [[ ${LSS_FILE_CHANGED} == 0 ]] || web_reload + return +} + # web_admin_write NAME PATH php|proxy|strip TARGET: publishes an admin interface below /PATH/ on the admin server # (WEB_ADMIN_PORT). TARGET is a directory with PHP files or an upstream URL. proxy passes the whole path on (the # upstream serves below /PATH/), strip removes /PATH first (for upstreams that only serve their root, such as Netdata). diff --git a/libexec/lss-app-backup b/libexec/lss-app-backup new file mode 100755 index 0000000..13e4012 --- /dev/null +++ b/libexec/lss-app-backup @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# lss app-backup: the data of the app stacks (Docker Compose, lib/apps.sh) in a form restic can back up consistently. +# Installed by the app modules as /usr/local/lib/lss/app-backup; lss-backup runs it before every backup, and lss +# verify of an app runs it for that app (app-backup NAME...). +# +# Every app registers itself in /etc/lss/app-backup/NAME.conf (KEY=value lines, parsed, never sourced): +# PROJECT the Compose project (lss-NAME) COMPOSE its compose.yaml +# DB postgres, mariadb or none: the service "db" is dumped (pg_dumpall, mariadb-dump of every database) +# DB_USER the database user of the PostgreSQL service (its superuser in the official image) +# COPY volumes copied while the service "app" is paused for a moment (SQLite databases next to their files) +# LIVE volumes backed up where they are (files only; the database of the stack is in the dump) +# PATHS host directories of the app (native installations) +# The dumps and copies go to /var/backups/lss-apps/NAME (root only), replaced on every run; restic keeps the history. +# The paths restic backs up (that directory, the LIVE volumes, PATHS) are listed in /var/lib/lss/app-backup/paths, +# and the result of each app in /var/lib/lss/app-backup/status. +set -Eeuo pipefail +shopt -s inherit_errexit + +REGISTRY=${LSS_APP_BACKUP_REGISTRY:-/etc/lss/app-backup} +OUT_ROOT=${LSS_APP_BACKUP_DIR:-/var/backups/lss-apps} +STATE=${LSS_APP_BACKUP_STATE:-/var/lib/lss/app-backup} +DOCKER=${LSS_DOCKER:-docker} + +say() { + printf 'lss-app-backup: %s\n' "$*" + return +} + +# setting FILE KEY [DEFAULT]: the value of KEY in the registration FILE. +setting() { + local file=$1 key=$2 default=${3-} line + while IFS= read -r line || [[ -n ${line} ]]; do + if [[ ${line} == "${key}="* ]]; then + printf '%s' "${line#*=}" + return 0 + fi + done <"${file}" + printf '%s' "${default}" + return +} + +# compose FILE ARGS...: docker compose for the stack of the registration FILE. +compose() { + local file=$1 + shift + "${DOCKER}" compose --project-name "$(setting "${file}" PROJECT)" -f "$(setting "${file}" COMPOSE)" "$@" + return +} + +# mountpoint FILE VOLUME: the directory of the named volume VOLUME of the stack on the host. +mountpoint() { + local file=$1 volume=$2 path + path=$("${DOCKER}" volume inspect --format '{{.Mountpoint}}' "$(setting "${file}" PROJECT)_${volume}") || return 1 + [[ ${path} == /* && -d ${path} ]] || return 1 + printf '%s' "${path}" + return +} + +# dump_db FILE OUT: the database of the stack as SQL. The MariaDB root password goes to the dump tool through its +# environment inside the container, never on a command line. +dump_db() { + local file=$1 out=$2 + case $(setting "${file}" DB none) in + postgres) + compose "${file}" exec -T db pg_dumpall --username="$(setting "${file}" DB_USER)" --clean --if-exists >"${out}" + ;; + mariadb) + # shellcheck disable=SC2016 # expanded inside the container + compose "${file}" exec -T db sh -c 'MYSQL_PWD=$(cat /run/secrets/db-root-password) exec mariadb-dump --user=root \ + --all-databases --single-transaction --routines --events --triggers --hex-blob' >"${out}" + ;; + *) return 1 ;; + esac + return +} + +# copy_volumes FILE DIR: the COPY volumes into DIR/VOLUME, with the stack paused so that files and SQLite databases +# match; the stack runs on even when a copy fails. +copy_volumes() { + local file=$1 dir=$2 volume source attempt status=0 + local -a volumes=() + read -ra volumes <<<"$(setting "${file}" COPY)" + ((${#volumes[@]} > 0)) || return 0 + # Only the app pauses; its database (if any) is dumped, and the one-shot "init" service has exited. + compose "${file}" pause app >/dev/null 2>&1 || return 1 + for volume in "${volumes[@]}"; do + if source=$(mountpoint "${file}" "${volume}"); then + rm -rf -- "${dir}/${volume}.part" + if cp -a -- "${source}" "${dir}/${volume}.part"; then + rm -rf -- "${dir:?}/${volume}" + mv -f -- "${dir}/${volume}.part" "${dir}/${volume}" + else + rm -rf -- "${dir}/${volume}.part" + status=1 + fi + else + status=1 + fi + done + for attempt in 1 2 3; do + compose "${file}" unpause app >/dev/null 2>&1 && break + ((attempt < 3)) || { + say "the app of $(setting "${file}" PROJECT) stays paused; run docker compose unpause app." >&2 + status=1 + } + sleep 2 + done + return "${status}" +} + +# backup_app FILE: the dump, the copies and the paths of one app; prints the paths for restic. +backup_app() { + local file=$1 name dir out volume path failed=0 + local -a volumes=() paths=() + name=$(basename "${file}" .conf) + dir=${OUT_ROOT}/${name} + mkdir -p "${dir}" + chmod 0700 "${dir}" + if [[ $(setting "${file}" DB none) != none ]]; then + out=${dir}/db.sql + # The dump goes to a temporary file first, so a failed run never replaces the last good dump. + if dump_db "${file}" "${out}.part" && [[ -s ${out}.part ]]; then + mv -f -- "${out}.part" "${out}" + else + rm -f -- "${out}.part" + say "the database dump of ${name} failed; ${out} keeps the previous one." >&2 + failed=1 + fi + fi + if ! copy_volumes "${file}" "${dir}"; then + say "the copy of the data of ${name} failed; ${dir} keeps the previous one." >&2 + failed=1 + fi + read -ra volumes <<<"$(setting "${file}" LIVE)" + for volume in "${volumes[@]}"; do + if path=$(mountpoint "${file}" "${volume}"); then + printf '%s\n' "${path}" + else + say "the volume ${volume} of ${name} is missing." >&2 + failed=1 + fi + done + read -ra paths <<<"$(setting "${file}" PATHS)" + for path in "${paths[@]}"; do + [[ ! -e ${path} ]] || printf '%s\n' "${path}" + done + return "${failed}" +} + +# run_backups [NAME...]: every registered app (or the ones named); fails when one of them failed. +run_backups() { + local file name failed=0 count=0 + local -a files=() + if (($# > 0)); then + for name in "$@"; do + [[ ${name} =~ ^[a-z0-9][a-z0-9-]*$ && -f ${REGISTRY}/${name}.conf ]] || { + say "${name} is not registered in ${REGISTRY}." >&2 + return 1 + } + files+=("${REGISTRY}/${name}.conf") + done + else + for file in "${REGISTRY}"/*.conf; do + [[ -f ${file} ]] && files+=("${file}") + done + fi + mkdir -p "${OUT_ROOT}" "${STATE}" + chmod 0700 "${OUT_ROOT}" "${STATE}" + : >"${STATE}/paths.tmp" + : >"${STATE}/status.tmp" + for file in "${files[@]}"; do + name=$(basename "${file}" .conf) + count=$((count + 1)) + if backup_app "${file}" >>"${STATE}/paths.tmp"; then + printf '%s ok %s\n' "${name}" "$(date +%s)" >>"${STATE}/status.tmp" + say "saved the data of ${name}." + else + printf '%s failed %s\n' "${name}" "$(date +%s)" >>"${STATE}/status.tmp" + failed=1 + fi + done + # A run for some apps only (lss verify) leaves the list of the full run alone. + if (($# == 0)); then + if ((count > 0)); then + printf '%s\n' "${OUT_ROOT}" >>"${STATE}/paths.tmp" + fi + mv -f -- "${STATE}/paths.tmp" "${STATE}/paths" + mv -f -- "${STATE}/status.tmp" "${STATE}/status" + ((count > 0)) || say "0 apps are registered here; nothing to save." + else + rm -f -- "${STATE}/paths.tmp" "${STATE}/status.tmp" + fi + return "${failed}" +} + +main() { + [[ ${EUID} -eq 0 ]] || { + say "must run as root." >&2 + exit 1 + } + umask 077 + run_backups "$@" +} + +# The unit tests source this file without running it. +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + main "$@" +fi diff --git a/libexec/lss-backup b/libexec/lss-backup index 42aa9db..f4403b9 100755 --- a/libexec/lss-backup +++ b/libexec/lss-backup @@ -9,7 +9,8 @@ # of the restic process alone, never through its command line; the database dumps and every other child never see # them. # -# backup run database dumps (db-dumps module), backup, forget and prune by the retention, check +# backup run database dumps (db-dumps module) and app data (app modules), backup, forget and prune by +# the retention, check # backup init creates the repository unless it exists # backup restore-test restores the canary file of the latest backup and compares it, then reads a part of the data # backup probe a small backup of the canary file and its restore test (lss verify, before the first run) @@ -26,6 +27,8 @@ STATE=${LSS_BACKUP_STATE:-/var/lib/lss/restic} TEXTFILE_DIR=${LSS_TEXTFILE_DIR:-/var/lib/prometheus/node-exporter} DB_DUMPS=${LSS_DB_DUMPS:-/usr/local/lib/lss/db-dumps} DB_DUMPS_CONF=${LSS_DB_DUMPS_CONF:-/etc/lss/db-dumps.conf} +APP_BACKUP=${LSS_APP_BACKUP:-/usr/local/lib/lss/app-backup} +APP_BACKUP_PATHS=${LSS_APP_BACKUP_STATE:-/var/lib/lss/app-backup}/paths CACHE=${LSS_BACKUP_CACHE:-/var/cache/restic} RESTIC=${LSS_RESTIC:-restic} CANARY_DIR=${STATE}/canary @@ -169,11 +172,15 @@ metrics() { return } -# backup_paths: the configured paths that exist (restic counts a missing one as an error). +# backup_paths: the configured paths and those of the app stacks (lss-app-backup) that exist (restic counts a missing +# one as an error). backup_paths() { local path local -a paths=() read -ra paths <<<"$(setting BACKUP_PATHS /etc)" + if [[ -f ${APP_BACKUP_PATHS} ]]; then + mapfile -t -O "${#paths[@]}" paths <"${APP_BACKUP_PATHS}" + fi for path in "${paths[@]}"; do if [[ -e ${path} ]]; then printf '%s\n' "${path}" @@ -216,6 +223,12 @@ cmd_run() { say "the database dumps failed; the backup goes on with the files." } fi + if [[ -x ${APP_BACKUP} ]]; then + "${APP_BACKUP}" || { + status=1 + say "saving the data of the apps failed; the backup goes on with the rest." + } + fi prepare cmd_init mapfile -t paths < <(backup_paths) diff --git a/lss b/lss index 532763c..68652c2 100755 --- a/lss +++ b/lss @@ -14,7 +14,7 @@ LSS_ROOT=$(cd -- "$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")" && pwd) LSS_VERSION=$(<"${LSS_ROOT}/VERSION") export LSS_ROOT LSS_VERSION -for library in log os config files template secrets state pkg repo fetch service firewall container web monitoring module ui; do +for library in log os config files template secrets state pkg repo fetch service firewall container web monitoring apps module ui; do # shellcheck source=/dev/null source "${LSS_ROOT}/lib/${library}.sh" done diff --git a/modules/apps/collabora.sh b/modules/apps/collabora.sh new file mode 100644 index 0000000..02cc5aa --- /dev/null +++ b/modules/apps/collabora.sh @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="collabora" +MOD_GROUP="apps" +MOD_DESCRIPTION="Collabora Online (CODE 26.04) as a Docker Compose stack behind the web server, the office suite of Nextcloud Office; only the Nextcloud host names may open documents" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Collabora site, separated by spaces; empty: collabora. +port|8082|Port of the Collabora container on 127.0.0.1, which the web server forwards to +memory|2g|Memory limit of the Collabora container +dictionaries|de_DE en_GB en_US|Spell checking languages, separated by spaces" +MOD_SECRETS="" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Collabora Online Development Edition 26.04.5.1.1, the multi-architecture index (amd64 and arm64), checked 2026-10-15. +# The image is distroless (no shell) and runs as cool (1001). +COOL_IMAGE=docker.io/collabora/code@sha256:bb8f3c9c5c9016045f25fccb5f0649be94469ebc7c7cc379f3576e470cccf33d + +mod_check() { + local languages='^[a-zA-Z_ -]*$' + app_check_domains + app_check_port port + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_COLLABORA_MEMORY must be a size such as 2g or 1536m." + [[ $(opt dictionaries) =~ ${languages} ]] || die "LSS_COLLABORA_DICTIONARIES must be language codes such as de_DE en_US." + return +} + +mod_install() { + app_compose_ready + return +} + +# _cool_wopi_hosts: the sites allowed to open documents (WOPI hosts): the host names of Nextcloud, as an alias group +# (the first one is the host, the others its aliases). Nothing else may use this server. +_cool_wopi_hosts() { + local domain list='' + for domain in $(app_domains_of nextcloud); do + list+=${list:+,}https://${domain}:443 + done + printf '%s' "${list}" + return +} + +# _cool_compose: the stack. Collabora runs without capabilities and without new privileges, so it cannot build its +# chroot jails; COOL_FORCE_LANDLOCK makes it isolate the document processes with Landlock instead. The root file +# system stays writable because the jails are created below /opt/cool. The admin console is off: it would need a +# password in the environment. Collabora always checks the certificate of Nextcloud (storage.ssl.as_scheme: an https +# WOPI address means a checked TLS connection), against the CAs of the image and app_collabora_trust. +_cool_compose() { + local domain params + domain=$(app_domain) + params="--o:ssl.enable=false --o:ssl.termination=true --o:admin_console.enable=false" + params+=" --o:ssl.ssl_verification=true --o:storage.ssl.as_scheme=true" + params+=" --o:storage.ssl.ca_file_path=/etc/coolwsd/lss-ca.pem --o:logging.level=warning" + cat </dev/null || die "collabora: https://$(app_domain)/ does not answer." + if state_installed nextcloud; then + app_nextcloud_office + fi + return +} + +mod_verify() { + local page probe pdf status answer waf + app_verify_stack + page=$(app_wait '' 200) || die "collabora: https://$(app_domain)/ does not answer." + [[ ${page%$'\n'*} == OK ]] || die "collabora: https://$(app_domain)/ answers '${page%$'\n'*}' instead of OK." + page=$(app_wait hosting/discovery 200 60) || die "collabora: the WOPI discovery does not answer." + grep -q 'urlsrc=' <<<"${page}" || die "collabora: the WOPI discovery lists no editors." + # Used, not only running: Collabora converts a text document to PDF. + probe=$(mktemp) + pdf=$(mktemp) + printf 'lss verify %s\n' "$(date +%s)" >"${probe}" + status=$(app_status "$(app_get cool/convert-to/pdf -F "data=@${probe};filename=lss-verify.txt" -o "${pdf}" --max-time 120)") + if [[ ${status} != 200 || $(head -c 4 "${pdf}") != %PDF ]]; then + # The answer tells Collabora's own refusal from one of the web server or its firewall. + answer=$(head -c 300 "${pdf}" | tr -c '[:print:]' ' ' | sed 's/<[^>]*>/ /g; s/ */ /g') + rm -f -- "${probe}" "${pdf}" + waf=$(app_waf_hits) + die "collabora: converting a document to PDF failed (HTTP ${status}: ${answer:-empty answer})${waf:+; ModSecurity: ${waf}}." + fi + rm -f -- "${probe}" "${pdf}" + if state_installed nextcloud; then + log_command _app_nc_occ richdocuments:activate-config || + die "collabora: Nextcloud Office cannot reach Collabora Online; see the log." + app_collabora_check + fi + return +} + +mod_remove() { + # Nextcloud stops offering Nextcloud Office; installing the module again turns it back on. + if state_installed nextcloud && command -v docker >/dev/null 2>&1; then + log_command _app_nc_occ app:disable richdocuments || + log_warn "collabora: Nextcloud Office could not be turned off in Nextcloud." + fi + app_remove + return +} diff --git a/modules/apps/forgejo.sh b/modules/apps/forgejo.sh new file mode 100644 index 0000000..68a501d --- /dev/null +++ b/modules/apps/forgejo.sh @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="forgejo" +MOD_GROUP="apps" +MOD_DESCRIPTION="Forgejo 16 (Git hosting) as a rootless Docker Compose stack behind the web server, with SQLite, Git over HTTPS only and no open registration" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Forgejo site, separated by spaces; empty: forgejo. +port|8083|Port of the Forgejo container on 127.0.0.1, which the web server forwards to +admin_user|lssadmin|Name of the first administrator (the password is the secret forgejo-admin-password) +admin_email||E-mail address of the first administrator; empty: ADMIN_USER@ +memory|1g|Memory limit of the Forgejo container" +MOD_SECRETS="forgejo-admin-password forgejo-secret-key forgejo-internal-token" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Forgejo 16.0.6 rootless (runs as git, 1000:1000), the multi-architecture index (amd64 and arm64), checked +# 2026-10-15. A new release means a new digest here; Forgejo migrates its database when it starts. +FJ_IMAGE=codeberg.org/forgejo/forgejo@sha256:51b47921660e1ec7127f094208fe06901ba57df4fe6d59fcccc3a2419514a77f +FJ_USER=1000:1000 + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt admin_user) =~ ^[a-z][a-z0-9._-]{1,38}$ && $(opt admin_user) != admin ]] || + die "LSS_FORGEJO_ADMIN_USER must be a lowercase login name other than admin (reserved by Forgejo)." + [[ -z $(opt admin_email) || $(opt admin_email) =~ ^[^@[:space:]]+@[^@[:space:]]+\.[a-z]+$ ]] || + die "LSS_FORGEJO_ADMIN_EMAIL must be an e-mail address." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_FORGEJO_MEMORY must be a size such as 1g or 768m." + return +} + +mod_install() { + app_compose_ready + return +} + +# _fj_email: the e-mail address of the first administrator. +_fj_email() { + local email + email=$(opt admin_email) + [[ -n ${email} ]] || email=$(opt admin_user)@$(app_domain) + printf '%s' "${email}" + return +} + +# _fj_compose: the stack. Forgejo reads its keys from files (the *_URI settings), so they are neither in app.ini nor +# in the environment. SSH stays off: Git works over HTTPS through the web server. +_fj_compose() { + local domain + domain=$(app_domain) + cat </dev/null | awk 'NR > 1 { print $2 }' | grep -qx -- "${user}"; then + secret_exists forgejo-admin-password || + log_warn "forgejo: the administrator ${user} exists, but the secret forgejo-admin-password is missing." + return 0 + fi + output=$(_fj admin user create --username "${user}" --email "$(_fj_email)" --admin --random-password \ + --random-password-length 32 --must-change-password=false 2>&1) || { + log_error "forgejo: $(grep -v 'generated random password' <<<"${output}")" + die "forgejo: the administrator ${user} cannot be created." + } + password=$(sed -n "s/^generated random password is '\(.*\)'$/\1/p" <<<"${output}") + [[ -n ${password} ]] || die "forgejo: Forgejo did not report the password of ${user}." + secret_set forgejo-admin-password "${password}" + log_info "forgejo: created the administrator ${user}; password in $(secret_path forgejo-admin-password)" + files_note_change "created the Forgejo administrator ${user}" + return +} + +mod_configure() { + app_secret forgejo-secret-key "${FJ_USER}" 64 secret-key + app_secret forgejo-internal-token "${FJ_USER}" 64 internal-token + _fj_compose | app_compose + app_backup COPY=data + app_up + app_site "$(opt_int port)" + app_wait api/healthz 200 >/dev/null || die "forgejo: https://$(app_domain)/api/healthz does not answer." + _fj_admin + return +} + +# _fj_api METHOD PATH [CURL OPTION...]: an API request as the administrator; prints the body and the status. +_fj_api() { + local method=$1 path=$2 netrc page + shift 2 + netrc=$(app_netrc "$(opt admin_user)" forgejo-admin-password) + page=$(app_get "api/v1/${path}" --netrc-file "${netrc}" -X "${method}" -H 'Content-Type: application/json' "$@") || true + rm -f -- "${netrc}" + printf '%s' "${page}" + return +} + +mod_verify() { + local page repo user + user=$(opt admin_user) + app_verify_stack + app_verify_backup + page=$(app_wait api/healthz 200) || die "forgejo: https://$(app_domain)/api/healthz does not answer." + [[ $(jq -r '.status' <<<"${page%$'\n'*}" 2>/dev/null) =~ ^(pass|warn)$ ]] || die "forgejo: the health check fails." + page=$(app_wait user/login 200 60) || die "forgejo: the login page does not answer." + grep -q 'name="password"' <<<"${page}" || die "forgejo: https://$(app_domain)/user/login shows no login." + # Registration is off: the sign-up page has no form to repeat a password. + page=$(app_get user/sign_up) || true + ! grep -q 'name="retype"' <<<"${page}" || die "forgejo: anybody can register at https://$(app_domain)/user/sign_up." + # Used, not only running: the administrator creates a repository through the API, reads it and deletes it. + repo="lss-verify-$(date +%s)" + page=$(_fj_api POST user/repos -d "{\"name\":\"${repo}\",\"private\":true,\"auto_init\":true}") + [[ $(app_status "${page}") == 201 ]] || die "forgejo: creating a repository as ${user} answers $(app_status "${page}")." + page=$(_fj_api GET "repos/${user}/${repo}/contents/README.md") + [[ $(app_status "${page}") == 200 ]] || die "forgejo: the new repository has no README.md ($(app_status "${page}"))." + page=$(_fj_api DELETE "repos/${user}/${repo}") + [[ $(app_status "${page}") == 204 ]] || die "forgejo: deleting the test repository answers $(app_status "${page}")." + return +} + +mod_remove() { + app_remove + return +} diff --git a/modules/apps/keycloak.sh b/modules/apps/keycloak.sh new file mode 100644 index 0000000..a26fcdd --- /dev/null +++ b/modules/apps/keycloak.sh @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="keycloak" +MOD_GROUP="apps" +MOD_DESCRIPTION="Keycloak 26 (single sign-on with OpenID Connect and SAML) with PostgreSQL as a Docker Compose stack behind the web server" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Keycloak site, separated by spaces; empty: keycloak. +port|8086|Port of the Keycloak container on 127.0.0.1, which the web server forwards to +admin_user|admin|Name of the first administrator of the master realm (the password is the secret keycloak-admin-password) +memory|1536m|Memory limit of the Keycloak container" +MOD_SECRETS="keycloak-admin-password keycloak-db-password" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Keycloak 26.8.1 (runs as keycloak, 1000:0), the multi-architecture index (amd64 and arm64), checked 2026-10-15. +KC_IMAGE=quay.io/keycloak/keycloak@sha256:79c623a22abacf3fe6c00a5fb30f5b7843d717b234033b80995891c06116d244 +KC_USER=1000:0 + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt admin_user) =~ ^[a-z][a-z0-9._-]{1,31}$ ]] || die "LSS_KEYCLOAK_ADMIN_USER must be a lowercase login name." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_KEYCLOAK_MEMORY must be a size such as 1536m or 2g." + return +} + +mod_install() { + app_compose_ready + return +} + +# _kc_compose: the stack. The passwords reach Keycloak in a configuration file (KC_CONFIG_FILE), not in the +# environment. "start" builds the server for PostgreSQL and the health checks on the first start and writes below +# /opt/keycloak, so the root file system stays writable. +_kc_compose() { + cat </dev/null || + die "keycloak: the master realm at https://$(app_domain)/realms/master does not answer." + log_info "keycloak: the admin console is https://$(app_domain)/admin/; user $(opt admin_user), password in $(secret_path keycloak-admin-password)" + return +} + +# _kc_token: an access token of the administrator (admin-cli, password grant); the password goes to curl as a file. +_kc_token() { + local page + page=$(app_get realms/master/protocol/openid-connect/token -X POST -d grant_type=password -d client_id=admin-cli \ + --data-urlencode "username=$(opt admin_user)" --data-urlencode "password@$(secret_path keycloak-admin-password)") || true + [[ $(app_status "${page}") == 200 ]] || return 1 + jq -r '.access_token // empty' <<<"${page%$'\n'*}" + return +} + +mod_verify() { + local page token headers user id status + app_verify_stack + app_verify_backup + page=$(app_wait realms/master/.well-known/openid-configuration 200) || die "keycloak: the master realm does not answer." + [[ $(jq -r '.issuer' <<<"${page%$'\n'*}" 2>/dev/null) == "https://$(app_domain)/realms/master" ]] || + die "keycloak: the issuer is not https://$(app_domain)/realms/master (KC_HOSTNAME or the proxy headers are wrong)." + # Used, not only running: the administrator logs in, creates a user through the admin API and deletes it. + token=$(_kc_token) || die "keycloak: $(opt admin_user) cannot log in to the master realm." + [[ -n ${token} ]] || die "keycloak: the login of $(opt admin_user) returned no token." + headers=$(mktemp) + printf 'Authorization: Bearer %s\nContent-Type: application/json\n' "${token}" >"${headers}" + user="lss-verify-$(date +%s)" + status=$(app_status "$(app_get admin/realms/master/users -X POST -H "@${headers}" -d "{\"username\":\"${user}\",\"enabled\":false}")") + if [[ ${status} != 201 ]]; then + rm -f -- "${headers}" + die "keycloak: creating a user answers ${status}." + fi + page=$(app_get "admin/realms/master/users?exact=true&username=${user}" -H "@${headers}") + id=$(jq -r '.[0].id // empty' <<<"${page%$'\n'*}" 2>/dev/null) || id='' + if [[ -z ${id} ]]; then + rm -f -- "${headers}" + die "keycloak: the new user is missing." + fi + status=$(app_status "$(app_get "admin/realms/master/users/${id}" -X DELETE -H "@${headers}")") + rm -f -- "${headers}" + [[ ${status} == 204 ]] || die "keycloak: deleting the test user answers ${status}." + return +} + +mod_remove() { + app_remove + return +} diff --git a/modules/apps/matomo.sh b/modules/apps/matomo.sh new file mode 100644 index 0000000..e25d226 --- /dev/null +++ b/modules/apps/matomo.sh @@ -0,0 +1,232 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="matomo" +MOD_GROUP="apps" +MOD_DESCRIPTION="Matomo 5 (web analytics) with MariaDB as a Docker Compose stack behind the web server, installed with a super user and a first website, reports archived by a systemd timer" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Matomo site, separated by spaces; empty: matomo. +port|8087|Port of the Matomo container on 127.0.0.1, which the web server forwards to +admin_user|admin|Name of the super user (the password is the secret matomo-admin-password) +admin_email||E-mail address of the super user; empty: ADMIN_USER@ +site_url||Address of the first website Matomo measures; empty: https:// +memory|1g|Memory limit of the Matomo container" +MOD_SECRETS="matomo-admin-password matomo-db-password matomo-db-root-password matomo-api-token" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Matomo 5.14.0 (apache), the multi-architecture index (amd64 and arm64), checked 2026-10-15. +MT_IMAGE=docker.io/library/matomo@sha256:1e00192627025a9eb9108858a0d2fcedb495958adf8ee12cd2b2e4ec306efae4 +# www-data in the image; the container runs as this user from the start, so it needs no capabilities. +MT_USER=33:33 +# A browser user agent, so Matomo does not drop the test visit of mod_verify as a bot. +MT_AGENT='Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0' + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt admin_user) =~ ^[a-zA-Z0-9._@-]{2,100}$ ]] || die "LSS_MATOMO_ADMIN_USER must be a login name." + [[ -z $(opt admin_email) || $(opt admin_email) =~ ^[^@[:space:]]+@[^@[:space:]]+\.[a-z]+$ ]] || + die "LSS_MATOMO_ADMIN_EMAIL must be an e-mail address." + [[ -z $(opt site_url) || $(opt site_url) =~ ^https?://[A-Za-z0-9.-]+(:[0-9]+)?(/[^[:space:]]*)?$ ]] || + die "LSS_MATOMO_SITE_URL must be an address such as https://www.example.com." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_MATOMO_MEMORY must be a size such as 1g or 768m." + return +} + +mod_install() { + app_compose_ready + return +} + +# _mt_compose: the stack. The installer of Matomo takes the database password from MATOMO_DATABASE_PASSWORD_FILE. +_mt_compose() { + cat </dev/null + return +} + +# _mt_step JAR STEP [CURL OPTION...]: one step of the web installer; prints the body and the status. Creating the +# tables takes long on a busy server, so a step may take up to five minutes (the later --max-time wins in curl). +_mt_step() { + local jar=$1 step=$2 code=0 + shift 2 + app_get "index.php?module=Installation&action=${step}" -b "${jar}" -c "${jar}" --max-time 300 --show-error "$@" || code=$? + ((code == 0)) || log_error "matomo: the installer step ${step:-welcome} gets no answer (curl exit status ${code})." + return "${code}" +} + +# _mt_post JAR STEP FIELD...: posts the form of STEP; fails unless Matomo moves on (redirect) or shows a page without a +# form error. +_mt_post() { + local jar=$1 step=$2 page status + shift 2 + page=$(_mt_step "${jar}" "${step}" -X POST "$@") || true + status=$(app_status "${page}") + if [[ ${status} != 302 && ${status} != 200 ]] || grep -q 'class="[^"]*form-errors\|class="[^"]*alert-danger' <<<"${page}"; then + log_error "matomo: the installer step ${step} answers ${status}: $(sed -n 's/.*\(alert-danger\|form-errors\)[^>]*>\([^<]*\).*/\2/p' <<<"${page}" | head -3)" + return 1 + fi + return 0 +} + +# _mt_install: the web installer, step by step (Matomo has no installer on the command line). The passwords go to +# curl as files, so they never show up in the process list; "**********" makes the installer use the database +# password from MATOMO_DATABASE_PASSWORD_FILE. +_mt_install() { + local jar email site domain waf + _mt_installed && return 0 + domain=$(app_domain) + email=$(opt admin_email) + [[ -n ${email} ]] || email=$(opt admin_user)@${domain} + site=$(opt site_url) + [[ -n ${site} ]] || site=https://${domain} + secret_get matomo-admin-password 32 >/dev/null || die "matomo: the password of the super user cannot be created." + log_info "matomo: installing Matomo (super user $(opt admin_user), password in $(secret_path matomo-admin-password))" + jar=$(mktemp) + if ! { + _mt_step "${jar}" '' >/dev/null && + _mt_step "${jar}" systemCheck >/dev/null && + _mt_post "${jar}" databaseSetup -d host=db -d username=matomo -d 'password=**********' -d dbname=matomo \ + -d tables_prefix=matomo_ --data-urlencode 'adapter=PDO\MYSQL' -d schema=Mariadb && + _mt_step "${jar}" tablesCreation >/dev/null && + _mt_post "${jar}" setupSuperUser --data-urlencode "login=$(opt admin_user)" \ + --data-urlencode "password@$(secret_path matomo-admin-password)" \ + --data-urlencode "password_bis@$(secret_path matomo-admin-password)" --data-urlencode "email=${email}" && + _mt_post "${jar}" firstWebsiteSetup --data-urlencode "siteName=${domain}" --data-urlencode "url=${site}" \ + -d timezone=UTC -d ecommerce=0 && + _mt_step "${jar}" trackingCode >/dev/null && + _mt_post "${jar}" finished -d anonymise_ip=1 + }; then + rm -f -- "${jar}" + waf=$(app_waf_hits) + die "matomo: the installation failed; see above${waf:+ (ModSecurity: ${waf})}." + fi + rm -f -- "${jar}" + _mt_installed || die "matomo: Matomo is not installed after the web installer." + files_note_change "installed Matomo" + return +} + +# _mt_set SECTION KEY VALUE: a setting of config.ini.php, changed only when it differs. +_mt_set() { + local section=$1 key=$2 value=$3 + # Matomo's ini writer puts strings in quotes (the console sets "0"), the installer writes plain numbers. + if ! app_exec app grep -Eqx "${key} = \"?${value}\"?" /var/www/html/config/config.ini.php 2>/dev/null; then + log_command _mt_console config:set --section="${section}" --key="${key}" --value="${value}" || + die "matomo: the console cannot set ${section}.${key}." + files_note_change "matomo setting ${section}.${key}" + fi + return +} + +# _mt_token: an app token of the super user for the reporting API (the secret matomo-api-token). +_mt_token() { + local page token + secret_exists matomo-api-token && return 0 + page=$(app_get index.php -X POST -d module=API -d method=UsersManager.createAppSpecificTokenAuth -d format=json \ + --data-urlencode "userLogin=$(opt admin_user)" --data-urlencode "passwordConfirmation@$(secret_path matomo-admin-password)" \ + -d description=lss) || true + token=$(jq -r '.value // empty' <<<"${page%$'\n'*}" 2>/dev/null) || token='' + [[ -n ${token} ]] || die "matomo: the reporting API gives the super user no token (HTTP $(app_status "${page}"))." + secret_set matomo-api-token "${token}" + files_note_change "created the Matomo API token" + return +} + +mod_configure() { + app_db_secrets mariadb "${MT_USER}" + _mt_compose | app_compose + app_backup DB=mariadb LIVE=html + app_up + app_site "$(opt_int port)" + # The system check of the installer contains PHP code, which rule 953110 of the Core Rule Set takes for leaked + # source code; ModSecurity then cuts the answer off. + app_waf_exclude 953110 + app_wait matomo.js 200 >/dev/null || die "matomo: https://$(app_domain)/matomo.js does not answer." + _mt_install + _mt_set General force_ssl 1 + _mt_set General assume_secure_protocol 1 + _mt_set General enable_browser_archiving_triggering 0 + _mt_token + app_timer matomo-archive 1h "Matomo report archiving" app php /var/www/html/console core:archive --url="https://$(app_domain)/" + return +} + +# _mt_api METHOD [CURL OPTION...]: a call of the reporting API with the token (in the request body, not the URL). +_mt_api() { + local method=$1 file + shift + file=$(secret_path matomo-api-token) + app_get index.php -X POST -d module=API -d "method=${method}" -d format=json --data-urlencode "token_auth@${file}" "$@" + return +} + +mod_verify() { + local page action + app_verify_stack + app_verify_backup + page=$(app_wait 'index.php?module=Login' 200) || die "matomo: the login page does not answer." + grep -q 'form_password\|name="form_password"' <<<"${page}" || die "matomo: https://$(app_domain)/index.php?module=Login shows no login." + # Used, not only running: a page view goes in through the tracker and comes out of the reporting API. + action="lss-verify-$(date +%s)" + page=$(app_get "matomo.php?idsite=1&rec=1&send_image=0&action_name=${action}&url=https%3A%2F%2F$(app_domain)%2F${action}" -A "${MT_AGENT}") + [[ $(app_status "${page}") =~ ^(200|204)$ ]] || die "matomo: the tracker answers $(app_status "${page}")." + page=$(_mt_api Live.getLastVisitsDetails -d idSite=1 -d period=day -d date=today -d filter_limit=20) + [[ $(app_status "${page}") == 200 ]] || die "matomo: the reporting API answers $(app_status "${page}")." + grep -q "${action}" <<<"${page}" || die "matomo: the test page view does not show up in the reporting API." + svc_is_enabled lss-matomo-archive.timer || die "matomo: the timer lss-matomo-archive for the report archiving is off." + timeout 600 systemctl start lss-matomo-archive.service || die "matomo: the report archiving fails (journalctl -u lss-matomo-archive)." + return +} + +mod_remove() { + app_timer_remove matomo-archive + app_remove + return +} diff --git a/modules/apps/minio.sh b/modules/apps/minio.sh new file mode 100644 index 0000000..fdeedfd --- /dev/null +++ b/modules/apps/minio.sh @@ -0,0 +1,272 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="minio" +MOD_GROUP="apps" +MOD_DESCRIPTION="S3-compatible object storage with the MinIO server code (PGSTY Silo, the maintained fork, since MinIO archived its community edition) behind the web server: the S3 API and the web console as two HTTPS sites" +MOD_REQUIRES="system nginx|apache|caddy" +MOD_MODES="native docker" +MOD_OPTIONS="domains||Host names of the S3 API, separated by spaces; empty: minio. +console_domains||Host names of the web console, separated by spaces; empty: minio-console. +api_port|8091|Port of the S3 API on 127.0.0.1, which the web server forwards to +console_port|8092|Port of the web console on 127.0.0.1, which the web server forwards to +root_user|lssadmin|Name of the root user (the password is the secret minio-root-password) +memory|1g|Memory limit of the container (docker mode)" +MOD_SECRETS="minio-root-password" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Silo RELEASE.2026-09-16T00-00-00Z: the image (multi-architecture index, amd64 and arm64) for the docker mode, the +# packages of the same release (data/checksums.txt) for the native mode. Checked 2026-10-15. +SILO_IMAGE=docker.io/pgsty/silo@sha256:635197cb9f36d01bee221d34d1c7d7960f6a95c48b0b6c01d99cd13bdae51a46 +SILO_VERSION=20260916000000 +# Not the root user of the image: an unprivileged user that owns only the data volume. +SILO_USER=1000:1000 +SILO_ENV=/etc/default/silo +SILO_ETC=/etc/silo +SILO_DATA=/var/lib/silo +# Silo falls back to the well-known minioadmin:minioadmin when it finds no credentials; mod_verify makes sure it did not. +SILO_DEFAULT_LOGIN=minioadmin + +mod_check() { + local domain + app_check_domains + for domain in $(opt console_domains); do + app_valid_domain "${domain}" || die "LSS_MINIO_CONSOLE_DOMAINS: '${domain}' is not a host name like minio-console.example.com." + done + app_check_port api_port + app_check_port console_port + [[ $(opt_int api_port) != "$(opt_int console_port)" ]] || die "LSS_MINIO_API_PORT and LSS_MINIO_CONSOLE_PORT must differ." + [[ $(opt root_user) =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]{2,63}$ && $(opt root_user) != "${SILO_DEFAULT_LOGIN}" ]] || + die "LSS_MINIO_ROOT_USER must be 3 to 64 letters, digits, dots, dashes or underscores, other than ${SILO_DEFAULT_LOGIN}." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_MINIO_MEMORY must be a size such as 1g or 768m." + return +} + +_minio_docker() { + [[ $(module_mode minio) == docker ]] + return +} + +# _minio_console_domains: the host names of the web console. +_minio_console_domains() { + local domains host + domains=$(opt console_domains) + if [[ -z ${domains} ]]; then + host=$(hostname -f 2>/dev/null || true) + [[ -n ${host} ]] || host=$(hostname) + domains=minio-console.${host,,} + fi + printf '%s' "${domains}" + return +} + +_minio_console_domain() { + local first + read -r first _ <<<"$(_minio_console_domains)" + printf '%s' "${first}" + return +} + +# _minio_package_installed: the pinned Silo release is installed. +_minio_package_installed() { + if os_is_debian; then + [[ $(dpkg-query -W -f='${Status} ${Version}' silo 2>/dev/null) == "install ok installed ${SILO_VERSION}"* ]] + else + [[ $(rpm -q --qf '%{VERSION}' silo 2>/dev/null) == "${SILO_VERSION}"* ]] + fi + return +} + +mod_install() { + local tmp + if _minio_docker; then + app_compose_ready + return 0 + fi + _minio_package_installed && return 0 + tmp=$(mktemp -d) + if os_is_debian; then + fetch_verified silo-deb "${tmp}/silo.deb" + pkg_update_once + log_command _pkg_apt install --no-install-recommends "${tmp}/silo.deb" || die "minio: apt cannot install Silo; see the log." + else + fetch_verified silo-rpm "${tmp}/silo.rpm" + log_command _pkg_dnf install "${tmp}/silo.rpm" || die "minio: dnf cannot install Silo; see the log." + fi + rm -rf -- "${tmp}" + _minio_package_installed || die "minio: Silo ${SILO_VERSION} is not installed after the package installation." + files_note_change "installed Silo ${SILO_VERSION}" + return +} + +# _minio_compose: the stack of the docker mode. The image reads the root user and password from the files +# /run/secrets/access_key and /run/secrets/secret_key (MINIO_ROOT_USER_FILE and MINIO_ROOT_PASSWORD_FILE). +_minio_compose() { + cat </dev/null 2>&1 || die "minio: the package did not create the user silo." + install -d -m 0750 -o root -g silo "${SILO_ETC}" + install -d -m 0750 -o silo -g silo "${SILO_DATA}" + file_write 0440 root:silo "${SILO_ETC}/root-password" <<<"${password}" + [[ ${LSS_FILE_CHANGED} == 0 ]] || changed=1 + file_write 0644 root:root "${SILO_ENV}" </dev/null || die "minio: https://$(app_domain)/minio/health/ready does not answer." + log_info "minio: S3 API https://$(app_domain), console https://$(_minio_console_domain); user $(opt root_user), password in $(secret_path minio-root-password)" + return +} + +# _minio_s3 USER SECRET METHOD PATH [CURL OPTION...]: an S3 request signed with AWS Signature V4 by curl; the login +# comes from a netrc file. SECRET empty: the default login of the server. +_minio_s3() { + local user=$1 secret=$2 method=$3 path=$4 netrc page + shift 4 + if [[ -n ${secret} ]]; then + netrc=$(app_netrc "${user}" "${secret}") + else + netrc=$(mktemp) + chmod 600 "${netrc}" + printf 'machine %s login %s password %s\n' "$(app_domain)" "${SILO_DEFAULT_LOGIN}" "${SILO_DEFAULT_LOGIN}" >"${netrc}" + fi + page=$(app_get "${path}" --netrc-file "${netrc}" --aws-sigv4 aws:amz:us-east-1:s3 -X "${method}" "$@") || true + rm -f -- "${netrc}" + printf '%s' "${page}" + return +} + +# _minio_error PAGE: the S3 error code in the XML answer PAGE (such as SignatureDoesNotMatch). +_minio_error() { + local page=$1 + sed -n 's:.*\([A-Za-z]*\).*:\1:p' <<<"${page}" | head -n 1 + return +} + +mod_verify() { + local page bucket probe user status + user=$(opt root_user) + app_verify_backup + if _minio_docker; then + app_verify_stack + else + svc_is_active silo.service || die "minio: silo.service is not running." + [[ $(stat -c '%a %U:%G' "${SILO_ETC}/root-password") == '440 root:silo' ]] || + die "minio: ${SILO_ETC}/root-password must be mode 440, owned by root:silo." + page=$(app_get '' --head) || true + grep -qi '^strict-transport-security:' <<<"${page}" || die "minio: https://$(app_domain)/ sends no Strict-Transport-Security header." + fi + page=$(app_wait minio/health/ready 200) || die "minio: the S3 API does not answer." + page=$(APP_GET_DOMAIN=$(_minio_console_domain) app_wait '' 200 60) || die "minio: the web console does not answer." + grep -qi '"${probe}" + status=$(app_status "$(_minio_s3 "${user}" minio-root-password PUT "${bucket}")") + [[ ${status} == 200 ]] || die "minio: creating the bucket ${bucket} answers ${status}." + # The hash of the body goes along explicitly: curl before 8.x leaves x-amz-content-sha256 out, and the server then + # checks the signature against the hash of an empty body. + page=$(_minio_s3 "${user}" minio-root-password PUT "${bucket}/probe.txt" --data-binary "@${probe}" \ + -H "x-amz-content-sha256: $(sha256sum "${probe}" | cut -d ' ' -f 1)") + rm -f -- "${probe}" + status=$(app_status "${page}") + [[ ${status} == 200 ]] || die "minio: storing an object answers ${status} ($(_minio_error "${page}"))." + page=$(_minio_s3 "${user}" minio-root-password GET "${bucket}/probe.txt") + [[ ${page%$'\n'*} == "${bucket}" ]] || die "minio: the stored object does not come back." + status=$(app_status "$(_minio_s3 "${user}" minio-root-password DELETE "${bucket}/probe.txt")") + [[ ${status} == 204 ]] || die "minio: deleting the object answers ${status}." + status=$(app_status "$(_minio_s3 "${user}" minio-root-password DELETE "${bucket}")") + [[ ${status} == 204 ]] || die "minio: deleting the bucket answers ${status}." + return +} + +mod_remove() { + app_site_remove minio-console + if _minio_docker; then + app_remove + return 0 + fi + app_site_remove minio + app_backup_remove + svc_disable silo.service + svc_dropin_remove silo.service lss-sandbox + file_remove "${SILO_ETC}" + file_remove "${SILO_ENV}" + pkg_remove silo + if [[ ${LSS_PURGE:-0} == 1 ]]; then + file_remove "${SILO_DATA}" + secret_remove minio-root-password + fi + return +} diff --git a/modules/apps/n8n.sh b/modules/apps/n8n.sh new file mode 100644 index 0000000..2fe648d --- /dev/null +++ b/modules/apps/n8n.sh @@ -0,0 +1,199 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="n8n" +MOD_GROUP="apps" +MOD_DESCRIPTION="n8n 2 (workflow automation) with PostgreSQL as a Docker Compose stack behind the web server, with the owner account already created and telemetry off" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the n8n site, separated by spaces; empty: n8n. +port|8089|Port of the n8n container on 127.0.0.1, which the web server forwards to +owner_email||E-mail address of the owner account (its login); empty: admin@ +memory|1g|Memory limit of the n8n container" +MOD_SECRETS="n8n-owner-password n8n-encryption-key n8n-db-password" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# n8n 2.44.2 (runs as node, 1000:1000), the multi-architecture index (amd64 and arm64), checked 2026-10-15. +N8N_IMAGE=docker.io/n8nio/n8n@sha256:b57e7a9c8e5ab1a8fe9b03fff3158052aec65f571ed1f759897a20f306ad329d +N8N_USER=1000:1000 +# n8n binds a browser session to this header; the requests of mod_verify send the same one throughout. +N8N_BROWSER_ID=lss-verify + +mod_check() { + app_check_domains + app_check_port port + [[ -z $(opt owner_email) || $(opt owner_email) =~ ^[^@[:space:]]+@[^@[:space:]]+\.[a-z]+$ ]] || + die "LSS_N8N_OWNER_EMAIL must be an e-mail address." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_N8N_MEMORY must be a size such as 1g or 768m." + return +} + +mod_install() { + app_compose_ready + return +} + +_n8n_email() { + local email + email=$(opt owner_email) + [[ -n ${email} ]] || email=admin@$(app_domain) + printf '%s' "${email}" + return +} + +# _n8n_compose: the stack. Every secret reaches n8n as a file (*_FILE); the frontend cache and /tmp are the only +# writable paths besides the volume. +_n8n_compose() { + local domain + domain=$(app_domain) + cat <"${file}" + return +} + +# _n8n_owner: the owner account (POST /rest/owner/setup, possible once, before anybody logs in). +_n8n_owner() { + local body page status + _n8n_password + body=$(mktemp) + chmod 600 "${body}" + # shellcheck disable=SC2016 # jq variables + _n8n_body "${body}" '{email: $email, firstName: "Server", lastName: "Admin", password: $password}' + page=$(app_get rest/owner/setup -X POST -H 'Content-Type: application/json' --data-binary "@${body}") || true + rm -f -- "${body}" + status=$(app_status "${page}") + if [[ ${status} == 200 ]]; then + log_info "n8n: created the owner $(_n8n_email); password in $(secret_path n8n-owner-password)" + files_note_change "created the n8n owner" + elif [[ ${status} != 400 ]] || ! grep -qi 'already setup' <<<"${page}"; then + die "n8n: the owner cannot be created (HTTP ${status})." + fi + return +} + +mod_configure() { + app_db_secrets postgres "${N8N_USER}" + app_secret n8n-encryption-key "${N8N_USER}" 48 encryption-key + _n8n_compose | app_compose + app_backup DB=postgres DB_USER=n8n LIVE=data + app_up + app_site "$(opt_int port)" + app_wait healthz/readiness 200 >/dev/null || die "n8n: https://$(app_domain)/healthz/readiness does not answer." + _n8n_owner + return +} + +# _n8n_rest JAR PATH [CURL OPTION...]: a request to the REST API of the editor in the session of JAR. +_n8n_rest() { + local jar=$1 path=$2 + shift 2 + app_get "rest/${path}" -b "${jar}" -c "${jar}" -H "browser-id: ${N8N_BROWSER_ID}" -H 'Content-Type: application/json' "$@" + return +} + +mod_verify() { + local page jar body id name + app_verify_stack + app_verify_backup + page=$(app_wait healthz/readiness 200) || die "n8n: https://$(app_domain)/healthz/readiness does not answer." + page=$(app_wait signin 200 60) || die "n8n: the sign-in page does not answer." + grep -qi 'n8n' <<<"${page}" || die "n8n: https://$(app_domain)/signin is not n8n." + # Used, not only running: the owner logs in, creates a workflow, reads it back and deletes it. + jar=$(mktemp) + body=$(mktemp) + chmod 600 "${body}" + # shellcheck disable=SC2016 # jq variables + _n8n_body "${body}" '{emailOrLdapLoginId: $email, email: $email, password: $password}' + page=$(_n8n_rest "${jar}" login -X POST --data-binary "@${body}") || true + rm -f -- "${body}" + if [[ $(app_status "${page}") != 200 ]]; then + rm -f -- "${jar}" + die "n8n: the owner $(_n8n_email) cannot log in (HTTP $(app_status "${page}"))." + fi + name="lss-verify-$(date +%s)" + page=$(_n8n_rest "${jar}" workflows -X POST -d "{\"name\":\"${name}\",\"nodes\":[],\"connections\":{},\"settings\":{}}") + id=$(jq -r '.data.id // empty' <<<"${page%$'\n'*}" 2>/dev/null) || id='' + if [[ -z ${id} ]]; then + rm -f -- "${jar}" + die "n8n: creating a workflow answers $(app_status "${page}")." + fi + page=$(_n8n_rest "${jar}" "workflows/${id}") + if [[ $(jq -r '.data.name // empty' <<<"${page%$'\n'*}" 2>/dev/null) != "${name}" ]]; then + rm -f -- "${jar}" + die "n8n: the new workflow does not come back." + fi + # n8n 2 deletes only archived workflows. + _n8n_rest "${jar}" "workflows/${id}/archive" -X POST -d '{}' >/dev/null || true + page=$(_n8n_rest "${jar}" "workflows/${id}" -X DELETE) + rm -f -- "${jar}" + [[ $(app_status "${page}") == 200 ]] || die "n8n: deleting the test workflow answers $(app_status "${page}")." + return +} + +mod_remove() { + app_remove + return +} diff --git a/modules/apps/nextcloud.sh b/modules/apps/nextcloud.sh new file mode 100644 index 0000000..1de73bc --- /dev/null +++ b/modules/apps/nextcloud.sh @@ -0,0 +1,219 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="nextcloud" +MOD_GROUP="apps" +MOD_DESCRIPTION="Nextcloud Hub 34 with PostgreSQL as a Docker Compose stack behind the web server, background jobs by a systemd timer, linked to Collabora Online when the collabora module is installed" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Nextcloud site, separated by spaces; empty: nextcloud. +port|8081|Port of the Nextcloud container on 127.0.0.1, which the web server forwards to +admin_user|admin|Name of the first administrator (the password is the secret nextcloud-admin-password) +memory|1g|Memory limit of the Nextcloud container" +MOD_SECRETS="nextcloud-admin-password nextcloud-db-password" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Nextcloud 34.0.4 (apache), the multi-architecture index (amd64 and arm64), checked 2026-10-15. A new release means a +# new digest here; the container upgrades the installation when it starts. +NC_IMAGE=docker.io/library/nextcloud@sha256:0e5688ba0988b9112cf03f9c6bab53eccb466d278435f6596f16c1f25acacf74 +# www-data in the image; the container runs as this user from the start, so it needs no capabilities. +NC_USER=33:33 + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt admin_user) =~ ^[a-z][a-z0-9._-]{1,31}$ ]] || die "LSS_NEXTCLOUD_ADMIN_USER must be a lowercase login name." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_NEXTCLOUD_MEMORY must be a size such as 1g or 768m." + return +} + +mod_install() { + app_compose_ready + return +} + +# _nc_compose: the stack. Nextcloud runs as www-data on a read-only root file system; its code, configuration and +# files are in the volume "html". The secrets reach it only as files; occ reads the passwords for the installation +# from stdin (_nc_install). +_nc_compose() { + local domain + domain=$(app_domain) + cat </dev/null | jq -r '.installed // false' 2>/dev/null) == true ]] + return +} + +# _nc_started: the entrypoint of the image has finished. On the first start it copies Nextcloud into the volume +# (custom_apps, config and version.php come last) and only then starts Apache; occ in between finds a half-copied tree +# ("App directory custom_apps not found"). Through docker-proxy the port takes connections from the start, so the +# test is an HTTP answer from Apache itself (any status; curl reports 000 for none). +_nc_started() { + [[ $(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://127.0.0.1:$(opt_int port)/status.php" 2>/dev/null) != 000 ]] + return +} + +# _nc_install: the first installation. occ asks for the database and the admin password when they are not on its +# command line; both come from the mounted secret files through a pipe, so they never appear in a process list. +_nc_install() { + local i + for ((i = 0; i < 100; i++)); do + _nc_started && _nc_occ status >/dev/null 2>&1 && break + sleep 3 + done + _nc_installed && return 0 + log_info "nextcloud: installing Nextcloud (admin user $(opt admin_user), password in $(secret_path nextcloud-admin-password))" + # shellcheck disable=SC2016 # expanded by the shell in the container + log_command app_exec -e SHELL_INTERACTIVE=1 app sh -c 'printf "%s\n%s\n" "$(cat /run/secrets/db-password)" "$(cat /run/secrets/admin-password)" | + php occ maintenance:install --database pgsql --database-host db --database-name nextcloud --database-user nextcloud \ + --admin-user "$1" --data-dir /var/www/html/data' sh "$(opt admin_user)" || + die "nextcloud: the installation failed; see the log." + _nc_installed || die "nextcloud: Nextcloud is not installed after occ maintenance:install." + files_note_change "installed Nextcloud" + return +} + +# _nc_set KEY VALUE: a system setting, changed only when it differs. +_nc_set() { + local key=$1 value=$2 index=${3:-} + local -a keys=("${key}") + [[ -z ${index} ]] || keys+=("${index}") + if [[ $(_nc_occ config:system:get "${keys[@]}" 2>/dev/null) != "${value}" ]]; then + log_command _nc_occ config:system:set "${keys[@]}" --value="${value}" || die "nextcloud: occ cannot set ${key}." + files_note_change "nextcloud setting ${key}" + fi + return +} + +# _nc_settings: host names, URLs and defaults after the installation. +_nc_settings() { + local domain index=1 + _nc_set trusted_domains localhost 0 + for domain in $(app_domains); do + _nc_set trusted_domains "${domain}" "${index}" + index=$((index + 1)) + done + _nc_set overwrite.cli.url "https://$(app_domain)" + _nc_set overwriteprotocol https + _nc_set maintenance_window_start 1 + if [[ $(_nc_occ config:app:get core backgroundjobs_mode 2>/dev/null) != cron ]]; then + log_command _nc_occ background:cron || die "nextcloud: occ cannot switch background jobs to cron." + files_note_change "nextcloud background jobs by cron" + fi + return +} + +mod_configure() { + app_db_secrets postgres "${NC_USER}" + app_secret nextcloud-admin-password "${NC_USER}" 32 admin-password + _nc_compose | app_compose + app_backup DB=postgres DB_USER=nextcloud LIVE=html + app_up + _nc_install + _nc_settings + app_site "$(opt_int port)" + app_nextcloud_office + app_timer nextcloud-cron 5min "Nextcloud background jobs" app php -f /var/www/html/cron.php + return +} + +# _nc_dav METHOD PATH [CURL OPTION...]: a WebDAV request as the administrator; prints the body and the status. +_nc_dav() { + local method=$1 path=$2 netrc page + shift 2 + netrc=$(app_netrc "$(opt admin_user)" nextcloud-admin-password) + page=$(app_get "remote.php/dav/files/$(opt admin_user)/${path}" --netrc-file "${netrc}" -X "${method}" "$@") || true + rm -f -- "${netrc}" + printf '%s' "${page}" + return +} + +mod_verify() { + local page status probe + app_verify_stack + app_verify_backup + page=$(app_wait status.php 200) || die "nextcloud: https://$(app_domain)/status.php does not answer." + [[ $(jq -r '.installed' <<<"${page%$'\n'*}" 2>/dev/null) == true ]] || die "nextcloud: Nextcloud is not installed." + [[ $(jq -r '.maintenance' <<<"${page%$'\n'*}" 2>/dev/null) == false ]] || die "nextcloud: Nextcloud is in maintenance mode." + page=$(app_wait login 200 60) || die "nextcloud: the login page does not answer." + grep -q 'name="password"\|id="password"\|login' <<<"${page}" || die "nextcloud: https://$(app_domain)/login shows no login." + # Used, not only running: the administrator stores a file over WebDAV, reads it back and deletes it. + probe="lss-verify-$(date +%s)" + printf '%s' "${probe}" >"${TMPDIR:-/tmp}/${probe}.txt" + status=$(app_status "$(_nc_dav PUT "${probe}.txt" --data-binary "@${TMPDIR:-/tmp}/${probe}.txt")") + rm -f -- "${TMPDIR:-/tmp}/${probe}.txt" + [[ ${status} == 201 || ${status} == 204 ]] || die "nextcloud: WebDAV upload as $(opt admin_user) answers ${status}." + page=$(_nc_dav GET "${probe}.txt") + [[ ${page%$'\n'*} == "${probe}" ]] || die "nextcloud: the uploaded file does not come back over WebDAV." + status=$(app_status "$(_nc_dav DELETE "${probe}.txt")") + [[ ${status} == 204 ]] || die "nextcloud: deleting the test file answers ${status}." + svc_is_enabled lss-nextcloud-cron.timer || die "nextcloud: the timer lss-nextcloud-cron for the background jobs is off." + timeout 600 systemctl start lss-nextcloud-cron.service || die "nextcloud: the background jobs fail (journalctl -u lss-nextcloud-cron)." + if state_installed collabora; then + log_command _nc_occ richdocuments:activate-config || die "nextcloud: Nextcloud Office cannot reach Collabora Online." + app_collabora_check + fi + return +} + +mod_remove() { + app_timer_remove nextcloud-cron + app_remove + return +} diff --git a/modules/apps/self-updater.sh b/modules/apps/self-updater.sh new file mode 100644 index 0000000..910bfcd --- /dev/null +++ b/modules/apps/self-updater.sh @@ -0,0 +1,256 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="self-updater" +MOD_GROUP="apps" +MOD_DESCRIPTION="Self-Updater 4.2 (VectoDE): keeps a checkout of a Git repository on this server current, restarts its service, checks its health and rolls back on failure" +MOD_REQUIRES="system node" +MOD_MODES="native" +MOD_OPTIONS="repo||Repository to deploy (HTTPS URL, git@host:owner/repo.git or a local path); required +branch|main|Branch to follow (track branch) +track|branch|branch (deploy the newest commit of the branch) or tag (deploy the newest release tag matching v*) +path|/srv/app|Directory of the deployed checkout; created on the first run, owned by the user self-updater +service|none|What restarts after a deployment: none, or systemd:UNIT (a sudo rule allows exactly that restart) +health_url||URL that must answer 200 after a deployment, otherwise the previous commit comes back; empty: no check +interval|60|Seconds between two polls of the repository (at least 15) +port|8093|Port of the webhook receiver (/hooks/self-updater, signed with the secret self-updater-webhook-secret) and the status endpoint (/status) on 127.0.0.1 +token||Access token for a private HTTPS repository (GitHub: fine-grained, Contents read), stored in the lss secrets (self-updater-token)" +MOD_SECRETS="self-updater-webhook-secret self-updater-token" +MOD_PORTS="none (the status endpoint and the webhook receiver listen on 127.0.0.1)" + +# A new version is a new entry self-updater- in data/checksums.txt and a new SU_VERSION. +SU_VERSION=4.2.1 +SU_USER=self-updater +SU_HOME=/var/lib/self-updater +SU_CONFIG_DIR=/etc/self-updater +SU_CONFIG=${SU_CONFIG_DIR}/updater.config.json +SU_ENV=${SU_CONFIG_DIR}/.env +SU_UNIT=self-updater +SU_SUDOERS=/etc/sudoers.d/lss-self-updater +SU_STATUS_PATH=/status + +mod_check() { + local repo path service interval + repo=$(opt repo) + [[ -n ${repo} ]] || die "LSS_SELF_UPDATER_REPO must name the repository to deploy (https://github.com/owner/repo.git)." + [[ ${repo} =~ ^(https://[A-Za-z0-9.-]+(:[0-9]+)?/[A-Za-z0-9._/~-]+|git@[A-Za-z0-9.-]+:[A-Za-z0-9._/~-]+|/[A-Za-z0-9._/-]+)$ ]] || + die "LSS_SELF_UPDATER_REPO: '${repo}' is not an HTTPS URL, git@host:owner/repo.git or an absolute path." + [[ $(opt branch) =~ ^[A-Za-z0-9._/-]+$ ]] || die "LSS_SELF_UPDATER_BRANCH: '$(opt branch)' is not a branch name." + [[ $(opt track) =~ ^(branch|tag)$ ]] || die "LSS_SELF_UPDATER_TRACK must be branch or tag." + path=$(opt path) + [[ ${path} =~ ^/[A-Za-z0-9._/-]+$ && /${path}/ != */../* ]] || die "LSS_SELF_UPDATER_PATH must be an absolute path." + case ${path%/} in + / | /etc* | /usr* | /bin* | /sbin* | /lib* | /boot* | /proc* | /sys* | /dev* | /root* | /var/lib/lss* | /opt/lss*) + die "LSS_SELF_UPDATER_PATH must not be a system directory (${path})." + ;; + *) ;; + esac + service=$(opt service) + [[ ${service} == none || ${service} =~ ^systemd:[A-Za-z0-9@._-]+$ ]] || + die "LSS_SELF_UPDATER_SERVICE must be none or systemd:UNIT." + [[ -z $(opt health_url) || $(opt health_url) =~ ^https?://[A-Za-z0-9.:\[\]-]+(/[A-Za-z0-9._/?=\&%~-]*)?$ ]] || + die "LSS_SELF_UPDATER_HEALTH_URL: '$(opt health_url)' is not an http(s) URL." + interval=$(opt_int interval) + ((interval >= 15)) || die "LSS_SELF_UPDATER_INTERVAL must be at least 15 seconds." + [[ $(opt token) != *[[:space:]]* ]] || die "LSS_SELF_UPDATER_TOKEN must not contain spaces." + app_check_port port + fetch_entry "self-updater-${SU_VERSION}" >/dev/null || + die "self-updater: data/checksums.txt has no checksum for Self-Updater ${SU_VERSION}." + return +} + +# _su_installed: the version of the installed self-updater, or nothing. +_su_installed() { + local package + package="$(npm root --global 2>/dev/null)/@vectode/self-updater/package.json" + [[ -f ${package} ]] || return 0 + jq -r '.version // empty' "${package}" 2>/dev/null || true + return +} + +# The release archive from GitHub, checked against data/checksums.txt and installed with npm (no install scripts; the +# package has no dependencies). Node.js comes from the node module. +mod_install() { + local tmp + pkg_install git jq sudo + if [[ $(_su_installed) != "${SU_VERSION}" ]]; then + tmp=$(mktemp -d) + fetch_verified "self-updater-${SU_VERSION}" "${tmp}/self-updater.tgz" + log_command npm install --global --ignore-scripts --no-audit --no-fund "${tmp}/self-updater.tgz" || + die "self-updater: npm cannot install Self-Updater ${SU_VERSION}." + rm -rf -- "${tmp}" + files_note_change "installed Self-Updater ${SU_VERSION}" + fi + mon_user "${SU_USER}" "${SU_HOME}" + return +} + +# _su_token_flag: true when the repository needs the token. +_su_token_flag() { + if [[ -n $(opt token) ]]; then printf 'true'; else printf 'false'; fi + return +} + +# _su_config: updater.config.json. Secrets stay in .env (${SELF_UPDATER_WEBHOOK_SECRET}, ${GITHUB_TOKEN}). +_su_config() { + local service unit='' + service=$(opt service) + [[ ${service} == none ]] || unit=${service#systemd:} + jq -n --arg repo "$(opt repo)" --arg branch "$(opt branch)" --arg track "$(opt track)" --arg path "$(opt path)" \ + --arg unit "${unit}" --arg health "$(opt health_url)" --argjson interval "$(opt_int interval)" \ + --argjson port "$(opt_int port)" --arg status "${SU_STATUS_PATH}" --arg token "$(_su_token_flag)" ' + { + version: 3, + repo: ({url: $repo, branch: $branch} + (if $token == "true" then {token: "${GITHUB_TOKEN}"} else {} end)), + track: {mode: $track, tagPattern: "v*"}, + workspace: {path: $path}, + service: (if $unit == "" then {type: "none"} else {type: "systemd", name: $unit, sudo: true} end), + rollback: {enabled: true}, + schedule: {enabled: true, intervalSeconds: $interval, jitterSeconds: 10}, + webhook: {enabled: true, host: "127.0.0.1", port: $port, statusPath: $status, secret: "${SELF_UPDATER_WEBHOOK_SECRET}"}, + logging: {level: "info", format: "text"} + } + (if $health == "" then {} else {healthCheck: {url: $health}} end)' + return +} + +# _su_env: the environment file with the secrets the configuration refers to. +_su_env() { + local secret token + secret_load self-updater-webhook-secret secret 48 + printf 'SELF_UPDATER_WEBHOOK_SECRET=%s\n' "${secret}" + token=$(opt token) + if [[ -n ${token} ]]; then + log_mask "${token}" + if ! secret_exists self-updater-token || [[ $(secret_get self-updater-token) != "${token}" ]]; then + secret_set self-updater-token "${token}" + fi + printf 'GITHUB_TOKEN=%s\n' "${token}" + fi + return +} + +_su_unit() { + local path nnp=yes bin + path=$(opt path) + # The directory as its real path: with /bin a link to /usr/bin, command -v answers either, depending on PATH. + bin=$(command -v self-updater || printf '/usr/bin/self-updater') + bin=$(readlink -f "$(dirname "${bin}")")/self-updater + # sudo for the restart of a systemd unit needs new privileges. + [[ $(opt service) == none ]] || nnp=no + cat </dev/null || die "self-updater: visudo rejects ${SU_SUDOERS}." + fi + log_command runuser -u "${SU_USER}" -- env SELF_UPDATER_CONFIG="${SU_CONFIG}" HOME="${SU_HOME}" \ + self-updater validate --config "${SU_CONFIG}" --env-file "${SU_ENV}" || + die "self-updater: the configuration does not validate; see the log." + _su_unit | svc_unit "${SU_UNIT}.service" + local -a sandbox=("ReadWritePaths=${SU_CONFIG_DIR}" "ReadWritePaths=${path}" "ReadWritePaths=${SU_HOME}") + if [[ ${service} != none ]]; then + # sudo needs its setuid bit and to read /etc/sudoers.d. + sandbox+=(NoNewPrivileges=no RestrictSUIDSGID=no) + fi + svc_sandbox "${SU_UNIT}.service" "${sandbox[@]}" + # The backup takes the deployed checkout and the state of the updater (its home) along. + app_backup "PATHS=${path} ${SU_HOME}" + svc_enable "${SU_UNIT}" + if [[ $(files_changes_of self-updater) != "${before}" ]]; then + svc_restart "${SU_UNIT}" || die "self-updater: ${SU_UNIT} does not start; see journalctl -u ${SU_UNIT}." + fi + return +} + +# _su_status: the JSON of the status endpoint, or nothing. +_su_status() { + curl -fs --max-time 10 "http://127.0.0.1:$(opt_int port)${SU_STATUS_PATH}" 2>/dev/null || true + return +} + +mod_verify() { + local status='' i commit + [[ $(_su_installed) == "${SU_VERSION}" ]] || die "self-updater: Self-Updater ${SU_VERSION} is not installed." + svc_is_active "${SU_UNIT}" || die "self-updater: ${SU_UNIT} is not running; see journalctl -u ${SU_UNIT}." + app_verify_backup + [[ $(stat -c '%a %U:%G' "${SU_ENV}") == "640 root:${SU_USER}" ]] || die "self-updater: ${SU_ENV} must be mode 640, root:${SU_USER}." + # The first deployment runs right after the start (--immediate): a clone of the repository into the workspace. + for ((i = 0; i < 100; i++)); do + status=$(_su_status) + commit=$(jq -r '.workspaces[0].deployed.commit // empty' <<<"${status}" 2>/dev/null) || commit='' + [[ -z ${commit} ]] || break + sleep 3 + done + [[ -n ${status} ]] || die "self-updater: the status endpoint http://127.0.0.1:$(opt_int port)${SU_STATUS_PATH} does not answer." + [[ -n ${commit} ]] || + die "self-updater: nothing is deployed from $(opt repo) yet (last attempt: $(jq -c '.workspaces[0].lastAttempt' <<<"${status}" 2>/dev/null))." + [[ $(runuser -u "${SU_USER}" -- git -C "$(opt path)" rev-parse HEAD 2>/dev/null) == "${commit}"* ]] || + die "self-updater: $(opt path) is not at the deployed commit ${commit}." + return +} + +mod_remove() { + svc_disable "${SU_UNIT}" + svc_dropin_remove "${SU_UNIT}.service" lss-sandbox + file_remove "/etc/systemd/system/${SU_UNIT}.service" + svc_daemon_reload + file_remove "${SU_SUDOERS}" + file_remove "${SU_CONFIG}" + file_remove "${SU_ENV}" + app_backup_remove + if [[ -n $(_su_installed) ]]; then + log_command npm uninstall --global @vectode/self-updater || die "self-updater: npm cannot remove Self-Updater." + files_note_change "removed Self-Updater" + fi + # The deployed checkout and the state of the updater stay unless --purge. + [[ ${LSS_PURGE:-0} == 1 ]] || return 0 + rm -rf -- "${SU_CONFIG_DIR}" "${SU_HOME}" + secret_remove self-updater-webhook-secret + secret_remove self-updater-token + files_note_change "deleted the state of Self-Updater" + return +} diff --git a/modules/apps/uptime-kuma.sh b/modules/apps/uptime-kuma.sh new file mode 100644 index 0000000..a156dfa --- /dev/null +++ b/modules/apps/uptime-kuma.sh @@ -0,0 +1,161 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="uptime-kuma" +MOD_GROUP="apps" +MOD_DESCRIPTION="Uptime Kuma 2 (status pages and uptime checks) as a rootless Docker Compose stack behind the web server, with SQLite and the first administrator already created" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Uptime Kuma site, separated by spaces; empty: uptime-kuma. +port|8084|Port of the Uptime Kuma container on 127.0.0.1, which the web server forwards to +admin_user|admin|Name of the first administrator (the password is the secret uptime-kuma-admin-password) +memory|512m|Memory limit of the Uptime Kuma container" +MOD_SECRETS="uptime-kuma-admin-password" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Uptime Kuma 2.5.6, the slim rootless variant (runs as node, 1000:1000; no bundled Chromium or MariaDB), the +# multi-architecture index (amd64 and arm64), checked 2026-10-15. +KUMA_IMAGE=docker.io/louislam/uptime-kuma@sha256:6eb4e96a7cbdb8a5d6779aa26dce2cb97dcea65b486856b9b2280f73a1d0adf4 +KUMA_USER=1000:1000 + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt admin_user) =~ ^[a-zA-Z][a-zA-Z0-9._-]{1,31}$ ]] || die "LSS_UPTIME_KUMA_ADMIN_USER must be a login name." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_UPTIME_KUMA_MEMORY must be a size such as 512m or 1g." + return +} + +mod_install() { + app_compose_ready + return +} + +_kuma_compose() { + cat < { console.error(message); process.exit(1); }; +const call = (event, ...args) => new Promise((resolve) => socket.emit(event, ...args, resolve)); +// The server registers its handlers only after it has answered the connection, so the first question is asked +// again until an answer comes. +const ask = async (event, ...args) => { + for (let attempt = 0; attempt < 20; attempt++) { + const answer = await Promise.race([call(event, ...args), new Promise((resolve) => setTimeout(resolve, 3000))]); + if (answer !== undefined) return answer; + } + fail(`no answer to ${event}`); +}; +setTimeout(() => fail("timeout"), 120000); +socket.on("connect_error", (error) => fail(`connect: ${error.message}`)); +socket.on("connect", async () => { + if (action === "setup") { + const needed = await ask("needSetup"); + if (!needed) { console.log("exists"); process.exit(0); } + const result = await call("setup", username, password); + if (!result.ok) fail(`setup: ${result.msg}`); + console.log("created"); + process.exit(0); + } + const login = await ask("login", { username, password, token: "" }); + if (!login.ok) fail(`login: ${login.msg}`); + const added = await call("add", { + type: "http", name: "lss verify", url: "http://127.0.0.1:3001/", method: "GET", interval: 20, + retryInterval: 20, resendInterval: 0, maxretries: 0, timeout: 16, maxredirects: 10, + accepted_statuscodes: ["200-399"], notificationIDList: {}, ignoreTls: false, upsideDown: false, + expiryNotification: false, kafkaProducerBrokers: [], kafkaProducerSaslOptions: { mechanism: "None" }, + conditions: [], rabbitmqNodes: [], active: true, + }); + if (!added.ok) fail(`add monitor: ${added.msg}`); + socket.on("heartbeat", async (beat) => { + if (beat.monitorID !== added.monitorID) return; + const removed = await call("deleteMonitor", added.monitorID, false); + if (!removed.ok) fail(`delete monitor: ${removed.msg}`); + if (beat.status !== 1) fail(`the monitor reports down: ${beat.msg}`); + console.log("up"); + process.exit(0); + }); +}); +EOF + return +} + +# _kuma ACTION: runs the client in the container. +_kuma() { + local action=$1 + app_exec app node /opt/lss/lss-kuma.js "${action}" + return +} + +mod_configure() { + local result + app_secret uptime-kuma-admin-password "${KUMA_USER}" 32 admin-password + _kuma_script | app_file config/lss-kuma.js "${KUMA_USER}" 0444 + _kuma_compose | app_compose + app_backup COPY=data + app_up + app_site "$(opt_int port)" + app_wait '' 302 >/dev/null || die "uptime-kuma: https://$(app_domain)/ does not answer." + result=$(_kuma setup 2>&1) || die "uptime-kuma: the administrator cannot be created: ${result}" + if [[ ${result} == *created* ]]; then + log_info "uptime-kuma: created the administrator $(opt admin_user); password in $(secret_path uptime-kuma-admin-password)" + files_note_change "created the Uptime Kuma administrator" + fi + return +} + +mod_verify() { + local page result + app_verify_stack + app_verify_backup + page=$(app_wait '' 302) || die "uptime-kuma: https://$(app_domain)/ does not answer." + page=$(app_wait dashboard 200 60) || die "uptime-kuma: the dashboard does not answer." + grep -qi 'uptime kuma' <<<"${page}" || die "uptime-kuma: https://$(app_domain)/dashboard is not Uptime Kuma." + # Used, not only running: the administrator logs in and a monitor checks Uptime Kuma itself. + result=$(_kuma check 2>&1) || die "uptime-kuma: the check as $(opt admin_user) failed: ${result}" + [[ ${result} == *up* ]] || die "uptime-kuma: the check as $(opt admin_user) failed: ${result}" + return +} + +mod_remove() { + app_remove + return +} diff --git a/modules/apps/vaultwarden.sh b/modules/apps/vaultwarden.sh new file mode 100644 index 0000000..b0bdc5f --- /dev/null +++ b/modules/apps/vaultwarden.sh @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="vaultwarden" +MOD_GROUP="apps" +MOD_DESCRIPTION="Vaultwarden 1.37 (password manager, compatible with the Bitwarden apps) as a Docker Compose stack behind the web server; no open registration, users are invited from the admin page" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the Vaultwarden site, separated by spaces; empty: vaultwarden. +port|8085|Port of the Vaultwarden container on 127.0.0.1, which the web server forwards to +memory|512m|Memory limit of the Vaultwarden container" +MOD_SECRETS="vaultwarden-admin-token" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# Vaultwarden 1.37.4, the multi-architecture index (amd64 and arm64), checked 2026-10-15. +VW_IMAGE=docker.io/vaultwarden/server@sha256:efb3cde962015fcc036b2ea625242248611943b27212ebf4392841fb30fad055 +# Not the root user of the image: an unprivileged user that owns only the data volume. +VW_USER=1000:1000 + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_VAULTWARDEN_MEMORY must be a size such as 512m or 1g." + return +} + +mod_install() { + app_compose_ready + # argon2 hashes the admin token (EPEL on the RHEL family). + pkg_enable_epel + pkg_install argon2 + return +} + +_vw_compose() { + cat </dev/null || die "vaultwarden: https://$(app_domain)/alive does not answer." + log_info "vaultwarden: the admin page is https://$(app_domain)/admin; token in $(secret_path vaultwarden-admin-token)" + return +} + +# _vw_admin JAR PATH [CURL OPTION...]: a request to the admin page with the session in the cookie file JAR. +_vw_admin() { + local jar=$1 path=$2 + shift 2 + app_get "admin/${path}" -b "${jar}" -c "${jar}" "$@" + return +} + +mod_verify() { + local page jar email id status + app_verify_stack + app_verify_backup + page=$(app_wait alive 200) || die "vaultwarden: https://$(app_domain)/alive does not answer." + page=$(app_wait '' 200 60) || die "vaultwarden: the web vault does not answer." + grep -qi 'bitwarden\|vaultwarden' <<<"${page}" || die "vaultwarden: https://$(app_domain)/ is not the web vault." + # Used, not only running: log in to the admin page with the token, invite a user, find and delete it. + jar=$(mktemp) + email="lss-verify-$(date +%s)@example.com" + status=$(app_status "$(_vw_admin "${jar}" '' -X POST --data-urlencode "token@$(secret_path vaultwarden-admin-token)")") + if [[ ${status} != 200 && ${status} != 303 ]] || ! grep -q 'VW_ADMIN' "${jar}"; then + rm -f -- "${jar}" + die "vaultwarden: the admin token is not accepted (HTTP ${status})." + fi + page=$(_vw_admin "${jar}" invite -X POST -H 'Content-Type: application/json' -d "{\"email\":\"${email}\"}") + if [[ $(app_status "${page}") != 200 ]]; then + rm -f -- "${jar}" + die "vaultwarden: inviting a user answers $(app_status "${page}")." + fi + page=$(_vw_admin "${jar}" users -H 'Accept: application/json') + id=$(jq -r --arg email "${email}" '.[] | select(.email == $email) | .id' <<<"${page%$'\n'*}" 2>/dev/null) || id='' + if [[ -z ${id} ]]; then + rm -f -- "${jar}" + die "vaultwarden: the invited user is missing from the user list." + fi + status=$(app_status "$(_vw_admin "${jar}" "users/${id}/delete" -X POST -H 'Content-Type: application/json')") + rm -f -- "${jar}" + [[ ${status} == 200 ]] || die "vaultwarden: deleting the invited user answers ${status}." + return +} + +mod_remove() { + app_remove + [[ ${LSS_PURGE:-0} != 1 ]] || state_set vaultwarden token_sum '' + return +} diff --git a/modules/apps/wordpress.sh b/modules/apps/wordpress.sh new file mode 100644 index 0000000..1471f72 --- /dev/null +++ b/modules/apps/wordpress.sh @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +MOD_NAME="wordpress" +MOD_GROUP="apps" +MOD_DESCRIPTION="WordPress 7.1 with MariaDB as a Docker Compose stack behind the web server, installed with a first administrator; the theme and plugin editor is off" +MOD_REQUIRES="docker nginx|apache|caddy" +MOD_MODES="docker" +MOD_OPTIONS="domains||Host names of the WordPress site, separated by spaces; empty: wordpress. +port|8088|Port of the WordPress container on 127.0.0.1, which the web server forwards to +title|WordPress|Title of the site +admin_user|lssadmin|Name of the first administrator (the password is the secret wordpress-admin-password) +admin_email||E-mail address of the first administrator; empty: ADMIN_USER@ +memory|768m|Memory limit of the WordPress container" +MOD_SECRETS="wordpress-admin-password wordpress-db-password wordpress-db-root-password" +MOD_PORTS="none of its own (the HTTPS port of the web server)" + +# WordPress 7.1.3 with PHP 8.4 (apache), the multi-architecture index (amd64 and arm64), checked 2026-10-15. +WP_IMAGE=docker.io/library/wordpress@sha256:309307430abf80009378a63a79df73554a6449b700fac59a991b4959e6dda8ca +# www-data in the image; the container runs as this user from the start, so it needs no capabilities. +WP_USER=33:33 + +mod_check() { + app_check_domains + app_check_port port + [[ $(opt admin_user) =~ ^[a-z][a-z0-9._-]{2,59}$ && $(opt admin_user) != admin ]] || + die "LSS_WORDPRESS_ADMIN_USER must be a lowercase login name other than admin (the first name attackers try)." + [[ -z $(opt admin_email) || $(opt admin_email) =~ ^[^@[:space:]]+@[^@[:space:]]+\.[a-z]+$ ]] || + die "LSS_WORDPRESS_ADMIN_EMAIL must be an e-mail address." + [[ -n $(opt title) && $(opt title) != *[\'\"\\]* ]] || die "LSS_WORDPRESS_TITLE must not be empty or contain quotes or backslashes." + [[ $(opt memory) =~ ^[1-9][0-9]*[mg]$ ]] || die "LSS_WORDPRESS_MEMORY must be a size such as 768m or 1g." + return +} + +mod_install() { + app_compose_ready + return +} + +# _wp_compose: the stack. wp-config.php reads the database password from WORDPRESS_DB_PASSWORD_FILE on every request; +# the extra settings fix the address, force HTTPS for the login and turn off the file editor of the dashboard. +_wp_compose() { + local domain + domain=$(app_domain) + cat </dev/null || die "wordpress: the password of the administrator cannot be created." + log_info "wordpress: installing WordPress (administrator $(opt admin_user), password in $(secret_path wordpress-admin-password))" + page=$(app_get 'wp-admin/install.php?step=2' -X POST --data-urlencode "weblog_title=$(opt title)" \ + --data-urlencode "user_name=$(opt admin_user)" --data-urlencode "admin_password@$(secret_path wordpress-admin-password)" \ + --data-urlencode "admin_password2@$(secret_path wordpress-admin-password)" --data-urlencode "admin_email=${email}" \ + -d blog_public=1) || true + [[ $(app_status "${page}") == 200 ]] || die "wordpress: the installer answers $(app_status "${page}")." + _wp_installed || die "wordpress: WordPress is not installed after the installer." + files_note_change "installed WordPress" + return +} + +mod_configure() { + app_db_secrets mariadb "${WP_USER}" + _wp_compose | app_compose + app_backup DB=mariadb LIVE=html + app_up + app_site "$(opt_int port)" + app_wait wp-includes/css/buttons.min.css 200 >/dev/null || die "wordpress: https://$(app_domain)/ does not answer." + _wp_install + return +} + +mod_verify() { + local page jar status home + app_verify_stack + app_verify_backup + page=$(app_wait wp-login.php 200) || die "wordpress: the login page does not answer." + grep -q 'name="pwd"' <<<"${page}" || die "wordpress: https://$(app_domain)/wp-login.php shows no login." + # rest_route works with plain permalinks too; /wp-json/ needs pretty permalinks, which a fresh site lacks. + page=$(app_get 'index.php?rest_route=/') || true + home=$(jq -r '.home // empty' <<<"${page%$'\n'*}" 2>/dev/null) || home='' + [[ $(app_status "${page}") == 200 && ${home} == "https://$(app_domain)" ]] || + die "wordpress: the REST API answers $(app_status "${page}") with the address '${home}' instead of https://$(app_domain)." + # Used, not only running: the administrator logs in and opens the dashboard. Without testcookie WordPress skips its + # check for a browser that blocks cookies; the session cookies land in the jar. + jar=$(mktemp) + page=$(app_get wp-login.php -b "${jar}" -c "${jar}" -X POST \ + --data-urlencode "log=$(opt admin_user)" --data-urlencode "pwd@$(secret_path wordpress-admin-password)" \ + --data-urlencode "redirect_to=https://$(app_domain)/wp-admin/") || true + status=$(app_status "${page}") + if [[ ${status} != 302 ]]; then + rm -f -- "${jar}" + die "wordpress: $(opt admin_user) cannot log in (HTTP ${status})." + fi + page=$(app_get wp-admin/ -b "${jar}") || true + rm -f -- "${jar}" + if [[ $(app_status "${page}") != 200 ]] || ! grep -q 'id="wpadminbar"' <<<"${page}"; then + die "wordpress: the dashboard does not open after the login." + fi + return +} + +mod_remove() { + app_remove + return +} diff --git a/modules/backup/restic.sh b/modules/backup/restic.sh index 185dff5..869511c 100644 --- a/modules/backup/restic.sh +++ b/modules/backup/restic.sh @@ -121,12 +121,14 @@ mod_configure() { fi file_write 0755 root:root "${BACKUP_SCRIPT}" <"${LSS_ROOT}/libexec/lss-backup" _restic_conf | file_write 0600 root:root "${BACKUP_CONF}" - # Reads everything, writes only its state, its cache, the dumps, the metrics and a local repository. + # Reads everything, writes only its state, its cache, the dumps (of the databases and of the app stacks, which it + # reaches through the Docker socket), the metrics and a local repository. svc_unit "${BACKUP_UNIT}.service" < <( printf '# Managed by lss (module restic).\n[Unit]\nDescription=lss backup with restic\n' printf 'After=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\n' printf 'ExecStart=%s run\nNice=10\nIOSchedulingClass=idle\n' "${BACKUP_SCRIPT}" - svc_scanner_sandbox strict "${BACKUP_STATE}" "${BACKUP_CACHE}" /var/backups/lss-db "${MON_TEXTFILE_DIR}" \ + svc_scanner_sandbox strict "${BACKUP_STATE}" "${BACKUP_CACHE}" /var/backups/lss-db /var/backups/lss-apps \ + /var/lib/lss/app-backup "${MON_TEXTFILE_DIR}" \ ${writable:+"${writable}"} ) svc_unit "${BACKUP_UNIT}.timer" </dev/null) == Y ]]; then + # dockerd confines containers with AppArmor where the kernel has it and its interface (securityfs) is mounted, + # which a container (Docker in a test container) does not see. + if os_is_debian && [[ $(cat /sys/module/apparmor/parameters/enabled 2>/dev/null) == Y ]] && + [[ -d /sys/kernel/security/apparmor ]]; then [[ ${options} == *name=apparmor* ]] || die "docker: containers run without AppArmor (${options})." fi # The API is root on the host: only the Unix socket, never TCP. dockerd itself holds the published ports of the diff --git a/modules/mail/mail-relay.sh b/modules/mail/mail-relay.sh index 7fc1dbb..7e8f5e0 100644 --- a/modules/mail/mail-relay.sh +++ b/modules/mail/mail-relay.sh @@ -213,6 +213,8 @@ mod_remove() { svc_disable "$(mail_postfix_unit)" svc_disable postfix svc_dropin_remove "${unit}" lss-sandbox + svc_selinux_spawner_remove postfix + svc_selinux_spawner_remove postfix_start file_remove "${RELAY_ALIASES}" file_remove "${RELAY_SASL}" file_remove "${RELAY_GENERIC}" diff --git a/modules/mail/mailserver.sh b/modules/mail/mailserver.sh index 51e8bc7..a36bd3d 100644 --- a/modules/mail/mailserver.sh +++ b/modules/mail/mailserver.sh @@ -380,6 +380,8 @@ _ms_dovecot() { # Dovecot puts its sockets for Postfix there; Postfix 3.10 creates the directory only on its first start, which # comes later, and the sandbox needs it to exist. Postfix's own owner and mode. [[ -d /var/spool/postfix/private ]] || install -d -m 0700 -o postfix -g root /var/spool/postfix/private + # Dovecot starts its auth and delivery processes in their own SELinux domains (svc_selinux_spawner). + svc_selinux_spawner dovecot "$(type -P dovecot || echo /usr/sbin/dovecot)" /usr/libexec/dovecot /usr/lib/dovecot svc_sandbox dovecot.service "ReadWritePaths=${MS_VMAIL}" ReadWritePaths=-/var/lib/dovecot \ ReadWritePaths=/var/spool/postfix/private svc_enable dovecot @@ -926,6 +928,9 @@ mod_remove() { svc_dropin_remove "${unit}" lss-sandbox svc_dropin_remove dovecot.service lss-sandbox svc_dropin_remove rspamd.service lss-sandbox + svc_selinux_spawner_remove postfix + svc_selinux_spawner_remove postfix_start + svc_selinux_spawner_remove dovecot local file for file in "${MS_MAILBOXES}" "${MS_VIRTUAL}" "${MS_SENDERS}" "${MS_ALIASES}" "${MS_RECIPIENT_ACCESS}" "${MS_USERS}" \ "${MS_DOVECOT_CONF}" "${MS_DNS_FILE}" "${MS_RSPAMD}/worker-normal.inc" "${MS_RSPAMD}/worker-proxy.inc" \ diff --git a/modules/security/firewall.sh b/modules/security/firewall.sh index ecf29ce..5ed4553 100644 --- a/modules/security/firewall.sh +++ b/modules/security/firewall.sh @@ -147,7 +147,7 @@ mod_verify() { mod_remove() { fw_rollback_cancel if [[ $(fw_backend) == ufw ]]; then - if command -v ufw >/dev/null 2>&1; then + if type -P ufw >/dev/null; then ufw --force disable >/dev/null fi svc_disable ufw diff --git a/modules/web/caddy.sh b/modules/web/caddy.sh index 14626fc..f9a5e93 100644 --- a/modules/web/caddy.sh +++ b/modules/web/caddy.sh @@ -78,11 +78,12 @@ $(_caddy_headers) } } -# Dot files and directories (.git, .env, .htpasswd) are never served; ACME challenges below .well-known are. +# Dot files and directories (.git, .env, .htpasswd) are never served; .well-known paths (ACME challenges, OpenID and +# similar discovery documents of proxied apps) are, as with nginx and Apache. (lss_hidden) { @lss_hidden { path */.* - not path /.well-known/* + not path */.well-known/* } respond @lss_hidden 404 } diff --git a/modules/web/modsecurity.sh b/modules/web/modsecurity.sh index dc69d2e..f71681b 100644 --- a/modules/web/modsecurity.sh +++ b/modules/web/modsecurity.sh @@ -479,7 +479,7 @@ mod_install() { # _modsecurity_engine_conf SERVER: the engine settings after modsecurity.conf-recommended of ModSecurity. _modsecurity_engine_conf() { - local server=$1 bytes nofiles json=0 + local server=$1 bytes nofiles json=0 unmatched bytes=$(_modsecurity_bytes "$(_modsecurity_max_body)") # Bodies without uploads (forms, JSON) up to 1 MiB: the recommended 128 KiB turns away ordinary JSON documents. nofiles=$((bytes < 1048576 ? bytes : 1048576)) @@ -507,7 +507,17 @@ EOF cat <<'EOF' SecRule REQBODY_ERROR "!@eq 0" "id:'200002',phase:2,t:none,log,deny,status:400,msg:'Failed to parse request body.',logdata:'%{reqbody_error_msg}',severity:2" SecRule MULTIPART_STRICT_ERROR "!@eq 0" "id:'200003',phase:2,t:none,log,deny,status:400,msg:'Multipart request body failed strict validation: PE %{REQBODY_PROCESSOR_ERROR}, BQ %{MULTIPART_BOUNDARY_QUOTED}, BW %{MULTIPART_BOUNDARY_WHITESPACE}, DB %{MULTIPART_DATA_BEFORE}, DA %{MULTIPART_DATA_AFTER}, HF %{MULTIPART_HEADER_FOLDING}, LF %{MULTIPART_LF_LINE}, SM %{MULTIPART_MISSING_SEMICOLON}, IQ %{MULTIPART_INVALID_QUOTING}, IP %{MULTIPART_INVALID_PART}, IH %{MULTIPART_INVALID_HEADER_FOLDING}, FL %{MULTIPART_FILE_LIMIT_EXCEEDED}'" -SecRule MULTIPART_UNMATCHED_BOUNDARY "!@eq 0" "id:'200004',phase:2,t:none,log,deny,msg:'Multipart parser detected a possible unmatched boundary.'" +EOF + # ModSecurity 3 (nginx) sets MULTIPART_UNMATCHED_BOUNDARY to 2 for every well-formed body that ends with its final + # boundary and to 1 only for a possible unmatched one; "!@eq 0" of ModSecurity 2 would turn away every upload, so + # libmodsecurity's own recommendation tests for 1. + if [[ ${server} == nginx ]]; then + unmatched='@eq 1' + else + unmatched='!@eq 0' + fi + printf '%s\n' "SecRule MULTIPART_UNMATCHED_BOUNDARY \"${unmatched}\" \"id:'200004',phase:2,t:none,log,deny,msg:'Multipart parser detected a possible unmatched boundary.'\"" + cat <<'EOF' SecPcreMatchLimit 1000 SecPcreMatchLimitRecursion 1000 SecRule TX:/^MSC_/ "!@streq 0" "id:'200005',phase:2,t:none,log,deny,msg:'ModSecurity internal error flagged: %{MATCHED_VAR_NAME}'" @@ -594,6 +604,8 @@ _modsecurity_main_conf() { printf 'Include %s\n' "${dir}/modsecurity.conf" "${dir}/crs-setup.conf" _modsecurity_plugin_includes "${dir}" config _modsecurity_plugin_includes "${dir}" before + # The exclusions of the routes (web_waf_exclude); the file exists, empty or not. + printf 'Include %s\n' "${dir}/sites.conf" find "$(_modsecurity_crs_dir)/rules" -maxdepth 1 -type f -name '*.conf' | LC_ALL=C sort | sed 's/^/Include /' _modsecurity_plugin_includes "${dir}" after return @@ -707,6 +719,7 @@ mod_configure() { file_write 0644 root:root "${dir}/modsecurity.conf" < <(_modsecurity_engine_conf "${server}") file_write 0644 root:root "${dir}/crs-setup.conf" < <(_modsecurity_crs_setup) _modsecurity_plugins "${dir}" + file_write 0644 root:root "${dir}/sites.conf" < <(web_waf_rules) file_write 0644 root:root "${dir}/main.conf" < <(_modsecurity_main_conf "${dir}") file_write 0644 root:root /etc/logrotate.d/lss-modsecurity < <(_modsecurity_logrotate "${log}") if [[ ${server} == apache ]]; then diff --git a/profiles/all.conf b/profiles/all.conf index 249fae7..467f481 100644 --- a/profiles/all.conf +++ b/profiles/all.conf @@ -6,5 +6,6 @@ # # For the nightly test and as a reference. ssh-2fa, clamav and usbguard stay opt-in as in minimal-secure. k3s # conflicts with docker (both bring their own containerd and claim the forwarding rules) and has its own profile, -# k8s-node; podman is the alternative to docker. -LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls vhost haproxy modsecurity php composer node python ruby java go dotnet mariadb postgresql redis mongodb phpmyadmin adminer pgadmin docker portainer restic db-dumps wireguard dns-resolver ntp-server mailserver webmail node-exporter prometheus alertmanager grafana loki alloy netdata smartd +# k8s-node; podman is the alternative to docker. The apps need about 4 GB of memory on top of the rest, and +# self-updater needs the repository it deploys (LSS_SELF_UPDATER_REPO). +LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls vhost haproxy modsecurity php composer node python ruby java go dotnet mariadb postgresql redis mongodb phpmyadmin adminer pgadmin docker portainer restic db-dumps wireguard dns-resolver ntp-server mailserver webmail node-exporter prometheus alertmanager grafana loki alloy netdata smartd nextcloud collabora forgejo uptime-kuma vaultwarden keycloak matomo wordpress n8n minio self-updater diff --git a/profiles/devops.conf b/profiles/devops.conf index 5a53f83..7dd0c14 100644 --- a/profiles/devops.conf +++ b/profiles/devops.conf @@ -2,9 +2,9 @@ # # SPDX-License-Identifier: MIT -# Profile: DevOps server: minimal-secure plus Docker and the monitoring stack (Prometheus, Alertmanager, Grafana, Loki) +# Profile: DevOps server: minimal-secure plus Docker, the Git forge Forgejo behind nginx and the monitoring stack (Prometheus, Alertmanager, Grafana, Loki) # # Every interface stays on 127.0.0.1 (SSH tunnel) until LSS__ACCESS=vpn with the wireguard module or -# LSS__WEB_ADMIN=true with a web server says otherwise. netdata and restic are opt-in. A Git forge (Forgejo) -# and CI runners join this profile with their modules in a later stage. -LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard docker node-exporter prometheus alertmanager loki alloy grafana smartd +# LSS__WEB_ADMIN=true with a web server says otherwise; Forgejo is the one site on port 443 +# (LSS_FORGEJO_DOMAINS, by default forgejo. in front of the server's name). netdata and restic are opt-in. +LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard docker nginx forgejo node-exporter prometheus alertmanager loki alloy grafana smartd diff --git a/profiles/office.conf b/profiles/office.conf index 007fa7e..1b90819 100644 --- a/profiles/office.conf +++ b/profiles/office.conf @@ -2,10 +2,12 @@ # # SPDX-License-Identifier: MIT -# Profile: Office server: minimal-secure plus a mail server with webmail behind nginx, certificates and backups +# Profile: Office server: minimal-secure plus mail with webmail, Nextcloud with Collabora Online, Vaultwarden, certificates and backups # # LSS_MAILSERVER_DOMAINS (and LSS_MAILSERVER_USERS) must be set; lss prints the DNS records to set and keeps them in # /etc/lss/mailserver-dns.txt. With LSS_TLS_DOMAINS naming the mail host name, LSS_MAILSERVER_CERTIFICATE=default -# gives Postfix and Dovecot the Let's Encrypt certificate. restic backs up the mailboxes, the DKIM keys and the -# webmail database daily. Nextcloud, Collabora and Vaultwarden join this profile with the application modules. -LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls php mailserver webmail restic +# gives Postfix and Dovecot the Let's Encrypt certificate. Nextcloud, Collabora Online and Vaultwarden run as Docker +# Compose stacks behind nginx, each on its own host name (LSS_NEXTCLOUD_DOMAINS, LSS_COLLABORA_DOMAINS, +# LSS_VAULTWARDEN_DOMAINS; by default nextcloud., collabora. and vaultwarden. in front of the server's name), which must +# point to the server. restic backs up the mailboxes, the DKIM keys and the webmail database daily. +LSS_MODULES=system time swap admin-user tools logging ssh-hardening firewall fail2ban kernel-hardening mac auditd auto-updates password-policy aide rkhunter lynis guard nginx tls php mailserver webmail docker nextcloud collabora vaultwarden restic diff --git a/tests/integration/pebble.sh b/tests/integration/pebble.sh index 53beb6f..198cbba 100644 --- a/tests/integration/pebble.sh +++ b/tests/integration/pebble.sh @@ -16,6 +16,8 @@ c87fb918d9bac8db11aa54493b5f0cf6e0725a3fe73f617d1c64737e1ab4caf9 test/certs/loca 0977979255b0e0721c17335c056b627f66a43cf56e5a52f3cda81fb111ecf00e test/certs/localhost/key.pem" # Where the target keeps Pebble; LSS_TLS_CA_BUNDLE in settings.conf points at the CA certificate here. PEBBLE_DIR=/etc/lss-pebble +# The CA bundle LSS_TLS_CA_BUNDLE points at: the minica certificate of the API and the roots Pebble issues from. +PEBBLE_BUNDLE=${PEBBLE_DIR}/ca-bundle.pem pebble_start() { local arch sum work file @@ -61,5 +63,19 @@ EOF --setenv=PEBBLE_WFE_NONCEREJECT=0 ${PEBBLE_DIR}/pebble -config ${PEBBLE_DIR}/config.json for i in \$(seq 30); do curl -fsS --cacert ${PEBBLE_DIR}/pebble.minica.pem -o /dev/null https://localhost:14000/dir && exit 0; sleep 1; done journalctl -u lss-pebble --no-pager; exit 1" + pebble_bundle + return +} + +# pebble_bundle: PEBBLE_BUNDLE, the CA bundle of settings.conf. Pebble serves its API with the minica certificate but +# signs the certificates it issues with a root it creates at every start; apps that check each other's certificates +# (Nextcloud and Collabora) need that root, as they would need the root of a step-ca. Roots of earlier starts stay in +# the bundle, so certificates issued before a restart keep their anchor. +pebble_bundle() { + in_target "set -e + [ -s ${PEBBLE_BUNDLE} ] || cp ${PEBBLE_DIR}/pebble.minica.pem ${PEBBLE_BUNDLE} + curl -fsS --retry 3 --cacert ${PEBBLE_DIR}/pebble.minica.pem -o ${PEBBLE_DIR}/root.pem https://localhost:15000/roots/0 + grep -q 'BEGIN CERTIFICATE' ${PEBBLE_DIR}/root.pem + grep -qxF \"\$(sed -n 2p ${PEBBLE_DIR}/root.pem)\" ${PEBBLE_BUNDLE} || { echo; cat ${PEBBLE_DIR}/root.pem; } >>${PEBBLE_BUNDLE}" return } diff --git a/tests/integration/run.sh b/tests/integration/run.sh index 737568c..404e963 100755 --- a/tests/integration/run.sh +++ b/tests/integration/run.sh @@ -3,15 +3,17 @@ # # SPDX-License-Identifier: MIT -# Integration test: tests/integration/run.sh SYSTEM|host MODULE... +# Integration test: tests/integration/run.sh SYSTEM|host [--base MODULE,...] MODULE... # tests/integration/run.sh SYSTEM|host --profile NAME [LYNIS_MIN_INDEX] # # For every module: install (with its requirements), verify, install again (every module in the run must report # "unchanged"), and remove. A comma joins modules that are tested together (nginx,tls,vhost: installed at once, -# removed in reverse order). With --profile: install the whole profile, verify, install again, then run the Lynis -# audit and fail below LYNIS_MIN_INDEX. SYSTEM is a directory in tests/integration/images (a systemd container), -# vm-NAME for a KVM virtual machine from tests/integration/vm-images.conf, or "host" for the machine itself (modules -# that need a real kernel; used on throwaway CI virtual machines only). +# removed in reverse order). --base installs modules the others need first (docker and a web server for the apps), +# verifies them and removes them after the last one, so the figures of each module leave them out. With --profile: +# install the whole profile, verify, install again, then run the Lynis audit and fail below LYNIS_MIN_INDEX. SYSTEM +# is a directory in tests/integration/images (a systemd container), vm-NAME for a KVM virtual machine from +# tests/integration/vm-images.conf, or "host" for the machine itself (modules that need a real kernel; used on +# throwaway CI virtual machines only). set -Eeuo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../.." @@ -23,8 +25,9 @@ shift } settings=tests/integration/settings.conf container="lss-test-${system}" -# A run that hangs ends after 40 minutes, so the diagnostics below still show what it waited for. -LSS_RUN="timeout --kill-after=60 2400 lss" +# A run that hangs ends after 40 minutes (LSS_TEST_TIMEOUT seconds), so the diagnostics below still show what it waited +# for. The profile with every module takes almost that long on its own. +LSS_RUN="timeout --kill-after=60 ${LSS_TEST_TIMEOUT:-2400} lss" # shellcheck source=tests/integration/vm.sh source tests/integration/vm.sh # shellcheck source=tests/integration/pebble.sh @@ -67,15 +70,21 @@ start_target() { docker build -q -f "tests/integration/images/${system}/Containerfile" -t "lss-test:${system}" \ "tests/integration/images/${system}" >/dev/null docker rm -f "${container}" >/dev/null 2>&1 || true + # The storage of Podman and of Docker inside the container is on volumes: overlayfs does not stack on the + # overlayfs of the container itself. docker run -d --name "${container}" --hostname lss-test --privileged --cgroupns=host \ -v /sys/fs/cgroup:/sys/fs/cgroup:rw --tmpfs /run --tmpfs /run/lock --tmpfs /tmp:exec \ - -v /var/lib/containers \ + -v /var/lib/containers -v /var/lib/docker -v /var/lib/containerd \ -v "${PWD}:/src:ro" "lss-test:${system}" >/dev/null in_target 'for i in $(seq 60); do s=$(systemctl is-system-running 2>/dev/null || true); [[ $s == running || $s == degraded ]] && exit 0; sleep 1; done; exit 1' in_target 'cp -a /src /opt/lss-src' fi in_target 'ln -sf /opt/lss-src/lss /usr/local/sbin/lss && mkdir -p /etc/lss' in_target "cat > /etc/lss/lss.conf" <"${settings}" + # Settings of one job only (one per line), such as the docker mode of a module that runs natively elsewhere. + if [[ -n ${LSS_TEST_SETTINGS:-} ]]; then + in_target "cat >> /etc/lss/lss.conf" <<<"${LSS_TEST_SETTINGS}" + fi # A setting that no module reads is a typo in the test settings. if in_target 'lss --no-color list 2>&1' | grep 'is not a setting'; then exit 1 @@ -250,6 +259,20 @@ if [[ " $* " == *[\ ,]tls[\ ,]* ]]; then group_end fi +base=() +if [[ $1 == --base ]]; then + IFS=',' read -ra base <<<"${2:?usage: run.sh SYSTEM|host --base MODULE,... MODULE...}" + shift 2 + group_start "${system}: base ${base[*]}" + before=$(resources_snapshot) + started=${SECONDS} + memory_watch_start + in_target "${LSS_RUN} --yes --no-color install ${base[*]}" + in_target "${LSS_RUN} --yes --no-color verify" + resources_report "${system} ${base[*]}" "${before}" "${started}" + group_end +fi + for item in "$@"; do IFS=',' read -ra set <<<"${item}" group_start "${system}: ${set[*]}" @@ -269,4 +292,12 @@ for item in "$@"; do fi group_end done -echo "${system}: ${*} passed" +if [[ ${#base[@]} -gt 0 && ${LSS_TEST_KEEP:-0} != 1 ]]; then + group_start "${system}: remove base ${base[*]}" + install_again "${base[@]}" + for ((i = ${#base[@]} - 1; i >= 0; i--)); do + in_target "${LSS_RUN} --yes --no-color remove ${base[i]}" + done + group_end +fi +echo "${system}: ${base[*]} ${*} passed" diff --git a/tests/integration/settings.conf b/tests/integration/settings.conf index 7f10778..65aec54 100644 --- a/tests/integration/settings.conf +++ b/tests/integration/settings.conf @@ -12,7 +12,7 @@ LSS_SYSTEM_UPGRADE=false LSS_AIDE_EXCLUDE=/opt /usr/local /usr/share/dotnet /usr/share/swift /usr/share/miniconda /usr/lib/jvm /usr/lib/google-cloud-sdk /home /snap /var/lib/snapd # Web group: certificates from Pebble inside the target (tests/integration/pebble.sh), which accepts any host name. LSS_TLS_SERVER=https://localhost:14000/dir -LSS_TLS_CA_BUNDLE=/etc/lss-pebble/pebble.minica.pem +LSS_TLS_CA_BUNDLE=/etc/lss-pebble/ca-bundle.pem LSS_TLS_DOMAINS=lss-test.example.com LSS_VHOST_SITES=demo:static:demo.example.com,www.demo.example.com app:proxy:app.example.com:http://127.0.0.1:9 LSS_HAPROXY_SERVICES=echo:9190/local:127.0.0.1:9,127.0.0.1:10 @@ -53,3 +53,20 @@ LSS_MAILSERVER_ALIASES=info:alice,bob@example.org # so mail-relay names it lss-test.localdomain). LSS_GUARD_EMAIL=root@lss-test.localdomain LSS_GUARD_DAILY_SUMMARY=true + +# Apps: each on a host name of its own below example.com (certificates from Pebble where tls is installed; the tests +# reach the sites through --resolve, the containers through extra_hosts). Self-Updater deploys its own repository by +# release tag. +LSS_NEXTCLOUD_DOMAINS=cloud.example.com +LSS_COLLABORA_DOMAINS=office.example.com +LSS_FORGEJO_DOMAINS=git.example.com +LSS_UPTIME_KUMA_DOMAINS=status.example.com +LSS_VAULTWARDEN_DOMAINS=vault.example.com +LSS_KEYCLOAK_DOMAINS=sso.example.com +LSS_MATOMO_DOMAINS=stats.example.com +LSS_WORDPRESS_DOMAINS=blog.example.com +LSS_N8N_DOMAINS=n8n.example.com +LSS_MINIO_DOMAINS=s3.example.com +LSS_MINIO_CONSOLE_DOMAINS=s3-console.example.com +LSS_SELF_UPDATER_REPO=https://github.com/VectoDE/Self-Updater.git +LSS_SELF_UPDATER_TRACK=tag diff --git a/tests/integration/vm.sh b/tests/integration/vm.sh index 2603702..aa74ef9 100644 --- a/tests/integration/vm.sh +++ b/tests/integration/vm.sh @@ -9,6 +9,11 @@ VM_DIR='' VM_PORT=${LSS_VM_SSH_PORT:-2222} +# Size of the VM: the app stacks and the profile all need more than the default (LSS_VM_MEMORY in MiB, LSS_VM_CPUS, +# LSS_VM_DISK as for qemu-img). +VM_MEMORY=${LSS_VM_MEMORY:-4096} +VM_CPUS=${LSS_VM_CPUS:-2} +VM_DISK=${LSS_VM_DISK:-20G} vm_ssh() { ssh -i "${VM_DIR}/key" -p "${VM_PORT}" -o BatchMode=yes -o StrictHostKeyChecking=no \ @@ -51,7 +56,7 @@ vm_start() { read -r image group <<<"$(vm_image "${name}")" [[ -n ${image} ]] || exit 1 VM_DIR=$(mktemp -d) - qemu-img create -q -f qcow2 -b "${image}" -F qcow2 "${VM_DIR}/disk.qcow2" 20G + qemu-img create -q -f qcow2 -b "${image}" -F qcow2 "${VM_DIR}/disk.qcow2" "${VM_DISK}" ssh-keygen -q -t ed25519 -N '' -C lss-vm-test -f "${VM_DIR}/key" cat >"${VM_DIR}/user-data" <"${VM_DIR}/meta-data" cloud-localds "${VM_DIR}/seed.iso" "${VM_DIR}/user-data" "${VM_DIR}/meta-data" - qemu-system-x86_64 -enable-kvm -cpu host -machine q35 -m 4096 -smp 2 \ + qemu-system-x86_64 -enable-kvm -cpu host -machine q35 -m "${VM_MEMORY}" -smp "${VM_CPUS}" \ -drive "file=${VM_DIR}/disk.qcow2,if=virtio" -drive "file=${VM_DIR}/seed.iso,if=virtio,format=raw" \ -nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${VM_PORT}-:22" \ -device qemu-xhci -device usb-kbd \ diff --git a/tests/unit/apps.bats b/tests/unit/apps.bats new file mode 100644 index 0000000..955facf --- /dev/null +++ b/tests/unit/apps.bats @@ -0,0 +1,575 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2025-2026 VectoDE +# +# SPDX-License-Identifier: MIT + +# lib/apps.sh: the Compose stacks of the app modules (files, secrets for a container user, start, site, checks, +# removal) against a stub docker. + +setup() { + load helpers + lss_test_setup + use_os debian-12 + shopt -s lastpipe + LSS_MODULE=demo + MOD_OPTIONS="domains||Host names +port|8081|Port on 127.0.0.1" + CTR_CONFIG_DIR=${BATS_TEST_TMPDIR}/containers + CTR_PULL_DELAY=0 + WEB_WAF_DIR=${BATS_TEST_TMPDIR}/waf + APP_DOCKER_CONFIG=${BATS_TEST_TMPDIR}/daemon.json + APP_SUBUID=${BATS_TEST_TMPDIR}/subuid + APP_SUBGID=${BATS_TEST_TMPDIR}/subgid + printf '{"icc": false}\n' >"${APP_DOCKER_CONFIG}" + ENGINE_LOG=${BATS_TEST_TMPDIR}/engine.log + : >"${ENGINE_LOG}" + APP_BACKUP_SCRIPT=${BATS_TEST_TMPDIR}/app-backup + APP_BACKUP_REGISTRY=${BATS_TEST_TMPDIR}/app-backup.d + APP_BACKUP_DIR=${BATS_TEST_TMPDIR}/lss-apps + WRITES=${BATS_TEST_TMPDIR}/writes + : >"${WRITES}" + # file_write as the test user; the owner it was asked for goes to $WRITES ("== MODE OWNER DEST"). + eval "_lss_file_write() $(declare -f file_write | tail -n +2)" + # shellcheck disable=SC2329 # called by the functions under test + file_write() { + printf '== %s %s %s\n' "$1" "$2" "$3" >>"${WRITES}" + _lss_file_write "$1" "${OWNER}" "$3" + return + } +} + +# docker stub: records every call; "compose ... config --services" prints $SERVICES, "compose ... ps" prints +# $RUNNING, "compose ... config --format json" an empty stack, "compose version" and "info" succeed. +docker() { + printf 'docker %s\n' "$*" >>"${ENGINE_LOG}" + case $* in + *"config --services"*) [[ -z ${SERVICES:-} ]] || tr ' ' '\n' <<<"${SERVICES}" ;; + *"ps --status running --services"*) [[ -z ${RUNNING:-} ]] || tr ' ' '\n' <<<"${RUNNING}" ;; + *"config --format json"*) printf '{"services": {}}\n' ;; + *) ;; + esac + return "${DOCKER_EXIT:-0}" +} + +@test "the stack of a module has its project name and directory" { + [ "$(app_project)" = lss-demo ] + [ "$(app_dir)" = "${CTR_CONFIG_DIR}/lss-demo" ] + [ "$(app_compose_file)" = "${CTR_CONFIG_DIR}/lss-demo/compose.yaml" ] +} + +@test "host names and ports are checked" { + app_valid_domain cloud.example.com + run app_valid_domain localhost + [ "${status}" -eq 1 ] + run app_valid_domain Cloud.Example.com + [ "${status}" -eq 1 ] + config_set LSS_DEMO_DOMAINS 'a.example.com b.example.org' + app_check_domains + config_set LSS_DEMO_DOMAINS 'a.example.com bad_name' + run app_check_domains + [ "${status}" -eq 1 ] + [[ ${output} == *"LSS_DEMO_DOMAINS: 'bad_name' is not a host name like demo.example.com"* ]] + app_check_port port + config_set LSS_DEMO_PORT 80 + run app_check_port port + [[ ${output} == *"must be a port from 1024 to 65535"* ]] + config_set LSS_DEMO_PORT "${WEB_ADMIN_PORT}" + run app_check_port port + [[ ${output} == *"must differ from the admin port"* ]] +} + +@test "without host names the app is named after the module below the server's name" { + # shellcheck disable=SC2329 # called by app_domains + hostname() { + printf 'Server1.Example.com\n' + } + [ "$(app_domains)" = demo.server1.example.com ] + config_set LSS_DEMO_DOMAINS 'cloud.example.com www.example.com' + [ "$(app_domains)" = 'cloud.example.com www.example.com' ] + [ "$(app_domain)" = cloud.example.com ] +} + +@test "container users are mapped into Docker's user namespace" { + [ "$(app_host_owner 1000)" = 1000:1000 ] + [ "$(app_host_owner 33:44)" = 33:44 ] + printf '{"userns-remap": "default"}\n' >"${APP_DOCKER_CONFIG}" + printf 'admin:100000:65536\ndockremap:231072:65536\n' >"${APP_SUBUID}" + printf 'dockremap:231072:65536\n' >"${APP_SUBGID}" + [ "$(app_host_owner 0)" = 231072:231072 ] + [ "$(app_host_owner 70:71)" = 231142:231143 ] + printf '{"userns-remap": "remap:remap"}\n' >"${APP_DOCKER_CONFIG}" + run app_host_owner 0 + [ "${status}" -eq 1 ] + [[ ${output} == *"has no ID range for remap"* ]] + run app_host_owner www-data + [[ ${output} == *"'www-data' is not UID or UID:GID"* ]] +} + +@test "secrets are copied for the container user without a final newline" { + printf '{"userns-remap": "default"}\n' >"${APP_DOCKER_CONFIG}" + printf 'dockremap:100000:65536\n' | tee "${APP_SUBUID}" >"${APP_SUBGID}" + app_secret demo-db-password 999:70 40 + local copy=${CTR_CONFIG_DIR}/lss-demo/secrets/demo-db-password + grep -qxF "== 0400 100999:100070 ${copy}" "${WRITES}" + [ "$(stat -c '%a' "${copy}")" = 400 ] + cmp -s "${copy}" "${LSS_SECRETS_DIR}/demo-db-password" + [ "$(wc -c <"${copy}")" -eq 40 ] + [ "${APP_CHANGED}" = 1 ] + APP_CHANGED=0 + app_secret demo-db-password 999:70 40 + [ "${APP_CHANGED}" = 0 ] + app_secret demo-db-password 999:70 40 db-password.app + [ -f "${CTR_CONFIG_DIR}/lss-demo/secrets/db-password.app" ] + run app_file ../escape 0 0400 /dev/null || skip "jq is not installed" + LSS_MODULE=docker + state_set docker status installed + LSS_MODULE=demo + printf 'services: {}\n' | app_compose + [ "${APP_COMPOSE_CHANGED}" = 1 ] + grep -qxF "== 0600 root:root $(app_compose_file)" "${WRITES}" + SERVICES='app db' RUNNING='app db' + app_up + grep -q -- "up --detach --remove-orphans --quiet-pull$" "${ENGINE_LOG}" + [ "${APP_COMPOSE_CHANGED}" = 0 ] + : >"${ENGINE_LOG}" + app_up + run ! grep -q -- " up " "${ENGINE_LOG}" + RUNNING='db' + app_up + grep -q -- " up --detach" "${ENGINE_LOG}" + : >"${ENGINE_LOG}" + RUNNING='app db' + APP_CHANGED=1 + app_up + grep -q -- "up --detach --remove-orphans --quiet-pull --force-recreate$" "${ENGINE_LOG}" + [ "${APP_CHANGED}" = 0 ] + DOCKER_EXIT=1 run app_up + [[ ${output} == *"docker compose is missing"* ]] +} + +@test "app_running needs every service of the stack" { + SERVICES='' RUNNING='' + run app_running + [ "${status}" -eq 1 ] + SERVICES='db app' RUNNING='app db' + app_running + SERVICES='db app' RUNNING='app' + run app_running + [ "${status}" -eq 1 ] + app_exec app occ status + grep -qxF "docker compose --project-name lss-demo -f ${CTR_CONFIG_DIR}/lss-demo/compose.yaml exec -T app occ status" "${ENGINE_LOG}" +} + +@test "pages come from the web server of this server, with their status" { + config_set LSS_DEMO_DOMAINS cloud.example.com + # shellcheck disable=SC2329 # called by app_get + curl() { + printf 'curl %s\n' "$*" >>"${STUB_LOG}" + printf 'login\n%s' "${CURL_STATUS:-200}" + return 0 + } + local page + page=$(app_get /status.php) + [ "$(app_status "${page}")" = 200 ] + grep -qF -- "--resolve cloud.example.com:443:127.0.0.1 https://cloud.example.com/status.php" "${STUB_LOG}" + page=$(app_wait login 200 5) + [[ ${page} == *login* ]] + # shellcheck disable=SC2329 # called by app_wait + sleep() { + return 0 + } + CURL_STATUS=502 run app_wait login 200 1 + [ "${status}" -eq 1 ] + [[ ${output} == *"https://cloud.example.com/login answers 502 instead of 200"* ]] +} + +@test "the site proxies to 127.0.0.1 for every host name and gets a certificate with tls" { + config_set LSS_DEMO_DOMAINS 'cloud.example.com www.example.com' + # shellcheck disable=SC2329 # called by app_site + web_cert_placeholder() { + printf 'placeholder %s\n' "$*" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_site + web_site_write() { + printf 'site %s\n' "$*" >>"${STUB_LOG}" + LSS_FILE_CHANGED=1 + } + # shellcheck disable=SC2329 # called by app_site + web_reload() { + printf 'reload\n' >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_site + web_cert_issue() { + printf 'issue %s\n' "$*" >>"${STUB_LOG}" + } + app_site 8081 + grep -qxF 'placeholder demo cloud.example.com www.example.com' "${STUB_LOG}" + grep -qxF 'site demo proxy http://127.0.0.1:8081 cloud.example.com www.example.com' "${STUB_LOG}" + grep -qxF reload "${STUB_LOG}" + run ! grep -q '^issue' "${STUB_LOG}" + state_set tls status installed + app_site 8081 + grep -qxF 'issue demo cloud.example.com www.example.com' "${STUB_LOG}" +} + +@test "verify checks the files, the rules, the containers and the headers of the site" { + command -v jq >/dev/null || skip "jq is not installed" + # shellcheck disable=SC2329 # called by app_verify_stack + stat() { + printf '600 root\n' + } + run app_verify_stack + [[ ${output} == *"compose.yaml is missing"* ]] + printf 'services: {}\n' | app_compose + SERVICES='app' RUNNING='' + run app_verify_stack + [[ ${output} == *"a container of lss-demo is not running"* ]] + RUNNING='app' + # shellcheck disable=SC2329 # called by app_verify_stack + curl() { + printf 'HTTP/2 200\r\n%s\r\n\n200' "${HEADER:-}" + return 0 + } + run app_verify_stack + [[ ${output} == *"sends no Strict-Transport-Security header"* ]] + HEADER='strict-transport-security: max-age=63072000' + app_verify_stack +} + +@test "removal stops the stack and keeps the data unless --purge" { + MOD_SECRETS="demo-db-password" + app_secret demo-db-password 0 + printf 'services: {}\n' | app_compose + # shellcheck disable=SC2329 # called by app_remove + web_server() { + printf 'nginx' + } + # shellcheck disable=SC2329 # called by app_remove + web_site_remove() { + printf 'site-remove %s\n' "$*" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_remove + web_reload() { + return 0 + } + app_remove + grep -q -- "down --remove-orphans$" "${ENGINE_LOG}" + grep -qxF 'site-remove demo' "${STUB_LOG}" + [ ! -e "$(app_dir)" ] + secret_exists demo-db-password + printf 'services: {}\n' | app_compose + LSS_PURGE=1 app_remove + grep -q -- "down --remove-orphans --volumes$" "${ENGINE_LOG}" + run secret_exists demo-db-password + [ "${status}" -eq 1 ] +} + +@test "database services run unprivileged on the internal network, with their password from a file" { + run app_postgres cloud + [[ ${output} == *"image: ${APP_POSTGRES_IMAGE}"* ]] + [[ ${output} == *'user: "70:70"'* ]] + [[ ${output} == *"POSTGRES_USER: cloud"*"POSTGRES_PASSWORD_FILE: /run/secrets/db-password"* ]] + [[ ${output} == *"secrets/db-password.db:/run/secrets/db-password:ro,z"* ]] + [[ ${output} == *"networks: [backend]"*"pg_isready"* ]] + [[ ${output} == *"cap_drop: [ALL]"*"read_only: true"* ]] + run app_mariadb shop + [[ ${output} == *'user: "999:999"'* ]] + [[ ${output} == *"MARIADB_PASSWORD_FILE: /run/secrets/db-password"*"MARIADB_ROOT_PASSWORD_FILE: /run/secrets/db-root-password"* ]] + [[ ${output} == *"healthcheck.sh"* ]] + run app_networks + [[ ${output} == *"backend:"*"internal: true"* ]] + run app_init data /data 1000:1000 + [[ ${output} == *'entrypoint: ["chown", "1000:1000", "/data"]'* ]] + # Shared SELinux label: the app mounts the volume with another MCS category than init. + [[ ${output} == *"- data:/data:z"* ]] + [[ ${output} == *"network_mode: none"*"cap_drop: [ALL]"*"cap_add: [CHOWN]"*"read_only: true"* ]] + [[ ${output} == *'restart: "no"'* ]] +} + +@test "database secrets go to the database container and the app, each for its own user" { + app_db_secrets postgres 33:33 + local dir=${CTR_CONFIG_DIR}/lss-demo/secrets + grep -qxF "== 0400 70:70 ${dir}/db-password.db" "${WRITES}" + grep -qxF "== 0400 33:33 ${dir}/db-password.app" "${WRITES}" + cmp -s "${dir}/db-password.db" "${LSS_SECRETS_DIR}/demo-db-password" + run ! grep -q db-root-password "${WRITES}" + app_db_secrets mariadb 1000 app-db + grep -qxF "== 0400 999:999 ${dir}/db-root-password.db" "${WRITES}" + grep -qxF "== 0400 1000:1000 ${dir}/app-db" "${WRITES}" + secret_exists demo-db-root-password +} + +@test "the one-shot init service does not count as a stopped service" { + SERVICES='app init' RUNNING='app' + app_running + SERVICES='app db init' RUNNING='db' + run app_running + [ "${status}" -eq 1 ] +} + +@test "logins for curl go through a private netrc file" { + config_set LSS_DEMO_DOMAINS cloud.example.com + secret_set demo-admin-password 'S3cret' + local file + file=$(app_netrc admin demo-admin-password) + [ "$(stat -c '%a' "${file}")" = 600 ] + [ "$(cat "${file}")" = 'machine cloud.example.com login admin password S3cret' ] + rm -f -- "${file}" +} + +@test "the host names of another app come from its settings or the server's name" { + # shellcheck disable=SC2329 # called by app_domains_of + hostname() { + printf 'srv.example.com\n' + } + [ "$(app_domains_of collabora)" = collabora.srv.example.com ] + config_set LSS_COLLABORA_DOMAINS 'office.example.com docs.example.com' + [ "$(app_domains_of collabora)" = 'office.example.com docs.example.com' ] +} + +@test "Nextcloud Office is linked once both apps are there, trusting the certificate of the collabora site" { + config_set LSS_COLLABORA_DOMAINS office.example.com + # shellcheck disable=SC2329 # called by app_collabora_check + curl() { + printf 'curl %s\n' "$*" >>"${STUB_LOG}" + printf '{"status": "Ok"}\n200' + } + LSS_MODULE=collabora + app_nextcloud_office + [ ! -s "${ENGINE_LOG}" ] + state_set nextcloud status installed + WEB_TLS_DIR=${BATS_TEST_TMPDIR}/tls + mkdir -p "${WEB_TLS_DIR}/collabora" + printf 'CERT\n' >"${WEB_TLS_DIR}/collabora/fullchain.pem" + app_nextcloud_office + local occ="docker compose --project-name lss-nextcloud -f ${CTR_CONFIG_DIR}/lss-nextcloud/compose.yaml exec -T app php occ" + grep -qxF "${occ} app:install richdocuments" "${ENGINE_LOG}" + grep -qxF "${occ} config:app:set richdocuments wopi_url --value=https://office.example.com" "${ENGINE_LOG}" + grep -qF "exec -T app sh -c cat >/tmp/lss-web.pem" "${ENGINE_LOG}" + grep -qxF "${occ} security:certificates:import /tmp/lss-web.pem" "${ENGINE_LOG}" + grep -qxF "${occ} richdocuments:activate-config" "${ENGINE_LOG}" + grep -q '^curl .*--resolve office.example.com:443:127.0.0.1 -X POST .*https://office.example.com/hosting/wopiAccessCheck$' "${STUB_LOG}" + [ -n "$(state_get nextcloud trusted_cert)" ] + : >"${ENGINE_LOG}" + app_nextcloud_office + run ! grep -q 'certificates:import' "${ENGINE_LOG}" + # With the tls module, only a private CA needs trusting. + state_set tls status installed + [ -z "$(_app_route_trust collabora)" ] + printf 'CA\n' >"${BATS_TEST_TMPDIR}/ca.pem" + config_set LSS_TLS_CA_BUNDLE "${BATS_TEST_TMPDIR}/ca.pem" + [ "$(_app_route_trust collabora)" = "${BATS_TEST_TMPDIR}/ca.pem" ] + DOCKER_EXIT=1 run app_nextcloud_office + [ "${status}" -eq 1 ] + [[ ${output} == *"Nextcloud Office (richdocuments) cannot be installed"* ]] +} + +@test "timers run a command in a container of the stack and go away on removal" { + # shellcheck disable=SC2329 # called by app_timer + svc_unit() { + printf -- '--- %s\n' "$1" >>"${STUB_LOG}" + cat >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_timer + svc_enable() { + printf 'enable %s\n' "$1" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_timer_remove + svc_disable() { + printf 'disable %s\n' "$1" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_timer_remove + file_remove() { + printf 'remove %s\n' "$1" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_timer_remove + svc_daemon_reload() { + return 0 + } + app_timer demo-cron 5min "Demo jobs" app php -f cron.php + grep -qxF -- '--- lss-demo-cron.service' "${STUB_LOG}" + grep -qxF "ExecStart=docker compose --project-name lss-demo -f ${CTR_CONFIG_DIR}/lss-demo/compose.yaml exec -T app php -f cron.php" "${STUB_LOG}" + grep -qxF 'OnUnitActiveSec=5min' "${STUB_LOG}" + grep -qxF 'enable lss-demo-cron.timer' "${STUB_LOG}" + app_timer_remove demo-cron + grep -qxF 'disable lss-demo-cron.timer' "${STUB_LOG}" + grep -qxF 'remove /etc/systemd/system/lss-demo-cron.service' "${STUB_LOG}" +} + +@test "a second site of an app has its own name and host names, and goes away with its certificate" { + # shellcheck disable=SC2329 # called by app_site + web_cert_placeholder() { + return 0 + } + # shellcheck disable=SC2329 # called by app_site + web_site_write() { + printf 'site %s\n' "$*" >>"${STUB_LOG}" + LSS_FILE_CHANGED=0 + } + app_site 8092 demo-console console.example.com + grep -qxF 'site demo-console proxy http://127.0.0.1:8092 console.example.com' "${STUB_LOG}" + # shellcheck disable=SC2329 # called by app_site_remove + web_server() { + printf 'caddy' + } + # shellcheck disable=SC2329 # called by app_site_remove + web_site_remove() { + printf 'site-remove %s\n' "$*" >>"${STUB_LOG}" + } + # shellcheck disable=SC2329 # called by app_site_remove + web_reload() { + return 0 + } + # shellcheck disable=SC2329 # called by app_site_remove + certbot() { + printf 'certbot %s\n' "$*" >>"${STUB_LOG}" + } + WEB_TLS_DIR=${BATS_TEST_TMPDIR}/tls + WEB_CERT_LIVE=${BATS_TEST_TMPDIR}/live + mkdir -p "${WEB_CERT_LIVE}/demo-console" + app_site_remove demo-console + grep -qxF 'site-remove demo-console' "${STUB_LOG}" + grep -qxF 'certbot delete --non-interactive --cert-name demo-console' "${STUB_LOG}" +} + +@test "pages of a second host name of the app" { + config_set LSS_DEMO_DOMAINS cloud.example.com + # shellcheck disable=SC2329 # called by app_get + curl() { + printf 'curl %s\n' "$*" >>"${STUB_LOG}" + printf 'ok\n200' + } + APP_GET_DOMAIN=console.example.com app_get login >/dev/null + grep -qF -- "--resolve console.example.com:443:127.0.0.1 https://console.example.com/login" "${STUB_LOG}" +} + +@test "Collabora trusts the host CAs and the Nextcloud route, and starts again when Nextcloud comes second" { + config_set LSS_NEXTCLOUD_DOMAINS cloud.example.com + config_set LSS_COLLABORA_DOMAINS office.example.com + WEB_TLS_DIR=${BATS_TEST_TMPDIR}/tls + mkdir -p "${WEB_TLS_DIR}/nextcloud" "${WEB_TLS_DIR}/default" + printf 'DEFAULT\n' >"${WEB_TLS_DIR}/default/fullchain.pem" + local trust="${CTR_CONFIG_DIR}/lss-collabora/trust/ca.pem" + # Collabora alone: the CAs of the host (or the default certificate on a host without a bundle). + app_collabora_trust + [ -s "${trust}" ] + grep -qxF "== 0444 root:root ${trust}" "${WRITES}" + run ! grep -qxF NEXTCLOUD "${trust}" + # With the Nextcloud route: its self-signed certificate too. + printf 'NEXTCLOUD\n' >"${WEB_TLS_DIR}/nextcloud/fullchain.pem" + app_collabora_trust + [ "${LSS_FILE_CHANGED}" = 1 ] + grep -qxF NEXTCLOUD "${trust}" + app_collabora_trust + [ "${LSS_FILE_CHANGED}" = 0 ] + # Nextcloud second: Collabora gets the new certificate and starts again before the access check. + # shellcheck disable=SC2329 # called by app_nextcloud_office + curl() { + printf 'curl %s\n' "$*" >>"${STUB_LOG}" + printf '{"status": "Ok"}\n200' + } + command -v jq >/dev/null || skip "jq is not installed" + state_set docker status installed + state_set collabora status installed + printf 'services: {}\n' >"${CTR_CONFIG_DIR}/lss-collabora/compose.yaml" + printf 'NEXTCLOUD 2\n' >"${WEB_TLS_DIR}/nextcloud/fullchain.pem" + LSS_MODULE=nextcloud + app_nextcloud_office + grep -q "compose --project-name lss-collabora .* up .*--force-recreate" "${ENGINE_LOG}" + grep -q '^curl .* https://office.example.com/hosting/wopiAccessCheck$' "${STUB_LOG}" + grep -qxF 'NEXTCLOUD 2' "${trust}" +} + +@test "the WOPI access check of Collabora must reach Nextcloud with a valid certificate" { + config_set LSS_NEXTCLOUD_DOMAINS cloud.example.com + config_set LSS_COLLABORA_DOMAINS office.example.com + # shellcheck disable=SC2329 # called by app_collabora_check + curl() { + printf 'curl %s\n' "$*" >>"${STUB_LOG}" + printf '%s\n%s' "${ANSWER}" "${CODE:-200}" + } + ANSWER='{"status": "Ok"}' app_collabora_check + grep -qF -- '--data {"callbackUrl":"https://cloud.example.com/status.php"}' "${STUB_LOG}" + ANSWER='{"status": "SelfSignedCertificate"}' run app_collabora_check + [ "${status}" -eq 1 ] + [[ ${output} == *'cannot reach Nextcloud at https://cloud.example.com with a valid certificate ({"status": "SelfSignedCertificate"}'* ]] + ANSWER='{"status": "Ok"}' CODE=502 run app_collabora_check + [ "${status}" -eq 1 ] +} + +@test "an app registers its data for the backup, which verify saves and removal forgets" { + app_backup DB=postgres DB_USER=demo LIVE=html 'COPY=data cache' + local conf="${APP_BACKUP_REGISTRY}/demo.conf" + grep -qxF "== 0755 root:root ${APP_BACKUP_SCRIPT}" "${WRITES}" + cmp "${REPO_ROOT}/libexec/lss-app-backup" "${APP_BACKUP_SCRIPT}" + grep -qxF "== 0600 root:root ${conf}" "${WRITES}" + grep -qxF PROJECT=lss-demo "${conf}" + grep -qxF "COMPOSE=${CTR_CONFIG_DIR}/lss-demo/compose.yaml" "${conf}" + grep -qxF DB=postgres "${conf}" + grep -qxF DB_USER=demo "${conf}" + grep -qxF 'COPY=data cache' "${conf}" + grep -qxF LIVE=html "${conf}" + grep -qxF PATHS= "${conf}" + run app_backup DATABASE=postgres + [[ ${output} == *"app_backup does not know 'DATABASE=postgres'"* ]] + run app_backup 'PATHS=/srv;rm' + [ "${status}" -eq 1 ] + # verify runs the script for this app; a database app needs a dump. + printf '#!/bin/sh\necho "save $*" >>"%s"\n' "${STUB_LOG}" >"${APP_BACKUP_SCRIPT}" + chmod +x "${APP_BACKUP_SCRIPT}" + run app_verify_backup + [ "${status}" -eq 1 ] + [[ ${output} == *"the database dump for the backup is empty"* ]] + grep -qxF 'save demo' "${STUB_LOG}" + mkdir -p "${APP_BACKUP_DIR}/demo" + printf 'SQL\n' >"${APP_BACKUP_DIR}/demo/db.sql" + app_verify_backup + printf '#!/bin/sh\nexit 1\n' >"${APP_BACKUP_SCRIPT}" + run app_verify_backup + [[ ${output} == *"saving the data of the app for the backup fails"* ]] + # Removal keeps the saved data unless --purge; the script goes with the last app. + app_backup_remove + [ ! -e "${conf}" ] + [ ! -e "${APP_BACKUP_REGISTRY}" ] + [ ! -e "${APP_BACKUP_SCRIPT}" ] + [ -s "${APP_BACKUP_DIR}/demo/db.sql" ] + run app_verify_backup + [[ ${output} == *"the backup would leave the app out"* ]] + app_backup COPY=data + LSS_PURGE=1 app_backup_remove + [ ! -e "${APP_BACKUP_DIR}/demo" ] +} + +@test "app_waf_hits names the ModSecurity rules of the latest audit entries" { + run app_waf_hits + [ "${status}" -eq 0 ] + [ -z "${output}" ] + state_set nginx status installed + state_set modsecurity status installed + # shellcheck disable=SC2329 # called by app_waf_hits + web_log_dir() { printf '%s' "${BATS_TEST_TMPDIR}/log"; } + mkdir -p "${BATS_TEST_TMPDIR}/log/modsecurity" + printf '%s\n' '---H---' 'ModSecurity: Warning. [file "x.conf"] [line "1"] [id "932180"] [rev ""] [msg "Restricted File Upload Attempt"]' \ + 'ModSecurity: Access denied with code 403 (phase 2). [id "949110"] [msg "Inbound Anomaly Score Exceeded (Total Score: 5)"]' \ + >"${BATS_TEST_TMPDIR}/log/modsecurity/audit.log" + run app_waf_hits + [ "${output}" = '[id "932180"] [msg "Restricted File Upload Attempt"] [id "949110"] [msg "Inbound Anomaly Score Exceeded (Total Score: 5)"]' ] +} diff --git a/tests/unit/backup.bats b/tests/unit/backup.bats index 472da90..ef3ee51 100644 --- a/tests/unit/backup.bats +++ b/tests/unit/backup.bats @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: MIT -# The backup scripts (libexec/lss-backup, libexec/lss-db-dumps) against a restic stub, and the checks of the restic +# The backup scripts (libexec/lss-backup, libexec/lss-db-dumps, libexec/lss-app-backup) against a restic stub, and the checks of the restic # module. setup() { @@ -19,6 +19,10 @@ setup() { export LSS_DB_DUMPS_CONF=${BATS_TEST_TMPDIR}/db-dumps.conf export LSS_DB_DUMPS_STATE=${BATS_TEST_TMPDIR}/db-dumps-state export LSS_RESTIC=restic_stub + export LSS_APP_BACKUP=${BATS_TEST_TMPDIR}/app-backup + export LSS_APP_BACKUP_STATE=${BATS_TEST_TMPDIR}/app-backup-state + export LSS_APP_BACKUP_REGISTRY=${BATS_TEST_TMPDIR}/app-backup.d + export LSS_APP_BACKUP_DIR=${BATS_TEST_TMPDIR}/lss-apps mkdir -p "${LSS_TEXTFILE_DIR}" "${LSS_SECRETS_DIR}" printf 'BACKUP_REPOSITORY=/srv/repo\nBACKUP_PATHS=/etc /does-not-exist\nBACKUP_EXCLUDE=*.tmp\n' >"${LSS_BACKUP_CONF}" printf 'password' >"${LSS_SECRETS_DIR}/restic-password" @@ -334,3 +338,149 @@ load_dumps() { [ "${status}" -eq 1 ] [ ! -e "${BATS_TEST_TMPDIR}/w.rdb" ] } + +@test "a run saves the data of the apps first and backs up the paths they list" { + load_backup + mkdir -p "${LSS_APP_BACKUP_STATE}" "${BATS_TEST_TMPDIR}/volume" + printf '#!/bin/sh\necho apps >>"%s"\n' "${STUB_LOG}" >"${LSS_APP_BACKUP}" + chmod +x "${LSS_APP_BACKUP}" + printf '%s\n' "${BATS_TEST_TMPDIR}/volume" /does-not-exist-either >"${LSS_APP_BACKUP_STATE}/paths" + cmd_run + grep -qxF apps "${STUB_LOG}" + grep -qxF "restic backup --tag lss --exclude-caches --exclude *.tmp /etc ${BATS_TEST_TMPDIR}/volume ${LSS_BACKUP_STATE}/canary" "${STUB_LOG}" + printf '#!/bin/sh\nexit 1\n' >"${LSS_APP_BACKUP}" + run cmd_run + [ "${status}" -eq 1 ] + [[ ${output} == *"saving the data of the apps failed; the backup goes on with the rest."* ]] +} + +load_apps() { + # shellcheck source=/dev/null + source "${REPO_ROOT}/libexec/lss-app-backup" + DOCKER=docker_stub + mkdir -p "${REGISTRY}" "${BATS_TEST_TMPDIR}/volumes/lss-demo_data" "${BATS_TEST_TMPDIR}/volumes/lss-demo_html" + printf 'app.db\n' >"${BATS_TEST_TMPDIR}/volumes/lss-demo_data/app.db" + return +} + +# docker stub for lss-app-backup: records the call; volume inspect prints the directory below volumes/ (missing +# ones fail), exec prints $DUMP (fails with $DUMP_EXIT), pause and unpause fail with $PAUSE_EXIT. +docker_stub() { + printf 'docker %s\n' "$*" >>"${STUB_LOG}" + case $* in + 'volume inspect'*) + [[ -d ${BATS_TEST_TMPDIR}/volumes/${*: -1} ]] || return 1 + printf '%s\n' "${BATS_TEST_TMPDIR}/volumes/${*: -1}" + ;; + *' exec '*) + printf '%s' "${DUMP:-}" + return "${DUMP_EXIT:-0}" + ;; + *' pause '* | *' unpause '*) return "${PAUSE_EXIT:-0}" ;; + *) ;; + esac + return 0 +} + +register() { + printf 'PROJECT=lss-%s\nCOMPOSE=/etc/lss/containers/lss-%s/compose.yaml\n%s\n' "$1" "$1" "$2" >"${REGISTRY}/$1.conf" + return +} + +@test "app data: database dumps, copies of paused apps and the paths restic takes as they are" { + load_apps + register demo $'DB=postgres\nDB_USER=demo\nCOPY=data\nLIVE=html\nPATHS=/etc /does-not-exist' + DUMP='SQL' run_backups + local compose='docker compose --project-name lss-demo -f /etc/lss/containers/lss-demo/compose.yaml' + grep -qxF "${compose} exec -T db pg_dumpall --username=demo --clean --if-exists" "${STUB_LOG}" + grep -qxF "${compose} pause app" "${STUB_LOG}" + grep -qxF "${compose} unpause app" "${STUB_LOG}" + [ "$(cat "${OUT_ROOT}/demo/db.sql")" = SQL ] + [ "$(cat "${OUT_ROOT}/demo/data/app.db")" = app.db ] + [ "$(stat -c '%a' "${OUT_ROOT}/demo")" = 700 ] + [ "$(cat "${STATE}/paths")" = "$(printf '%s\n' "${BATS_TEST_TMPDIR}/volumes/lss-demo_html" /etc "${OUT_ROOT}")" ] + grep -q '^demo ok [0-9]*$' "${STATE}/status" + # MariaDB: mariadb-dump as root with the password from the secret file, inside the container. + register shop $'DB=mariadb' + DUMP='SQL' run_backups shop + grep -qF 'compose --project-name lss-shop -f /etc/lss/containers/lss-shop/compose.yaml exec -T db sh -c MYSQL_PWD=$(cat /run/secrets/db-root-password) exec mariadb-dump --user=root' "${STUB_LOG}" + [ -s "${OUT_ROOT}/shop/db.sql" ] +} + +@test "a failed dump or copy keeps the previous one, unpauses the app and fails the run" { + load_apps + register demo $'DB=postgres\nDB_USER=demo\nCOPY=data' + mkdir -p "${OUT_ROOT}/demo/data" + printf 'OLD\n' >"${OUT_ROOT}/demo/db.sql" + printf 'OLD\n' >"${OUT_ROOT}/demo/data/app.db" + # shellcheck disable=SC2329 # called by the functions under test + cp() { + return 1 + } + DUMP='PART' DUMP_EXIT=1 run run_backups + [ "${status}" -eq 1 ] + [[ ${output} == *"the database dump of demo failed; ${OUT_ROOT}/demo/db.sql keeps the previous one."* ]] + [[ ${output} == *"the copy of the data of demo failed; ${OUT_ROOT}/demo keeps the previous one."* ]] + [ "$(cat "${OUT_ROOT}/demo/db.sql")" = OLD ] + [ "$(cat "${OUT_ROOT}/demo/data/app.db")" = OLD ] + [ ! -e "${OUT_ROOT}/demo/db.sql.part" ] + grep -q 'unpause app' "${STUB_LOG}" + grep -q '^demo failed ' "${STATE}/status" +} + +@test "an app that cannot be paused or unpaused, and a missing volume, fail the run" { + load_apps + # shellcheck disable=SC2329 # called by the functions under test + sleep() { + : + } + register demo $'COPY=data' + PAUSE_EXIT=1 run run_backups demo + [ "${status}" -eq 1 ] + [[ ${output} == *"the copy of the data of demo failed"* ]] + run ! grep -q 'unpause' "${STUB_LOG}" + register demo $'COPY=data\nLIVE=gone' + # Pausing works, unpausing never does. + # shellcheck disable=SC2329 # called by the functions under test + compose() { + [[ $2 != unpause ]] + } + run run_backups demo + [[ ${output} == *"the app of lss-demo stays paused; run docker compose unpause app."* ]] + [[ ${output} == *"the volume gone of demo is missing."* ]] + register demo $'COPY=gone' + run run_backups demo + [ "${status}" -eq 1 ] +} + +@test "a run for named apps leaves the full list alone; unknown names and an empty registry are reported" { + load_apps + register demo $'LIVE=html' + run_backups + local before + before=$(cat "${STATE}/paths") + register other $'LIVE=data' + mkdir -p "${BATS_TEST_TMPDIR}/volumes/lss-other_data" + run_backups other + [ "$(cat "${STATE}/paths")" = "${before}" ] + run run_backups '../etc' + [ "${status}" -eq 1 ] + [[ ${output} == *"../etc is not registered"* ]] + rm -f -- "${REGISTRY}"/*.conf + run run_backups + [ "${status}" -eq 0 ] + [[ ${output} == *"0 apps are registered here; nothing to save."* ]] + [ ! -s "${STATE}/paths" ] + register bad $'DB=sqlite' + run run_backups bad + [ "${status}" -eq 1 ] +} + +@test "the app backup script needs root" { + if [[ ${EUID} -eq 0 ]]; then + skip "runs as root" + fi + run bash "${REPO_ROOT}/libexec/lss-app-backup" + [ "${status}" -eq 1 ] + [[ ${output} == *"must run as root."* ]] +} diff --git a/tests/unit/container.bats b/tests/unit/container.bats index 0394e32..de67287 100644 --- a/tests/unit/container.bats +++ b/tests/unit/container.bats @@ -395,12 +395,50 @@ record_writes() { run ctr_compose_up 'Bad Name' "${file}" [ "${status}" -eq 1 ] ctr_compose_up shop "${file}" + grep -qxF "docker compose --project-name shop -f ${file} pull --quiet --policy missing" "${BATS_TEST_TMPDIR}/engine.log" grep -qxF "docker compose --project-name shop -f ${file} up --detach --remove-orphans --quiet-pull" "${BATS_TEST_TMPDIR}/engine.log" LSS_MODULE=podman run ctr_compose_up shop "${file}" [[ ${output} == *"need the docker module"* ]] } +@test "Compose projects retry a failed image pull and give up after the last attempt" { + command -v jq >/dev/null || skip "jq is not installed" + local file=${BATS_TEST_TMPDIR}/compose.yaml + : >"${file}" + CTR_PULL_DELAY=0 + printf '1 +' >"${BATS_TEST_TMPDIR}/fails" + # shellcheck disable=SC2329 # called by ctr_compose_up + docker() { + local left + printf 'docker %s +' "$*" >>"${BATS_TEST_TMPDIR}/engine.log" + if [[ $* == *config* ]]; then + printf '{"services": {}} +' + elif [[ $* == *" pull "* ]]; then + left=$(cat "${BATS_TEST_TMPDIR}/fails") + ((left > 0)) || return 0 + printf '%s +' $((left - 1)) >"${BATS_TEST_TMPDIR}/fails" + return 1 + fi + return 0 + } + ctr_compose_up shop "${file}" + [ "$(grep -c ' pull ' "${BATS_TEST_TMPDIR}/engine.log")" -eq 2 ] + grep -q ' up --detach' "${BATS_TEST_TMPDIR}/engine.log" + : >"${BATS_TEST_TMPDIR}/engine.log" + printf '9 +' >"${BATS_TEST_TMPDIR}/fails" + run ctr_compose_up shop "${file}" + [ "${status}" -eq 1 ] + [[ ${output} == *"images of shop cannot be pulled"* ]] + [ "$(grep -c ' pull ' "${BATS_TEST_TMPDIR}/engine.log")" -eq "${CTR_PULL_TRIES}" ] + run ! grep -q ' up --detach' "${BATS_TEST_TMPDIR}/engine.log" +} + @test "Trivy is kept when it is current and removed with the last engine" { CTR_TRIVY=${BATS_TEST_TMPDIR}/trivy CTR_TRIVY_CACHE=${BATS_TEST_TMPDIR}/trivy-cache diff --git a/tests/unit/helpers.bash b/tests/unit/helpers.bash index 028bb71..0a067a1 100644 --- a/tests/unit/helpers.bash +++ b/tests/unit/helpers.bash @@ -22,7 +22,7 @@ lss_test_setup() { export OWNER : >"${STUB_LOG}" local library - for library in log os config files template secrets state pkg repo fetch service firewall container web mail monitoring module ui; do + for library in log os config files template secrets state pkg repo fetch service firewall container web mail monitoring apps module ui; do # shellcheck source=/dev/null source "${REPO_ROOT}/lib/${library}.sh" done diff --git a/tests/unit/mail.bats b/tests/unit/mail.bats index 22cc077..862cb7d 100644 --- a/tests/unit/mail.bats +++ b/tests/unit/mail.bats @@ -67,9 +67,25 @@ load_module() { @test "the Postfix sandbox keeps /dev and opens only the queue, the state and what the module adds" { # shellcheck disable=SC2329 # called by mail_postfix_sandbox svc_sandbox() { printf '%s\n' "$@"; } + # shellcheck disable=SC2329 # called by mail_postfix_sandbox + svc_selinux_spawner() { printf 'spawner %s\n' "$*" >>"${STUB_LOG}"; } + # shellcheck disable=SC2329 # called by mail_postfix_sandbox + postconf() { + case $2 in + sendmail_path) printf '/usr/sbin/sendmail.postfix\n' ;; + *) printf '/usr/lib/postfix/sbin\n' ;; + esac + } + # shellcheck disable=SC2329 # called by mail_postfix_sandbox + svc_exec_programs() { printf '%s\n' /usr/sbin/restorecon /usr/sbin/postfix; } run mail_postfix_sandbox ReadWritePaths=/var/mail [ "${lines[0]}" = 'postfix@.service' ] [[ ${output} == *$'PrivateDevices=no\nReadWritePaths=/var/spool/postfix\nReadWritePaths=/var/lib/postfix\nReadWritePaths=/var/mail'* ]] + # NoNewPrivileges stays as svc_sandbox sets it; under SELinux the master starts its daemons in their own domains + # through a policy module. + run ! grep -q NoNewPrivileges <<<"${output}" + grep -qxF 'spawner postfix /usr/lib/postfix/sbin/master /usr/lib/postfix/sbin' "${STUB_LOG}" + grep -qxF 'spawner postfix_start init_t /usr/sbin/restorecon /usr/sbin/postfix /usr/sbin/sendmail.postfix' "${STUB_LOG}" } @test "Postfix starts once, restarts only after a change and stops the run on a configuration it rejects" { diff --git a/tests/unit/system.bats b/tests/unit/system.bats index 30baf05..f26fff2 100644 --- a/tests/unit/system.bats +++ b/tests/unit/system.bats @@ -42,6 +42,30 @@ calls() { [ "${output}" = 'Setting up nginx (1.24.0)' ] } +@test "apt is tried again when a download fails or an index cannot be fetched, and gives up after three attempts" { + use_os ubuntu-22.04 + PKG_APT_DELAY=0 + export STUB_OUT_apt_get='E: Failed to fetch https://ppa.launchpadcontent.net/x.deb 503 Service Unavailable' + export STUB_EXIT_apt_get=100 + run _pkg_apt_retry install dotnet-sdk-10.0 + [ "${status}" -eq 100 ] + [ "$(grep -c '^apt-get .* install dotnet-sdk-10.0' "${STUB_LOG}")" -eq 3 ] + [[ ${output} == *'failed (attempt 2 of 3)'* ]] + : >"${STUB_LOG}" + # An index that stays out of reach is only a warning of apt-get update, after the last attempt as well. + export STUB_OUT_apt_get='W: Failed to fetch https://ppa.launchpadcontent.net/dists/jammy/InRelease 503' + export STUB_EXIT_apt_get=0 + run _pkg_apt_retry update + [ "${status}" -eq 0 ] + [ "$(grep -c '^apt-get .* update' "${STUB_LOG}")" -eq 3 ] + : >"${STUB_LOG}" + export STUB_OUT_apt_get='Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease' + run _pkg_apt_retry update + [ "${status}" -eq 0 ] + [ "$(grep -c '^apt-get .* update' "${STUB_LOG}")" -eq 1 ] + [ "${output}" = 'Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease' ] +} + @test "dnf is used on the RHEL family and family lists pick the right names" { use_os rocky-9 export STUB_EXIT_rpm=1 @@ -274,7 +298,7 @@ time.sleep(30)' >"${port_file}" 2>/dev/null 3>&- & @test "without a firewall tool rules are only recorded and applied later" { use_os debian-12 PATH=$(printf '%s' "${PATH}" | tr ':' '\n' | grep -v fixtures/bin | paste -sd:) - command -v ufw >/dev/null && skip "ufw is installed on this machine" + type -P ufw >/dev/null && skip "ufw is installed on this machine" LSS_MODULE=web fw_allow 443/tcp [ "$(cat "${LSS_STATE_DIR}/firewall/web")" = 443/tcp ] @@ -656,6 +680,99 @@ EOF run ! grep -qxF 'NoNewPrivileges=yes' <<<"${output}" } +@test "svc_selinux_spawner allows the domains of a master's workers with NoNewPrivileges, once, and removes them" { + export SVC_SELINUX_DIR=${BATS_TEST_TMPDIR}/selinux + local dir=${BATS_TEST_TMPDIR}/libexec modules='' program + mkdir -p "${dir}" + for program in master qmgr smtpd trivial-rewrite; do + printf '#!/bin/sh +' >"${dir}/${program}" + chmod 0755 "${dir}/${program}" + done + install -m 0644 /dev/null "${dir}/qmgr.conf" + eval "_lss_file_write() $(declare -f file_write | tail -n +2)" + # shellcheck disable=SC2329 # called by the function under test + file_write() { _lss_file_write "$1" "${OWNER}" "$3"; } + # shellcheck disable=SC2329 # called by the function under test + selinuxexeccon() { + case ${1##*/}:$2 in + master:*:init_t:*) printf 'system_u:system_r:postfix_master_t:s0\n' ;; + qmgr:*:postfix_master_t:* | qmgr.conf:*) printf 'system_u:system_r:postfix_qmgr_t:s0\n' ;; + smtpd:*:postfix_master_t:*) printf 'system_u:system_r:postfix_smtpd_t:s0\n' ;; + unknown:*) return 1 ;; + *) printf '%s\n' "$2" ;; + esac + } + # shellcheck disable=SC2329 + semodule() { + if [[ $1 == -l ]]; then + printf '%s\n' "${modules}" + else + printf 'semodule %s %s\n' "$1" "${2##*/}" >>"${STUB_LOG}" + fi + } + # Without SELinux there is nothing to allow. + export STUB_EXIT_selinuxenabled=1 + svc_selinux_spawner postfix "${dir}/master" "${dir}" + run ! grep -q semodule "${STUB_LOG}" + [ ! -e "${SVC_SELINUX_DIR}" ] + # Only the workers that change their domain, from executable files; the master's own programs stay out. + export STUB_EXIT_selinuxenabled=0 + svc_selinux_spawner postfix "${dir}/master" "${dir}" "${BATS_TEST_TMPDIR}/missing" + grep -qxF 'semodule -i lss_spawn_postfix.cil' "${STUB_LOG}" + [ "$(cat "${SVC_SELINUX_DIR}/lss_spawn_postfix.cil")" = "$(printf '%s\n' \ + '(allow postfix_master_t postfix_qmgr_t (process2 (nnp_transition nosuid_transition)))' \ + '(allow postfix_master_t postfix_smtpd_t (process2 (nnp_transition nosuid_transition)))')" ] + # Loaded and unchanged: nothing to do. Gone from the policy: loaded again. + : >"${STUB_LOG}" + modules=lss_spawn_postfix + svc_selinux_spawner postfix "${dir}/master" "${dir}" + run ! grep -q semodule "${STUB_LOG}" + modules='' + svc_selinux_spawner postfix "${dir}/master" "${dir}" + grep -qxF 'semodule -i lss_spawn_postfix.cil' "${STUB_LOG}" + : >"${STUB_LOG}" + modules=lss_spawn_postfix + svc_selinux_spawner_remove postfix + grep -qxF 'semodule -r lss_spawn_postfix' "${STUB_LOG}" + [ ! -e "${SVC_SELINUX_DIR}/lss_spawn_postfix.cil" ] + # A master that SELinux does not know stops the run. + run svc_selinux_spawner postfix "${BATS_TEST_TMPDIR}/unknown" "${dir}" + [ "${status}" -ne 0 ] + # A domain as the source and single programs: the helpers systemd runs for a unit. + : >"${STUB_LOG}" + modules='' + printf '#!/bin/sh\n' >"${BATS_TEST_TMPDIR}/restorecon" + chmod 0755 "${BATS_TEST_TMPDIR}/restorecon" + # shellcheck disable=SC2329 + selinuxexeccon() { + if [[ ${1##*/}:$2 == restorecon:*:init_t:* ]]; then + printf 'system_u:system_r:setfiles_t:s0\n' + else + printf '%s\n' "$2" + fi + } + svc_selinux_spawner postfix_start init_t "${BATS_TEST_TMPDIR}/restorecon" "${dir}/qmgr" + grep -qxF 'semodule -i lss_spawn_postfix_start.cil' "${STUB_LOG}" + [ "$(cat "${SVC_SELINUX_DIR}/lss_spawn_postfix_start.cil")" = \ + '(allow init_t setfiles_t (process2 (nnp_transition nosuid_transition)))' ] +} + +@test "svc_exec_programs lists the programs systemd runs for a unit" { + # shellcheck disable=SC2329 # called by the function under test + systemctl() { + case $3 in + ExecStartPre) printf '%s\n' '{ path=/usr/sbin/restorecon ; argv[]=/usr/sbin/restorecon -R /var/spool/postfix/pid ; ignore_errors=yes }' \ + '{ path=/usr/libexec/postfix/aliasesdb ; argv[]=/usr/libexec/postfix/aliasesdb ; ignore_errors=no }' ;; + ExecStart) printf '%s\n' '{ path=/usr/sbin/postfix ; argv[]=/usr/sbin/postfix start ; ignore_errors=no }' ;; + *) ;; + esac + } + run svc_exec_programs postfix.service + [ "${status}" -eq 0 ] + [ "${output}" = "$(printf '%s\n' /usr/sbin/restorecon /usr/libexec/postfix/aliasesdb /usr/sbin/postfix)" ] +} + @test "svc_sandbox writes the baseline and the service's own lines, without new privileges outside SELinux" { # shellcheck disable=SC2329 # called by the function under test svc_dropin() { @@ -793,3 +910,39 @@ EOF grep -qxF 'Suites: trixie/' "${BATS_TEST_TMPDIR}/writes" run ! grep -q '^Components:' "${BATS_TEST_TMPDIR}/writes" } + +@test "ufw runs again when it loses the iptables race (initcaps) and passes other failures through" { + local bin=${BATS_TEST_TMPDIR}/racebin + mkdir -p "${bin}" + FW_UFW_DELAY=0 + printf '1\n' >"${BATS_TEST_TMPDIR}/fails" + cat >"${bin}/ufw" <