diff --git a/api/config.py b/api/config.py
index d776479..74c7ca3 100644
--- a/api/config.py
+++ b/api/config.py
@@ -9,6 +9,11 @@ class Settings:
PG_USER: str = os.environ.get("PGUSER", "postgres")
PG_PASSWORD: str = os.environ.get("PGPASSWORD", "")
PG_DATABASE: str = os.environ.get("PGDATABASE", "te_mgmt_dev")
+ # M3d (#107): the runtime role the API's request pool connects as. It may only
+ # read and insert in the append-only tables. PG_USER stays the owner, used at
+ # startup for migrations and grants. Unset: the pool connects as PG_USER.
+ APP_DB_USER: str = os.environ.get("APP_DB_USER", "")
+ APP_DB_PASSWORD: str = os.environ.get("APP_DB_PASSWORD", "")
# Schema where dynamically created per-CSV tables + the registry live.
UPLOADS_SCHEMA: str = os.environ.get("CSV_UPLOADS_SCHEMA", "csv_uploads")
diff --git a/api/db.py b/api/db.py
index 9afca94..27f267c 100644
--- a/api/db.py
+++ b/api/db.py
@@ -2,6 +2,7 @@
import logging
import time
+from contextlib import contextmanager
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout
from pathlib import Path
@@ -39,8 +40,8 @@ def init_pool(max_attempts: int = 30, base_delay: float = 1.0) -> None:
maxconn=_MAXCONN,
host=settings.PG_HOST,
port=settings.PG_PORT,
- user=settings.PG_USER,
- password=settings.PG_PASSWORD,
+ user=settings.APP_DB_USER or settings.PG_USER,
+ password=settings.APP_DB_PASSWORD if settings.APP_DB_USER else settings.PG_PASSWORD,
dbname=settings.PG_DATABASE,
)
if attempt > 1:
@@ -176,8 +177,67 @@ def bootstrap() -> None:
settings.PG_USER, settings.PG_HOST, settings.PG_PORT,
settings.PG_DATABASE)
command.upgrade(cfg, "head")
+ with owner_connect() as conn, conn.cursor() as cur:
+ for schema, create in ((settings.UPLOADS_SCHEMA, True), (settings.CONTROL_SCHEMA, False),
+ (settings.TE_SCHEMA, False)):
+ grant_runtime(cur, schema, create=create)
_migrate_existing_tenants()
logger.info("Alembic migrations complete")
+ if not settings.APP_DB_USER:
+ logger.warning("APP_DB_USER is not set: the API connects as the owner %r, so the database "
+ "doesn't stop it changing the audit log (M3d). See docs/QUICKSTART.md.",
+ settings.PG_USER)
+
+
+@contextmanager
+def owner_connect():
+ """A connection as the owner (PG_USER): for migrations, provisioning and grants, never requests.
+ Closed on exit; the caller commits."""
+ conn = psycopg2.connect(host=settings.PG_HOST, port=settings.PG_PORT, user=settings.PG_USER,
+ password=settings.PG_PASSWORD, dbname=settings.PG_DATABASE)
+ try:
+ yield conn
+ finally:
+ conn.close()
+
+
+# Tables the runtime role may only read and insert (M3, #107). Triggers also stop the owner.
+APPEND_ONLY = ("audit_log", "uploads", "upload_blobs", "source_rows", "alembic_version")
+
+
+def grant_runtime(cur, schema: str, create: bool) -> None:
+ """Give the runtime role (APP_DB_USER) what requests need in one schema (M3d, #107).
+
+ Read and write on ordinary tables; read and insert only on APPEND_ONLY.
+ ``create`` lets it create, and so own and drop, the per-upload csv_* tables;
+ ones the owner created before M3d are handed over. Commits. No-op without
+ APP_DB_USER or if the schema doesn't exist.
+ """
+ role = settings.APP_DB_USER
+ if not role:
+ return
+ cur.execute("SELECT 1 FROM pg_namespace WHERE nspname = %s", (schema,))
+ if cur.fetchone() is None:
+ return
+ s, r = sql.Identifier(schema), sql.Identifier(role)
+ cur.execute(sql.SQL("GRANT USAGE{} ON SCHEMA {} TO {}").format(
+ sql.SQL(", CREATE") if create else sql.SQL(""), s, r))
+ cur.execute(sql.SQL("GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA {} TO {}").format(s, r))
+ cur.execute(sql.SQL("GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA {} TO {}").format(s, r))
+ cur.execute("SELECT relname FROM pg_class WHERE relnamespace = %s::regnamespace AND relname = ANY(%s)",
+ (schema, list(APPEND_ONLY)))
+ for (table,) in cur.fetchall():
+ cur.execute(sql.SQL("REVOKE UPDATE, DELETE, TRUNCATE ON {}.{} FROM {}").format(
+ s, sql.Identifier(table), r))
+ cur.execute("SELECT to_regclass(%s)", (f'"{schema}".csv_files',))
+ if create and cur.fetchone()[0] is not None:
+ cur.execute(sql.SQL(
+ "SELECT c.relname FROM {}.csv_files f JOIN pg_class c ON c.relname = f.table_name "
+ "AND c.relnamespace = %s::regnamespace WHERE f.mode = 'dynamic' "
+ "AND pg_get_userbyid(c.relowner) <> %s").format(s), (schema, role))
+ for (table,) in cur.fetchall():
+ cur.execute(sql.SQL("ALTER TABLE {}.{} OWNER TO {}").format(s, sql.Identifier(table), r))
+ cur.connection.commit()
def alembic_ini_path() -> Path:
@@ -188,7 +248,7 @@ def _migrate_existing_tenants() -> None:
"""Bring every tenant schema to head too (decision 6: migrations run across N schemas)."""
from api.tenancy import migrate_tenant
- with Conn() as conn, conn.cursor() as cur:
+ with owner_connect() as conn, conn.cursor() as cur:
cur.execute(sql.SQL("SELECT slug FROM {}.tenants ORDER BY id").format(
sql.Identifier(settings.CONTROL_SCHEMA)))
slugs = [r[0] for r in cur.fetchall()]
diff --git a/api/tenancy.py b/api/tenancy.py
index e08f1f2..e4a5163 100644
--- a/api/tenancy.py
+++ b/api/tenancy.py
@@ -66,12 +66,13 @@ def migrate_tenant(slug: str) -> None:
command.upgrade(cfg, "head")
finally:
settings.UPLOADS_SCHEMA = original
- from api.db import Conn
+ from api.db import grant_runtime, owner_connect
- with Conn() as conn:
+ with owner_connect() as conn:
with conn.cursor() as cur:
provision_te_tables(cur, schema_for(slug), settings.TE_SCHEMA)
- conn.commit()
+ conn.commit()
+ grant_runtime(cur, schema_for(slug), create=True)
def provision_te_tables(cur, schema: str, template: str) -> int:
diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md
index 91df9b4..6a3240e 100644
--- a/docs/QUICKSTART.md
+++ b/docs/QUICKSTART.md
@@ -118,6 +118,25 @@ It prompts for a password of at least 12 characters, or reads it from
`TE_ADMIN_PASSWORD`. Sign in with tenant `dev`, username `admin` and that
password.
+### Set up the runtime database role (once, M3d)
+
+The API answers requests as a separate database role, `te_app`. It can only read
+and add to the audit log and the stored uploads, never change or delete them
+(#107). The owner (`PGUSER`, usually `postgres`) is used only at startup. Stop the
+API first, then run this once from the same folder:
+
+```powershell
+cd "C:\Users\User\OneDrive\Desktop\Migration using ai\PostgreDataMigrationAppWithCSVLoader"
+python scripts\create_app_role.py
+```
+
+It connects as the owner with the `PG*` values from step 3, so run it in a
+window where they're set (`PGPORT` 5433 and `PGPASSWORD` for PostgreSQL 18).
+It creates the role with a random password and writes the role's settings to
+`%USERPROFILE%\.te-migration\app-db.env`. It never prints the password.
+`start-api.ps1` loads that file. Until you run it, the API prints
+`APP_DB_USER not set` and runs as the owner. Running it again sets a new password.
+
### Change or reset a password (#117)
- **Your own password:** sign in, then click the key icon in the header.
diff --git a/docs/VCRM.md b/docs/VCRM.md
index ef5f876..d3f8a84 100644
--- a/docs/VCRM.md
+++ b/docs/VCRM.md
@@ -143,7 +143,7 @@ Status as assessed on 2026-10-26 at `main` 032b522. None of these is "Verified"
| ID | Status | Reason | Planned verification | Milestone |
|----|--------|--------|----------------------|-----------|
| BR-23 | ✅ | Every upload attempt gets an `uploads` row (M3b, #107): id, original filename, SHA-256, uploader, timestamp and outcome (imported, partly_imported, failed, duplicate, rejected). The tenant is the schema the row lives in (decision 6), not a column. Registry UNIQUE hash and name from M1 (#87). `test_preserved_uploads.py` | Integration + DB constraint tests | M1, M3 |
-| BR-24 | ⚠️ Partial | Original bytes (`upload_blobs`) and parsed rows (`source_rows`) kept for every upload, including failed ones and files later overwritten or deleted (M3b, #107). Triggers reject UPDATE, DELETE and TRUNCATE (`test_preserved_uploads.py`, 6 tests). Still to do: the least-privilege runtime role (M3d); edits kept separate arrive with M4 | DB negative tests | M3 |
+| BR-24 | ⚠️ Partial | Original bytes (`upload_blobs`) and parsed rows (`source_rows`) kept for every upload, including failed ones and files later overwritten or deleted (M3b, #107). Triggers reject UPDATE, DELETE and TRUNCATE (`test_preserved_uploads.py`, 7 tests), and the API's runtime role has no UPDATE, DELETE or TRUNCATE grant on them (M3d, `test_runtime_role.py`). Still to do: show edits kept separate once they exist (M4) | DB negative tests | M3 |
| BR-25 | ⚠️ Partial | 10-row preview sample; 200-row cap on table view | API paging + e2e | M4 |
| BR-26 | ❌ Not started | No code | API + e2e | M4 |
| BR-27 | ❌ Not started | No code | API + DB tests | M4 |
@@ -154,7 +154,7 @@ Status as assessed on 2026-10-26 at `main` 032b522. None of these is "Verified"
| BR-32 | ❌ Not started | No code | API + e2e | M6 |
| BR-33 | ⚠️ Partial | Single-transaction load; no staging or lineage | Integration incl. failure injection | M6 |
| BR-34 | ⚠️ Partial | Total, inserted, duplicate and failed counts persisted per import (M1: #90, `tests/test_import_counts.py`); an upload where every row fails is reported as failed and saves nothing, and partial failures are shown on the files list (#123, `tests/test_failed_uploads.py`); over-wide rows reported, not dropped (M1: #88, `tests/test_ragged_rows.py`); excluded and edited rows not yet tracked (M6) | Integration | M1, M6 |
-| BR-35 | ⚠️ Partial | Export formula injection neutralised (M1: #86, `export.test.ts`). Audit log append-only (M3a, #107): triggers reject UPDATE, DELETE and TRUNCATE for every role; actor recorded; hash chain checked by `GET /api/audit/verify` (`test_audit_append_only.py`, 7 tests). Still to do: the least-privilege runtime role (M3d) and report filters (M7) | DB negative tests + unit | M1, M3, M7 |
+| BR-35 | ⚠️ Partial | Export formula injection neutralised (M1: #86, `export.test.ts`). Audit log append-only (M3a, #107): triggers reject UPDATE, DELETE and TRUNCATE for every role; actor recorded; hash chain checked by `GET /api/audit/verify` (`test_audit_append_only.py`, 7 tests). The API's runtime role can only read and insert it, and can't disable its triggers (M3d, `test_runtime_role.py`). Still to do: report filters, including stored failure reasons (M7) | DB negative tests + unit | M1, M3, M7 |
| BR-36 | ⚠️ Partial | Done in M2:
• local accounts with lockout (#98, #103)
• OIDC (#105)
• per-tenant schemas, T&E tables included (#100, #104)
• role matrix in `docs/ROLES.md` (#101)
Tests: `tests/test_accounts.py`, `test_tenant_isolation.py`, `test_roles.py`, `test_oidc.py`, `test_password_reset.py`. Each failed with its control removed.
• frontend sign-in; the browser no longer holds the shared key (#115)
• password change, admin reset and break-glass reset (#117; `tests/test_password_reset.py`)
Open: the shared key still grants non-tenant access to scripts and services; security review (#112); real-IdP OIDC check (#113) | Cross-tenant negative tests; role matrix tests | M2 |
| BR-37 | ⚠️ Partial | No external calls found; no automated check; no offline doc (#60) | Egress-blocked run + offline dry run | M7 |
| BR-38 | ❌ Not started | No container bundle for the application yet (`infra/Dockerfile` is the schema and evals runner only) | Offline smoke test 7/7 on Windows, macOS and Linux | #128 |
diff --git a/docs/offline-install.md b/docs/offline-install.md
index 2c5b97a..231767c 100644
--- a/docs/offline-install.md
+++ b/docs/offline-install.md
@@ -66,8 +66,8 @@ The script runs six steps:
1. Checks every file against `SHA256SUMS`.
2. Installs the Ubuntu packages, using the bundle as the only apt source. The VM's own apt sources are ignored.
3. Copies the app to `/opt/te-migration`, creates the `te-app` system user, and builds a Python virtual environment from `wheels/` without network access.
-4. Deploys database `te_mgmt_prod` with schema `te_prod` and its login role `te_prod_user`, using a generated password.
-5. Writes `/opt/te-migration/env` (mode 600) with that password and a generated API key, and starts the `te-api` (port 8000) and `te-frontend` (port 3000) services under systemd. The API key is for scripts only. The frontend holds no key; people sign in.
+4. Deploys database `te_mgmt_prod` with schema `te_prod` and its owner role `te_prod_user`. It also creates the runtime role `te_prod_runtime`, which serves requests and can only read and insert the audit and upload tables (M3d, #107). Both get generated passwords.
+5. Writes `/opt/te-migration/env` (mode 600) with both passwords and a generated API key, and starts the `te-api` (port 8000) and `te-frontend` (port 3000) services under systemd. The API key is for scripts only. The frontend holds no key; people sign in.
6. Creates the first tenant and its tenant admin, then runs the smoke test.
During step 2, apt may print `W: Download is performed unsandboxed as root`. This only means apt's sandbox user can't read the bundle folder, and it's harmless.
diff --git a/scripts/create_app_role.py b/scripts/create_app_role.py
new file mode 100644
index 0000000..b69c8e2
--- /dev/null
+++ b/scripts/create_app_role.py
@@ -0,0 +1,71 @@
+"""Create or update the API's runtime database role (M3d, #107).
+
+Run once from the repo root, with the owner's PG* environment set (the same
+settings start-api.ps1 uses):
+ python scripts/create_app_role.py
+
+What it does:
+- creates the login role (default te_app) with a new random password, or gives
+ an existing one a new password;
+- lets the owner hand tables to it (GRANT te_app TO );
+- brings every schema to the current migration and applies the grants: read and
+ write on ordinary tables, read and insert only on the audit and upload tables;
+- writes APP_DB_USER and APP_DB_PASSWORD to an env file only you can read
+ (default ~/.te-migration/app-db.env), which start-api.ps1 loads.
+
+The password is never printed. Set TE_APP_DB_PASSWORD to choose it yourself.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import secrets
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from psycopg2 import sql # noqa: E402
+
+from api import db # noqa: E402
+from api.config import settings # noqa: E402
+
+DEFAULT_ENV_FILE = Path.home() / ".te-migration" / "app-db.env"
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ ap.add_argument("--role", default="te_app")
+ ap.add_argument("--env-file", type=Path, default=DEFAULT_ENV_FILE)
+ args = ap.parse_args()
+ if args.role == settings.PG_USER:
+ print("The runtime role must differ from the owner (PGUSER).", file=sys.stderr)
+ return 2
+ password = os.environ.get("TE_APP_DB_PASSWORD") or secrets.token_urlsafe(32)
+
+ with db.owner_connect() as conn, conn.cursor() as cur:
+ role, pw = sql.Identifier(args.role), sql.Literal(password)
+ cur.execute("SELECT 1 FROM pg_roles WHERE rolname = %s", (args.role,))
+ verb = "ALTER" if cur.fetchone() else "CREATE"
+ cur.execute(sql.SQL(verb + " ROLE {} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE PASSWORD {}")
+ .format(role, pw))
+ cur.execute(sql.SQL("GRANT {} TO {}").format(role, sql.Identifier(settings.PG_USER)))
+ cur.execute(sql.SQL("GRANT CONNECT ON DATABASE {} TO {}").format(
+ sql.Identifier(settings.PG_DATABASE), role))
+ conn.commit()
+
+ settings.APP_DB_USER, settings.APP_DB_PASSWORD = args.role, password
+ db.bootstrap() # migrations, then grants on every schema, tenants included
+
+ args.env_file.parent.mkdir(parents=True, exist_ok=True)
+ fd = os.open(args.env_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
+ f.write(f"APP_DB_USER={args.role}\nAPP_DB_PASSWORD={password}\n")
+ print(f"Role '{args.role}' is ready ({'created' if verb == 'CREATE' else 'new password set'}). Its settings are in {args.env_file}; "
+ "start-api.ps1 loads them. Restart the API.")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/offline/install.sh b/scripts/offline/install.sh
index a0beed5..1393908 100755
--- a/scripts/offline/install.sh
+++ b/scripts/offline/install.sh
@@ -43,6 +43,14 @@ sed "s/__INJECT_AT_DEPLOY__/$DB_PASS/" "$DEST/app/build/environments/env_prod.ex
> "$DEST/app/build/environments/env_prod.sql"
( cd "$DEST/app" && sudo -u postgres env PGHOST=/var/run/postgresql bash build/deploy_all.sh prod >/dev/null )
rm "$DEST/app/build/environments/env_prod.sql" # it holds the password
+# M3d (#107): a separate runtime role for requests; it can only read and insert the
+# audit and upload tables. te_prod_user stays the owner and runs migrations at startup.
+APP_DB_PASS="$(openssl rand -hex 24)"
+sudo -u postgres psql -q -v ON_ERROR_STOP=1 -d te_mgmt_prod -c "SET client_min_messages = warning" \
+ -c "DO \$\$BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'te_prod_runtime') THEN CREATE ROLE te_prod_runtime LOGIN; END IF; END\$\$" \
+ -c "ALTER ROLE te_prod_runtime LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE PASSWORD '$APP_DB_PASS'" \
+ -c "GRANT te_prod_runtime TO te_prod_user" \
+ -c "GRANT CONNECT ON DATABASE te_mgmt_prod TO te_prod_runtime" >/dev/null
echo "== 5/6 configuration and services"
cat > "$DEST/env" <