diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 821ae9a..4499cbb 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -81,7 +81,8 @@ jobs: PGPORT: "5432" PGUSER: postgres PGPASSWORD: postgres - PGDATABASE: postgres + PGDATABASE: te_mgmt_dev + API_ALLOW_DESTRUCTIVE: "true" steps: - name: Checkout uses: actions/checkout@v7 @@ -142,27 +143,21 @@ jobs: fi done - # Tier E deploys all four environments and compares their schemas, so it - # runs here where the postgres service exists. Order matters: E deploys - # every environment, which the cross-env parity pytest markers then rely on. - # Tier X is excluded per ADR-0001 — see the free-tier job's note. - - name: Evals — Tiers P, I, S, E - run: python3 evals/runner.py --tiers p,i,s,e --verbose - - - name: Deploy all environments (for cross-env parity) + - name: Deploy all environments run: | for env in dev test staging prod; do bash build/deploy_all.sh "$env" done + # Tier E compares the four deployed schemas column by column. + # Tier X is excluded per ADR-0001 — see the free-tier job's note. + - name: Evals — Tiers P, I, S, E + run: python3 evals/runner.py --tiers p,i,s,e --verbose + # Prints a final block accounting for every test: PASSED / FAILED / # ERROR / SKIPPED (with reasons) / NOT RUN (deselected). --strict fails # the build on any skip. - # The API reads PGDATABASE (api/config.py). The schema lives in - # te_mgmt_dev, not the job-level default "postgres" (issue #45). - name: Full test suite — final result with skip accounting - env: - PGDATABASE: te_mgmt_dev run: python3 scripts/test_report.py --strict - name: Upload eval reports diff --git a/evals/datasets/tier_e/02_empty_schema_is_detected/NOTES.txt b/evals/datasets/tier_e/02_empty_schema_is_detected/NOTES.txt new file mode 100644 index 0000000..9ca13ba --- /dev/null +++ b/evals/datasets/tier_e/02_empty_schema_is_detected/NOTES.txt @@ -0,0 +1,7 @@ +Negative control — no input data required. + +This scenario creates a temporary empty schema (_eval_empty_probe) in the +dev database, verifies that _column_signature returns zero tables, and drops +the schema. It exists to prevent structural comparison from silently passing +when a schema is missing or empty — the kind of false green that masked the +original CI failure (issue #45 follow-up). diff --git a/evals/expected/tier_e/02_empty_schema_is_detected.json b/evals/expected/tier_e/02_empty_schema_is_detected.json new file mode 100644 index 0000000..56af256 --- /dev/null +++ b/evals/expected/tier_e/02_empty_schema_is_detected.json @@ -0,0 +1,7 @@ +{ + "scenario": "02_empty_schema_is_detected", + "description": "Negative control: an empty schema must return 0 tables from _column_signature, proving that structural comparison is not silently succeeding on missing data.", + "expected": { + "table_count": 0 + } +} diff --git a/evals/runner.py b/evals/runner.py index 5a9c0a0..ffed487 100644 --- a/evals/runner.py +++ b/evals/runner.py @@ -676,6 +676,8 @@ def run_tier_e_scenario(scenario_dir: Path) -> ScenarioResult: if name == "01_envs_have_identical_structure": return _run_envs_have_identical_structure(result, expected) + if name == "02_empty_schema_is_detected": + return _run_empty_schema_is_detected(result, expected) result.errors.append("Unknown tier-E scenario: " + name) return result @@ -748,6 +750,46 @@ def _run_envs_have_identical_structure( return result +def _run_empty_schema_is_detected( + result: ScenarioResult, expected: Dict[str, Any] +) -> ScenarioResult: + """Negative control: _column_signature on a schema with no tables returns + an empty dict, and the eval correctly flags it as having 0 tables. + + Creates a disposable schema in the dev database, queries it, asserts the + expected zero-table result, and drops the schema unconditionally. + """ + db = "te_mgmt_dev" + tmp_schema = "_eval_empty_probe" + env = _pg_env() + try: + subprocess.run( + ["psql", "-v", "ON_ERROR_STOP=1", "-d", db, "-c", + "CREATE SCHEMA IF NOT EXISTS " + tmp_schema], + env=env, capture_output=True, text=True, timeout=10, + ) + sig = _column_signature(db, tmp_schema) + table_count = len(sig) + result.actual = {"table_count": table_count, "signature": sig} + exp = expected.get("expected", {}) + if table_count != exp.get("table_count", 0): + result.errors.append( + "expected " + str(exp.get("table_count", 0)) + + " tables but _column_signature returned " + + str(table_count) + ) + except Exception as exc: + result.errors.append("empty-schema probe failed: " + str(exc)) + finally: + subprocess.run( + ["psql", "-d", db, "-c", + "DROP SCHEMA IF EXISTS " + tmp_schema + " CASCADE"], + env=env, capture_output=True, text=True, timeout=10, + ) + result.passed = not result.errors + return result + + # --------------------------------------------------------------------------- # Tier X — Cross-engine structural coverage (VCRM BR-02) #