Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 8 additions & 13 deletions .github/workflows/quality-gate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,8 @@ jobs:
PGPORT: "5432"
PGUSER: postgres
PGPASSWORD: postgres
PGDATABASE: postgres
PGDATABASE: te_mgmt_dev
API_ALLOW_DESTRUCTIVE: "true"
steps:
- name: Checkout
uses: actions/checkout@v7
Expand Down Expand Up @@ -142,27 +143,21 @@ jobs:
fi
done

# Tier E deploys all four environments and compares their schemas, so it
# runs here where the postgres service exists. Order matters: E deploys
# every environment, which the cross-env parity pytest markers then rely on.
# Tier X is excluded per ADR-0001 — see the free-tier job's note.
- name: Evals — Tiers P, I, S, E
run: python3 evals/runner.py --tiers p,i,s,e --verbose

- name: Deploy all environments (for cross-env parity)
- name: Deploy all environments
run: |
for env in dev test staging prod; do
bash build/deploy_all.sh "$env"
done

# Tier E compares the four deployed schemas column by column.
# Tier X is excluded per ADR-0001 — see the free-tier job's note.
- name: Evals — Tiers P, I, S, E
run: python3 evals/runner.py --tiers p,i,s,e --verbose

# Prints a final block accounting for every test: PASSED / FAILED /
# ERROR / SKIPPED (with reasons) / NOT RUN (deselected). --strict fails
# the build on any skip.
# The API reads PGDATABASE (api/config.py). The schema lives in
# te_mgmt_dev, not the job-level default "postgres" (issue #45).
- name: Full test suite — final result with skip accounting
env:
PGDATABASE: te_mgmt_dev
run: python3 scripts/test_report.py --strict

- name: Upload eval reports
Expand Down
7 changes: 7 additions & 0 deletions evals/datasets/tier_e/02_empty_schema_is_detected/NOTES.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Negative control — no input data required.

This scenario creates a temporary empty schema (_eval_empty_probe) in the
dev database, verifies that _column_signature returns zero tables, and drops
the schema. It exists to prevent structural comparison from silently passing
when a schema is missing or empty — the kind of false green that masked the
original CI failure (issue #45 follow-up).
7 changes: 7 additions & 0 deletions evals/expected/tier_e/02_empty_schema_is_detected.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"scenario": "02_empty_schema_is_detected",
"description": "Negative control: an empty schema must return 0 tables from _column_signature, proving that structural comparison is not silently succeeding on missing data.",
"expected": {
"table_count": 0
}
}
42 changes: 42 additions & 0 deletions evals/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -676,6 +676,8 @@ def run_tier_e_scenario(scenario_dir: Path) -> ScenarioResult:

if name == "01_envs_have_identical_structure":
return _run_envs_have_identical_structure(result, expected)
if name == "02_empty_schema_is_detected":
return _run_empty_schema_is_detected(result, expected)

result.errors.append("Unknown tier-E scenario: " + name)
return result
Expand Down Expand Up @@ -748,6 +750,46 @@ def _run_envs_have_identical_structure(
return result


def _run_empty_schema_is_detected(
result: ScenarioResult, expected: Dict[str, Any]
) -> ScenarioResult:
"""Negative control: _column_signature on a schema with no tables returns
an empty dict, and the eval correctly flags it as having 0 tables.

Creates a disposable schema in the dev database, queries it, asserts the
expected zero-table result, and drops the schema unconditionally.
"""
db = "te_mgmt_dev"
tmp_schema = "_eval_empty_probe"
env = _pg_env()
try:
subprocess.run(
["psql", "-v", "ON_ERROR_STOP=1", "-d", db, "-c",
"CREATE SCHEMA IF NOT EXISTS " + tmp_schema],
env=env, capture_output=True, text=True, timeout=10,
)
sig = _column_signature(db, tmp_schema)
table_count = len(sig)
result.actual = {"table_count": table_count, "signature": sig}
exp = expected.get("expected", {})
if table_count != exp.get("table_count", 0):
result.errors.append(
"expected " + str(exp.get("table_count", 0))
+ " tables but _column_signature returned "
+ str(table_count)
)
except Exception as exc:
result.errors.append("empty-schema probe failed: " + str(exc))
finally:
subprocess.run(
["psql", "-d", db, "-c",
"DROP SCHEMA IF EXISTS " + tmp_schema + " CASCADE"],
env=env, capture_output=True, text=True, timeout=10,
)
result.passed = not result.errors
return result


# ---------------------------------------------------------------------------
# Tier X — Cross-engine structural coverage (VCRM BR-02)
#
Expand Down
Loading