diff --git a/.github/workflows/python-quality.yml b/.github/workflows/python-quality.yml index 41506df..0e446d7 100644 --- a/.github/workflows/python-quality.yml +++ b/.github/workflows/python-quality.yml @@ -14,6 +14,8 @@ on: - "data_quality/**" - "docs/**" - "examples/**" + - "notebooks/**" + - "reporting/**" - "tests/**" push: branches: @@ -28,6 +30,8 @@ on: - "data_quality/**" - "docs/**" - "examples/**" + - "notebooks/**" + - "reporting/**" - "tests/**" workflow_dispatch: @@ -42,7 +46,7 @@ jobs: quality: name: Python 3.12 · ${{ matrix.os }} runs-on: ${{ matrix.os }} - timeout-minutes: 15 + timeout-minutes: 20 strategy: fail-fast: false @@ -78,14 +82,14 @@ jobs: python -m pip install -e ".[dev,ml,notebook]" - name: Compile Python sources - run: python -m compileall -q main.py data_quality examples tests + run: python -m compileall -q main.py data_quality reporting examples tests - name: Run Ruff lint checks id: ruff-lint continue-on-error: true shell: pwsh run: | - $output = & python -m ruff check main.py data_quality examples tests 2>&1 + $output = & python -m ruff check main.py data_quality reporting examples tests 2>&1 $exitCode = $LASTEXITCODE $output | Tee-Object -FilePath ruff-lint.txt exit $exitCode @@ -95,7 +99,7 @@ jobs: continue-on-error: true shell: pwsh run: | - $output = & python -m ruff format --check main.py data_quality examples tests 2>&1 + $output = & python -m ruff format --check main.py data_quality reporting examples tests 2>&1 $exitCode = $LASTEXITCODE $output | Tee-Object -FilePath ruff-format.txt exit $exitCode @@ -130,20 +134,64 @@ jobs: throw "Unexpected exact duplicate count." } + - name: Run reporting workflow + run: >- + python -m reporting + --input .ci-output/data-quality + --output .ci-output/reporting + + - name: Verify reporting control totals + shell: pwsh + run: | + $report = Get-Content ".ci-output/reporting/reporting_summary.json" -Raw | ConvertFrom-Json + if ($report.kpi_reconciliation -ne "passed") { throw "KPI reconciliation failed." } + if ($report.module_count -ne 4) { throw "Unexpected module count." } + if ($report.result_count -ne 8) { throw "Unexpected reporting result count." } + if ($report.rejected_row_count -ne 7) { throw "Unexpected rejected row count." } + if ($report.overall_average_score_percentage -ne 70.0) { + throw "Unexpected overall average score." + } + if ($report.overall_pass_rate_percentage -ne 62.5) { + throw "Unexpected overall pass rate." + } + if ($report.rejection_reason_count -ne 7) { + throw "Unexpected rejection reason count." + } + if (-not (Test-Path ".ci-output/reporting/average_score_by_module.svg")) { + throw "Average-score chart is missing." + } + if (-not (Test-Path ".ci-output/reporting/pass_rate_by_module.svg")) { + throw "Pass-rate chart is missing." + } + - name: Run optional ML example run: python examples/optional/logistic_regression_basics.py - - name: Execute clean notebook copy + - name: Execute environment notebook run: | python -c "from pathlib import Path; Path('.ci-output').mkdir(exist_ok=True)" jupyter nbconvert --to notebook --execute dataspell_test.ipynb --output environment-check.executed.ipynb --output-dir .ci-output --ExecutePreprocessor.timeout=120 - - name: Upload data-quality outputs + - name: Execute reporting verification notebook + run: >- + jupyter nbconvert + --to notebook + --execute notebooks/reporting_verification.ipynb + --output reporting-verification.executed.ipynb + --output-dir .ci-output + --ExecutePreprocessor.timeout=120 + + - name: Upload verified workflow outputs if: always() uses: actions/upload-artifact@v7 with: - name: data-quality-output-${{ matrix.os }} - path: .ci-output/data-quality + name: verified-reporting-${{ matrix.os }} + path: | + .ci-output/data-quality + .ci-output/reporting + .ci-output/reporting-notebook + .ci-output/environment-check.executed.ipynb + .ci-output/reporting-verification.executed.ipynb if-no-files-found: ignore retention-days: 3 diff --git a/README.md b/README.md index c4bc633..d82d7b4 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ [![Python quality](https://github.com/DataTideHH/python-data-basics/actions/workflows/python-quality.yml/badge.svg)](https://github.com/DataTideHH/python-data-basics/actions/workflows/python-quality.yml) -**Python 3.12 · pandas · data quality · pytest · Ruff · Jupyter · GitHub Actions** +**Python 3.12 · pandas · data quality · reporting · matplotlib · pytest · Ruff · Jupyter · GitHub Actions** -This repository is a compact, tested foundation for reproducible Python Data/BI workflows. It combines environment setup, deterministic pandas transformations, a small auditable data-quality workflow, notebook hygiene and cross-platform CI. +This repository is a compact, tested foundation for reproducible Python Data/BI workflows. It combines environment setup, deterministic pandas transformations, an auditable data-quality workflow, reconciled reporting outputs, clean notebooks and cross-platform CI. -It is part of my DataTideHH portfolio during the IHK retraining program in Data and Process Analysis. The scope remains deliberately bounded: this is a reusable learning baseline, not a production ETL platform, predictive-model showcase or finished business analysis. +It is part of my DataTideHH portfolio during the IHK retraining program in Data and Process Analysis. The scope remains deliberately bounded: this is a reusable learning and verification baseline, not a production ETL platform, predictive-model showcase or enterprise reporting solution. --- @@ -16,35 +16,46 @@ It is part of my DataTideHH portfolio during the IHK retraining program in Data |---|---| | Python baseline | Python 3.12-compatible environment and deterministic pandas sanity check | | Dependency model | Runtime, notebook, optional ML and development groups in `pyproject.toml` | -| Data-quality workflow | CSV input, schema checks, type conversion, row-level rejection, cleaning, KPIs and export | -| Testing | pytest coverage for baseline logic, data-quality rules, notebook hygiene and optional ML | +| Data-quality workflow | CSV input, schema checks, controlled conversion, rejection reasons, cleaning, KPIs and export | +| Reporting workflow | KPI reconciliation, rejection-reason summary, control totals and Matplotlib SVG charts | +| Notebook verification | Clean reporting notebook with tables, charts and explicit assertions | +| Testing | pytest coverage for baseline logic, quality rules, reporting reconciliation and notebook hygiene | | Code quality | Ruff linting and formatting plus Python bytecode compilation | -| Continuous integration | Ubuntu 24.04 and Windows 2025 matrix with Python 3.12 | -| Credential safety | Synthetic/public-safe inputs; local environments and secrets excluded | +| Continuous integration | Complete Ubuntu 24.04 and Windows 2025 matrix with Python 3.12 | +| Credential safety | Synthetic/public-safe inputs; local environments, secrets and generated outputs excluded | -## What This Repository Demonstrates +## End-to-End Flow -The repository focuses on small tasks that recur in Data/BI work: - -- create an isolated Python environment -- define direct dependencies separately from development tooling -- read raw CSV data as text before controlled conversion -- enforce required columns and explicit value rules -- preserve rejected records with reason codes -- normalise identifiers and text fields -- derive analysis-ready columns -- aggregate deterministic module KPIs -- export cleaned data, rejected rows, KPIs and a JSON quality report -- test positive and negative data-quality cases -- run the same checks on Windows and Linux +```text +data/raw/training_results.csv + │ + ▼ +data-quality validation and cleaning + │ + ├── cleaned_results.csv + ├── rejected_results.csv + ├── module_kpis.csv + └── quality_report.json + │ + ▼ +reporting reconciliation and presentation + │ + ├── rejection_reason_summary.csv + ├── reporting_summary.json + ├── average_score_by_module.svg + └── pass_rate_by_module.svg + │ + ▼ +clean reporting notebook with repeated control assertions +``` -More complete analyses remain in separate repositories. This project provides tested building blocks underneath them. +The reporting layer does not trust a persisted KPI file blindly. It recalculates module KPIs from the cleaned records and fails when values differ. --- ## Quick Start -Detailed setup instructions are in [`docs/setup.md`](docs/setup.md). +Detailed platform instructions are in [`docs/setup.md`](docs/setup.md). ### Windows PowerShell @@ -72,15 +83,13 @@ On the Intel iMac used for local portfolio work, Python 3.12 is available at `/u ## Data-Quality Workflow -The main portfolio increment in this repository is the workflow in [`data_quality/`](data_quality/). - -It processes the synthetic raw file: +The tested workflow in [`data_quality/`](data_quality/) processes: ```text data/raw/training_results.csv ``` -Run it from the repository root: +### Run on Windows PowerShell ```powershell python -m data_quality ` @@ -88,7 +97,7 @@ python -m data_quality ` --output ".ci-output/data-quality" ``` -Equivalent macOS/Linux command: +### Run on macOS or Linux ```bash python -m data_quality \ @@ -96,7 +105,7 @@ python -m data_quality \ --output .ci-output/data-quality ``` -The workflow writes: +### Generated files ```text .ci-output/data-quality/ @@ -106,72 +115,128 @@ The workflow writes: └── quality_report.json ``` -### Validation rules - -Required fields: - -```text -result_id -learner_id -module -assessment_date -score -max_score -pass_score -``` - -Implemented checks include: +### Implemented validation rules - required-column validation - missing-value detection - strict ISO date parsing -- numeric conversion +- controlled numeric conversion - positive `max_score` - non-negative score and pass threshold - score not above maximum - pass threshold not above maximum - exact duplicate removal - conflicting duplicate rejection -- whitespace and identifier normalisation - -Invalid rows are not silently dropped. They are exported with explicit pipe-separated rejection reasons. +- identifier and whitespace normalisation -### Derived fields +Invalid rows are not silently discarded. They remain auditable in `rejected_results.csv` with explicit pipe-separated reason codes. Accepted records receive: -- `source_row` for lineage back to the raw CSV +- `source_row` for lineage to the raw CSV - `score_percentage` -- Boolean `passed` +- non-null Boolean `passed` + +The committed fixture contains 15 raw rows and produces: + +```text +accepted rows: 8 +rejected rows: 7 +exact duplicate rows removed: 1 +``` -### KPI output +Detailed rules and boundaries are documented in [`docs/data-quality-workflow.md`](docs/data-quality-workflow.md). -The workflow aggregates one row per module with: +--- -- result count -- distinct learner count -- average score percentage -- passed count -- failed count -- pass-rate percentage +## Reporting Workflow -The committed fixture contains 15 raw rows. The expected control totals are: +Run reporting after the data-quality output exists. + +### Windows PowerShell + +```powershell +python -m reporting ` + --input ".ci-output/data-quality" ` + --output ".ci-output/reporting" +``` + +### macOS and Linux + +```bash +python -m reporting \ + --input .ci-output/data-quality \ + --output .ci-output/reporting +``` + +### Generated files ```text -accepted rows: 8 -rejected rows: 7 -exact duplicate rows removed: 1 +.ci-output/reporting/ +├── average_score_by_module.svg +├── pass_rate_by_module.svg +├── rejection_reason_summary.csv +└── reporting_summary.json ``` -Detailed rules, expected module values and scope boundaries are documented in [`docs/data-quality-workflow.md`](docs/data-quality-workflow.md). +### Verified control totals + +| Control | Expected value | +|---|---:| +| Modules | 4 | +| Accepted results | 8 | +| Rejected rows | 7 | +| Overall average score | 70.00% | +| Overall pass rate | 62.50% | +| Distinct rejection reasons | 7 | +| KPI reconciliation | passed | + +The reporting workflow stops with a non-zero exit code when an expected input file is missing, a required column is absent or persisted module KPIs differ from the fresh calculation. + +Detailed behaviour is documented in [`docs/reporting-notebook.md`](docs/reporting-notebook.md). + +--- + +## Verified Reference Charts + +The repository includes compact SVG snapshots for the committed synthetic fixture. The CI workflow generates fresh Matplotlib SVG files from the current outputs on every run. + +### Average score + +![Average score by module](docs/assets/average-score-by-module.svg) + +### Pass rate + +![Pass rate by module](docs/assets/pass-rate-by-module.svg) + +These charts are descriptive controls for a small synthetic dataset, not statistical evidence about real learners or business operations. + +--- + +## Reporting Notebook + +[`notebooks/reporting_verification.ipynb`](notebooks/reporting_verification.ipynb) reads the generated data-quality outputs and performs the same reporting checks through the reusable package. + +It contains: + +1. verified module KPI table +2. rejection-reason summary +3. average-score chart +4. pass-rate chart +5. explicit assertions for the expected control totals +6. final `Reporting notebook verification passed.` marker + +The committed notebook contains no outputs, execution counts, local paths or IDE timestamps. GitHub Actions executes a temporary copy only after the data-quality and reporting command-line workflows succeed. + +[`dataspell_test.ipynb`](dataspell_test.ipynb) remains the smaller environment and import check. --- ## Baseline Entry Point -[`main.py`](main.py) remains a separate deterministic environment and pandas sanity check. It reports the active interpreter and direct runtime package versions, validates a tiny DataFrame and calculates a stable category summary. +[`main.py`](main.py) is a separate deterministic environment and pandas sanity check. It reports the active interpreter and direct runtime package versions, validates a tiny DataFrame and calculates a stable category summary. -This keeps environment verification separate from the larger row-level data-quality workflow. +This separates environment verification from the row-level data-quality and reporting workflows. --- @@ -185,32 +250,25 @@ This keeps environment verification separate from the larger row-level data-qual --- -## Notebook Hygiene - -[`dataspell_test.ipynb`](dataspell_test.ipynb) verifies core package imports without committing: - -- cell outputs -- execution counts -- IDE execution timestamps -- absolute local paths -- incorrect legacy Python metadata - -GitHub Actions executes a temporary copy into `.ci-output/` and leaves the committed notebook unchanged. - ---- - ## Dependency Model `pyproject.toml` is the source of truth. | Installation | Included scope | |---|---| -| `python -m pip install -e .` | Runtime baseline and data-quality package | +| `python -m pip install -e .` | Runtime baseline, data quality and reporting | | `python -m pip install -e ".[notebook]"` | Runtime plus Jupyter | | `python -m pip install -e ".[ml]"` | Runtime plus optional scikit-learn example | | `python -m pip install -e ".[dev]"` | Runtime plus pytest and Ruff | | `python -m pip install -r requirements-dev.txt` | Complete CI-equivalent environment | +Installed command-line entry points: + +```text +python-data-quality +python-data-reporting +``` + The requirement files remain small wrappers rather than machine-specific freezes of every transitive package. --- @@ -218,15 +276,23 @@ The requirement files remain small wrappers rather than machine-specific freezes ## Local Quality Checks ```bash -python -m compileall -q main.py data_quality examples tests -python -m ruff check main.py data_quality examples tests -python -m ruff format --check main.py data_quality examples tests +python -m compileall -q main.py data_quality reporting examples tests +python -m ruff check main.py data_quality reporting examples tests +python -m ruff format --check main.py data_quality reporting examples tests python -m pytest python main.py python -m data_quality --input data/raw/training_results.csv --output .ci-output/data-quality +python -m reporting --input .ci-output/data-quality --output .ci-output/reporting python examples/optional/logistic_regression_basics.py ``` +Execute the notebooks separately: + +```bash +jupyter nbconvert --to notebook --execute dataspell_test.ipynb --output environment-check.executed.ipynb --output-dir .ci-output --ExecutePreprocessor.timeout=120 +jupyter nbconvert --to notebook --execute notebooks/reporting_verification.ipynb --output reporting-verification.executed.ipynb --output-dir .ci-output --ExecutePreprocessor.timeout=120 +``` + --- ## Continuous Integration @@ -242,13 +308,16 @@ Each matrix job: 1. installs the project and optional quality groups 2. compiles Python sources 3. runs Ruff lint and format checks -4. runs pytest +4. runs the complete pytest suite 5. executes `main.py` -6. executes the complete data-quality workflow -7. validates the expected row counts -8. executes the optional ML example -9. executes a clean notebook copy -10. uploads short-lived generated workflow outputs and Ruff diagnostics +6. executes the full data-quality workflow +7. verifies the 15/8/7 source control totals +8. executes the reporting workflow +9. verifies KPI reconciliation and reporting totals +10. executes the optional ML example +11. executes both clean notebooks +12. uploads short-lived verified data, reporting, chart and notebook artefacts +13. enforces the final quality gate The workflow is quality assurance, not deployment or release automation. @@ -264,17 +333,26 @@ python-data-basics/ │ ├── __init__.py │ ├── __main__.py │ └── workflow.py +├── reporting/ +│ ├── __init__.py +│ ├── __main__.py +│ └── workflow.py +├── notebooks/ +│ └── reporting_verification.ipynb ├── docs/ -│ ├── api-json-oauth2-notes.md +│ ├── assets/ +│ │ ├── average-score-by-module.svg +│ │ └── pass-rate-by-module.svg │ ├── data-quality-workflow.md -│ ├── ollama-local-api-notes.md +│ ├── reporting-notebook.md │ └── setup.md ├── examples/ ├── tests/ │ ├── test_data_quality_workflow.py │ ├── test_main.py │ ├── test_notebook_hygiene.py -│ └── test_optional_ml.py +│ ├── test_optional_ml.py +│ └── test_reporting_workflow.py ├── dataspell_test.ipynb ├── main.py ├── pyproject.toml @@ -289,7 +367,7 @@ python-data-basics/ Only synthetic learning data and public endpoints belong in this repository. The committed training-results file contains no real learners or personal information. -Excluded content includes local environments, `.env` files, API keys, OAuth tokens, credential downloads, personal/customer data, IDE metadata, caches and generated workflow outputs. +Excluded content includes local environments, `.env` files, API keys, OAuth tokens, credential downloads, personal/customer data, IDE metadata, caches, executed notebooks and generated workflow outputs. --- @@ -297,12 +375,13 @@ Excluded content includes local environments, `.env` files, API keys, OAuth toke This repository does not claim: -- a production Python package or ETL platform +- a production Python package, ETL platform or semantic model - streaming or distributed processing -- production orchestration or observability +- production orchestration, observability or publication - regulatory data validation - a validated predictive model -- a complete business analysis or dashboard +- an interactive dashboard or Power BI report +- statistical inference from the synthetic fixture - deployment or cloud infrastructure -The workflow is intentionally small enough to inspect, run, test and explain in an interview or technical review. +The complete workflow remains small enough to inspect, run, test and explain in an interview or technical review. diff --git a/docs/assets/average-score-by-module.svg b/docs/assets/average-score-by-module.svg new file mode 100644 index 0000000..2c3193c --- /dev/null +++ b/docs/assets/average-score-by-module.svg @@ -0,0 +1,40 @@ + + Average score by module + Data Quality 71.50 percent, Process Analysis 55.00 percent, Python Basics 75.00 percent, SQL Basics 77.33 percent. + + Average score by module + + + + 0% + 25% + 50% + 75% + 100% + + + + + + + + + + + + + + + 71.50% + 55.00% + 75.00% + 77.33% + + + Data Quality + Process Analysis + Python Basics + SQL Basics + + Verified synthetic fixture · 8 accepted assessment results + diff --git a/docs/assets/pass-rate-by-module.svg b/docs/assets/pass-rate-by-module.svg new file mode 100644 index 0000000..274c2a1 --- /dev/null +++ b/docs/assets/pass-rate-by-module.svg @@ -0,0 +1,40 @@ + + Pass rate by module + Data Quality 50.00 percent, Process Analysis 50.00 percent, Python Basics 100.00 percent, SQL Basics 66.67 percent. + + Pass rate by module + + + + 0% + 25% + 50% + 75% + 100% + + + + + + + + + + + + + + + 50.00% + 50.00% + 100.00% + 66.67% + + + Data Quality + Process Analysis + Python Basics + SQL Basics + + Verified synthetic fixture · 5 of 8 accepted results passed + diff --git a/docs/reporting-notebook.md b/docs/reporting-notebook.md new file mode 100644 index 0000000..d47dc46 --- /dev/null +++ b/docs/reporting-notebook.md @@ -0,0 +1,156 @@ +# Reporting and Notebook Verification + +## Purpose + +This increment adds a compact reporting layer on top of the tested data-quality workflow. It does not recalculate business results from the raw CSV independently. Instead, it consumes the four generated data-quality outputs, verifies that the persisted module KPIs still match a fresh calculation from the cleaned records, and then produces presentation-ready reporting artefacts. + +The reporting sequence is: + +```text +raw CSV +→ tested data-quality workflow +→ cleaned and rejected records +→ persisted module KPIs +→ independent KPI reconciliation +→ rejection-reason summary +→ charts and notebook verification +``` + +## Required inputs + +Run the data-quality workflow first: + +```bash +python -m data_quality \ + --input data/raw/training_results.csv \ + --output .ci-output/data-quality +``` + +The reporting workflow expects: + +```text +.ci-output/data-quality/ +├── cleaned_results.csv +├── rejected_results.csv +├── module_kpis.csv +└── quality_report.json +``` + +A missing file, missing required column or KPI mismatch stops reporting with a non-zero exit code. + +## Reporting command + +### Windows PowerShell + +```powershell +python -m reporting ` + --input ".ci-output/data-quality" ` + --output ".ci-output/reporting" +``` + +### macOS and Linux + +```bash +python -m reporting \ + --input .ci-output/data-quality \ + --output .ci-output/reporting +``` + +The command writes: + +```text +.ci-output/reporting/ +├── average_score_by_module.svg +├── pass_rate_by_module.svg +├── rejection_reason_summary.csv +└── reporting_summary.json +``` + +## Verified control totals + +The committed synthetic fixture produces these reporting controls: + +| Control | Expected value | +|---|---:| +| Reconciled modules | 4 | +| Accepted results | 8 | +| Rejected rows | 7 | +| Overall average score | 70.00% | +| Overall pass rate | 62.50% | +| Distinct rejection reasons | 7 | +| Rejection-reason occurrences | 9 | +| KPI reconciliation | passed | + +The reporting workflow recalculates module KPIs from `cleaned_results.csv` and compares them with `module_kpis.csv`. The comparison permits harmless dtype differences introduced by CSV persistence but not value differences. + +## Rejection-reason reporting + +`rejected_results.csv` retains pipe-separated reason codes per rejected source row. Reporting explodes those codes and creates one deterministic count per reason. + +A rejected row can violate more than one rule. The fixture has seven rejected rows but nine reason occurrences: + +| Rejection reason | Occurrences | +|---|---:| +| `pass_score_above_max_score` | 2 | +| `score_above_max_score` | 2 | +| `duplicate_exact` | 1 | +| `invalid_assessment_date` | 1 | +| `max_score_not_positive` | 1 | +| `missing_learner_id` | 1 | +| `missing_score` | 1 | + +This distinction is intentional. A non-positive maximum score can also cause the score and pass threshold to exceed that maximum, so suppressing secondary violations would hide useful quality information. + +## Notebook + +[`notebooks/reporting_verification.ipynb`](../notebooks/reporting_verification.ipynb) reads the generated data-quality files and performs the same reconciliation through the reusable `reporting` package. + +The notebook contains: + +1. verified module KPI table +2. rejection-reason summary +3. average-score chart +4. pass-rate chart +5. explicit control-total assertions +6. final `Reporting notebook verification passed.` marker + +The notebook resolves the repository root from its current working directory, so it behaves consistently when launched from the repository root, an IDE or `nbconvert` from the `notebooks/` directory. + +The committed notebook contains no outputs, execution counts, local paths or IDE timestamps. CI executes a temporary copy after the data-quality and reporting command-line workflows have succeeded. + +## Reference charts + +The README embeds two small SVG reference charts for the committed synthetic fixture. Fresh Matplotlib SVG files are generated on every CI run and uploaded as short-lived workflow artefacts. + +The committed reference files are: + +- [`docs/assets/average-score-by-module.svg`](assets/average-score-by-module.svg) +- [`docs/assets/pass-rate-by-module.svg`](assets/pass-rate-by-module.svg) + +## Automated verification + +The pytest suite covers: + +- expected reporting control totals +- exact generated file set +- complete and deterministic rejection-reason counts +- rejection of a deliberately modified KPI value +- clear failure for a missing reporting input +- non-empty, labelled SVG output +- stable empty-summary schemas +- hygiene checks for both committed notebooks + +GitHub Actions runs the complete chain on Ubuntu 24.04 and Windows 2025 with Python 3.12. + +## Scope boundary + +This is a learning-grade reporting and verification layer. It does not claim: + +- a production semantic model +- a Power BI report +- scheduled orchestration +- interactive dashboard filtering +- production publication or access control +- statistical inference from the small synthetic fixture + +Its purpose is to demonstrate a controlled transition from validated tabular data to reproducible reporting evidence. diff --git a/notebooks/reporting_verification.ipynb b/notebooks/reporting_verification.ipynb new file mode 100644 index 0000000..7282ce5 --- /dev/null +++ b/notebooks/reporting_verification.ipynb @@ -0,0 +1,194 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Reporting Verification\n", + "\n", + "This notebook reads the generated data-quality outputs, reconciles persisted module KPIs against a fresh calculation, summarises rejection reasons and renders two reporting charts. The committed notebook contains no execution output.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "from IPython.display import display\n", + "from matplotlib import pyplot as plt\n", + "\n", + "from reporting import (\n", + " build_rejection_reason_summary,\n", + " build_reporting_summary,\n", + " create_metric_figure,\n", + " load_reporting_inputs,\n", + " reconcile_module_kpis,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def find_repository_root(start: Path) -> Path:\n", + " for candidate in (start, *start.parents):\n", + " if (candidate / \"pyproject.toml\").is_file() and (candidate / \"data_quality\").is_dir():\n", + " return candidate\n", + " raise FileNotFoundError(\"Could not locate the repository root.\")\n", + "\n", + "REPOSITORY_ROOT = find_repository_root(Path.cwd())\n", + "DATA_QUALITY_DIR = REPOSITORY_ROOT / \".ci-output/data-quality\"\n", + "NOTEBOOK_OUTPUT_DIR = REPOSITORY_ROOT / \".ci-output/reporting-notebook\"\n", + "NOTEBOOK_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "inputs = load_reporting_inputs(DATA_QUALITY_DIR)\n", + "module_kpis = reconcile_module_kpis(inputs.cleaned, inputs.module_kpis)\n", + "rejection_summary = build_rejection_reason_summary(inputs.rejected)\n", + "reporting_summary = build_reporting_summary(\n", + " inputs,\n", + " module_kpis,\n", + " rejection_summary,\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Verified module KPIs\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "display(module_kpis)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Rejection reason summary\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "display(rejection_summary)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Average score by module\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "average_figure = create_metric_figure(\n", + " module_kpis,\n", + " \"average_score_percentage\",\n", + " \"Average score by module\",\n", + " \"Average score (%)\",\n", + ")\n", + "average_figure.savefig(\n", + " NOTEBOOK_OUTPUT_DIR / \"average_score_by_module.svg\",\n", + " format=\"svg\",\n", + " metadata={\"Date\": None},\n", + ")\n", + "display(average_figure)\n", + "plt.close(average_figure)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pass rate by module\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pass_rate_figure = create_metric_figure(\n", + " module_kpis,\n", + " \"pass_rate_percentage\",\n", + " \"Pass rate by module\",\n", + " \"Pass rate (%)\",\n", + ")\n", + "pass_rate_figure.savefig(\n", + " NOTEBOOK_OUTPUT_DIR / \"pass_rate_by_module.svg\",\n", + " format=\"svg\",\n", + " metadata={\"Date\": None},\n", + ")\n", + "display(pass_rate_figure)\n", + "plt.close(pass_rate_figure)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Control totals\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert reporting_summary[\"kpi_reconciliation\"] == \"passed\"\n", + "assert reporting_summary[\"module_count\"] == 4\n", + "assert reporting_summary[\"result_count\"] == 8\n", + "assert reporting_summary[\"rejected_row_count\"] == 7\n", + "assert reporting_summary[\"overall_average_score_percentage\"] == 70.0\n", + "assert reporting_summary[\"overall_pass_rate_percentage\"] == 62.5\n", + "assert reporting_summary[\"rejection_reason_count\"] == 7\n", + "\n", + "print(\"Reporting notebook verification passed.\")\n", + "reporting_summary\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.12", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index 9c052a3..2129d99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python-data-basics" -version = "0.2.0" +version = "0.3.0" description = "A compact, tested Python 3.12 foundation for reproducible Data and BI workflows." readme = "README.md" requires-python = ">=3.12" @@ -20,6 +20,7 @@ dependencies = [ [project.scripts] python-data-quality = "data_quality.__main__:main" +python-data-reporting = "reporting.__main__:main" [project.optional-dependencies] notebook = [ @@ -35,7 +36,7 @@ dev = [ [tool.setuptools] py-modules = ["main"] -packages = ["data_quality"] +packages = ["data_quality", "reporting"] [tool.pytest.ini_options] addopts = "-ra --strict-config --strict-markers" diff --git a/reporting/__init__.py b/reporting/__init__.py new file mode 100644 index 0000000..a0a79c6 --- /dev/null +++ b/reporting/__init__.py @@ -0,0 +1,25 @@ +"""Verified reporting helpers for data-quality outputs.""" + +from reporting.workflow import ( + ReportingError, + ReportingInputs, + ReportingResult, + build_rejection_reason_summary, + build_reporting_summary, + create_metric_figure, + load_reporting_inputs, + reconcile_module_kpis, + run_reporting, +) + +__all__ = [ + "ReportingError", + "ReportingInputs", + "ReportingResult", + "build_rejection_reason_summary", + "build_reporting_summary", + "create_metric_figure", + "load_reporting_inputs", + "reconcile_module_kpis", + "run_reporting", +] diff --git a/reporting/__main__.py b/reporting/__main__.py new file mode 100644 index 0000000..ada613a --- /dev/null +++ b/reporting/__main__.py @@ -0,0 +1,49 @@ +"""Command-line entry point for verified reporting outputs.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from reporting.workflow import ReportingError, run_reporting + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Reconcile data-quality outputs and generate reporting tables and charts." + ) + parser.add_argument( + "--input", + type=Path, + required=True, + help="Directory containing cleaned_results.csv and the other data-quality outputs.", + ) + parser.add_argument( + "--output", + type=Path, + required=True, + help="Directory for verified reporting outputs.", + ) + return parser + + +def main() -> int: + args = build_parser().parse_args() + try: + result = run_reporting(args.input, args.output) + except ReportingError as exc: + print(f"Reporting workflow failed: {exc}") + return 1 + + print("Reporting workflow passed.") + print(f"Modules: {result.summary['module_count']}") + print(f"Results: {result.summary['result_count']}") + print(f"Rejected rows: {result.summary['rejected_row_count']}") + print(f"Overall average score: {result.summary['overall_average_score_percentage']:.2f}%") + print(f"Overall pass rate: {result.summary['overall_pass_rate_percentage']:.2f}%") + print(f"Output directory: {result.output_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/reporting/workflow.py b/reporting/workflow.py new file mode 100644 index 0000000..a8b6317 --- /dev/null +++ b/reporting/workflow.py @@ -0,0 +1,265 @@ +"""Reporting, reconciliation and chart generation for verified data-quality outputs.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import matplotlib +import pandas as pd +from matplotlib import pyplot as plt +from pandas.testing import assert_frame_equal + +from data_quality import build_module_kpis + +plt.switch_backend("Agg") +matplotlib.rcParams["svg.fonttype"] = "none" +matplotlib.rcParams["svg.hashsalt"] = "python-data-basics-reporting" + +CLEANED_COLUMNS = { + "result_id", + "learner_id", + "module", + "assessment_date", + "score", + "max_score", + "pass_score", + "score_percentage", + "passed", +} +REJECTED_COLUMNS = {"result_id", "rejection_reasons"} +KPI_COLUMNS = [ + "module", + "result_count", + "learner_count", + "average_score_percentage", + "passed_count", + "failed_count", + "pass_rate_percentage", +] + + +class ReportingError(ValueError): + """Raised when reporting inputs are missing or fail reconciliation.""" + + +@dataclass(frozen=True) +class ReportingInputs: + """Verified files loaded from one data-quality workflow output directory.""" + + cleaned: pd.DataFrame + rejected: pd.DataFrame + module_kpis: pd.DataFrame + quality_report: dict[str, Any] + + +@dataclass(frozen=True) +class ReportingResult: + """In-memory and persisted outputs from one reporting run.""" + + module_kpis: pd.DataFrame + rejection_reason_summary: pd.DataFrame + summary: dict[str, Any] + output_dir: Path + + +def _require_columns(frame: pd.DataFrame, required: set[str], label: str) -> None: + missing = sorted(required - set(frame.columns)) + if missing: + raise ReportingError(f"{label} is missing required columns: {', '.join(missing)}") + + +def load_reporting_inputs(data_quality_dir: str | Path) -> ReportingInputs: + """Load and validate the four files produced by the data-quality workflow.""" + + input_dir = Path(data_quality_dir) + paths = { + "cleaned": input_dir / "cleaned_results.csv", + "rejected": input_dir / "rejected_results.csv", + "module_kpis": input_dir / "module_kpis.csv", + "quality_report": input_dir / "quality_report.json", + } + missing_files = sorted(path.name for path in paths.values() if not path.is_file()) + if missing_files: + raise ReportingError(f"Missing reporting input files: {', '.join(missing_files)}") + + cleaned = pd.read_csv(paths["cleaned"]) + rejected = pd.read_csv(paths["rejected"]) + module_kpis = pd.read_csv(paths["module_kpis"]) + quality_report = json.loads(paths["quality_report"].read_text(encoding="utf-8")) + + _require_columns(cleaned, CLEANED_COLUMNS, "cleaned_results.csv") + _require_columns(rejected, REJECTED_COLUMNS, "rejected_results.csv") + _require_columns(module_kpis, set(KPI_COLUMNS), "module_kpis.csv") + + return ReportingInputs( + cleaned=cleaned, + rejected=rejected, + module_kpis=module_kpis.loc[:, KPI_COLUMNS].copy(), + quality_report=quality_report, + ) + + +def reconcile_module_kpis(cleaned: pd.DataFrame, persisted_kpis: pd.DataFrame) -> pd.DataFrame: + """Recalculate module KPIs and fail when they differ from the persisted CSV.""" + + recalculated = build_module_kpis(cleaned).loc[:, KPI_COLUMNS] + recalculated = recalculated.sort_values("module", kind="stable").reset_index(drop=True) + persisted = ( + persisted_kpis.loc[:, KPI_COLUMNS] + .sort_values("module", kind="stable") + .reset_index(drop=True) + ) + + try: + assert_frame_equal( + recalculated, + persisted, + check_dtype=False, + check_exact=False, + rtol=1e-9, + atol=1e-9, + ) + except AssertionError as exc: + raise ReportingError("Persisted module KPIs do not match recalculated values.") from exc + + return recalculated + + +def build_rejection_reason_summary(rejected: pd.DataFrame) -> pd.DataFrame: + """Count pipe-separated rejection reason codes in deterministic order.""" + + _require_columns(rejected, REJECTED_COLUMNS, "rejected results") + if rejected.empty: + return pd.DataFrame(columns=["rejection_reason", "rejected_row_count"]) + + reasons = rejected["rejection_reasons"].astype("string").str.split("|").explode().str.strip() + reasons = reasons[reasons.ne("") & reasons.notna()] + summary = ( + reasons.value_counts() + .rename_axis("rejection_reason") + .reset_index(name="rejected_row_count") + ) + summary = summary.sort_values( + ["rejected_row_count", "rejection_reason"], + ascending=[False, True], + kind="stable", + ).reset_index(drop=True) + summary["rejected_row_count"] = summary["rejected_row_count"].astype("int64") + return summary + + +def build_reporting_summary( + inputs: ReportingInputs, + reconciled_kpis: pd.DataFrame, + rejection_summary: pd.DataFrame, +) -> dict[str, Any]: + """Build compact machine-readable reporting control totals.""" + + passed = inputs.cleaned["passed"].astype("boolean") + top_reason = None + if not rejection_summary.empty: + top_reason = str(rejection_summary.loc[0, "rejection_reason"]) + + return { + "kpi_reconciliation": "passed", + "source_quality_status": inputs.quality_report.get("quality_status"), + "module_count": int(len(reconciled_kpis)), + "result_count": int(len(inputs.cleaned)), + "rejected_row_count": int(len(inputs.rejected)), + "overall_average_score_percentage": round( + float(inputs.cleaned["score_percentage"].mean()), 2 + ), + "overall_pass_rate_percentage": round(float(passed.mean() * 100), 2), + "rejection_reason_count": int(len(rejection_summary)), + "top_rejection_reason": top_reason, + "generated_files": [ + "average_score_by_module.svg", + "pass_rate_by_module.svg", + "rejection_reason_summary.csv", + "reporting_summary.json", + ], + } + + +def create_metric_figure( + module_kpis: pd.DataFrame, + metric: str, + title: str, + y_label: str, +) -> plt.Figure: + """Create a bounded percentage bar chart for one verified module KPI.""" + + if metric not in module_kpis.columns: + raise ReportingError(f"Unknown KPI metric: {metric}") + + figure, axis = plt.subplots(figsize=(8, 4.5)) + bars = axis.bar(module_kpis["module"], module_kpis[metric]) + axis.set_ylim(0, 100) + axis.set_ylabel(y_label) + axis.set_title(title) + axis.tick_params(axis="x", rotation=20) + + for bar, value in zip(bars, module_kpis[metric], strict=True): + axis.text( + bar.get_x() + bar.get_width() / 2, + float(value) + 2, + f"{float(value):.2f}%", + ha="center", + va="bottom", + ) + + figure.tight_layout() + return figure + + +def _save_svg(figure: plt.Figure, output_path: Path) -> None: + figure.savefig(output_path, format="svg", metadata={"Date": None}) + plt.close(figure) + + +def run_reporting(data_quality_dir: str | Path, output_dir: str | Path) -> ReportingResult: + """Reconcile source KPIs and export reporting tables, charts and control totals.""" + + inputs = load_reporting_inputs(data_quality_dir) + reconciled_kpis = reconcile_module_kpis(inputs.cleaned, inputs.module_kpis) + rejection_summary = build_rejection_reason_summary(inputs.rejected) + summary = build_reporting_summary(inputs, reconciled_kpis, rejection_summary) + + destination = Path(output_dir) + destination.mkdir(parents=True, exist_ok=True) + + rejection_summary.to_csv( + destination / "rejection_reason_summary.csv", + index=False, + lineterminator="\n", + ) + (destination / "reporting_summary.json").write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + average_figure = create_metric_figure( + reconciled_kpis, + "average_score_percentage", + "Average score by module", + "Average score (%)", + ) + _save_svg(average_figure, destination / "average_score_by_module.svg") + + pass_rate_figure = create_metric_figure( + reconciled_kpis, + "pass_rate_percentage", + "Pass rate by module", + "Pass rate (%)", + ) + _save_svg(pass_rate_figure, destination / "pass_rate_by_module.svg") + + return ReportingResult( + module_kpis=reconciled_kpis, + rejection_reason_summary=rejection_summary, + summary=summary, + output_dir=destination, + ) diff --git a/tests/test_notebook_hygiene.py b/tests/test_notebook_hygiene.py index afa9851..16118b5 100644 --- a/tests/test_notebook_hygiene.py +++ b/tests/test_notebook_hygiene.py @@ -3,15 +3,22 @@ import json from pathlib import Path -NOTEBOOK_PATH = Path(__file__).resolve().parents[1] / "dataspell_test.ipynb" +import pytest +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +NOTEBOOK_PATHS = [ + REPOSITORY_ROOT / "dataspell_test.ipynb", + REPOSITORY_ROOT / "notebooks" / "reporting_verification.ipynb", +] -def load_notebook() -> dict[str, object]: - return json.loads(NOTEBOOK_PATH.read_text(encoding="utf-8")) +def load_notebook(notebook_path: Path) -> dict[str, object]: + return json.loads(notebook_path.read_text(encoding="utf-8")) -def test_notebook_has_no_stored_execution_output() -> None: - notebook = load_notebook() + +@pytest.mark.parametrize("notebook_path", NOTEBOOK_PATHS) +def test_notebook_has_no_stored_execution_output(notebook_path: Path) -> None: + notebook = load_notebook(notebook_path) for cell in notebook["cells"]: if cell["cell_type"] == "code": @@ -20,8 +27,9 @@ def test_notebook_has_no_stored_execution_output() -> None: assert "ExecuteTime" not in cell.get("metadata", {}) -def test_notebook_uses_python_3_12_metadata() -> None: - notebook = load_notebook() +@pytest.mark.parametrize("notebook_path", NOTEBOOK_PATHS) +def test_notebook_uses_python_3_12_metadata(notebook_path: Path) -> None: + notebook = load_notebook(notebook_path) metadata = notebook["metadata"] assert metadata["kernelspec"]["display_name"] == "Python 3.12" @@ -30,8 +38,17 @@ def test_notebook_uses_python_3_12_metadata() -> None: assert metadata["language_info"]["pygments_lexer"] == "ipython3" -def test_notebook_contains_no_local_absolute_paths() -> None: - serialized = NOTEBOOK_PATH.read_text(encoding="utf-8") +@pytest.mark.parametrize("notebook_path", NOTEBOOK_PATHS) +def test_notebook_contains_no_local_absolute_paths(notebook_path: Path) -> None: + serialized = notebook_path.read_text(encoding="utf-8") assert "/Users/" not in serialized assert "C:\\Users\\" not in serialized + + +def test_reporting_notebook_contains_explicit_verification_marker() -> None: + reporting_notebook = NOTEBOOK_PATHS[1].read_text(encoding="utf-8") + + assert "Reporting notebook verification passed." in reporting_notebook + assert "reconcile_module_kpis" in reporting_notebook + assert ".ci-output/data-quality" in reporting_notebook diff --git a/tests/test_reporting_workflow.py b/tests/test_reporting_workflow.py new file mode 100644 index 0000000..b99ca22 --- /dev/null +++ b/tests/test_reporting_workflow.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pandas as pd +import pytest + +from data_quality import run_workflow +from reporting import ( + ReportingError, + build_rejection_reason_summary, + load_reporting_inputs, + reconcile_module_kpis, + run_reporting, +) + +SAMPLE_INPUT = Path("data/raw/training_results.csv") + + +def prepare_data_quality_outputs(tmp_path: Path) -> Path: + output_dir = tmp_path / "data-quality" + run_workflow(SAMPLE_INPUT, output_dir) + return output_dir + + +def test_reporting_outputs_match_verified_control_totals(tmp_path: Path) -> None: + data_quality_dir = prepare_data_quality_outputs(tmp_path) + result = run_reporting(data_quality_dir, tmp_path / "reporting") + + assert result.summary["kpi_reconciliation"] == "passed" + assert result.summary["source_quality_status"] == "passed_with_rejections" + assert result.summary["module_count"] == 4 + assert result.summary["result_count"] == 8 + assert result.summary["rejected_row_count"] == 7 + assert result.summary["overall_average_score_percentage"] == 70.0 + assert result.summary["overall_pass_rate_percentage"] == 62.5 + assert result.summary["rejection_reason_count"] == 7 + + expected_files = { + "average_score_by_module.svg", + "pass_rate_by_module.svg", + "rejection_reason_summary.csv", + "reporting_summary.json", + } + assert {path.name for path in result.output_dir.iterdir()} == expected_files + + persisted_summary = json.loads( + (result.output_dir / "reporting_summary.json").read_text(encoding="utf-8") + ) + assert persisted_summary == result.summary + + +def test_rejection_reason_summary_is_complete_and_deterministic(tmp_path: Path) -> None: + data_quality_dir = prepare_data_quality_outputs(tmp_path) + inputs = load_reporting_inputs(data_quality_dir) + + summary = build_rejection_reason_summary(inputs.rejected) + + assert list(summary.columns) == ["rejection_reason", "rejected_row_count"] + assert dict(zip(summary["rejection_reason"], summary["rejected_row_count"], strict=True)) == { + "pass_score_above_max_score": 2, + "score_above_max_score": 2, + "duplicate_exact": 1, + "invalid_assessment_date": 1, + "max_score_not_positive": 1, + "missing_learner_id": 1, + "missing_score": 1, + } + assert list(summary["rejection_reason"]) == [ + "pass_score_above_max_score", + "score_above_max_score", + "duplicate_exact", + "invalid_assessment_date", + "max_score_not_positive", + "missing_learner_id", + "missing_score", + ] + + +def test_persisted_kpi_mismatch_is_rejected(tmp_path: Path) -> None: + data_quality_dir = prepare_data_quality_outputs(tmp_path) + inputs = load_reporting_inputs(data_quality_dir) + modified = inputs.module_kpis.copy() + modified.loc[modified["module"] == "SQL Basics", "pass_rate_percentage"] = 99.0 + + with pytest.raises(ReportingError, match="do not match"): + reconcile_module_kpis(inputs.cleaned, modified) + + +def test_missing_reporting_input_file_raises_clear_error(tmp_path: Path) -> None: + data_quality_dir = prepare_data_quality_outputs(tmp_path) + (data_quality_dir / "module_kpis.csv").unlink() + + with pytest.raises(ReportingError, match="module_kpis.csv"): + load_reporting_inputs(data_quality_dir) + + +def test_generated_svg_charts_are_nonempty_and_labeled(tmp_path: Path) -> None: + data_quality_dir = prepare_data_quality_outputs(tmp_path) + result = run_reporting(data_quality_dir, tmp_path / "reporting") + + average_svg = (result.output_dir / "average_score_by_module.svg").read_text(encoding="utf-8") + pass_rate_svg = (result.output_dir / "pass_rate_by_module.svg").read_text(encoding="utf-8") + + assert " None: + rejected = pd.DataFrame(columns=["result_id", "rejection_reasons"]) + + summary = build_rejection_reason_summary(rejected) + + assert summary.empty + assert list(summary.columns) == ["rejection_reason", "rejected_row_count"]