diff --git a/.github/workflows/python-quality.yml b/.github/workflows/python-quality.yml new file mode 100644 index 0000000..e73e422 --- /dev/null +++ b/.github/workflows/python-quality.yml @@ -0,0 +1,131 @@ +name: Python quality + +on: + pull_request: + branches: + - main + paths: + - ".github/workflows/python-quality.yml" + - "*.py" + - "*.ipynb" + - "pyproject.toml" + - "requirements*.txt" + - "examples/**" + - "tests/**" + push: + branches: + - main + paths: + - ".github/workflows/python-quality.yml" + - "*.py" + - "*.ipynb" + - "pyproject.toml" + - "requirements*.txt" + - "examples/**" + - "tests/**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: python-quality-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + quality: + name: Python 3.12 · ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + + strategy: + fail-fast: false + matrix: + os: + - ubuntu-24.04 + - windows-2025 + + env: + PYTHONUTF8: "1" + + steps: + - name: Check out repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python 3.12 + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Show Python and pip versions + run: | + python --version + python -m pip --version + + - name: Install project and quality dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev,ml,notebook]" + + - name: Compile Python sources + run: python -m compileall -q main.py examples tests + + - name: Run Ruff lint checks + id: ruff-lint + continue-on-error: true + shell: pwsh + run: | + $output = & python -m ruff check main.py examples tests 2>&1 + $exitCode = $LASTEXITCODE + $output | Tee-Object -FilePath ruff-lint.txt + exit $exitCode + + - name: Verify Ruff formatting + id: ruff-format + continue-on-error: true + shell: pwsh + run: | + $output = & python -m ruff format --check main.py examples tests 2>&1 + $exitCode = $LASTEXITCODE + $output | Tee-Object -FilePath ruff-format.txt + exit $exitCode + + - name: Run pytest suite + run: python -m pytest + + - name: Run baseline entry point + run: python main.py + + - name: Run optional ML example + run: python examples/optional/logistic_regression_basics.py + + - name: Execute clean notebook copy + run: | + python -c "from pathlib import Path; Path('.ci-output').mkdir(exist_ok=True)" + jupyter nbconvert --to notebook --execute dataspell_test.ipynb --output environment-check.executed.ipynb --output-dir .ci-output --ExecutePreprocessor.timeout=120 + + - name: Upload quality diagnostics + if: always() + uses: actions/upload-artifact@v7 + with: + name: quality-diagnostics-${{ matrix.os }} + path: | + ruff-lint.txt + ruff-format.txt + if-no-files-found: ignore + retention-days: 3 + + - name: Enforce Ruff quality gate + if: always() + shell: pwsh + run: | + $lintOutcome = "${{ steps.ruff-lint.outcome }}" + $formatOutcome = "${{ steps.ruff-format.outcome }}" + + if ($lintOutcome -ne "success" -or $formatOutcome -ne "success") { + throw "Ruff quality gate failed. Download the quality diagnostics artifact for details." + } diff --git a/.gitignore b/.gitignore index 023163c..78fddbf 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ __pycache__/ # Jupyter .ipynb_checkpoints/ +.ci-output/ # Test / tooling caches .pytest_cache/ @@ -29,6 +30,8 @@ dist/ # macOS .DS_Store + +# Local configuration and credentials .env *.env credentials.json diff --git a/README.md b/README.md index 995182f..4c8db1d 100644 --- a/README.md +++ b/README.md @@ -1,304 +1,281 @@ # Python Data Basics -**Python 3.12 data environment · pandas · NumPy · matplotlib · scikit-learn · Jupyter · JSON · API basics · OAuth2 notes · optional Ollama local API** +[![Python quality](https://github.com/DataTideHH/python-data-basics/actions/workflows/python-quality.yml/badge.svg)](https://github.com/DataTideHH/python-data-basics/actions/workflows/python-quality.yml) -This repository documents a working Python 3.12 data environment and selected Python fundamentals for Data/BI work. +**Python 3.12 · pandas · NumPy · matplotlib · pytest · Ruff · Jupyter · GitHub Actions** -It is part of my broader **DataTideHH portfolio** and supports my learning path toward **Data/BI Analyst** roles with a focus on Python, pandas, SQL, Power BI, Microsoft Fabric/Azure fundamentals, API data workflows and reproducible analysis. +This repository is a compact, tested foundation for reproducible Python Data/BI workflows. It demonstrates a clean project environment, deterministic tabular transformations, explicit dependency groups, public notebook hygiene and cross-platform quality checks. -The purpose is not to collect random scripts. This repository is intended as a small, understandable foundation for learning and documenting practical Python data workflows before building larger analysis and BI-related projects. +It is part of my DataTideHH portfolio during the IHK retraining program in Data and Process Analysis. The scope is deliberately bounded: this is a reusable learning baseline, not a production package, a machine-learning showcase or a substitute for the larger analysis projects in the portfolio. --- -## Why This Repository Matters for Data/BI +## Project at a Glance -Many Data/BI workflows start with basic Python tasks: - -- read structured data -- inspect and clean tabular data -- work with CSV files -- understand JSON responses -- request data from APIs -- transform nested data into tables -- document assumptions and limitations -- keep credentials and tokens out of Git -- prepare clean outputs for reporting or further analysis - -This repository is the technical foundation for those skills. More complete project work is documented in separate repositories such as API-based analysis projects, public-data analysis projects and SQL/database projects. - ---- - -## Current Scope - -This repository currently focuses on: - -- Python 3.12 environment setup -- project-specific virtual environment usage -- PyCharm and DataSpell workflow -- pandas, NumPy, matplotlib and scikit-learn basics -- Jupyter Notebook / DataSpell validation -- a minimal logistic regression example in `main.py` -- CSV and DataFrame basics -- JSON parsing basics -- basic public API request workflow -- OAuth2 concept notes -- optional local Ollama API example -- reproducible and safe learning examples - -It is deliberately small. The goal is to keep the basics understandable and reusable. - ---- - -## Tested Environment - -This repository has been tested on the following local setup: - -| Area | Tested setup | +| Area | Current implementation | |---|---| -| Device | iMac Retina 4K, 21.5-inch, Late 2015 | -| Architecture | Intel x86_64 | -| Operating system | macOS Sonoma 14.8.7 via OpenCore Legacy Patcher | -| Python IDE | PyCharm via JetBrains Toolbox | -| Notebook IDE | DataSpell via JetBrains Toolbox | -| Python version | Python 3.12.13 | -| Environment | Project-specific `.venv` | -| Core packages | pandas, NumPy, matplotlib, scikit-learn | -| Notebook stack | Jupyter Notebook | -| Version control | Git / GitHub | - -This repository also documents that the Python data stack works on a legacy Intel Mac setup used as a stable learning and development machine. +| Python baseline | Python 3.12-compatible environment and deterministic pandas sanity check | +| Dependency model | Direct runtime, notebook, optional ML and development groups in `pyproject.toml` | +| Data handling | Small CSV, JSON and public API examples with synthetic or public-safe inputs | +| Testing | pytest coverage for the baseline transformation, notebook hygiene and optional ML example | +| Code quality | Ruff linting and formatting checks plus Python bytecode compilation | +| Notebook hygiene | Cleared outputs, neutral metadata and tests against committed local paths | +| Continuous integration | Matrix workflow for Ubuntu 24.04 and Windows 2025 with Python 3.12 | +| Credential safety | Local environments, tokens, secrets and machine-specific files remain excluded | + +## What This Repository Demonstrates + +The repository focuses on foundational tasks that recur in Data/BI work: + +- create an isolated and reproducible Python environment +- define direct dependencies separately from development tooling +- build and validate small pandas transformations +- parse nested JSON into tabular structures +- call a public API without embedding credentials +- keep notebook outputs and local paths out of version control +- run syntax, lint, formatting and unit checks automatically +- use the same entry points on Windows, macOS, Linux and GitHub-hosted runners + +More complete business analyses remain in separate repositories. This project provides the tested building blocks underneath them. --- -## Setup +## Quick Start -Create the virtual environment with Python 3.12: +The detailed platform-specific procedure is documented in [`docs/setup.md`](docs/setup.md). -```bash -/usr/local/bin/python3.12 -m venv .venv +### Windows PowerShell + +```powershell +py -3.12 -m venv .venv +& ".\.venv\Scripts\Activate.ps1" +python -m pip install --upgrade pip +python -m pip install -r requirements-dev.txt +python main.py ``` -Activate it: +### macOS and Linux ```bash +python3.12 -m venv .venv source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install -r requirements-dev.txt +python main.py ``` -Install the exact tested dependency set: +On the Intel iMac used for local portfolio work, Python 3.12 is currently available at `/usr/local/bin/python3.12`. -```bash -python -m pip install -r requirements.txt -``` +Expected baseline output contains: -Alternatively, install only the core packages: - -```bash -python -m pip install -r requirements-core.txt -``` +- Python implementation and version +- pandas, NumPy and matplotlib versions +- a deterministic two-row category summary +- `Baseline check passed.` --- -## Run the Python Baseline Example +## Dependency Model -Run: +`pyproject.toml` is the source of truth. -```bash -python main.py -``` - -The script verifies the interpreter and package versions, creates a small example DataFrame and trains a minimal logistic regression model on synthetic learning data. - -Expected output includes: - -- Python data environment check -- Python version -- pandas version -- NumPy version -- matplotlib availability -- scikit-learn version -- minimal logistic regression example +| Installation | Included scope | +|---|---| +| `python -m pip install -e .` | pandas, NumPy and matplotlib runtime baseline | +| `python -m pip install -e ".[notebook]"` | runtime baseline plus Jupyter | +| `python -m pip install -e ".[ml]"` | runtime baseline plus scikit-learn example | +| `python -m pip install -e ".[dev]"` | runtime baseline plus pytest and Ruff | +| `python -m pip install -r requirements.txt` | complete local learning environment | +| `python -m pip install -r requirements-dev.txt` | complete development and CI-equivalent environment | -The exact package versions may differ on other machines if the environment is recreated with a different dependency set. +The requirement files are intentionally small wrappers. They no longer contain a machine-specific freeze of every transitive Jupyter dependency. --- -## DataSpell Notebook +## Baseline Entry Point -The notebook +[`main.py`](main.py) performs two bounded checks: -```text -dataspell_test.ipynb -``` - -verifies that DataSpell uses the same project-specific Python 3.12 virtual environment and can import the core data stack. +1. reports the active interpreter and direct runtime package versions +2. creates a deterministic DataFrame, validates its required columns and values, and calculates one summary row per category -This is useful because notebook environments can easily point to a different interpreter than the main project. The notebook documents that the local DataSpell setup is aligned with the repository environment. - ---- +The script exits with an error when: -## Planned Learning Modules +- Python is older than 3.12 +- a required column is missing +- the value column contains missing values +- the deterministic transformation returns an unexpected shape -| Module | Status | Purpose | -|---|---|---| -| CSV / pandas basics | Added as example | Read small structured data and work with DataFrames | -| JSON basics | Added as example | Understand nested API-like data structures | -| API request basics | Added as example | Fetch public API data without credentials | -| OAuth2 concept notes | Added as documentation | Understand tokens, scopes and safe credential handling | -| Local Ollama API example | Added as optional example | Practice JSON request/response patterns against a local API | -| Data cleaning basics | Planned | Handle missing values, types and simple validation | -| Basic visualizations | Planned | Create simple charts for analysis and reporting | -| SQL-to-pandas workflow | Planned | Read database query results into pandas | -| Notebook workflow | Planned | Use notebooks for documented analysis steps | +This keeps the repository focused on an explainable Data/BI baseline rather than presenting a tiny synthetic model as the primary result. --- -## Repository Structure +## Example Modules -```text -python-data-basics/ -├── main.py -├── dataspell_test.ipynb -├── README.md -├── requirements.txt -├── requirements-core.txt -├── LICENSE -├── .editorconfig -├── .gitignore -├── examples/ -│ ├── 01_csv_pandas_basics.py -│ ├── 02_json_basics.py -│ ├── 03_api_request_basics.py -│ └── 04_ollama_local_api_basics.py -└── docs/ - ├── api-json-oauth2-notes.md - └── ollama-local-api-notes.md -``` +### CSV and pandas -Local virtual environments, IDE metadata, cache files, token files and machine-specific files are intentionally excluded from Git. +[`examples/01_csv_pandas_basics.py`](examples/01_csv_pandas_basics.py) demonstrates reading in-memory CSV data, filtering and grouped aggregation. -Examples: +### JSON normalization -```text -.venv/ -.idea/ -__pycache__/ -*.pyc -.DS_Store -.env -*.env -credentials.json -token.json -access_token -refresh_token -``` +[`examples/02_json_basics.py`](examples/02_json_basics.py) demonstrates nested dictionaries and lists and converts selected values into a tabular DataFrame. ---- +### Public API request -## Example Modules +[`examples/03_api_request_basics.py`](examples/03_api_request_basics.py) calls Open-Meteo for Hamburg using the Python standard library. It uses no API key or token and handles common network failures. -### CSV / pandas basics +### Local Ollama request -```text -examples/01_csv_pandas_basics.py -``` +[`examples/04_ollama_local_api_basics.py`](examples/04_ollama_local_api_basics.py) remains an optional localhost-only JSON request example. It is not part of the automated CI path because it requires a running local Ollama service and an installed model. -Demonstrates a small CSV-like dataset, loads it into pandas and calculates simple grouped results. +### Optional logistic regression -### JSON basics +[`examples/optional/logistic_regression_basics.py`](examples/optional/logistic_regression_basics.py) contains the former `main.py` model example. It now has an explicit optional dependency group and a clear limitation: the tiny synthetic dataset demonstrates scikit-learn API usage only and does not support a model-quality claim. -```text -examples/02_json_basics.py +Run it with: + +```bash +python examples/optional/logistic_regression_basics.py ``` -Demonstrates JSON parsing, nested dictionaries/lists and basic normalization into tabular data. +--- -### API request basics +## Notebook Hygiene -```text -examples/03_api_request_basics.py -``` +[`dataspell_test.ipynb`](dataspell_test.ipynb) verifies the project interpreter and core package imports without storing machine-specific evidence. -Demonstrates a public API request using Python standard-library tools. It uses a public endpoint and does not require credentials. +The committed notebook contains: -### OAuth2 concept notes +- no cell outputs +- no execution counts +- no IDE execution timestamps +- no absolute local interpreter path +- Python 3.12 kernel and language metadata -```text -docs/api-json-oauth2-notes.md -``` +The pytest suite checks these properties. GitHub Actions executes a temporary notebook copy into the ignored `.ci-output/` directory, leaving the committed notebook unchanged. -Explains API basics, JSON basics and OAuth2 concepts such as access tokens, refresh tokens, scopes, client IDs and client secrets. +--- -### Optional Ollama local API example +## Local Quality Checks -```text -examples/04_ollama_local_api_basics.py +Run the same core checks used in CI: + +```bash +python -m compileall -q main.py examples tests +python -m ruff check main.py examples tests +python -m ruff format --check main.py examples tests +python -m pytest +python main.py +python examples/optional/logistic_regression_basics.py ``` -Demonstrates a local JSON request/response workflow against an Ollama server on `localhost`. +Execute the notebook separately: -This example is optional. It only works if Ollama is installed, running locally and a model is available. +```bash +python -c "from pathlib import Path; Path('.ci-output').mkdir(exist_ok=True)" +jupyter nbconvert --to notebook --execute dataspell_test.ipynb --output environment-check.executed.ipynb --output-dir .ci-output --ExecutePreprocessor.timeout=120 +``` --- -## Credentials, Tokens and Secrets +## Continuous Integration -This repository must not contain real credentials, tokens or secrets. +The workflow in [`.github/workflows/python-quality.yml`](.github/workflows/python-quality.yml) uses a Python 3.12 matrix on: -Do not commit: +- Ubuntu 24.04 +- Windows 2025 -- API keys -- OAuth2 access tokens -- OAuth2 refresh tokens -- client secrets -- private `.env` files -- downloaded credential files -- personal data -- customer data +Each job: -If an example ever needs configuration, use documented placeholders or an `.env.example` file, not real secrets. +1. checks out the repository with read-only contents permission and without persisted credentials +2. installs the project and all optional quality groups +3. compiles Python sources +4. runs Ruff linting +5. verifies Ruff formatting +6. runs pytest +7. executes the baseline entry point +8. executes the optional ML example +9. executes the clean notebook into a temporary ignored directory -OAuth2 is currently documented conceptually only. This is intentional. +The workflow is an automated quality check, not a deployment or release pipeline. --- -## What This Demonstrates - -This repository demonstrates a working Python data baseline setup using: +## Repository Structure -- Python 3.12 -- a project-specific virtual environment -- pandas for tabular data handling -- NumPy for numerical work -- matplotlib availability for visualization -- scikit-learn for a minimal machine learning example -- Jupyter Notebook / DataSpell for notebook-based work -- PyCharm as the primary Python IDE -- Git and GitHub for version control -- JSON parsing basics -- public API request basics -- safe handling of OAuth2 concepts without committing secrets -- optional local API interaction through Ollama +```text +python-data-basics/ +├── .github/ +│ └── workflows/ +│ └── python-quality.yml +├── docs/ +│ ├── api-json-oauth2-notes.md +│ ├── ollama-local-api-notes.md +│ └── setup.md +├── examples/ +│ ├── __init__.py +│ ├── 01_csv_pandas_basics.py +│ ├── 02_json_basics.py +│ ├── 03_api_request_basics.py +│ ├── 04_ollama_local_api_basics.py +│ └── optional/ +│ ├── __init__.py +│ └── logistic_regression_basics.py +├── tests/ +│ ├── test_main.py +│ ├── test_notebook_hygiene.py +│ └── test_optional_ml.py +├── dataspell_test.ipynb +├── main.py +├── pyproject.toml +├── requirements.txt +├── requirements-dev.txt +├── .editorconfig +├── .gitignore +├── LICENSE +└── README.md +``` --- -## Relationship to Other Portfolio Projects +## Credentials and Data Safety + +Only synthetic learning data and public endpoints belong in this repository. -This repository is a foundation repository. +Excluded content includes: -More complete project examples are documented separately: +- `.env` files +- API keys and client secrets +- OAuth access and refresh tokens +- credential downloads +- personal or customer data +- local virtual environments +- IDE metadata and caches +- executed CI notebook copies -- `open-meteo-germany-weather-ranking` for an API-to-CSV scoring workflow -- `hamburg-district-data-basics` for public-data analysis and Power BI preparation -- `sql-server-docker-basics` for SQL Server and Data/BI database practice +OAuth2 remains conceptual documentation only. Any future authenticated example must use placeholders and local configuration rather than committed credentials. --- -## Notes and Limitations +## Relationship to Other Portfolio Projects + +This repository is the tested Python foundation beneath more specific projects: + +- [`open-meteo-germany-weather-ranking`](https://github.com/DataTideHH/open-meteo-germany-weather-ranking) — API-to-CSV scoring workflow +- [`hamburg-district-data-basics`](https://github.com/DataTideHH/hamburg-district-data-basics) — public-data analysis and Power BI preparation +- [`sql-server-docker-basics`](https://github.com/DataTideHH/sql-server-docker-basics) — SQL Server, relational integrity, star schema and CI +- [`flask-country-data-api`](https://github.com/DataTideHH/flask-country-data-api) — validated ingestion, persistence and API delivery -This repository is intentionally small. +## Current Boundaries -It is not intended to be a production application, a package or a complete API client library. The examples are intentionally small, readable and safe to run locally. +This repository does not claim: -The focus is on understanding basic building blocks that can later be used in practical Data/BI projects. +- a production Python package +- a production API client +- a validated predictive model +- a large business analysis +- a finished dashboard +- deployment or cloud infrastructure +- support for copying virtual environments between operating systems -No virtual environment, IDE metadata, cache files, token files, credentials or machine-specific files are committed. +The next useful increment is a small, tested data-quality workflow with explicit raw input, validation rules, cleaned output and KPI aggregation. diff --git a/dataspell_test.ipynb b/dataspell_test.ipynb index 689b940..113415d 100644 --- a/dataspell_test.ipynb +++ b/dataspell_test.ipynb @@ -1,69 +1,47 @@ { "cells": [ { - "cell_type": "code", - "id": "initial_id", - "metadata": { - "collapsed": true, - "ExecuteTime": { - "end_time": "2026-05-23T20:44:00.512493Z", - "start_time": "2026-05-23T20:43:49.393955Z" - } - }, + "cell_type": "markdown", + "id": "purpose", + "metadata": {}, "source": [ - "import sys\n", - "import pandas as pd\n", - "import numpy as np\n", - "import matplotlib\n", - "import sklearn\n", + "# Python data environment check\n", "\n", - "print(\"Executable:\", sys.executable)\n", - "print(\"pandas:\", pd.__version__)\n", - "print(\"numpy:\", np.__version__)\n", - "print(\"matplotlib:\", matplotlib.__version__)\n", - "print(\"scikit-learn:\", sklearn.__version__)" - ], - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Executable: /Users/tobiaswietelmann/Code/python-data-basics/.venv/bin/python\n", - "pandas: 3.0.3\n", - "numpy: 2.4.6\n", - "matplotlib: 3.10.9\n", - "scikit-learn: 1.8.0\n" - ] - } - ], - "execution_count": 1 + "This notebook verifies the project interpreter and direct runtime dependencies. Outputs are intentionally cleared before commit." + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "", - "id": "2c7a39eb2f34ea11" + "id": "environment-check", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "{\n", + " \"pandas\": pd.__version__,\n", + " \"numpy\": np.__version__,\n", + " \"matplotlib\": matplotlib.__version__,\n", + "}" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.12", "language": "python", "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.12" } }, "nbformat": 4, diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 0000000..3f79d1d --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,137 @@ +# Cross-platform setup + +This project targets Python 3.12 or newer and keeps all dependencies inside a project-specific virtual environment. + +`pyproject.toml` is the dependency source of truth. The requirement files are convenience wrappers: + +- `requirements.txt` installs the complete local learning environment +- `requirements-dev.txt` additionally installs pytest and Ruff for development and CI parity + +## Dependency groups + +| Group | Purpose | +|---|---| +| default | pandas, NumPy and matplotlib runtime baseline | +| `notebook` | Jupyter notebook execution | +| `ml` | optional scikit-learn learning example | +| `dev` | pytest and Ruff quality tooling | + +## Windows PowerShell + +Verify Python 3.12: + +```powershell +py -3.12 --version +``` + +Create and activate the virtual environment: + +```powershell +py -3.12 -m venv .venv +& ".\.venv\Scripts\Activate.ps1" +``` + +Install the complete development environment: + +```powershell +python -m pip install --upgrade pip +python -m pip install -r requirements-dev.txt +``` + +PowerShell activation is optional. When local policy prevents script activation, use the virtual-environment interpreter directly instead of changing machine-wide policy: + +```powershell +& ".\.venv\Scripts\python.exe" -m pip install --upgrade pip +& ".\.venv\Scripts\python.exe" -m pip install -r requirements-dev.txt +& ".\.venv\Scripts\python.exe" main.py +``` + +## Windows Command Prompt + +Create and activate the virtual environment: + +```bat +py -3.12 -m venv .venv +.venv\Scripts\activate.bat +``` + +Install the complete development environment: + +```bat +python -m pip install --upgrade pip +python -m pip install -r requirements-dev.txt +``` + +## macOS and Linux + +Verify that the intended interpreter is available: + +```bash +python3.12 --version +command -v python3.12 +``` + +Create and activate the virtual environment: + +```bash +python3.12 -m venv .venv +source .venv/bin/activate +``` + +On the Intel iMac used for local portfolio work, the explicit interpreter path is currently: + +```bash +/usr/local/bin/python3.12 -m venv .venv +``` + +Install the complete development environment: + +```bash +python -m pip install --upgrade pip +python -m pip install -r requirements-dev.txt +``` + +Do not copy a `.venv` directory between macOS, Linux, Windows, OneDrive locations or different CPU architectures. Recreate it from `pyproject.toml` or the requirement wrapper on each machine. + +## Minimal runtime installation + +For the baseline script without notebooks, machine learning or quality tools: + +```bash +python -m pip install -e . +``` + +For the complete local learning environment without developer tools: + +```bash +python -m pip install -r requirements.txt +``` + +## Local verification + +Run the same core checks used by GitHub Actions: + +```bash +python -m compileall -q main.py examples tests +python -m ruff check main.py examples tests +python -m ruff format --check main.py examples tests +python -m pytest +python main.py +python examples/optional/logistic_regression_basics.py +``` + +Execute the notebook into an ignored temporary directory without modifying the committed source notebook: + +```bash +python -c "from pathlib import Path; Path('.ci-output').mkdir(exist_ok=True)" +jupyter nbconvert --to notebook --execute dataspell_test.ipynb --output environment-check.executed.ipynb --output-dir .ci-output --ExecutePreprocessor.timeout=120 +``` + +## IDE interpreter selection + +Configure PyCharm, DataSpell, VS Code or Jupyter to use the interpreter inside this repository: + +- Windows: `.venv\Scripts\python.exe` +- macOS/Linux: `.venv/bin/python` + +Do not select a global interpreter when the repository-specific environment is available. diff --git a/examples/01_csv_pandas_basics.py b/examples/01_csv_pandas_basics.py index 511e777..16b4861 100644 --- a/examples/01_csv_pandas_basics.py +++ b/examples/01_csv_pandas_basics.py @@ -1,21 +1,16 @@ -""" -CSV / pandas basics. - -This example uses a small in-memory CSV-style dataset. -It demonstrates: -- reading structured data -- creating a pandas DataFrame -- deriving a calculated column -- grouping and aggregating data +"""CSV and pandas basics. -No external files are required. +This example uses a small in-memory CSV-style dataset. It demonstrates reading +structured data, creating a pandas DataFrame, filtering, grouping and +aggregation. No external files are required. """ +from __future__ import annotations + from io import StringIO import pandas as pd - CSV_DATA = """city,category,month,value Hamburg,weather,2026-06,18.5 Berlin,weather,2026-06,21.2 @@ -27,17 +22,17 @@ def main() -> None: - df = pd.read_csv(StringIO(CSV_DATA)) + frame = pd.read_csv(StringIO(CSV_DATA)) print("Raw data:") - print(df) + print(frame) print("\nAverage value by category:") - summary = df.groupby("category", as_index=False)["value"].mean() + summary = frame.groupby("category", as_index=False)["value"].mean() print(summary) print("\nRows for Hamburg:") - print(df[df["city"] == "Hamburg"]) + print(frame[frame["city"] == "Hamburg"]) if __name__ == "__main__": diff --git a/examples/02_json_basics.py b/examples/02_json_basics.py index ecce71f..4716081 100644 --- a/examples/02_json_basics.py +++ b/examples/02_json_basics.py @@ -1,19 +1,15 @@ -""" -JSON basics. - -This example demonstrates: -- parsing JSON text -- accessing nested dictionaries and lists -- converting selected values into a pandas DataFrame +"""JSON normalization basics. -The structure is similar to many API responses. +This example parses an API-like JSON document, reads nested dictionaries and +lists, and converts selected values into a tabular pandas DataFrame. """ +from __future__ import annotations + import json import pandas as pd - JSON_TEXT = """ { "source": "example-api", @@ -56,10 +52,10 @@ def main() -> None: } ) - df = pd.DataFrame(rows) + frame = pd.DataFrame(rows) print("Normalized table:") - print(df) + print(frame) if __name__ == "__main__": diff --git a/examples/03_api_request_basics.py b/examples/03_api_request_basics.py index f127234..89910cc 100644 --- a/examples/03_api_request_basics.py +++ b/examples/03_api_request_basics.py @@ -1,18 +1,15 @@ -""" -Basic public API request example. - -This example calls the Open-Meteo API for Hamburg. -It uses only Python standard-library modules for the HTTP request. +"""Basic public API request example. -No API key is required. -No credentials or tokens are used. +This module calls Open-Meteo for Hamburg with Python standard-library tools. +The endpoint requires no API key, credentials or tokens. """ +from __future__ import annotations + import json from urllib.error import HTTPError, URLError from urllib.request import urlopen - OPEN_METEO_URL = ( "https://api.open-meteo.com/v1/forecast" "?latitude=53.5503" diff --git a/examples/04_ollama_local_api_basics.py b/examples/04_ollama_local_api_basics.py index 8830890..1b86d6b 100644 --- a/examples/04_ollama_local_api_basics.py +++ b/examples/04_ollama_local_api_basics.py @@ -1,21 +1,15 @@ -""" -Optional Ollama local API example. - -This example demonstrates a local JSON request/response workflow. -It only works if: -- Ollama is installed -- Ollama is running locally -- the configured model is available +"""Optional Ollama localhost API example. -No cloud API key is used. -No credentials or tokens are used. +The request works only when Ollama is running locally and the configured model +is available. It uses no cloud API key, credentials or tokens. """ +from __future__ import annotations + import json from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen - OLLAMA_URL = "http://localhost:11434/api/generate" MODEL = "llama3.2" diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..2c36388 --- /dev/null +++ b/examples/__init__.py @@ -0,0 +1 @@ +"""Small, runnable learning examples for the Python data baseline.""" diff --git a/examples/optional/__init__.py b/examples/optional/__init__.py new file mode 100644 index 0000000..89cb949 --- /dev/null +++ b/examples/optional/__init__.py @@ -0,0 +1 @@ +"""Optional examples that require additional dependency groups.""" diff --git a/examples/optional/logistic_regression_basics.py b/examples/optional/logistic_regression_basics.py new file mode 100644 index 0000000..c8d7abf --- /dev/null +++ b/examples/optional/logistic_regression_basics.py @@ -0,0 +1,66 @@ +"""Optional logistic-regression example using a tiny synthetic dataset. + +Install the optional ML dependency group before running this module: + + python -m pip install -e ".[ml]" + +The example is intentionally small and demonstrates API usage only. It is not a +model-quality claim and is not suitable for real predictions. +""" + +from __future__ import annotations + +import pandas as pd +from sklearn.linear_model import LogisticRegression + + +def build_sample_data() -> pd.DataFrame: + """Return deterministic synthetic observations for the learning example.""" + return pd.DataFrame( + { + "hours_studied": [1, 2, 3, 4, 5, 6, 7, 8], + "practice_score": [45, 50, 55, 60, 66, 72, 78, 85], + "passed": [0, 0, 0, 0, 1, 1, 1, 1], + } + ) + + +def predict_pass_probability( + frame: pd.DataFrame, + *, + hours_studied: float, + practice_score: float, +) -> float: + """Fit the demonstration model and return one probability between zero and one.""" + features = frame[["hours_studied", "practice_score"]] + target = frame["passed"] + + model = LogisticRegression(max_iter=1_000, random_state=42) + model.fit(features, target) + + observation = pd.DataFrame( + { + "hours_studied": [hours_studied], + "practice_score": [practice_score], + } + ) + return float(model.predict_proba(observation)[0][1]) + + +def main() -> None: + sample_data = build_sample_data() + probability = predict_pass_probability( + sample_data, + hours_studied=5, + practice_score=70, + ) + + print("Optional logistic regression example") + print("------------------------------------") + print(sample_data.to_string(index=False)) + print(f"\nPredicted pass probability for the sample observation: {probability:.2%}") + print("This output is a learning example, not a validated predictive model.") + + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py index 76931c3..053649d 100644 --- a/main.py +++ b/main.py @@ -1,64 +1,82 @@ +from __future__ import annotations + +import platform import sys import matplotlib import numpy as np import pandas as pd -import sklearn -from sklearn.linear_model import LogisticRegression + +MINIMUM_PYTHON = (3, 12) +REQUIRED_COLUMNS = {"city", "category", "value"} -def print_environment() -> None: - print("Python data environment check") - print("-----------------------------") - print(f"Executable: {sys.executable}") - print(f"Python: {sys.version.split()[0]}") - print(f"pandas: {pd.__version__}") - print(f"NumPy: {np.__version__}") - print(f"matplotlib: {matplotlib.__version__}") - print(f"scikit-learn: {sklearn.__version__}") - print() +def collect_environment() -> dict[str, str]: + """Return the interpreter and direct runtime dependency versions.""" + return { + "python": platform.python_version(), + "implementation": platform.python_implementation(), + "pandas": pd.__version__, + "numpy": np.__version__, + "matplotlib": matplotlib.__version__, + } -def build_sample_data() -> pd.DataFrame: +def build_baseline_data() -> pd.DataFrame: + """Create deterministic synthetic data for a small pandas sanity check.""" return pd.DataFrame( { - "hours_studied": [1, 2, 3, 4, 5, 6, 7, 8], - "practice_score": [45, 50, 55, 60, 66, 72, 78, 85], - "passed": [0, 0, 0, 0, 1, 1, 1, 1], + "city": ["Hamburg", "Berlin", "Hamburg", "Berlin"], + "category": ["quality", "quality", "processing", "processing"], + "value": [92.0, 88.0, 71.0, 77.0], } ) -def run_logistic_regression(df: pd.DataFrame) -> None: - features = df[["hours_studied", "practice_score"]] - target = df["passed"] +def summarize_baseline_data(frame: pd.DataFrame) -> pd.DataFrame: + """Validate the input shape and calculate one summary row per category.""" + missing_columns = REQUIRED_COLUMNS.difference(frame.columns) + if missing_columns: + missing = ", ".join(sorted(missing_columns)) + raise ValueError(f"Missing required columns: {missing}") - model = LogisticRegression() - model.fit(features, target) + if frame.empty: + raise ValueError("Baseline data must contain at least one row.") - new_student = pd.DataFrame( - { - "hours_studied": [5], - "practice_score": [70], - } + if frame["value"].isna().any(): + raise ValueError("Baseline data contains missing values in 'value'.") + + summary = ( + frame.groupby("category", as_index=False) + .agg(row_count=("value", "size"), average_value=("value", "mean")) + .sort_values("category", ignore_index=True) ) + return summary + - probability = model.predict_proba(new_student)[0][1] +def run_baseline_check() -> pd.DataFrame: + """Run the deterministic DataFrame transformation used by local and CI checks.""" + if sys.version_info < MINIMUM_PYTHON: + required = ".".join(str(part) for part in MINIMUM_PYTHON) + raise RuntimeError(f"Python {required} or newer is required.") - print("Sample data") - print("-----------") - print(df.to_string(index=False)) - print() + summary = summarize_baseline_data(build_baseline_data()) + if summary["row_count"].sum() != 4 or len(summary) != 2: + raise RuntimeError("The baseline pandas transformation returned unexpected results.") - print("Minimal logistic regression example") - print("-----------------------------------") - print(f"Predicted pass probability for the sample student: {probability:.2%}") + return summary def main() -> None: - print_environment() - sample_data = build_sample_data() - run_logistic_regression(sample_data) + print("Python data baseline check") + print("--------------------------") + for name, version in collect_environment().items(): + print(f"{name}: {version}") + + print("\nDeterministic pandas summary") + print("----------------------------") + print(run_baseline_check().to_string(index=False)) + print("\nBaseline check passed.") if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..449b44d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,49 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "python-data-basics" +version = "0.1.0" +description = "A compact, tested Python 3.12 foundation for reproducible Data and BI workflows." +readme = "README.md" +requires-python = ">=3.12" +license = { file = "LICENSE" } +authors = [ + { name = "Tobias Wietelmann" }, +] +dependencies = [ + "matplotlib>=3.8,<4", + "numpy>=1.26,<3", + "pandas>=2.2,<4", +] + +[project.optional-dependencies] +notebook = [ + "jupyter>=1.1,<2", +] +ml = [ + "scikit-learn>=1.4,<2", +] +dev = [ + "pytest>=8,<10", + "ruff>=0.9,<1", +] + +[tool.setuptools] +py-modules = ["main"] + +[tool.pytest.ini_options] +addopts = "-ra --strict-config --strict-markers" +testpaths = ["tests"] + +[tool.ruff] +target-version = "py312" +line-length = 100 +extend-exclude = [".ci-output"] + +[tool.ruff.lint] +select = ["B", "E4", "E7", "E9", "F", "I", "UP"] + +[tool.ruff.format] +docstring-code-format = true diff --git a/requirements-core.txt b/requirements-core.txt deleted file mode 100644 index 4ed35d4..0000000 --- a/requirements-core.txt +++ /dev/null @@ -1,5 +0,0 @@ -pandas -numpy -matplotlib -scikit-learn -jupyter diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..423f47c --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,3 @@ +# Convenience wrapper for local development and CI parity. +# pyproject.toml remains the dependency source of truth. +-e .[dev,ml,notebook] diff --git a/requirements.txt b/requirements.txt index 4cf80ec..6f8363f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,111 +1,3 @@ -anyio==4.13.0 -appnope==0.1.4 -argon2-cffi==25.1.0 -argon2-cffi-bindings==25.1.0 -arrow==1.4.0 -asttokens==3.0.1 -async-lru==2.3.0 -attrs==26.1.0 -babel==2.18.0 -beautifulsoup4==4.14.3 -bleach==6.3.0 -certifi==2026.5.20 -cffi==2.0.0 -charset-normalizer==3.4.7 -comm==0.2.3 -contourpy==1.3.3 -cycler==0.12.1 -debugpy==1.8.20 -decorator==5.3.1 -defusedxml==0.7.1 -executing==2.2.1 -fastjsonschema==2.21.2 -fonttools==4.63.0 -fqdn==1.5.1 -h11==0.16.0 -httpcore==1.0.9 -httpx==0.28.1 -idna==3.16 -ipykernel==7.2.0 -ipython==9.13.0 -ipython_pygments_lexers==1.1.1 -ipywidgets==8.1.8 -isoduration==20.11.0 -jedi==0.20.0 -Jinja2==3.1.6 -joblib==1.5.3 -json5==0.14.0 -jsonpointer==3.1.1 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 -jupyter==1.1.1 -jupyter-console==6.6.3 -jupyter-events==0.12.1 -jupyter-lsp==2.3.1 -jupyter_client==8.8.0 -jupyter_core==5.9.1 -jupyter_server==2.18.2 -jupyter_server_terminals==0.5.4 -jupyterlab==4.5.7 -jupyterlab_pygments==0.3.0 -jupyterlab_server==2.28.0 -jupyterlab_widgets==3.0.16 -kiwisolver==1.5.0 -lark==1.3.1 -MarkupSafe==3.0.3 -matplotlib==3.10.9 -matplotlib-inline==0.2.2 -mistune==3.2.1 -nbclient==0.10.4 -nbconvert==7.17.1 -nbformat==5.10.4 -nest-asyncio==1.6.0 -notebook==7.5.6 -notebook_shim==0.2.4 -numpy==2.4.6 -packaging==26.2 -pandas==3.0.3 -pandocfilters==1.5.1 -parso==0.8.7 -pexpect==4.9.0 -pillow==12.2.0 -platformdirs==4.9.6 -prometheus_client==0.25.0 -prompt_toolkit==3.0.52 -psutil==7.2.2 -ptyprocess==0.7.0 -pure_eval==0.2.3 -pycparser==3.0 -Pygments==2.20.0 -pyparsing==3.3.2 -python-dateutil==2.9.0.post0 -python-json-logger==4.1.0 -PyYAML==6.0.3 -pyzmq==27.1.0 -referencing==0.37.0 -requests==2.34.2 -rfc3339-validator==0.1.4 -rfc3986-validator==0.1.1 -rfc3987-syntax==1.1.0 -rpds-py==0.30.0 -scikit-learn==1.8.0 -scipy==1.17.1 -Send2Trash==2.1.0 -setuptools==82.0.1 -six==1.17.0 -soupsieve==2.8.3 -stack-data==0.6.3 -terminado==0.18.1 -threadpoolctl==3.6.0 -tinycss2==1.4.0 -tornado==6.5.5 -traitlets==5.15.0 -typing_extensions==4.15.0 -tzdata==2026.2 -uri-template==1.3.0 -urllib3==2.7.0 -wcwidth==0.7.0 -webcolors==25.10.0 -webencodings==0.5.1 -websocket-client==1.9.0 -widgetsnbextension==4.0.15 +# Convenience wrapper for the complete local learning environment. +# pyproject.toml remains the dependency source of truth. +-e .[ml,notebook] diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..df7602e --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +from main import ( + build_baseline_data, + collect_environment, + run_baseline_check, + summarize_baseline_data, +) + + +def test_collect_environment_reports_direct_runtime_dependencies() -> None: + environment = collect_environment() + + assert set(environment) == { + "python", + "implementation", + "pandas", + "numpy", + "matplotlib", + } + assert all(environment.values()) + + +def test_build_baseline_data_returns_expected_shape() -> None: + frame = build_baseline_data() + + assert list(frame.columns) == ["city", "category", "value"] + assert frame.shape == (4, 3) + assert frame["value"].notna().all() + + +def test_summarize_baseline_data_returns_deterministic_results() -> None: + summary = summarize_baseline_data(build_baseline_data()) + + assert summary.to_dict(orient="records") == [ + {"category": "processing", "row_count": 2, "average_value": 74.0}, + {"category": "quality", "row_count": 2, "average_value": 90.0}, + ] + + +def test_summarize_baseline_data_rejects_missing_columns() -> None: + incomplete = pd.DataFrame({"category": ["quality"], "value": [90.0]}) + + with pytest.raises(ValueError, match="Missing required columns: city"): + summarize_baseline_data(incomplete) + + +def test_summarize_baseline_data_rejects_missing_values() -> None: + frame = build_baseline_data() + frame.loc[0, "value"] = None + + with pytest.raises(ValueError, match="contains missing values"): + summarize_baseline_data(frame) + + +def test_run_baseline_check_succeeds() -> None: + summary = run_baseline_check() + + assert summary["row_count"].sum() == 4 + assert len(summary) == 2 diff --git a/tests/test_notebook_hygiene.py b/tests/test_notebook_hygiene.py new file mode 100644 index 0000000..afa9851 --- /dev/null +++ b/tests/test_notebook_hygiene.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import json +from pathlib import Path + +NOTEBOOK_PATH = Path(__file__).resolve().parents[1] / "dataspell_test.ipynb" + + +def load_notebook() -> dict[str, object]: + return json.loads(NOTEBOOK_PATH.read_text(encoding="utf-8")) + + +def test_notebook_has_no_stored_execution_output() -> None: + notebook = load_notebook() + + for cell in notebook["cells"]: + if cell["cell_type"] == "code": + assert cell["execution_count"] is None + assert cell["outputs"] == [] + assert "ExecuteTime" not in cell.get("metadata", {}) + + +def test_notebook_uses_python_3_12_metadata() -> None: + notebook = load_notebook() + metadata = notebook["metadata"] + + assert metadata["kernelspec"]["display_name"] == "Python 3.12" + assert metadata["kernelspec"]["name"] == "python3" + assert metadata["language_info"]["version"] == "3.12" + assert metadata["language_info"]["pygments_lexer"] == "ipython3" + + +def test_notebook_contains_no_local_absolute_paths() -> None: + serialized = NOTEBOOK_PATH.read_text(encoding="utf-8") + + assert "/Users/" not in serialized + assert "C:\\Users\\" not in serialized diff --git a/tests/test_optional_ml.py b/tests/test_optional_ml.py new file mode 100644 index 0000000..5456abe --- /dev/null +++ b/tests/test_optional_ml.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from examples.optional.logistic_regression_basics import ( + build_sample_data, + predict_pass_probability, +) + + +def test_optional_ml_example_returns_probability() -> None: + sample_data = build_sample_data() + probability = predict_pass_probability( + sample_data, + hours_studied=5, + practice_score=70, + ) + + assert sample_data.shape == (8, 3) + assert 0.0 < probability < 1.0