From d94fcc68a6970b80a4697f8b04bdac51c12b08ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Hidalgo?= <47150974+IvanHCenalmor@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:24:20 +0200 Subject: [PATCH] Template 0.1.20: lock files, no lost workflow pushes, fresh JupyterLab launch Workflow race (update_on_notebook_change.yml / update_on_src_change.yaml): - Both workflows commit inline and, when the push is rejected because the branch moved, rebase onto it (git rebase --autostash) and push again (5 attempts). stefanzweifel/git-auto-commit-action is no longer used. - The revert of a user commit runs only when the conda test failed, on top of the latest branch, with -m 1 only for a merge commit. - Each workflow has its own concurrency group (a shared group kept only one waiting run, so a third push cancelled the other workflow's run). Lock files: - New .tools/python/lock_requirements.py (uv==0.13.3, uv pip compile --universal) writes requirements*.lock.txt next to each requirements file: every package with its exact version, versions only. Conda packages of environment.yaml are constraints, not emitted. A fingerprint header keeps the lock unchanged until its inputs change. A failure is a warning and removes the stale lock. - The notebook workflow runs it after the merge; test_conda_environment.sh, post_install.sh / .bat and bump_constructor.py use the lock when present. Launcher: - launch_jupyter.py asks to press Enter when JupyterLab stops (--no-wait to skip) and gives JupyterLab a new, empty workspaces_dir at every launch, so only the app's Welcome notebook opens. - notebook_launcher.json: the Windows command ends with "& exit" so the cmd window closes after Enter. Migration v0_1_19_to_v0_1_20.py (surgical, idempotent, skips customised files), tests (workflow steps run with real git against a moving origin, lock script, bump_constructor, launcher, migration, chain) and the design note .tools/docs/migration_0_1_20.md. Co-Authored-By: Claude Opus 5.5 --- .../workflows/update_on_notebook_change.yml | 75 +- .github/workflows/update_on_src_change.yaml | 45 +- .template_sync/manifest.yaml | 5 +- .../migrations/v0_1_19_to_v0_1_20.py | 356 +++++++++ .../workflows/update_on_notebook_change.yml | 211 ++++++ .../workflows/update_on_src_change.yaml | 66 ++ .../.tools/bash/test_conda_environment.sh | 99 +++ .../.tools/python/lock_requirements.py | 243 +++++++ .../app/python_scripts/launch_jupyter.py | 324 +++++++++ .../workflows/update_on_notebook_change.yml | 154 ++++ .../workflows/update_on_src_change.yaml | 41 ++ .../.tools/bash/test_conda_environment.sh | 86 +++ .../app/python_scripts/launch_jupyter.py | 286 ++++++++ .tools/bash/test_conda_environment.sh | 27 +- .tools/docs/README.md | 1 + .tools/docs/migration_0_1_20.md | 49 ++ .tools/docs/notebook_requirements.md | 12 + .tools/docs/notebook_usage.md | 2 +- .tools/python/bump_constructor.py | 32 + .tools/python/lock_requirements.py | 243 +++++++ .tools/requirements.txt | 4 +- app/bash_bat_scripts/post_install.bat | 10 +- app/bash_bat_scripts/post_install.sh | 10 +- app/menuinst/notebook_launcher.json | 9 +- app/python_scripts/launch_jupyter.py | 40 +- tests/_v0_1_20.py | 41 ++ tests/test_template_sync_0_1_17.py | 6 +- tests/test_template_sync_0_1_18.py | 11 +- tests/test_template_sync_0_1_19.py | 2 +- tests/test_template_sync_0_1_20.py | 686 ++++++++++++++++++ 30 files changed, 3135 insertions(+), 41 deletions(-) create mode 100644 .template_sync/migrations/v0_1_19_to_v0_1_20.py create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_notebook_change.yml create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_src_change.yaml create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/.tools/bash/test_conda_environment.sh create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/.tools/python/lock_requirements.py create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/app/python_scripts/launch_jupyter.py create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_notebook_change.yml create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_src_change.yaml create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/previous/.tools/bash/test_conda_environment.sh create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/previous/app/python_scripts/launch_jupyter.py create mode 100644 .tools/docs/migration_0_1_20.md create mode 100644 .tools/python/lock_requirements.py create mode 100644 tests/_v0_1_20.py create mode 100644 tests/test_template_sync_0_1_20.py diff --git a/.github/workflows/update_on_notebook_change.yml b/.github/workflows/update_on_notebook_change.yml index c5fae88..4a8c03b 100644 --- a/.github/workflows/update_on_notebook_change.yml +++ b/.github/workflows/update_on_notebook_change.yml @@ -9,7 +9,9 @@ on: - 'notebooks/**/requirements.yaml' workflow_dispatch: -# This enables to run a single workflow at a time +# One run of this workflow at a time per branch. The src workflow has its own group: the two can run at the same time, +# because each one rebases its commit onto the latest branch before it pushes (a shared group kept only one waiting run, so a +# third push cancelled the waiting run of the other workflow). concurrency: group: notebooks-global-lock-${{ github.ref }} cancel-in-progress: false @@ -98,6 +100,12 @@ jobs: .tools/python/merge_requirements.py \ python .tools/python/merge_requirements.py --source-dir notebooks --output requirements.txt + # requirements*.txt -> requirements*.lock.txt: every package the installer installs with pip, with its exact version (uv, valid on + # Windows, macOS and Linux). A lock file is rewritten only when its requirements file, Python or JupyterLab version changed. + - name: Lock requirements + run: | + bash .tools/bash/run_with_context.sh artifacts/error_log_document.log "lock_requirements.py" .tools/python/lock_requirements.py python .tools/python/lock_requirements.py + - name: Check for changes id: check-changes run: | @@ -138,17 +146,66 @@ jobs: - name: Commit updated files id: auto-commit if: steps.check-changes.outputs.has_changes == 'true' && steps.test-conda-env.outcome == 'success' - uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5 - with: - commit_message: GitHub Action - Updated files because of notebook and/or requirements changes - file_pattern: requirements*.txt environment.yaml notebooks/* setup.py + env: + BRANCH: ${{ github.ref_name }} + COMMIT_MESSAGE: GitHub Action - Updated files because of notebook and/or requirements changes + # The run checked out the commit that triggered it (the scripts above compare it with its parent); the branch may have moved + # since (another workflow committed, or a new push). A rejected push is therefore retried after a rebase onto the latest branch. + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git ls-files -z --modified --deleted --others --exclude-standard -- 'requirements*.txt' environment.yaml 'notebooks/*' setup.py | xargs -0 -r git add -A -- + if git diff --cached --quiet; then + echo "No changes to commit." + exit 0 + fi + git commit -m "$COMMIT_MESSAGE" + for attempt in 1 2 3 4 5; do + if git push origin "HEAD:refs/heads/$BRANCH"; then + exit 0 + fi + echo "The push was rejected (attempt $attempt of 5): rebasing onto the latest $BRANCH." + sleep $((attempt * 5)) + git fetch origin "$BRANCH" + if ! git rebase --autostash "origin/$BRANCH"; then + git rebase --abort || true + echo "::error::The commit of this run conflicts with the latest $BRANCH; run this workflow again (Actions > Run workflow)." + exit 1 + fi + done + echo "::error::The push was rejected 5 times; run this workflow again (Actions > Run workflow)." + exit 1 + # Only when the merged requirements could not be installed: a failure of any other step (a script, the push) never reverts a commit. - name: Revert user commit that broke requirements - if: failure() && github.event_name == 'push' + if: failure() && github.event_name == 'push' && steps.test-conda-env.outcome == 'failure' + env: + BRANCH: ${{ github.ref_name }} + BAD_COMMIT: ${{ github.sha }} run: | git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git reset --hard git clean -fd - git revert "${{ github.sha }}" -m 1 --no-edit - git push origin HEAD:${{ github.ref_name }} + for attempt in 1 2 3 4 5; do + git fetch origin "$BRANCH" + git checkout -B "$BRANCH" "origin/$BRANCH" + # A merge commit is reverted against its first parent (-m 1); any other commit is reverted as it is. + if [ "$(git rev-list --parents -n 1 "$BAD_COMMIT" | wc -w)" -gt 2 ]; then + REVERT_ARGS=(-m 1) + else + REVERT_ARGS=() + fi + if ! git revert --no-edit "${REVERT_ARGS[@]}" "$BAD_COMMIT"; then + git revert --abort || true + echo "::error::$BAD_COMMIT broke the requirements but could not be reverted automatically (later commits change the same lines): revert it by hand." + exit 1 + fi + if git push origin "HEAD:refs/heads/$BRANCH"; then + exit 0 + fi + echo "The push of the revert was rejected (attempt $attempt of 5): retrying on the latest $BRANCH." + sleep $((attempt * 5)) + done + echo "::error::The revert of $BAD_COMMIT could not be pushed: revert it by hand." + exit 1 diff --git a/.github/workflows/update_on_src_change.yaml b/.github/workflows/update_on_src_change.yaml index 76b90f0..2ab8870 100644 --- a/.github/workflows/update_on_src_change.yaml +++ b/.github/workflows/update_on_src_change.yaml @@ -7,9 +7,11 @@ on: - 'src/**' workflow_dispatch: -# This enables to run a single workflow at a time +# One run of this workflow at a time per branch. The notebook workflow has its own group: the two can run at the same time, +# because each one rebases its commit onto the latest branch before it pushes (a shared group kept only one waiting run, so a +# third push cancelled the waiting run of the other workflow). concurrency: - group: notebooks-global-lock-${{ github.ref }} + group: src-change-lock-${{ github.ref }} cancel-in-progress: false jobs: @@ -18,13 +20,13 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - + steps: - name: Checkout repository uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 # Fetch all history for all branches and tags - + - name: Update src change file run: | cat > .tools/meta/src_change.yaml < Run workflow)." + exit 1 + fi + done + echo "::error::The push was rejected 5 times; run this workflow again (Actions > Run workflow)." + exit 1 diff --git a/.template_sync/manifest.yaml b/.template_sync/manifest.yaml index ac8d395..43b057a 100644 --- a/.template_sync/manifest.yaml +++ b/.template_sync/manifest.yaml @@ -1,4 +1,4 @@ -template_version: 0.1.19 +template_version: 0.1.20 migrations: - from: 0.0.0 to: 0.0.1 @@ -63,3 +63,6 @@ migrations: - from: 0.1.18 to: 0.1.19 script: migrations/v0_1_18_to_v0_1_19.py + - from: 0.1.19 + to: 0.1.20 + script: migrations/v0_1_19_to_v0_1_20.py diff --git a/.template_sync/migrations/v0_1_19_to_v0_1_20.py b/.template_sync/migrations/v0_1_19_to_v0_1_20.py new file mode 100644 index 0000000..b077854 --- /dev/null +++ b/.template_sync/migrations/v0_1_19_to_v0_1_20.py @@ -0,0 +1,356 @@ +"""Template 0.1.19 -> 0.1.20: no lost pushes between the two workflows, lock files for the pip requirements, a launcher that starts fresh. + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_20.md + +What this changes in an app repository (every step is independent, idempotent and never deletes anything): + +1. Workflows. update_on_notebook_change.yml and update_on_src_change.yaml are replaced by the 0.1.20 files ONLY when the app's file is still + identical to the 0.1.19 template file (line endings are ignored when comparing and kept when writing). The new files rebase their + commit onto the latest branch and retry a rejected push, have one concurrency group each, revert a commit only when the conda test + failed (a merge commit with -m 1, any other commit as it is), and the notebook workflow runs lock_requirements.py. +2. Lock files. .tools/python/lock_requirements.py is created (requirements*.txt -> requirements*.lock.txt with uv), `uv==` is added to + .tools/requirements.txt, .tools/bash/test_conda_environment.sh (replaced when identical to 0.1.19) tests the lock file, + post_install.sh / .bat install the lock file when the installer bundles one (known blocks only), and bump_constructor.py gets the + function that bundles the lock files (inserted at two known lines). +3. Launcher. app/python_scripts/launch_jupyter.py (replaced when identical to the rendered 0.1.19 file) asks to press Enter when JupyterLab + stops and gives JupyterLab an empty workspace folder at every launch (only the Welcome notebook opens). app/menuinst/notebook_launcher.json + gets a Windows command that ends with `& exit`, so the terminal window closes after Enter (cmd /K kept it open). +4. Notebook usage guide. The sentence on stopping JupyterLab in .tools/docs/notebook_usage.md is replaced when it is still the 0.1.19 text. + +Rules this migration keeps: surgical (only known lines or known files), idempotent (a second run reports no changes), never raises for a +customised file (the sync pull request must still be created), the line endings of each file are kept, nothing is deleted. +""" +from __future__ import annotations + +import importlib.util +import json +import re +from pathlib import Path +from typing import Any + + +RESOURCE_ROOT = Path(__file__).resolve().parents[1] / "resources" / "v0_1_19_to_v0_1_20" +PREVIOUS_ROOT = RESOURCE_ROOT / "previous" +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_20.md" +TAG = "[migration 0.1.20]" + +NOTEBOOK_WORKFLOW = Path(".github/workflows/update_on_notebook_change.yml") +SRC_WORKFLOW = Path(".github/workflows/update_on_src_change.yaml") +TEST_CONDA = Path(".tools/bash/test_conda_environment.sh") +LAUNCHER = Path("app/python_scripts/launch_jupyter.py") +LOCK_SCRIPT = Path(".tools/python/lock_requirements.py") +TOOLS_REQUIREMENTS = Path(".tools/requirements.txt") +POST_INSTALL_SH = Path("app/bash_bat_scripts/post_install.sh") +POST_INSTALL_BAT = Path("app/bash_bat_scripts/post_install.bat") +BUMP_CONSTRUCTOR = Path(".tools/python/bump_constructor.py") +MENU = Path("app/menuinst/notebook_launcher.json") +USAGE_DOC = Path(".tools/docs/notebook_usage.md") + +# Files replaced as a whole when the app's file is still the (rendered) 0.1.19 template file. +REPLACED_FILES = (NOTEBOOK_WORKFLOW, SRC_WORKFLOW, TEST_CONDA, LAUNCHER) +# Resource files that are copies of the 0.1.20 template files (a test checks they stay identical to the template). +MANAGED_FILES = (*REPLACED_FILES, LOCK_SCRIPT) + +UV_PIN = "uv==0.13.3" +UV_LINE = re.compile(r"^\s*uv\s*(?:[=<>!~;\[]|$)", re.IGNORECASE) +UV_LINES = "# lock_requirements.py (requirements*.txt -> requirements*.lock.txt)\n" + UV_PIN + "\n" + +# post_install.sh: (0.1.19 text, 0.1.20 text), LF. +SH_BLOCKS = ( + ( + 'echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE"\n' + '"$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1\n', + "# A lock file (requirements*.lock.txt next to the requirements file: every package with its exact version, written by\n" + "# .tools/python/lock_requirements.py) is installed instead of the requirements file when the installer bundles one.\n" + "locked_requirements() {\n" + ' if [ -f "${1%.txt}.lock.txt" ]; then echo "${1%.txt}.lock.txt"; else echo "$1"; fi\n' + "}\n" + 'SELECTED_REQUIREMENTS="$(locked_requirements "$SELECTED_REQUIREMENTS")"\n' + 'echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE"\n' + '"$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1\n', + ), + ( + ' "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-macos.txt" >> "$LOG_FILE" 2>&1\n', + ' "$PYTHON_EXE" -m pip install -r "$(locked_requirements "$PROJECT_ROOT/requirements-macos.txt")" >> "$LOG_FILE" 2>&1\n', + ), + ( + ' "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-linux.txt" >> "$LOG_FILE" 2>&1\n', + ' "$PYTHON_EXE" -m pip install -r "$(locked_requirements "$PROJECT_ROOT/requirements-linux.txt")" >> "$LOG_FILE" 2>&1\n', + ), +) +SH_DONE = "locked_requirements()" + +BAT_BLOCKS = ( + ( + 'echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%"\n' + '"%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1\n', + "REM A lock file (requirements*.lock.txt next to the requirements file: every package with its exact version, written by\n" + "REM .tools/python/lock_requirements.py) is installed instead of the requirements file when the installer bundles one.\n" + 'SET "SELECTED_LOCK=%SELECTED_REQUIREMENTS:~0,-4%.lock.txt"\n' + 'IF EXIST "%SELECTED_LOCK%" SET "SELECTED_REQUIREMENTS=%SELECTED_LOCK%"\n' + 'echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%"\n' + '"%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1\n', + ), + ( + 'IF EXIST "%PROJECT_ROOT%\\requirements-windows.txt" (\n' + ' echo Installing Windows-specific requirements. >> "%LOG_FILE%"\n' + ' "%PYTHON_EXE%" -m pip install -r "%PROJECT_ROOT%\\requirements-windows.txt" >> "%LOG_FILE%" 2>&1\n', + 'SET "WINDOWS_REQUIREMENTS=%PROJECT_ROOT%\\requirements-windows.txt"\n' + 'IF EXIST "%PROJECT_ROOT%\\requirements-windows.lock.txt" SET "WINDOWS_REQUIREMENTS=%PROJECT_ROOT%\\requirements-windows.lock.txt"\n' + 'IF EXIST "%PROJECT_ROOT%\\requirements-windows.txt" (\n' + ' echo Installing Windows-specific requirements from "%WINDOWS_REQUIREMENTS%". >> "%LOG_FILE%"\n' + ' "%PYTHON_EXE%" -m pip install -r "%WINDOWS_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1\n', + ), +) +BAT_DONE = "SELECTED_LOCK=" + +BUMP_FUNCTION_ANCHOR = "def ensure_extra_files(" +BUMP_CALL_OLD = " ensure_requirements_in_extra_files(data)\n" +BUMP_CALL_NEW = " ensure_requirements_in_extra_files(data)\n ensure_lock_files_in_extra_files(data)\n" +BUMP_DONE = "def ensure_lock_files_in_extra_files(" +BUMP_NEEDS = ("def extract_project_folder(", "def debug(", "from pathlib import Path") + +WIN_EXIT = ["&", "exit"] + +USAGE_OLD = ( + "When you have, click on the terminal window and press `Ctrl+C` to stop the Jupyter Lab session and close the terminal." +) +USAGE_NEW = ( + "When you have, stop the Jupyter Lab session with **File > Shut Down** in Jupyter Lab (or click on the terminal window and press `Ctrl+C`). " + "The terminal then asks you to press `Enter`, and closes." +) + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def _load_helpers() -> Any: + """Reuse the helpers of the 0.1.10 migration (token replacement, newline detection, file IO).""" + path = Path(__file__).resolve().parent / "v0_1_9_to_v0_1_10.py" + spec = importlib.util.spec_from_file_location("template_sync_helpers_0_1_20", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _resource(root: Path, relative: Path) -> str: + source = root / relative + if not source.is_file(): + raise FileNotFoundError(f"Template resource not found: {source}") + return source.read_bytes().decode("utf-8").replace("\r\n", "\n") + + +def _render(helpers: Any, text: str, replacements: dict[str, str]) -> str: + return str(helpers.render_template(text, replacements, "\n")) + + +def _write_lf(helpers: Any, path: Path, text_lf: str, newline: str) -> None: + helpers.write_text(path, text_lf.replace("\n", newline)) + + +# -------------------------------------------------------------------------------------------------------------------------------- +# Whole files (workflows, conda test, launcher) +# -------------------------------------------------------------------------------------------------------------------------------- +def _replace_file(helpers: Any, repo_root: Path, relative: Path, replacements: dict[str, str]) -> str: + path = repo_root / relative + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + current_lf = current.replace("\r\n", "\n") + new = _render(helpers, _resource(RESOURCE_ROOT, relative), replacements) + if current_lf == new: + return "already the 0.1.20 template file: unchanged" + if current_lf != _render(helpers, _resource(PREVIOUS_ROOT, relative), replacements): + return f"SKIPPED: customised (not the 0.1.19 template file), so it was left unchanged; apply the 0.1.20 changes by hand, see {DOCS_URL}" + _write_lf(helpers, path, new, helpers.detect_newline(current)) + return "updated: replaced by the 0.1.20 template file" + + +# -------------------------------------------------------------------------------------------------------------------------------- +# Lock files +# -------------------------------------------------------------------------------------------------------------------------------- +def _create_lock_script(helpers: Any, repo_root: Path) -> str: + path = repo_root / LOCK_SCRIPT + new = _resource(RESOURCE_ROOT, LOCK_SCRIPT) + if path.is_file(): + if helpers.read_text(path).replace("\r\n", "\n") == new: + return "already the 0.1.20 template file: unchanged" + return "SKIPPED: the repository has its own lock_requirements.py, left as it is" + if not path.parent.is_dir(): + return "SKIPPED: the repository has no .tools/python folder (nothing was created)" + with path.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(new) + return "updated: created" + + +def _add_uv(helpers: Any, repo_root: Path) -> str: + path = repo_root / TOOLS_REQUIREMENTS + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + if any(UV_LINE.match(line) for line in current.splitlines()): + return "already installs uv: unchanged" + newline = helpers.detect_newline(current) + text = current if not current or current.endswith(("\n", "\r")) else current + newline + helpers.write_text(path, text + UV_LINES.replace("\n", newline)) + return f"updated: added {UV_PIN}" + + +def _replace_blocks(helpers: Any, path: Path, blocks: tuple[tuple[str, str], ...], done: str, what: str) -> str: + current = helpers.read_text(path) + lf = current.replace("\r\n", "\n") + if done in lf: + return f"already {what}: unchanged" + missing = [old.splitlines()[0].strip() for old, _ in blocks if lf.count(old) != 1] + if missing: + shown = "; ".join(missing) + return f"SKIPPED: customised, the known lines were not found once ({shown}); left unchanged, apply the change by hand, see {DOCS_URL}" + for old, new in blocks: + lf = lf.replace(old, new) + _write_lf(helpers, path, lf, helpers.detect_newline(current)) + return f"updated: {what}" + + +def _bump_constructor(helpers: Any, repo_root: Path) -> str: + path = repo_root / BUMP_CONSTRUCTOR + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + lf = current.replace("\r\n", "\n") + if BUMP_DONE in lf: + return "already bundles the lock files: unchanged" + if lf.count(BUMP_FUNCTION_ANCHOR) != 1 or lf.count(BUMP_CALL_OLD) != 1 or not all(need in lf for need in BUMP_NEEDS): + return f"SKIPPED: customised (the known lines were not found once), left unchanged; add ensure_lock_files_in_extra_files by hand, see {DOCS_URL}" + lf = lf.replace(BUMP_FUNCTION_ANCHOR, BUMP_SNIPPET + BUMP_FUNCTION_ANCHOR, 1).replace(BUMP_CALL_OLD, BUMP_CALL_NEW, 1) + _write_lf(helpers, path, lf, helpers.detect_newline(current)) + return "updated: bundles the lock files (ensure_lock_files_in_extra_files)" + + +# The LOCK_FILES tuple and ensure_lock_files_in_extra_files of the 0.1.20 bump_constructor.py, with the blank lines before the next function +# (a test checks that the template script holds this text once). +BUMP_SNIPPET = '''LOCK_FILES = ( + "requirements.lock.txt", + "requirements_gpu.lock.txt", + "requirements-linux.lock.txt", + "requirements-windows.lock.txt", + "requirements-macos.lock.txt", +) + + +def ensure_lock_files_in_extra_files(construct_data: dict) -> None: + """Bundle the lock files (requirements*.lock.txt, written by lock_requirements.py) next to their requirements files. + + post_install installs a lock file instead of its requirements file when the installer bundles one. A mapping whose lock file is gone + (the requirements could not be locked) is removed, because constructor fails on a missing file. + """ + extra_files = construct_data.setdefault("extra_files", []) + project_folder = extract_project_folder(extra_files) + for lock_file in LOCK_FILES: + mapped = [item for item in extra_files if isinstance(item, dict) and lock_file in item] + if Path(lock_file).exists(): + if not mapped: + extra_files.append({lock_file: f"{project_folder}/{lock_file}"}) + debug(f"Added mapping for {lock_file}") + else: + debug(f"{lock_file} mapping already present") + elif mapped: + for item in mapped: + extra_files.remove(item) + debug(f"Removed the mapping of {lock_file}: the file does not exist (the installer installs the requirements file instead)") + + +''' + + +# -------------------------------------------------------------------------------------------------------------------------------- +# Launcher shortcut (Windows: close the terminal after Enter) +# -------------------------------------------------------------------------------------------------------------------------------- +def _menu(helpers: Any, repo_root: Path) -> str: + path = repo_root / MENU + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + lf = current.replace("\r\n", "\n") + try: + data = json.loads(lf) + except json.JSONDecodeError as exc: + return f"SKIPPED: not valid JSON ({exc}), left unchanged" + items = data.get("menu_items") if isinstance(data, dict) else None + if not isinstance(items, list) or len(items) != 1 or not isinstance(items[0], dict): + return f"SKIPPED: not the template shape (one menu item), left unchanged; see {DOCS_URL}" + item = items[0] + command = item.get("command") + win = (item.get("platforms") or {}).get("win") + if not isinstance(command, list) or not isinstance(win, dict) or not item.get("terminal"): + return f"SKIPPED: not the template shape (a terminal command with a win platform), left unchanged; see {DOCS_URL}" + if "command" in win: + if win["command"] == command + WIN_EXIT: + return "already closes the Windows terminal after Enter: unchanged" + return "SKIPPED: has its own Windows command, left unchanged; end it with \"&\", \"exit\" to close the terminal after Enter" + if json.dumps(data, indent=2, ensure_ascii=False) + "\n" != lf: + return f"SKIPPED: formatted differently from the template, left unchanged; add the Windows command by hand, see {DOCS_URL}" + win["command"] = command + WIN_EXIT + _write_lf(helpers, path, json.dumps(data, indent=2, ensure_ascii=False) + "\n", helpers.detect_newline(current)) + return "updated: the Windows command ends with `& exit` (the terminal closes after Enter)" + + +def _usage_doc(helpers: Any, repo_root: Path) -> str: + path = repo_root / USAGE_DOC + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + if USAGE_NEW in current: + return "unchanged" + if current.count(USAGE_OLD) != 1: + return "SKIPPED: the 0.1.19 sentence on stopping Jupyter Lab was not found (customised), left unchanged" + helpers.write_text(path, current.replace(USAGE_OLD, USAGE_NEW)) + return "updated: how to stop Jupyter Lab (File > Shut Down, then Enter)" + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """Replace the two workflows and the launcher, add the lock files (idempotent, surgical, never raises for a customised file).""" + _ = context + helpers = _load_helpers() + try: + replacements = helpers.load_replacements(repo_root) + except (OSError, ValueError) as exc: + log(f"could not read the app name from construct.yaml ({exc}): the launcher is compared without app names.") + replacements = {} + + changed = 0 + + def report(name: Path, step: Any) -> None: + nonlocal changed + try: + status = step() + except (OSError, ValueError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + if status.startswith("updated"): + changed += 1 + log(f"{name}: {status}") + + for relative in REPLACED_FILES: + report(relative, lambda relative=relative: _replace_file(helpers, repo_root, relative, replacements)) + report(LOCK_SCRIPT, lambda: _create_lock_script(helpers, repo_root)) + report(TOOLS_REQUIREMENTS, lambda: _add_uv(helpers, repo_root)) + for relative, blocks, done in ((POST_INSTALL_SH, SH_BLOCKS, SH_DONE), (POST_INSTALL_BAT, BAT_BLOCKS, BAT_DONE)): + if not (repo_root / relative).is_file(): + log(f"{relative}: not present in this repository (nothing was created)") + continue + report(relative, lambda relative=relative, blocks=blocks, done=done: _replace_blocks( + helpers, repo_root / relative, blocks, done, "installs the lock file when the installer bundles one" + )) + report(BUMP_CONSTRUCTOR, lambda: _bump_constructor(helpers, repo_root)) + report(MENU, lambda: _menu(helpers, repo_root)) + report(USAGE_DOC, lambda: _usage_doc(helpers, repo_root)) + + if changed: + log( + f"done: {changed} file(s) changed. Review the pull request before merging. The lock files are written by the next run of " + "'Updates on notebook change' (Actions > Run workflow); the launcher changes reach users with the next release of the app." + ) + else: + log("done: no repository changes were required.") diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_notebook_change.yml b/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_notebook_change.yml new file mode 100644 index 0000000..4a8c03b --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_notebook_change.yml @@ -0,0 +1,211 @@ +name: Updates on notebook change + +on: + push: + branches: [ main, master ] + paths: + - 'notebooks/**/*.ipynb' + - 'notebooks/notebook_latest_versions.yaml' + - 'notebooks/**/requirements.yaml' + workflow_dispatch: + +# One run of this workflow at a time per branch. The src workflow has its own group: the two can run at the same time, +# because each one rebases its commit onto the latest branch before it pushes (a shared group kept only one waiting run, so a +# third push cancelled the waiting run of the other workflow). +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon notebook change, update manifests and changelogs + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@2d756ea4c53f7f6b397767d8723b3a10a9f35bf2 # v44 + with: + separator: "," + files: | + notebooks/**/*.ipynb + notebooks/notebook_latest_versions.yaml + notebooks/**/requirements.yaml + + - name: Syncronize notebook versions + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "sync_notebook_versions.py" \ + .tools/python/sync_notebook_versions.py \ + python .tools/python/sync_notebook_versions.py + + - name: Update CHANGELOG + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_changelog.py" \ + .tools/python/update_changelog.py \ + python .tools/python/update_changelog.py notebooks/notebook_latest_versions.yaml + + - name: Update last commit in notebook_latest_versions.yaml + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_version_last_commit.py" \ + .tools/python/update_version_last_commit.py \ + python .tools/python/update_version_last_commit.py notebooks/notebook_latest_versions.yaml + + - name: Hide code cells in notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "hide_code_cells.py" \ + app/python_scripts/hide_code_cells.py \ + python app/python_scripts/hide_code_cells.py . + + - name: Index notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_notebook_README.py" \ + .tools/python/bump_notebook_README.py \ + python .tools/python/bump_notebook_README.py + + - name: Run merge script + if: github.event_name == 'workflow_dispatch' || contains(steps.changed-files.outputs.all_changed_files, 'requirements.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "merge_requirements.py" \ + .tools/python/merge_requirements.py \ + python .tools/python/merge_requirements.py --source-dir notebooks --output requirements.txt + + # requirements*.txt -> requirements*.lock.txt: every package the installer installs with pip, with its exact version (uv, valid on + # Windows, macOS and Linux). A lock file is rewritten only when its requirements file, Python or JupyterLab version changed. + - name: Lock requirements + run: | + bash .tools/bash/run_with_context.sh artifacts/error_log_document.log "lock_requirements.py" .tools/python/lock_requirements.py python .tools/python/lock_requirements.py + + - name: Check for changes + id: check-changes + run: | + if [ -z "$(git status --porcelain)" ]; then + echo "has_changes=false" >> $GITHUB_OUTPUT + else + echo "has_changes=true" >> $GITHUB_OUTPUT + fi + + - name: Set up Miniconda for requirements test + if: steps.check-changes.outputs.has_changes == 'true' + uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + auto-activate: true + miniforge-version: latest + use-mamba: true + + - name: Verify merged requirements inside conda + id: test-conda-env + if: steps.check-changes.outputs.has_changes == 'true' + env: + SKIP_CONDA_ENV_TEST: ${{ vars.SKIP_CONDA_ENV_TEST }} + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "test_conda_environment.sh" \ + .tools/bash/test_conda_environment.sh \ + bash .tools/bash/test_conda_environment.sh + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: notebook-debug-bundle + path: artifacts/error_log_document.log + archive: false + + - name: Commit updated files + id: auto-commit + if: steps.check-changes.outputs.has_changes == 'true' && steps.test-conda-env.outcome == 'success' + env: + BRANCH: ${{ github.ref_name }} + COMMIT_MESSAGE: GitHub Action - Updated files because of notebook and/or requirements changes + # The run checked out the commit that triggered it (the scripts above compare it with its parent); the branch may have moved + # since (another workflow committed, or a new push). A rejected push is therefore retried after a rebase onto the latest branch. + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git ls-files -z --modified --deleted --others --exclude-standard -- 'requirements*.txt' environment.yaml 'notebooks/*' setup.py | xargs -0 -r git add -A -- + if git diff --cached --quiet; then + echo "No changes to commit." + exit 0 + fi + git commit -m "$COMMIT_MESSAGE" + for attempt in 1 2 3 4 5; do + if git push origin "HEAD:refs/heads/$BRANCH"; then + exit 0 + fi + echo "The push was rejected (attempt $attempt of 5): rebasing onto the latest $BRANCH." + sleep $((attempt * 5)) + git fetch origin "$BRANCH" + if ! git rebase --autostash "origin/$BRANCH"; then + git rebase --abort || true + echo "::error::The commit of this run conflicts with the latest $BRANCH; run this workflow again (Actions > Run workflow)." + exit 1 + fi + done + echo "::error::The push was rejected 5 times; run this workflow again (Actions > Run workflow)." + exit 1 + + # Only when the merged requirements could not be installed: a failure of any other step (a script, the push) never reverts a commit. + - name: Revert user commit that broke requirements + if: failure() && github.event_name == 'push' && steps.test-conda-env.outcome == 'failure' + env: + BRANCH: ${{ github.ref_name }} + BAD_COMMIT: ${{ github.sha }} + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git reset --hard + git clean -fd + for attempt in 1 2 3 4 5; do + git fetch origin "$BRANCH" + git checkout -B "$BRANCH" "origin/$BRANCH" + # A merge commit is reverted against its first parent (-m 1); any other commit is reverted as it is. + if [ "$(git rev-list --parents -n 1 "$BAD_COMMIT" | wc -w)" -gt 2 ]; then + REVERT_ARGS=(-m 1) + else + REVERT_ARGS=() + fi + if ! git revert --no-edit "${REVERT_ARGS[@]}" "$BAD_COMMIT"; then + git revert --abort || true + echo "::error::$BAD_COMMIT broke the requirements but could not be reverted automatically (later commits change the same lines): revert it by hand." + exit 1 + fi + if git push origin "HEAD:refs/heads/$BRANCH"; then + exit 0 + fi + echo "The push of the revert was rejected (attempt $attempt of 5): retrying on the latest $BRANCH." + sleep $((attempt * 5)) + done + echo "::error::The revert of $BAD_COMMIT could not be pushed: revert it by hand." + exit 1 diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_src_change.yaml b/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_src_change.yaml new file mode 100644 index 0000000..2ab8870 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_src_change.yaml @@ -0,0 +1,66 @@ +name: Updates on external code change + +on: + push: + branches: [ main, master ] + paths: + - 'src/**' + workflow_dispatch: + +# One run of this workflow at a time per branch. The notebook workflow has its own group: the two can run at the same time, +# because each one rebases its commit onto the latest branch before it pushes (a shared group kept only one waiting run, so a +# third push cancelled the waiting run of the other workflow). +concurrency: + group: src-change-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon external code update the code version + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Update src change file + run: | + cat > .tools/meta/src_change.yaml < Run workflow)." + exit 1 + fi + done + echo "::error::The push was rejected 5 times; run this workflow again (Actions > Run workflow)." + exit 1 diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/bash/test_conda_environment.sh b/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/bash/test_conda_environment.sh new file mode 100644 index 0000000..4742aca --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/bash/test_conda_environment.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Test script to validate conda environment creation and pip requirements installation +# This script creates a test conda environment from environment.yaml and installs pip requirements + +set -e # Exit on any error +set -u # Exit on undefined variables + +is_truthy() { + case "${1:-}" in + 1|true|TRUE|True|yes|YES|Yes|on|ON|On) + return 0 + ;; + *) + return 1 + ;; + esac +} + +if is_truthy "${SKIP_CONDA_ENV_TEST:-}"; then + echo "SKIP_CONDA_ENV_TEST is enabled; skipping conda environment validation." + exit 0 +fi + +echo "==========================================" +echo "Testing Conda Environment Creation" +echo "==========================================" + +# Define test environment name +TEST_ENV_NAME="potential-potato-test-env" + +# Clean up any existing test environment +echo "Cleaning up any existing test environment..." +mamba env remove -n "$TEST_ENV_NAME" -y 2>/dev/null || true + +# Create conda environment from environment.yaml +echo "" +echo "Creating conda environment from environment.yaml..." +if [ ! -f "environment.yaml" ]; then + echo "ERROR: environment.yaml not found!" + exit 1 +fi + +mamba env create -n "$TEST_ENV_NAME" -f environment.yaml -y + +# Activate the environment +echo "" +echo "Activating test environment..." +# Note: In CI, conda/mamba environments need to be activated via conda run +# or by sourcing the activation script + +# Test pip installation of requirements.txt, or of its lock file (requirements.lock.txt, written by +# .tools/python/lock_requirements.py) when there is one: the installer installs the lock file, so that is what is tested. +if [ ! -f "requirements.txt" ]; then + echo "ERROR: requirements.txt not found!" + mamba env remove -n "$TEST_ENV_NAME" -y + exit 1 +fi +REQUIREMENTS_FILE="requirements.txt" +if [ -f "requirements.lock.txt" ]; then + REQUIREMENTS_FILE="requirements.lock.txt" +fi +echo "" +echo "Installing requirements from $REQUIREMENTS_FILE..." + +# Check if the requirements file is not empty +if [ ! -s "$REQUIREMENTS_FILE" ]; then + echo "WARNING: $REQUIREMENTS_FILE is empty, skipping pip install" +else + echo "Installing packages with pip..." + mamba run -n "$TEST_ENV_NAME" pip install --prefer-binary -r "$REQUIREMENTS_FILE" +fi + +# Report dependency conflicts between the pip packages and the conda packages of the environment. Only a warning: a conflict +# among the conda packages themselves must not block the notebooks. +echo "" +echo "Checking the installed packages for conflicts (pip check)..." +if ! mamba run -n "$TEST_ENV_NAME" pip check; then + echo "::warning::pip check found dependency conflicts in the test environment (see the step log); the installer may behave differently than expected." +fi + +# Verify installation by checking Python and key packages +echo "" +echo "Verifying installation..." +echo "Python version:" +mamba run -n "$TEST_ENV_NAME" python --version + +echo "" +echo "Installed pip packages (sample):" +mamba run -n "$TEST_ENV_NAME" pip list | head -n 20 + +# Clean up test environment +echo "" +echo "Cleaning up test environment..." +mamba env remove -n "$TEST_ENV_NAME" -y + +echo "" +echo "==========================================" +echo "✓ Conda environment test completed successfully!" +echo "==========================================" diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/python/lock_requirements.py b/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/python/lock_requirements.py new file mode 100644 index 0000000..c5834a2 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/python/lock_requirements.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Lock the merged requirements files: requirements*.txt -> requirements*.lock.txt. + +merge_requirements.py writes the requirements of the notebooks (requirements.txt, requirements_gpu.txt and the per-OS +requirements-.txt). Those files name the packages the notebooks need, but not the packages these depend on, so two +installations of the same release could get different versions. This script writes, next to each file, a lock file that lists +every package pip installs, each with its exact version, and post_install installs the lock file when the installer bundles one. + +How a lock file is made: +* `uv pip compile --universal`: one lock file that is valid on Windows, macOS and Linux (a package that only one platform needs carries + an environment marker, e.g. `colorama==0.4.6 ; sys_platform == 'win32'`). +* `--python-version` is the Python of environment.yaml. uv reads it as the lowest Python of the resolution, so a few packages can appear + twice with a `python_full_version` marker: pip installs the line that matches the Python of the installer. +* The conda packages of environment.yaml (jupyterlab, pip, setuptools, certifi, ...) are installed by the installer itself, so they are + not written to the lock file (their own dependencies are), and a conda pin such as `jupyterlab=4.4.0` constrains the resolution. +* The per-OS files are resolved with requirements.lock.txt as constraints, so a package that both files need gets the same version. +* Pip options of the requirements file (`--extra-index-url` of the GPU file, ...) are copied to the top of the lock file. + +A lock file is written again only when its requirements file, the Python or JupyterLab version (or this script's lock format) changed: +its header holds a fingerprint of those inputs. Re-running the script without such a change rewrites nothing, so a new release +installs the same versions as the previous one. To move every package to its newest version, delete the lock files and run the +workflow "Updates on notebook change" by hand (Actions > Run workflow). + +If a file cannot be locked (uv is missing, or the requirements cannot be resolved), the script prints a warning, removes the +out-of-date lock file of that file (an installer must never install versions that no longer match the requirements) and goes on: +the installer then installs the plain requirements file. The script itself always exits 0 for such a failure. +""" +from __future__ import annotations + +import argparse +import hashlib +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + +# Bump when the uv arguments or the output change, so that every lock file is written again. +LOCK_FORMAT = "1" +BASE_INPUT = "requirements.txt" +GPU_INPUT = "requirements_gpu.txt" +OS_INPUTS = ("requirements-linux.txt", "requirements-windows.txt", "requirements-macos.txt") +INPUTS = (BASE_INPUT, GPU_INPUT, *OS_INPUTS) +FINGERPRINT_PREFIX = "# lock-input: sha256:" +# Pip options that choose where packages come from: copied to the top of the lock file, so pip finds the locked versions where uv found them. +INDEX_OPTION = re.compile(r"^\s*(--index-url|--extra-index-url|-i|--find-links|-f|--trusted-host)(\s|=)") +CONDA_SPEC = re.compile(r"^\s*([A-Za-z0-9_.-]+)\s*(?:(==?)\s*([0-9][A-Za-z0-9_.+!*-]*))?(?:=\S+)?\s*$") +PLACEHOLDER = re.compile(r"^[A-Z_]+$") + + +def lock_path(requirements: Path) -> Path: + """requirements.txt -> requirements.lock.txt (same folder).""" + return requirements.with_name(requirements.stem + ".lock.txt") + + +def read_environment(path: Path) -> tuple[str | None, list[str], list[str]]: + """(Python version, conda package names, pip constraints from exact conda pins) of environment.yaml.""" + if not path.is_file(): + return None, [], [] + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + python_version = None + names: list[str] = [] + constraints: list[str] = [] + for dependency in data.get("dependencies", []) or []: + if not isinstance(dependency, str): + continue # the `pip:` subsection of an environment file + match = CONDA_SPEC.match(dependency) + if match: + name, operator, version = match.group(1), match.group(2), match.group(3) + else: + # a spec with a range or alternatives (`pip>=24.2`): only its name is used + name, operator, version = re.split(r"[\s<>=!|,\[]", dependency.strip(), maxsplit=1)[0], None, None + if not name: + continue + if version and PLACEHOLDER.match(version): + version = None # the template itself (python=PYTHON_VERSION) + if name.lower() == "python": + python_version = version + continue + names.append(name) + if version: + # conda `name=1.2` means any 1.2.* release, `name==1.2` exactly 1.2 + constraints.append(f"{name}=={version}" if operator == "==" else f"{name}=={version.removesuffix('.*')}.*") + return python_version, sorted(set(names), key=str.lower), constraints + + +def normalized_requirements(text: str) -> str: + """The requirement and option lines of a requirements file (comments, such as the timestamp of merge_requirements.py, are ignored).""" + lines = [] + for line in text.replace("\r\n", "\n").split("\n"): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + lines.append(stripped) + return "\n".join(lines) + + +def fingerprint(requirements_text: str, python_version: str, constraints: list[str], no_emit: list[str], extra_constraints: str) -> str: + payload = "\n---\n".join( + [LOCK_FORMAT, normalized_requirements(requirements_text), python_version, "\n".join(constraints), "\n".join(no_emit), extra_constraints] + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def current_fingerprint(lock: Path) -> str | None: + if not lock.is_file(): + return None + for line in lock.read_text(encoding="utf-8").splitlines(): + if line.startswith(FINGERPRINT_PREFIX): + return line[len(FINGERPRINT_PREFIX):].strip() + return None + + +def run_uv(requirements: Path, python_version: str, constraint_files: list[Path], no_emit: list[str]) -> str: + """The resolution of `requirements` (stdout of uv pip compile); raises RuntimeError with uv's message when it fails.""" + command = [ + sys.executable, "-m", "uv", "pip", "compile", str(requirements), + "--universal", "--python-version", python_version, + "--no-header", "--no-annotate", "--no-config", "--quiet", + ] + for constraint in constraint_files: + command += ["--constraint", str(constraint)] + for name in no_emit: + command += ["--no-emit-package", name] + try: + result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", check=False) + except OSError as exc: + raise RuntimeError(f"uv could not be started: {exc}") from exc + if result.returncode != 0: + message = (result.stderr or result.stdout).strip() or f"exit code {result.returncode}" + raise RuntimeError(message) + return result.stdout + + +def uv_version() -> str: + try: + result = subprocess.run([sys.executable, "-m", "uv", "--version"], capture_output=True, text=True, check=False) + except OSError: + return "uv" + # "uv 0.13.3 (502f93ca6 2026-10-19 x86_64-pc-windows-msvc)" -> "uv 0.13.3": the build details differ between computers + return " ".join(result.stdout.split()[:2]) or "uv" + + +def render_lock(requirements: Path, resolved: str, digest: str, python_version: str, uv: str) -> str: + options = [line.strip() for line in requirements.read_text(encoding="utf-8").splitlines() if INDEX_OPTION.match(line)] + packages = [line.rstrip() for line in resolved.replace("\r\n", "\n").split("\n") if line.strip() and not line.lstrip().startswith("#")] + header = [ + f"# Lock file of {requirements.name}: every package the installer installs with pip, with its exact version.", + f"# Written by .tools/python/lock_requirements.py ({uv}, universal resolution for Python {python_version}); do not edit it by hand.", + f"# It is written again only when {requirements.name}, the Python or JupyterLab version changes (see the fingerprint below).", + f"{FINGERPRINT_PREFIX}{digest}", + ] + body = options + ([""] if options else []) + packages + return "\n".join(header + [""] + body) + "\n" + + +def warn(message: str) -> None: + # `::warning::` shows the message on the summary page of the GitHub Actions run. + print(f"::warning::{message}", flush=True) + + +def lock_file( + requirements: Path, + python_version: str, + constraints: list[str], + no_emit: list[str], + base_lock: Path | None, + work_dir: Path, + uv: str, +) -> str: + """Lock one requirements file; returns what happened ("unchanged", "written", "failed").""" + lock = lock_path(requirements) + extra = base_lock.read_text(encoding="utf-8") if base_lock is not None and base_lock.is_file() else "" + digest = fingerprint(requirements.read_text(encoding="utf-8"), python_version, constraints, no_emit, normalized_requirements(extra)) + if current_fingerprint(lock) == digest: + print(f"{lock}: up to date (unchanged)") + return "unchanged" + constraint_files = [] + if constraints: + conda_constraints = work_dir / "conda_constraints.txt" + conda_constraints.write_text("\n".join(constraints) + "\n", encoding="utf-8") + constraint_files.append(conda_constraints) + if extra: + # Only the package lines: the pip options of the base lock are not constraints. + base_constraints = work_dir / "base_constraints.txt" + base_constraints.write_text( + "\n".join(line for line in extra.splitlines() if line.strip() and not line.startswith(("#", "-"))) + "\n", encoding="utf-8" + ) + constraint_files.append(base_constraints) + try: + resolved = run_uv(requirements, python_version, constraint_files, no_emit) + except RuntimeError as exc: + warn(f"{requirements} could not be locked, so the installer installs it without a lock file: {exc}") + if lock.exists(): + lock.unlink() + print(f"{lock}: removed (it no longer matches {requirements})") + return "failed" + with lock.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(render_lock(requirements, resolved, digest, python_version, uv)) + print(f"{lock}: written") + return "written" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Lock requirements*.txt into requirements*.lock.txt (every package with its exact version).") + parser.add_argument("--root", type=Path, default=Path("."), help="Folder with the requirements files and environment.yaml") + parser.add_argument("--environment", type=Path, default=None, help="Conda environment file (default: /environment.yaml)") + args = parser.parse_args(argv) + root: Path = args.root + environment = args.environment or root / "environment.yaml" + + python_version, conda_names, constraints = read_environment(environment) + if not python_version: + print(f"{environment}: no exact Python version (python=X.Y.Z), so nothing is locked.") + return 0 + no_emit = [name for name in conda_names if name.lower() not in {"git"}] + + # A lock file whose requirements file is gone is removed (the installer must not install it). + for name in INPUTS: + requirements = root / name + if not requirements.is_file() and lock_path(requirements).is_file(): + lock_path(requirements).unlink() + print(f"{lock_path(requirements)}: removed ({requirements} does not exist)") + + present = [root / name for name in INPUTS if (root / name).is_file()] + if not present: + print("No requirements files found: nothing to lock.") + return 0 + uv = uv_version() + results = {} + with tempfile.TemporaryDirectory() as temporary: + work_dir = Path(temporary) + for requirements in present: + base_lock = lock_path(root / BASE_INPUT) if requirements.name in OS_INPUTS else None + results[requirements.name] = lock_file(requirements, python_version, constraints, no_emit, base_lock, work_dir, uv) + failed = [name for name, status in results.items() if status == "failed"] + print(f"Lock files: {len(results) - len(failed)} of {len(results)} requirements files locked" + (f"; not locked: {', '.join(failed)}" if failed else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/app/python_scripts/launch_jupyter.py b/.template_sync/resources/v0_1_19_to_v0_1_20/app/python_scripts/launch_jupyter.py new file mode 100644 index 0000000..f45ec5f --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/app/python_scripts/launch_jupyter.py @@ -0,0 +1,324 @@ +"""TLS-resilient JupyterLab launcher for PROJECT_NAME. + +Certificate handling is configured before JupyterLab imports Tornado. On +Windows this prevents one malformed certificate-store entry from terminating +the application. Certificate and hostname verification remain enabled in every +supported mode. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import re +import shutil +import ssl +import sys +import tempfile +import traceback +from datetime import datetime, timezone +from importlib import metadata +from pathlib import Path + +PROJECT_DISPLAY_NAME = "PROJECT_NAME" +SCRIPT_DIR = Path(__file__).resolve().parent +_INSTALLED_LAYOUT = (SCRIPT_DIR / "notebooks").is_dir() +if _INSTALLED_LAYOUT: + APP_ROOT = SCRIPT_DIR + PREFIX = APP_ROOT.parent +else: + # Repository layout: app/python_scripts/launch_jupyter.py + repository_root = SCRIPT_DIR.parents[1] + APP_ROOT = repository_root if (repository_root / "notebooks").is_dir() else SCRIPT_DIR + PREFIX = APP_ROOT +DEFAULT_NOTEBOOK = APP_ROOT / "notebooks" / "Welcome.ipynb" +LOG_PATH = PREFIX / "launcher_debug.log" + + +def _write_log(message: str, *, console: bool = True) -> None: + timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds") + line = f"[{timestamp}] {message}" + if console: + print(message, flush=True) + try: + with LOG_PATH.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + except OSError: + # Logging must never prevent the application from launching. + pass + + +def _package_version(name: str) -> str: + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return "not installed" + + +def _project_ca_bundle_environment_variable() -> str: + normalized = re.sub(r"[^A-Za-z0-9]+", "_", PROJECT_DISPLAY_NAME).strip("_").upper() + return f"{normalized or 'LABCONSTRICTOR_APP'}_CA_BUNDLE" + + +def _probe_ssl() -> None: + """Create a verified client context without making a network request.""" + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) + if context.verify_mode != ssl.CERT_REQUIRED: + raise RuntimeError("TLS certificate verification is unexpectedly disabled.") + if not context.check_hostname: + raise RuntimeError("TLS hostname verification is unexpectedly disabled.") + + +def _inject_truststore() -> None: + """Route Python SSL contexts through the operating-system trust API.""" + import truststore + + truststore.inject_into_ssl() + + +def _select_ca_bundle() -> Path: + """Choose an explicit CA bundle for verified installer and runtime fallback.""" + for variable in ( + _project_ca_bundle_environment_variable(), + "LABCONSTRICTOR_CA_BUNDLE", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CURL_CA_BUNDLE", + ): + candidate = os.environ.get(variable, "").strip() + if candidate: + path = Path(candidate).expanduser() + if path.is_file(): + return path.resolve() + + import certifi + + return Path(certifi.where()).resolve() + + +def _install_bundle_fallback(bundle: Path) -> None: + """Use a verified PEM bundle without enumerating the Windows cert store.""" + original_context_type = ssl.SSLContext + + def create_context( + purpose: ssl.Purpose = ssl.Purpose.SERVER_AUTH, + *, + cafile: str | os.PathLike[str] | None = None, + capath: str | os.PathLike[str] | None = None, + cadata: str | bytes | None = None, + ) -> ssl.SSLContext: + if purpose == ssl.Purpose.SERVER_AUTH: + context = original_context_type(ssl.PROTOCOL_TLS_CLIENT) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + elif purpose == ssl.Purpose.CLIENT_AUTH: + context = original_context_type(ssl.PROTOCOL_TLS_SERVER) + else: + raise ValueError(purpose) + + if cafile or capath or cadata: + context.load_verify_locations( + cafile=os.fspath(cafile) if cafile else None, + capath=os.fspath(capath) if capath else None, + cadata=cadata, + ) + else: + context.load_verify_locations(cafile=str(bundle)) + + keylogfile = os.environ.get("SSLKEYLOGFILE") + if keylogfile and hasattr(context, "keylog_filename"): + context.keylog_filename = keylogfile + return context + + ssl.create_default_context = create_context # type: ignore[assignment] + ssl._create_default_https_context = create_context # type: ignore[attr-defined] + os.environ["SSL_CERT_FILE"] = str(bundle) + os.environ.setdefault("REQUESTS_CA_BUNDLE", str(bundle)) + os.environ.setdefault("CURL_CA_BUNDLE", str(bundle)) + + +def configure_tls() -> tuple[str, list[str]]: + """Prepare TLS before Jupyter/Tornado imports and return mode plus notes.""" + notes: list[str] = [] + + try: + if "truststore" not in getattr(ssl.SSLContext, "__module__", ""): + _inject_truststore() + _probe_ssl() + return "native truststore", notes + except Exception as exc: + notes.append(f"Native truststore activation failed: {exc}") + + try: + bundle = _select_ca_bundle() + _install_bundle_fallback(bundle) + _probe_ssl() + notes.append(f"Using verified CA bundle fallback: {bundle}") + return "verified CA bundle fallback", notes + except Exception as exc: + notes.append(f"CA bundle fallback failed: {exc}") + raise RuntimeError("Unable to create a verified TLS context. " + " | ".join(notes)) from exc + + +def _diagnostics(tls_mode: str, notes: list[str]) -> dict[str, object]: + return { + "status": "ok", + "application": PROJECT_DISPLAY_NAME, + "tls_mode": tls_mode, + "notes": notes, + "ca_bundle_override_variable": _project_ca_bundle_environment_variable(), + "python": sys.version.split()[0], + "python_executable": sys.executable, + "openssl": ssl.OPENSSL_VERSION, + "platform": platform.platform(), + "jupyterlab": _package_version("jupyterlab"), + "jupyter_server": _package_version("jupyter-server"), + "tornado": _package_version("tornado"), + "truststore": _package_version("truststore"), + "pip_system_certs": _package_version("pip-system-certs"), + "certifi": _package_version("certifi"), + "prefix": str(PREFIX), + "log": str(LOG_PATH), + } + + +def _show_error_dialog(message: str) -> None: + if os.name != "nt": + return + try: + import tkinter as tk + from tkinter import messagebox + + root = tk.Tk() + root.withdraw() + messagebox.showerror(f"{PROJECT_DISPLAY_NAME} could not start", message) + root.destroy() + except Exception: + pass + + +def _wait_for_enter() -> None: + """Keep the terminal window open until the user has read it, then let it close. + + The menu shortcut runs the launcher in a terminal window; without this the window + closes (or stays open, on Windows) as soon as JupyterLab stops. + """ + stream = sys.stdin + if stream is None or not stream.isatty(): + return + try: + input("\nPress Enter to close this window...") + except (EOFError, KeyboardInterrupt): + pass + + +def run_jupyter(notebook: Path) -> int: + if not notebook.is_file(): + raise FileNotFoundError(f"Welcome notebook not found: {notebook}") + + # Import only after configure_tls() has completed. Tornado creates SSL + # contexts during import on some versions. + from jupyterlab.labapp import main + + home = Path.home().resolve() + try: + notebook.resolve().relative_to(home) + root_dir = home + except ValueError: + # All-users installations can live outside the home directory. + root_dir = PREFIX.resolve() + + # A fresh, empty workspace folder for every launch: JupyterLab restores the open + # tabs of its workspaces, and the default folder (~/.jupyter/lab/workspaces) is + # shared by every app, so notebooks of other apps (and of earlier sessions) would + # open next to the Welcome notebook. Settings and themes are kept: they are stored + # in the user settings folder, not in the workspaces. + workspaces_dir = Path(tempfile.mkdtemp(prefix="labconstrictor-workspaces-")) + sys.argv = [ + "jupyter-lab", + str(notebook.resolve()), + "--ServerApp.ip=127.0.0.1", + f"--ServerApp.root_dir={root_dir}", + "--ServerApp.shutdown_no_activity_timeout=30", + "--FileContentsManager.always_delete_dir=True", + f"--LabApp.workspaces_dir={workspaces_dir}", + ] + try: + result = main() + finally: + shutil.rmtree(workspaces_dir, ignore_errors=True) + return int(result or 0) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=f"Launch {PROJECT_DISPLAY_NAME} in its packaged JupyterLab environment safely." + ) + parser.add_argument("notebook", nargs="?", type=Path, default=DEFAULT_NOTEBOOK) + parser.add_argument( + "--self-test", + action="store_true", + help="Validate TLS and Jupyter imports without starting the server.", + ) + parser.add_argument( + "--print-ca-bundle", + action="store_true", + help="Print the selected verified CA bundle path and exit.", + ) + parser.add_argument( + "--no-wait", + action="store_true", + help="Exit when JupyterLab stops, without asking to press Enter.", + ) + args = parser.parse_args(argv) + result = _launch(args) + if not (args.no_wait or args.self_test or args.print_ca_bundle): + _wait_for_enter() + return result + + +def _launch(args: argparse.Namespace) -> int: + try: + if args.print_ca_bundle: + print(_select_ca_bundle(), flush=True) + return 0 + + _write_log("Starting application launcher") + tls_mode, notes = configure_tls() + _write_log(f"TLS mode: {tls_mode}") + for note in notes: + # Recovery details remain available in launcher_debug.log and + # self-test JSON without alarming users during a normal launch. + _write_log(note, console=False) + + import jupyterlab.labapp # noqa: F401 + import tornado.netutil # noqa: F401 + + diagnostics = _diagnostics(tls_mode, notes) + if args.self_test: + print(json.dumps(diagnostics, indent=2), flush=True) + _write_log("Launcher self-test completed successfully") + return 0 + + _write_log(f"Opening notebook: {args.notebook}") + return run_jupyter(args.notebook) + except KeyboardInterrupt: + _write_log("Launcher interrupted by user") + return 130 + except Exception as exc: + details = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) + _write_log(details.rstrip()) + friendly = ( + f"{PROJECT_DISPLAY_NAME} could not start, but no certificate-store changes are required.\n\n" + f"A diagnostic log was written to:\n{LOG_PATH}\n\n" + f"Technical message: {exc}" + ) + print("\n" + friendly, file=sys.stderr, flush=True) + _show_error_dialog(friendly) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_notebook_change.yml b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_notebook_change.yml new file mode 100644 index 0000000..c5fae88 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_notebook_change.yml @@ -0,0 +1,154 @@ +name: Updates on notebook change + +on: + push: + branches: [ main, master ] + paths: + - 'notebooks/**/*.ipynb' + - 'notebooks/notebook_latest_versions.yaml' + - 'notebooks/**/requirements.yaml' + workflow_dispatch: + +# This enables to run a single workflow at a time +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon notebook change, update manifests and changelogs + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@2d756ea4c53f7f6b397767d8723b3a10a9f35bf2 # v44 + with: + separator: "," + files: | + notebooks/**/*.ipynb + notebooks/notebook_latest_versions.yaml + notebooks/**/requirements.yaml + + - name: Syncronize notebook versions + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "sync_notebook_versions.py" \ + .tools/python/sync_notebook_versions.py \ + python .tools/python/sync_notebook_versions.py + + - name: Update CHANGELOG + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_changelog.py" \ + .tools/python/update_changelog.py \ + python .tools/python/update_changelog.py notebooks/notebook_latest_versions.yaml + + - name: Update last commit in notebook_latest_versions.yaml + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_version_last_commit.py" \ + .tools/python/update_version_last_commit.py \ + python .tools/python/update_version_last_commit.py notebooks/notebook_latest_versions.yaml + + - name: Hide code cells in notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "hide_code_cells.py" \ + app/python_scripts/hide_code_cells.py \ + python app/python_scripts/hide_code_cells.py . + + - name: Index notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_notebook_README.py" \ + .tools/python/bump_notebook_README.py \ + python .tools/python/bump_notebook_README.py + + - name: Run merge script + if: github.event_name == 'workflow_dispatch' || contains(steps.changed-files.outputs.all_changed_files, 'requirements.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "merge_requirements.py" \ + .tools/python/merge_requirements.py \ + python .tools/python/merge_requirements.py --source-dir notebooks --output requirements.txt + + - name: Check for changes + id: check-changes + run: | + if [ -z "$(git status --porcelain)" ]; then + echo "has_changes=false" >> $GITHUB_OUTPUT + else + echo "has_changes=true" >> $GITHUB_OUTPUT + fi + + - name: Set up Miniconda for requirements test + if: steps.check-changes.outputs.has_changes == 'true' + uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + auto-activate: true + miniforge-version: latest + use-mamba: true + + - name: Verify merged requirements inside conda + id: test-conda-env + if: steps.check-changes.outputs.has_changes == 'true' + env: + SKIP_CONDA_ENV_TEST: ${{ vars.SKIP_CONDA_ENV_TEST }} + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "test_conda_environment.sh" \ + .tools/bash/test_conda_environment.sh \ + bash .tools/bash/test_conda_environment.sh + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: notebook-debug-bundle + path: artifacts/error_log_document.log + archive: false + + - name: Commit updated files + id: auto-commit + if: steps.check-changes.outputs.has_changes == 'true' && steps.test-conda-env.outcome == 'success' + uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5 + with: + commit_message: GitHub Action - Updated files because of notebook and/or requirements changes + file_pattern: requirements*.txt environment.yaml notebooks/* setup.py + + - name: Revert user commit that broke requirements + if: failure() && github.event_name == 'push' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git reset --hard + git clean -fd + git revert "${{ github.sha }}" -m 1 --no-edit + git push origin HEAD:${{ github.ref_name }} diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_src_change.yaml b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_src_change.yaml new file mode 100644 index 0000000..76b90f0 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_src_change.yaml @@ -0,0 +1,41 @@ +name: Updates on external code change + +on: + push: + branches: [ main, master ] + paths: + - 'src/**' + workflow_dispatch: + +# This enables to run a single workflow at a time +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon external code update the code version + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Update src change file + run: | + cat > .tools/meta/src_change.yaml </dev/null || true + +# Create conda environment from environment.yaml +echo "" +echo "Creating conda environment from environment.yaml..." +if [ ! -f "environment.yaml" ]; then + echo "ERROR: environment.yaml not found!" + exit 1 +fi + +mamba env create -n "$TEST_ENV_NAME" -f environment.yaml -y + +# Activate the environment +echo "" +echo "Activating test environment..." +# Note: In CI, conda/mamba environments need to be activated via conda run +# or by sourcing the activation script + +# Test pip installation of requirements.txt +echo "" +echo "Installing requirements from requirements.txt..." +if [ ! -f "requirements.txt" ]; then + echo "ERROR: requirements.txt not found!" + mamba env remove -n "$TEST_ENV_NAME" -y + exit 1 +fi + +# Check if requirements.txt is not empty +if [ ! -s "requirements.txt" ]; then + echo "WARNING: requirements.txt is empty, skipping pip install" +else + echo "Installing packages with pip..." + mamba run -n "$TEST_ENV_NAME" pip install --prefer-binary -r requirements.txt +fi + +# Verify installation by checking Python and key packages +echo "" +echo "Verifying installation..." +echo "Python version:" +mamba run -n "$TEST_ENV_NAME" python --version + +echo "" +echo "Installed pip packages (sample):" +mamba run -n "$TEST_ENV_NAME" pip list | head -n 20 + +# Clean up test environment +echo "" +echo "Cleaning up test environment..." +mamba env remove -n "$TEST_ENV_NAME" -y + +echo "" +echo "==========================================" +echo "✓ Conda environment test completed successfully!" +echo "==========================================" diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/previous/app/python_scripts/launch_jupyter.py b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/app/python_scripts/launch_jupyter.py new file mode 100644 index 0000000..0f91726 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/app/python_scripts/launch_jupyter.py @@ -0,0 +1,286 @@ +"""TLS-resilient JupyterLab launcher for PROJECT_NAME. + +Certificate handling is configured before JupyterLab imports Tornado. On +Windows this prevents one malformed certificate-store entry from terminating +the application. Certificate and hostname verification remain enabled in every +supported mode. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import re +import ssl +import sys +import traceback +from datetime import datetime, timezone +from importlib import metadata +from pathlib import Path + +PROJECT_DISPLAY_NAME = "PROJECT_NAME" +SCRIPT_DIR = Path(__file__).resolve().parent +_INSTALLED_LAYOUT = (SCRIPT_DIR / "notebooks").is_dir() +if _INSTALLED_LAYOUT: + APP_ROOT = SCRIPT_DIR + PREFIX = APP_ROOT.parent +else: + # Repository layout: app/python_scripts/launch_jupyter.py + repository_root = SCRIPT_DIR.parents[1] + APP_ROOT = repository_root if (repository_root / "notebooks").is_dir() else SCRIPT_DIR + PREFIX = APP_ROOT +DEFAULT_NOTEBOOK = APP_ROOT / "notebooks" / "Welcome.ipynb" +LOG_PATH = PREFIX / "launcher_debug.log" + + +def _write_log(message: str, *, console: bool = True) -> None: + timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds") + line = f"[{timestamp}] {message}" + if console: + print(message, flush=True) + try: + with LOG_PATH.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + except OSError: + # Logging must never prevent the application from launching. + pass + + +def _package_version(name: str) -> str: + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return "not installed" + + +def _project_ca_bundle_environment_variable() -> str: + normalized = re.sub(r"[^A-Za-z0-9]+", "_", PROJECT_DISPLAY_NAME).strip("_").upper() + return f"{normalized or 'LABCONSTRICTOR_APP'}_CA_BUNDLE" + + +def _probe_ssl() -> None: + """Create a verified client context without making a network request.""" + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) + if context.verify_mode != ssl.CERT_REQUIRED: + raise RuntimeError("TLS certificate verification is unexpectedly disabled.") + if not context.check_hostname: + raise RuntimeError("TLS hostname verification is unexpectedly disabled.") + + +def _inject_truststore() -> None: + """Route Python SSL contexts through the operating-system trust API.""" + import truststore + + truststore.inject_into_ssl() + + +def _select_ca_bundle() -> Path: + """Choose an explicit CA bundle for verified installer and runtime fallback.""" + for variable in ( + _project_ca_bundle_environment_variable(), + "LABCONSTRICTOR_CA_BUNDLE", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CURL_CA_BUNDLE", + ): + candidate = os.environ.get(variable, "").strip() + if candidate: + path = Path(candidate).expanduser() + if path.is_file(): + return path.resolve() + + import certifi + + return Path(certifi.where()).resolve() + + +def _install_bundle_fallback(bundle: Path) -> None: + """Use a verified PEM bundle without enumerating the Windows cert store.""" + original_context_type = ssl.SSLContext + + def create_context( + purpose: ssl.Purpose = ssl.Purpose.SERVER_AUTH, + *, + cafile: str | os.PathLike[str] | None = None, + capath: str | os.PathLike[str] | None = None, + cadata: str | bytes | None = None, + ) -> ssl.SSLContext: + if purpose == ssl.Purpose.SERVER_AUTH: + context = original_context_type(ssl.PROTOCOL_TLS_CLIENT) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + elif purpose == ssl.Purpose.CLIENT_AUTH: + context = original_context_type(ssl.PROTOCOL_TLS_SERVER) + else: + raise ValueError(purpose) + + if cafile or capath or cadata: + context.load_verify_locations( + cafile=os.fspath(cafile) if cafile else None, + capath=os.fspath(capath) if capath else None, + cadata=cadata, + ) + else: + context.load_verify_locations(cafile=str(bundle)) + + keylogfile = os.environ.get("SSLKEYLOGFILE") + if keylogfile and hasattr(context, "keylog_filename"): + context.keylog_filename = keylogfile + return context + + ssl.create_default_context = create_context # type: ignore[assignment] + ssl._create_default_https_context = create_context # type: ignore[attr-defined] + os.environ["SSL_CERT_FILE"] = str(bundle) + os.environ.setdefault("REQUESTS_CA_BUNDLE", str(bundle)) + os.environ.setdefault("CURL_CA_BUNDLE", str(bundle)) + + +def configure_tls() -> tuple[str, list[str]]: + """Prepare TLS before Jupyter/Tornado imports and return mode plus notes.""" + notes: list[str] = [] + + try: + if "truststore" not in getattr(ssl.SSLContext, "__module__", ""): + _inject_truststore() + _probe_ssl() + return "native truststore", notes + except Exception as exc: + notes.append(f"Native truststore activation failed: {exc}") + + try: + bundle = _select_ca_bundle() + _install_bundle_fallback(bundle) + _probe_ssl() + notes.append(f"Using verified CA bundle fallback: {bundle}") + return "verified CA bundle fallback", notes + except Exception as exc: + notes.append(f"CA bundle fallback failed: {exc}") + raise RuntimeError("Unable to create a verified TLS context. " + " | ".join(notes)) from exc + + +def _diagnostics(tls_mode: str, notes: list[str]) -> dict[str, object]: + return { + "status": "ok", + "application": PROJECT_DISPLAY_NAME, + "tls_mode": tls_mode, + "notes": notes, + "ca_bundle_override_variable": _project_ca_bundle_environment_variable(), + "python": sys.version.split()[0], + "python_executable": sys.executable, + "openssl": ssl.OPENSSL_VERSION, + "platform": platform.platform(), + "jupyterlab": _package_version("jupyterlab"), + "jupyter_server": _package_version("jupyter-server"), + "tornado": _package_version("tornado"), + "truststore": _package_version("truststore"), + "pip_system_certs": _package_version("pip-system-certs"), + "certifi": _package_version("certifi"), + "prefix": str(PREFIX), + "log": str(LOG_PATH), + } + + +def _show_error_dialog(message: str) -> None: + if os.name != "nt": + return + try: + import tkinter as tk + from tkinter import messagebox + + root = tk.Tk() + root.withdraw() + messagebox.showerror(f"{PROJECT_DISPLAY_NAME} could not start", message) + root.destroy() + except Exception: + pass + + +def run_jupyter(notebook: Path) -> int: + if not notebook.is_file(): + raise FileNotFoundError(f"Welcome notebook not found: {notebook}") + + # Import only after configure_tls() has completed. Tornado creates SSL + # contexts during import on some versions. + from jupyterlab.labapp import main + + home = Path.home().resolve() + try: + notebook.resolve().relative_to(home) + root_dir = home + except ValueError: + # All-users installations can live outside the home directory. + root_dir = PREFIX.resolve() + + sys.argv = [ + "jupyter-lab", + str(notebook.resolve()), + "--ServerApp.ip=127.0.0.1", + f"--ServerApp.root_dir={root_dir}", + "--ServerApp.shutdown_no_activity_timeout=30", + "--FileContentsManager.always_delete_dir=True", + ] + result = main() + return int(result or 0) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=f"Launch {PROJECT_DISPLAY_NAME} in its packaged JupyterLab environment safely." + ) + parser.add_argument("notebook", nargs="?", type=Path, default=DEFAULT_NOTEBOOK) + parser.add_argument( + "--self-test", + action="store_true", + help="Validate TLS and Jupyter imports without starting the server.", + ) + parser.add_argument( + "--print-ca-bundle", + action="store_true", + help="Print the selected verified CA bundle path and exit.", + ) + args = parser.parse_args(argv) + + try: + if args.print_ca_bundle: + print(_select_ca_bundle(), flush=True) + return 0 + + _write_log("Starting application launcher") + tls_mode, notes = configure_tls() + _write_log(f"TLS mode: {tls_mode}") + for note in notes: + # Recovery details remain available in launcher_debug.log and + # self-test JSON without alarming users during a normal launch. + _write_log(note, console=False) + + import jupyterlab.labapp # noqa: F401 + import tornado.netutil # noqa: F401 + + diagnostics = _diagnostics(tls_mode, notes) + if args.self_test: + print(json.dumps(diagnostics, indent=2), flush=True) + _write_log("Launcher self-test completed successfully") + return 0 + + _write_log(f"Opening notebook: {args.notebook}") + return run_jupyter(args.notebook) + except KeyboardInterrupt: + _write_log("Launcher interrupted by user") + return 130 + except Exception as exc: + details = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) + _write_log(details.rstrip()) + friendly = ( + f"{PROJECT_DISPLAY_NAME} could not start, but no certificate-store changes are required.\n\n" + f"A diagnostic log was written to:\n{LOG_PATH}\n\n" + f"Technical message: {exc}" + ) + print("\n" + friendly, file=sys.stderr, flush=True) + _show_error_dialog(friendly) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.tools/bash/test_conda_environment.sh b/.tools/bash/test_conda_environment.sh index 1450f7e..4742aca 100644 --- a/.tools/bash/test_conda_environment.sh +++ b/.tools/bash/test_conda_environment.sh @@ -48,21 +48,34 @@ echo "Activating test environment..." # Note: In CI, conda/mamba environments need to be activated via conda run # or by sourcing the activation script -# Test pip installation of requirements.txt -echo "" -echo "Installing requirements from requirements.txt..." +# Test pip installation of requirements.txt, or of its lock file (requirements.lock.txt, written by +# .tools/python/lock_requirements.py) when there is one: the installer installs the lock file, so that is what is tested. if [ ! -f "requirements.txt" ]; then echo "ERROR: requirements.txt not found!" mamba env remove -n "$TEST_ENV_NAME" -y exit 1 fi +REQUIREMENTS_FILE="requirements.txt" +if [ -f "requirements.lock.txt" ]; then + REQUIREMENTS_FILE="requirements.lock.txt" +fi +echo "" +echo "Installing requirements from $REQUIREMENTS_FILE..." -# Check if requirements.txt is not empty -if [ ! -s "requirements.txt" ]; then - echo "WARNING: requirements.txt is empty, skipping pip install" +# Check if the requirements file is not empty +if [ ! -s "$REQUIREMENTS_FILE" ]; then + echo "WARNING: $REQUIREMENTS_FILE is empty, skipping pip install" else echo "Installing packages with pip..." - mamba run -n "$TEST_ENV_NAME" pip install --prefer-binary -r requirements.txt + mamba run -n "$TEST_ENV_NAME" pip install --prefer-binary -r "$REQUIREMENTS_FILE" +fi + +# Report dependency conflicts between the pip packages and the conda packages of the environment. Only a warning: a conflict +# among the conda packages themselves must not block the notebooks. +echo "" +echo "Checking the installed packages for conflicts (pip check)..." +if ! mamba run -n "$TEST_ENV_NAME" pip check; then + echo "::warning::pip check found dependency conflicts in the test environment (see the step log); the installer may behave differently than expected." fi # Verify installation by checking Python and key packages diff --git a/.tools/docs/README.md b/.tools/docs/README.md index 71a117f..2e1f377 100644 --- a/.tools/docs/README.md +++ b/.tools/docs/README.md @@ -31,6 +31,7 @@ This file temporarily lists all internal LabConstrictor documentation. After you - [Template 0.1.17: install Tools before the package import check](migration_0_1_17.md) - [Template 0.1.18: new Tools pin, pinned GitHub Actions, install rules](migration_0_1_18.md) - [Template 0.1.19: src_change.yaml in every installer, Python series in setup.py](migration_0_1_19.md) +- [Template 0.1.20: lock files, no lost workflow pushes, a launcher that starts fresh](migration_0_1_20.md) ## Notebook authoring tips diff --git a/.tools/docs/migration_0_1_20.md b/.tools/docs/migration_0_1_20.md new file mode 100644 index 0000000..3560756 --- /dev/null +++ b/.tools/docs/migration_0_1_20.md @@ -0,0 +1,49 @@ +# Template 0.1.20: lock files, no lost workflow pushes, a launcher that starts fresh + +Design note for the 0.1.20 migration (`.template_sync/migrations/v0_1_19_to_v0_1_20.py`). How a change reaches an installed copy is described in [migration_0_1_16.md](migration_0_1_16.md): a new installer is needed, nothing edits an existing install. + +## 1. What changes + +| Part | Problem | Change | +|---|---|---| +| `update_on_notebook_change.yml`, `update_on_src_change.yaml` | One push that changes both `src/` and `notebooks/` starts both workflows. They shared one concurrency group, so they ran one after the other, but each checks out the commit that triggered it: the second one committed on top of an old `main` and its push was rejected (`non-fast-forward`). Seen in CellMigrationLab/LabChronicle runs 41727822989 (notebook workflow) and 41800906840 (src workflow). The notebook workflow then ran "Revert user commit that broke requirements" (its condition was any failure), created a revert of the user's merge commit, and was stopped only because that push was rejected too. | The commit step commits, and when the push is rejected it rebases onto the latest branch (`git rebase --autostash`) and pushes again (5 attempts). The revert runs only when the conda test failed, on top of the latest branch, with `-m 1` only for a merge commit. Each workflow has its own concurrency group (a group keeps only one waiting run, so a third push cancelled the waiting run of the other workflow). The checkout stays on the triggering commit, because `update_changelog.py` and `update_version_last_commit.py` compare it with its parent. `stefanzweifel/git-auto-commit-action` is no longer used. | +| Requirements (`requirements*.txt`) | The installer ran `pip install -r requirements.txt`: the notebooks' packages are pinned, the packages they depend on are not, so two installations of one release could get different versions. | `.tools/python/lock_requirements.py` (run by the notebook workflow after the merge) writes `requirements*.lock.txt` next to each requirements file with `uv pip compile --universal`: every package with its exact version, for Windows, macOS and Linux. `post_install.sh` / `.bat` install the lock file when the installer bundles one (`bump_constructor.py` bundles it), and the conda test of the workflow installs it too, then runs `pip check` (a warning only). | +| `launch_jupyter.py` | When JupyterLab stopped, the terminal window closed at once (macOS, Linux) or stayed open with a prompt (Windows). Every app shares `~/.jupyter/lab/workspaces` and opens JupyterLab in the home folder, so a new session restored the notebooks of other apps (a NucleiSky notebook in a LabChronicle session). | The launcher asks to press Enter when JupyterLab stops (also after an error; not for `--self-test`, `--print-ca-bundle` or `--no-wait`). JupyterLab gets a new, empty workspace folder at every launch, removed when it stops: only the Welcome notebook opens. Settings and themes are kept (they are not in the workspaces). | +| `notebook_launcher.json` | On Windows, menuinst runs the shortcut with `cmd /K`, which keeps the window open after the command. | The Windows command ends with `& exit`, so the window closes after Enter. | +| `notebook_usage.md` | Said to stop JupyterLab with `Ctrl+C`. | Names **File > Shut Down** first, and the Enter prompt. | + +### How a lock file is made + +* One lock file per requirements file: `requirements.lock.txt`, `requirements_gpu.lock.txt`, `requirements-windows.lock.txt`, ... A package that only one platform needs carries an environment marker (`colorama==0.4.6 ; sys_platform == 'win32'`). +* `--python-version` is the Python of `environment.yaml`. uv reads it as the lowest Python, so a few packages can appear twice with a `python_full_version` marker; pip installs the line that matches the installer's Python. +* The conda packages of `environment.yaml` (`jupyterlab`, `pip`, `setuptools`, `certifi`, ...) are installed by the installer itself: they are not written to the lock file (their dependencies are), and a conda pin such as `jupyterlab=4.4.0` constrains the resolution. +* The per-OS files are resolved with `requirements.lock.txt` as constraints, so a package both files need gets one version. The GPU file is resolved on its own (its PyTorch comes from the CUDA index, whose `--extra-index-url` line is copied to the lock file). +* Versions only, no hashes. +* The header holds a fingerprint of the inputs (the requirement lines, the Python and JupyterLab versions, the lock format). The file is written again only when one of them changes, so a release without new requirements installs the same versions as the previous one. To move every package to its newest version, delete the lock files and run **Updates on notebook change** by hand. +* If a file cannot be locked (no solution, uv missing), the step prints a GitHub warning, removes the out-of-date lock file and goes on: the installer installs the plain requirements file. `bump_constructor.py` removes the mapping of a lock file that is gone, so the installer build does not fail on it. + +## 2. What the migration does + +Every step is independent, idempotent, never deletes, never raises for a customised file (the sync pull request is still created) and keeps the line endings of each file. Each file gets one line in the output: `updated`, `unchanged` or `SKIPPED` with the reason. + +* **The two workflows, `test_conda_environment.sh`, `launch_jupyter.py`.** Replaced by the 0.1.20 file only when the app's file is still the 0.1.19 template file (rendered with the app's name; line endings ignored). A customised file is `SKIPPED`: apply the change of section 1 by hand. +* **`lock_requirements.py`.** Created (it is new). An app that has its own file keeps it (`SKIPPED`). +* **`.tools/requirements.txt`.** `uv==0.13.3` is added, unless the file already names `uv`. +* **`post_install.sh` / `.bat`.** The known install lines are replaced by the lines that pick the lock file. If one of them is not found exactly once, the file is `SKIPPED`. +* **`bump_constructor.py`.** `ensure_lock_files_in_extra_files` is inserted before `def ensure_extra_files(` and called after `ensure_requirements_in_extra_files(data)`. Both lines must be found once (LabChronicle's own script qualifies). +* **`notebook_launcher.json`.** The Windows command is added (the shared command plus `"&", "exit"`) when the file has the template shape and formatting; a file with its own Windows command is `SKIPPED`. +* **`notebook_usage.md`.** The sentence on stopping JupyterLab is replaced when it is the 0.1.19 text. +* A file that is not in the repository is not created (except `lock_requirements.py`, when `.tools/python` exists). + +## 3. Adopting it + +1. Merge the synchronization pull request. +2. Run **Actions > Updates on notebook change > Run workflow** once: it writes the lock files (the next push of a `requirements.yaml` would too). Check that the run is green and committed `requirements*.lock.txt`. +3. Build and release a new installer: it bundles the lock files and the new launcher. Installed copies do not change: reinstall. + +## 4. Limitations and what is not verified here + +* **macOS terminal.** After Enter, Terminal.app closes the window only if its profile says so (*Settings > Profiles > Shell > When the shell exits*); otherwise it shows `[Process completed]`. Not tested on a Mac yet. +* **Windows and `Ctrl+C`.** Stopping JupyterLab with `Ctrl+C` makes `cmd` ask `Terminate batch job (Y/N)?` after Enter: `N` closes the window, `Y` leaves it open. **File > Shut Down** avoids the question. +* **The lock covers pip only.** Constructor already fixes the conda packages when the installer is built. +* **Tested here:** the commit and revert steps of the workflows run with bash and git against a local origin that moved during the run (a rebased push, a conflict that pushes nothing, the revert of a merge and of a plain commit); `lock_requirements.py` with a fake uv (fingerprint, constraints, index lines, failures); `bump_constructor.py`; the launcher's workspace folder and Enter prompt; the migration on a 0.1.19 app and on customised files. `uv pip compile --universal` itself was tried by hand on LabChronicle's requirements (CPU, GPU with the CUDA index, Windows file). Not tested: a real GitHub Actions run, an installer build, and a launch on Windows, macOS and Linux. diff --git a/.tools/docs/notebook_requirements.md b/.tools/docs/notebook_requirements.md index e568dd7..9fea2b4 100644 --- a/.tools/docs/notebook_requirements.md +++ b/.tools/docs/notebook_requirements.md @@ -66,6 +66,18 @@ Follow the instructions provided inside the notebook to validate your file. You --- +## What happens next: the lock file + +When you push a notebook with its `requirements.yaml`, the workflow **Updates on notebook change** merges the requirements of every notebook into `requirements.txt` (plus `requirements_gpu.txt` and, if a notebook needs them, `requirements-windows.txt`, `requirements-linux.txt` or `requirements-macos.txt`). + +It then writes a **lock file** next to each of them (`requirements.lock.txt`, ...). A lock file lists every package the installer installs, including the packages your packages depend on, each with its exact version, for Windows, macOS and Linux. The installer installs the lock file, so every installation of a release gets the same versions. + +* A lock file changes only when its requirements file, the Python version or the JupyterLab version changes. Pushing a notebook without new requirements does not move any version. +* To move every package to its newest version, delete the `requirements*.lock.txt` files and run **Updates on notebook change** by hand (**Actions > Updates on notebook change > Run workflow**). +* If the requirements cannot be locked, the workflow shows a warning and the installer installs the plain `requirements.txt` instead. + +--- +
[← Previous](accept_pull_request.md)            diff --git a/.tools/docs/notebook_usage.md b/.tools/docs/notebook_usage.md index f2a7b75..c4e2c01 100644 --- a/.tools/docs/notebook_usage.md +++ b/.tools/docs/notebook_usage.md @@ -10,7 +10,7 @@ To open the application, go to the Start Menu on Windows or Linux, or the Applic When you start the application, a terminal will pop up. -> Don't be scared by the terminal, it is normal: it keeps your Jupyter Lab session running. **Do not close the terminal** until you have finished working with your notebooks. When you have, click on the terminal window and press `Ctrl+C` to stop the Jupyter Lab session and close the terminal. +> Don't be scared by the terminal, it is normal: it keeps your Jupyter Lab session running. **Do not close the terminal** until you have finished working with your notebooks. When you have, stop the Jupyter Lab session with **File > Shut Down** in Jupyter Lab (or click on the terminal window and press `Ctrl+C`). The terminal then asks you to press `Enter`, and closes. After some seconds, Jupyter Lab will open in your default web browser. The welcome notebook will appear automatically: diff --git a/.tools/python/bump_constructor.py b/.tools/python/bump_constructor.py index 5b7cf74..dcbbbb5 100644 --- a/.tools/python/bump_constructor.py +++ b/.tools/python/bump_constructor.py @@ -138,6 +138,37 @@ def ensure_app_manifest_in_extra_files(construct_data: dict, manifest_path: Path debug(f"{APP_MANIFEST} not found: the app will be registered without hosts (listed in every host)") return False +LOCK_FILES = ( + "requirements.lock.txt", + "requirements_gpu.lock.txt", + "requirements-linux.lock.txt", + "requirements-windows.lock.txt", + "requirements-macos.lock.txt", +) + + +def ensure_lock_files_in_extra_files(construct_data: dict) -> None: + """Bundle the lock files (requirements*.lock.txt, written by lock_requirements.py) next to their requirements files. + + post_install installs a lock file instead of its requirements file when the installer bundles one. A mapping whose lock file is gone + (the requirements could not be locked) is removed, because constructor fails on a missing file. + """ + extra_files = construct_data.setdefault("extra_files", []) + project_folder = extract_project_folder(extra_files) + for lock_file in LOCK_FILES: + mapped = [item for item in extra_files if isinstance(item, dict) and lock_file in item] + if Path(lock_file).exists(): + if not mapped: + extra_files.append({lock_file: f"{project_folder}/{lock_file}"}) + debug(f"Added mapping for {lock_file}") + else: + debug(f"{lock_file} mapping already present") + elif mapped: + for item in mapped: + extra_files.remove(item) + debug(f"Removed the mapping of {lock_file}: the file does not exist (the installer installs the requirements file instead)") + + def ensure_extra_files(construct_data: dict, notebooks_root: Path, src_root: Path) -> int: extra_files = construct_data.get("extra_files") if extra_files is None: @@ -295,6 +326,7 @@ def main(): data = load_construct(construct_path) # Check if the requirements files are included in extra_files, if not, add them ensure_requirements_in_extra_files(data) + ensure_lock_files_in_extra_files(data) ensure_app_manifest_in_extra_files(data) # Update extra_files with notebooks and src files diff --git a/.tools/python/lock_requirements.py b/.tools/python/lock_requirements.py new file mode 100644 index 0000000..c5834a2 --- /dev/null +++ b/.tools/python/lock_requirements.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Lock the merged requirements files: requirements*.txt -> requirements*.lock.txt. + +merge_requirements.py writes the requirements of the notebooks (requirements.txt, requirements_gpu.txt and the per-OS +requirements-.txt). Those files name the packages the notebooks need, but not the packages these depend on, so two +installations of the same release could get different versions. This script writes, next to each file, a lock file that lists +every package pip installs, each with its exact version, and post_install installs the lock file when the installer bundles one. + +How a lock file is made: +* `uv pip compile --universal`: one lock file that is valid on Windows, macOS and Linux (a package that only one platform needs carries + an environment marker, e.g. `colorama==0.4.6 ; sys_platform == 'win32'`). +* `--python-version` is the Python of environment.yaml. uv reads it as the lowest Python of the resolution, so a few packages can appear + twice with a `python_full_version` marker: pip installs the line that matches the Python of the installer. +* The conda packages of environment.yaml (jupyterlab, pip, setuptools, certifi, ...) are installed by the installer itself, so they are + not written to the lock file (their own dependencies are), and a conda pin such as `jupyterlab=4.4.0` constrains the resolution. +* The per-OS files are resolved with requirements.lock.txt as constraints, so a package that both files need gets the same version. +* Pip options of the requirements file (`--extra-index-url` of the GPU file, ...) are copied to the top of the lock file. + +A lock file is written again only when its requirements file, the Python or JupyterLab version (or this script's lock format) changed: +its header holds a fingerprint of those inputs. Re-running the script without such a change rewrites nothing, so a new release +installs the same versions as the previous one. To move every package to its newest version, delete the lock files and run the +workflow "Updates on notebook change" by hand (Actions > Run workflow). + +If a file cannot be locked (uv is missing, or the requirements cannot be resolved), the script prints a warning, removes the +out-of-date lock file of that file (an installer must never install versions that no longer match the requirements) and goes on: +the installer then installs the plain requirements file. The script itself always exits 0 for such a failure. +""" +from __future__ import annotations + +import argparse +import hashlib +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + +# Bump when the uv arguments or the output change, so that every lock file is written again. +LOCK_FORMAT = "1" +BASE_INPUT = "requirements.txt" +GPU_INPUT = "requirements_gpu.txt" +OS_INPUTS = ("requirements-linux.txt", "requirements-windows.txt", "requirements-macos.txt") +INPUTS = (BASE_INPUT, GPU_INPUT, *OS_INPUTS) +FINGERPRINT_PREFIX = "# lock-input: sha256:" +# Pip options that choose where packages come from: copied to the top of the lock file, so pip finds the locked versions where uv found them. +INDEX_OPTION = re.compile(r"^\s*(--index-url|--extra-index-url|-i|--find-links|-f|--trusted-host)(\s|=)") +CONDA_SPEC = re.compile(r"^\s*([A-Za-z0-9_.-]+)\s*(?:(==?)\s*([0-9][A-Za-z0-9_.+!*-]*))?(?:=\S+)?\s*$") +PLACEHOLDER = re.compile(r"^[A-Z_]+$") + + +def lock_path(requirements: Path) -> Path: + """requirements.txt -> requirements.lock.txt (same folder).""" + return requirements.with_name(requirements.stem + ".lock.txt") + + +def read_environment(path: Path) -> tuple[str | None, list[str], list[str]]: + """(Python version, conda package names, pip constraints from exact conda pins) of environment.yaml.""" + if not path.is_file(): + return None, [], [] + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + python_version = None + names: list[str] = [] + constraints: list[str] = [] + for dependency in data.get("dependencies", []) or []: + if not isinstance(dependency, str): + continue # the `pip:` subsection of an environment file + match = CONDA_SPEC.match(dependency) + if match: + name, operator, version = match.group(1), match.group(2), match.group(3) + else: + # a spec with a range or alternatives (`pip>=24.2`): only its name is used + name, operator, version = re.split(r"[\s<>=!|,\[]", dependency.strip(), maxsplit=1)[0], None, None + if not name: + continue + if version and PLACEHOLDER.match(version): + version = None # the template itself (python=PYTHON_VERSION) + if name.lower() == "python": + python_version = version + continue + names.append(name) + if version: + # conda `name=1.2` means any 1.2.* release, `name==1.2` exactly 1.2 + constraints.append(f"{name}=={version}" if operator == "==" else f"{name}=={version.removesuffix('.*')}.*") + return python_version, sorted(set(names), key=str.lower), constraints + + +def normalized_requirements(text: str) -> str: + """The requirement and option lines of a requirements file (comments, such as the timestamp of merge_requirements.py, are ignored).""" + lines = [] + for line in text.replace("\r\n", "\n").split("\n"): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + lines.append(stripped) + return "\n".join(lines) + + +def fingerprint(requirements_text: str, python_version: str, constraints: list[str], no_emit: list[str], extra_constraints: str) -> str: + payload = "\n---\n".join( + [LOCK_FORMAT, normalized_requirements(requirements_text), python_version, "\n".join(constraints), "\n".join(no_emit), extra_constraints] + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def current_fingerprint(lock: Path) -> str | None: + if not lock.is_file(): + return None + for line in lock.read_text(encoding="utf-8").splitlines(): + if line.startswith(FINGERPRINT_PREFIX): + return line[len(FINGERPRINT_PREFIX):].strip() + return None + + +def run_uv(requirements: Path, python_version: str, constraint_files: list[Path], no_emit: list[str]) -> str: + """The resolution of `requirements` (stdout of uv pip compile); raises RuntimeError with uv's message when it fails.""" + command = [ + sys.executable, "-m", "uv", "pip", "compile", str(requirements), + "--universal", "--python-version", python_version, + "--no-header", "--no-annotate", "--no-config", "--quiet", + ] + for constraint in constraint_files: + command += ["--constraint", str(constraint)] + for name in no_emit: + command += ["--no-emit-package", name] + try: + result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", check=False) + except OSError as exc: + raise RuntimeError(f"uv could not be started: {exc}") from exc + if result.returncode != 0: + message = (result.stderr or result.stdout).strip() or f"exit code {result.returncode}" + raise RuntimeError(message) + return result.stdout + + +def uv_version() -> str: + try: + result = subprocess.run([sys.executable, "-m", "uv", "--version"], capture_output=True, text=True, check=False) + except OSError: + return "uv" + # "uv 0.13.3 (502f93ca6 2026-10-19 x86_64-pc-windows-msvc)" -> "uv 0.13.3": the build details differ between computers + return " ".join(result.stdout.split()[:2]) or "uv" + + +def render_lock(requirements: Path, resolved: str, digest: str, python_version: str, uv: str) -> str: + options = [line.strip() for line in requirements.read_text(encoding="utf-8").splitlines() if INDEX_OPTION.match(line)] + packages = [line.rstrip() for line in resolved.replace("\r\n", "\n").split("\n") if line.strip() and not line.lstrip().startswith("#")] + header = [ + f"# Lock file of {requirements.name}: every package the installer installs with pip, with its exact version.", + f"# Written by .tools/python/lock_requirements.py ({uv}, universal resolution for Python {python_version}); do not edit it by hand.", + f"# It is written again only when {requirements.name}, the Python or JupyterLab version changes (see the fingerprint below).", + f"{FINGERPRINT_PREFIX}{digest}", + ] + body = options + ([""] if options else []) + packages + return "\n".join(header + [""] + body) + "\n" + + +def warn(message: str) -> None: + # `::warning::` shows the message on the summary page of the GitHub Actions run. + print(f"::warning::{message}", flush=True) + + +def lock_file( + requirements: Path, + python_version: str, + constraints: list[str], + no_emit: list[str], + base_lock: Path | None, + work_dir: Path, + uv: str, +) -> str: + """Lock one requirements file; returns what happened ("unchanged", "written", "failed").""" + lock = lock_path(requirements) + extra = base_lock.read_text(encoding="utf-8") if base_lock is not None and base_lock.is_file() else "" + digest = fingerprint(requirements.read_text(encoding="utf-8"), python_version, constraints, no_emit, normalized_requirements(extra)) + if current_fingerprint(lock) == digest: + print(f"{lock}: up to date (unchanged)") + return "unchanged" + constraint_files = [] + if constraints: + conda_constraints = work_dir / "conda_constraints.txt" + conda_constraints.write_text("\n".join(constraints) + "\n", encoding="utf-8") + constraint_files.append(conda_constraints) + if extra: + # Only the package lines: the pip options of the base lock are not constraints. + base_constraints = work_dir / "base_constraints.txt" + base_constraints.write_text( + "\n".join(line for line in extra.splitlines() if line.strip() and not line.startswith(("#", "-"))) + "\n", encoding="utf-8" + ) + constraint_files.append(base_constraints) + try: + resolved = run_uv(requirements, python_version, constraint_files, no_emit) + except RuntimeError as exc: + warn(f"{requirements} could not be locked, so the installer installs it without a lock file: {exc}") + if lock.exists(): + lock.unlink() + print(f"{lock}: removed (it no longer matches {requirements})") + return "failed" + with lock.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(render_lock(requirements, resolved, digest, python_version, uv)) + print(f"{lock}: written") + return "written" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Lock requirements*.txt into requirements*.lock.txt (every package with its exact version).") + parser.add_argument("--root", type=Path, default=Path("."), help="Folder with the requirements files and environment.yaml") + parser.add_argument("--environment", type=Path, default=None, help="Conda environment file (default: /environment.yaml)") + args = parser.parse_args(argv) + root: Path = args.root + environment = args.environment or root / "environment.yaml" + + python_version, conda_names, constraints = read_environment(environment) + if not python_version: + print(f"{environment}: no exact Python version (python=X.Y.Z), so nothing is locked.") + return 0 + no_emit = [name for name in conda_names if name.lower() not in {"git"}] + + # A lock file whose requirements file is gone is removed (the installer must not install it). + for name in INPUTS: + requirements = root / name + if not requirements.is_file() and lock_path(requirements).is_file(): + lock_path(requirements).unlink() + print(f"{lock_path(requirements)}: removed ({requirements} does not exist)") + + present = [root / name for name in INPUTS if (root / name).is_file()] + if not present: + print("No requirements files found: nothing to lock.") + return 0 + uv = uv_version() + results = {} + with tempfile.TemporaryDirectory() as temporary: + work_dir = Path(temporary) + for requirements in present: + base_lock = lock_path(root / BASE_INPUT) if requirements.name in OS_INPUTS else None + results[requirements.name] = lock_file(requirements, python_version, constraints, no_emit, base_lock, work_dir, uv) + failed = [name for name, status in results.items() if status == "failed"] + print(f"Lock files: {len(results) - len(failed)} of {len(results)} requirements files locked" + (f"; not locked: {', '.join(failed)}" if failed else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.tools/requirements.txt b/.tools/requirements.txt index d06b04e..8492066 100644 --- a/.tools/requirements.txt +++ b/.tools/requirements.txt @@ -1,3 +1,5 @@ # Python 3.11 pyyaml==6.0.2 -nbformat==5.10.4 \ No newline at end of file +nbformat==5.10.4 +# lock_requirements.py (requirements*.txt -> requirements*.lock.txt) +uv==0.13.3 diff --git a/app/bash_bat_scripts/post_install.bat b/app/bash_bat_scripts/post_install.bat index 4dd92c6..0321391 100644 --- a/app/bash_bat_scripts/post_install.bat +++ b/app/bash_bat_scripts/post_install.bat @@ -67,6 +67,10 @@ IF EXIST "%GPU_REQUIREMENTS%" ( echo GPU requirements file not found, installing CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" ) +REM A lock file (requirements*.lock.txt next to the requirements file: every package with its exact version, written by +REM .tools/python/lock_requirements.py) is installed instead of the requirements file when the installer bundles one. +SET "SELECTED_LOCK=%SELECTED_REQUIREMENTS:~0,-4%.lock.txt" +IF EXIST "%SELECTED_LOCK%" SET "SELECTED_REQUIREMENTS=%SELECTED_LOCK%" echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" "%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1 IF ERRORLEVEL 1 ( @@ -74,9 +78,11 @@ IF ERRORLEVEL 1 ( GOTO :fail ) +SET "WINDOWS_REQUIREMENTS=%PROJECT_ROOT%\requirements-windows.txt" +IF EXIST "%PROJECT_ROOT%\requirements-windows.lock.txt" SET "WINDOWS_REQUIREMENTS=%PROJECT_ROOT%\requirements-windows.lock.txt" IF EXIST "%PROJECT_ROOT%\requirements-windows.txt" ( - echo Installing Windows-specific requirements. >> "%LOG_FILE%" - "%PYTHON_EXE%" -m pip install -r "%PROJECT_ROOT%\requirements-windows.txt" >> "%LOG_FILE%" 2>&1 + echo Installing Windows-specific requirements from "%WINDOWS_REQUIREMENTS%". >> "%LOG_FILE%" + "%PYTHON_EXE%" -m pip install -r "%WINDOWS_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1 IF ERRORLEVEL 1 ( SET "FAILURE_MESSAGE=Windows-specific requirements installation failed." GOTO :fail diff --git a/app/bash_bat_scripts/post_install.sh b/app/bash_bat_scripts/post_install.sh index ebcaf0b..0854061 100644 --- a/app/bash_bat_scripts/post_install.sh +++ b/app/bash_bat_scripts/post_install.sh @@ -43,18 +43,24 @@ else echo "GPU requirements file not found, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" fi +# A lock file (requirements*.lock.txt next to the requirements file: every package with its exact version, written by +# .tools/python/lock_requirements.py) is installed instead of the requirements file when the installer bundles one. +locked_requirements() { + if [ -f "${1%.txt}.lock.txt" ]; then echo "${1%.txt}.lock.txt"; else echo "$1"; fi +} +SELECTED_REQUIREMENTS="$(locked_requirements "$SELECTED_REQUIREMENTS")" echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE" "$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1 if [[ "${OSTYPE:-}" == "darwin"* ]]; then echo "Detected macOS platform" >> "$LOG_FILE" if [ -f "$PROJECT_ROOT/requirements-macos.txt" ]; then - "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-macos.txt" >> "$LOG_FILE" 2>&1 + "$PYTHON_EXE" -m pip install -r "$(locked_requirements "$PROJECT_ROOT/requirements-macos.txt")" >> "$LOG_FILE" 2>&1 fi elif [[ "${OSTYPE:-}" == "linux-gnu"* ]]; then echo "Detected Linux platform" >> "$LOG_FILE" if [ -f "$PROJECT_ROOT/requirements-linux.txt" ]; then - "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-linux.txt" >> "$LOG_FILE" 2>&1 + "$PYTHON_EXE" -m pip install -r "$(locked_requirements "$PROJECT_ROOT/requirements-linux.txt")" >> "$LOG_FILE" 2>&1 fi else echo "Unknown platform: ${OSTYPE:-unset}" >> "$LOG_FILE" diff --git a/app/menuinst/notebook_launcher.json b/app/menuinst/notebook_launcher.json index bcb2c44..fef18e0 100644 --- a/app/menuinst/notebook_launcher.json +++ b/app/menuinst/notebook_launcher.json @@ -15,7 +15,14 @@ ], "platforms": { "win": { - "icon": "ICON_ICO_IMAGE_PATH" + "icon": "ICON_ICO_IMAGE_PATH", + "command": [ + "{{ PYTHON }}", + "{{ PREFIX }}/PROJECT_NAME/launch_jupyter.py", + "{{ PREFIX }}/PROJECT_NAME/notebooks/Welcome.ipynb", + "&", + "exit" + ] }, "linux": { "icon": "ICON_IMAGE_PATH" diff --git a/app/python_scripts/launch_jupyter.py b/app/python_scripts/launch_jupyter.py index 0f91726..f45ec5f 100644 --- a/app/python_scripts/launch_jupyter.py +++ b/app/python_scripts/launch_jupyter.py @@ -13,8 +13,10 @@ import os import platform import re +import shutil import ssl import sys +import tempfile import traceback from datetime import datetime, timezone from importlib import metadata @@ -197,6 +199,21 @@ def _show_error_dialog(message: str) -> None: pass +def _wait_for_enter() -> None: + """Keep the terminal window open until the user has read it, then let it close. + + The menu shortcut runs the launcher in a terminal window; without this the window + closes (or stays open, on Windows) as soon as JupyterLab stops. + """ + stream = sys.stdin + if stream is None or not stream.isatty(): + return + try: + input("\nPress Enter to close this window...") + except (EOFError, KeyboardInterrupt): + pass + + def run_jupyter(notebook: Path) -> int: if not notebook.is_file(): raise FileNotFoundError(f"Welcome notebook not found: {notebook}") @@ -213,6 +230,12 @@ def run_jupyter(notebook: Path) -> int: # All-users installations can live outside the home directory. root_dir = PREFIX.resolve() + # A fresh, empty workspace folder for every launch: JupyterLab restores the open + # tabs of its workspaces, and the default folder (~/.jupyter/lab/workspaces) is + # shared by every app, so notebooks of other apps (and of earlier sessions) would + # open next to the Welcome notebook. Settings and themes are kept: they are stored + # in the user settings folder, not in the workspaces. + workspaces_dir = Path(tempfile.mkdtemp(prefix="labconstrictor-workspaces-")) sys.argv = [ "jupyter-lab", str(notebook.resolve()), @@ -220,8 +243,12 @@ def run_jupyter(notebook: Path) -> int: f"--ServerApp.root_dir={root_dir}", "--ServerApp.shutdown_no_activity_timeout=30", "--FileContentsManager.always_delete_dir=True", + f"--LabApp.workspaces_dir={workspaces_dir}", ] - result = main() + try: + result = main() + finally: + shutil.rmtree(workspaces_dir, ignore_errors=True) return int(result or 0) @@ -240,8 +267,19 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Print the selected verified CA bundle path and exit.", ) + parser.add_argument( + "--no-wait", + action="store_true", + help="Exit when JupyterLab stops, without asking to press Enter.", + ) args = parser.parse_args(argv) + result = _launch(args) + if not (args.no_wait or args.self_test or args.print_ca_bundle): + _wait_for_enter() + return result + +def _launch(args: argparse.Namespace) -> int: try: if args.print_ca_bundle: print(_select_ca_bundle(), flush=True) diff --git a/tests/_v0_1_20.py b/tests/_v0_1_20.py new file mode 100644 index 0000000..53e791d --- /dev/null +++ b/tests/_v0_1_20.py @@ -0,0 +1,41 @@ +"""Helper of the tests: a live template file as it was before template 0.1.20. + +The resources of the older migrations are frozen: they are what those migrations deliver. Their tests compare them with the live template +files, and since 0.1.20 those files carry the lock-file lines (post_install.sh / .bat) or are new files (the two workflows). `before_0_1_20` +gives the bytes of the live file without the 0.1.20 change, so those tests keep checking what they checked; the 0.1.20 changes have their +own tests (test_template_sync_0_1_20.py). +""" +from __future__ import annotations + +import importlib.util +from pathlib import Path +from types import ModuleType + +ROOT = Path(__file__).resolve().parents[1] +MIGRATION_PATH = ROOT / ".template_sync/migrations/v0_1_19_to_v0_1_20.py" + + +def _migration() -> ModuleType: + spec = importlib.util.spec_from_file_location("migration_0_1_20_for_older_tests", MIGRATION_PATH) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {MIGRATION_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def before_0_1_20(relative: Path | str, data: bytes) -> bytes: + """`data` (a live template file at `relative`) without the change 0.1.20 made to it; any other file is returned as it is.""" + migration = _migration() + relative = Path(relative) + if relative in migration.REPLACED_FILES: + return (migration.PREVIOUS_ROOT / relative).read_bytes() + blocks = {migration.POST_INSTALL_SH: migration.SH_BLOCKS, migration.POST_INSTALL_BAT: migration.BAT_BLOCKS}.get(relative) + if blocks is None: + return data + text = data.decode("utf-8") + newline = "\r\n" if "\r\n" in text else "\n" + text = text.replace("\r\n", "\n") + for old, new in blocks: + text = text.replace(new, old) + return text.replace("\n", newline).encode("utf-8") diff --git a/tests/test_template_sync_0_1_17.py b/tests/test_template_sync_0_1_17.py index 6022498..15ebca6 100644 --- a/tests/test_template_sync_0_1_17.py +++ b/tests/test_template_sync_0_1_17.py @@ -18,6 +18,7 @@ from pathlib import Path from _hosts import strip_hosts +from _v0_1_20 import before_0_1_20 ROOT = Path(__file__).resolve().parents[1] MIGRATION_PATH = ROOT / ".template_sync/migrations/v0_1_16_to_v0_1_17.py" @@ -111,11 +112,12 @@ def run_migration(self) -> str: class TemplateConsistencyTests(unittest.TestCase): def test_resources_are_identical_to_the_template_files(self) -> None: """Since template 0.1.18 the live files carry the new Tools pin, so the resources of 0.1.17 (frozen: they are what 0.1.17 delivers) - must equal the live files except for that pin and for the app-hosts lines (a 0.1.18 step, removed by tests/_hosts.py); the byte-for-byte check against the live files is made for the 0.1.18 resources.""" + must equal the live files except for that pin, for the app-hosts lines (a 0.1.18 step, removed by tests/_hosts.py) and for the lock-file + lines of 0.1.20 (removed by tests/_v0_1_20.py); the byte-for-byte check against the live files is made for the newest resources.""" migration = load(MIGRATION_PATH, "migration_0_1_17_resources") for relative in migration.MANAGED_FILES: with self.subTest(file=str(relative)): - live = strip_hosts(relative.name, (ROOT / relative).read_bytes()).replace(TEMPLATE_PIN.encode(), OLD_PIN.encode()) + live = strip_hosts(relative.name, before_0_1_20(relative, (ROOT / relative).read_bytes())).replace(TEMPLATE_PIN.encode(), OLD_PIN.encode()) self.assertEqual((migration.RESOURCE_ROOT / relative).read_bytes(), live) def test_the_manifest_lists_the_migration_and_the_new_version(self) -> None: diff --git a/tests/test_template_sync_0_1_18.py b/tests/test_template_sync_0_1_18.py index 4566178..a1c75c0 100644 --- a/tests/test_template_sync_0_1_18.py +++ b/tests/test_template_sync_0_1_18.py @@ -16,6 +16,7 @@ from types import ModuleType from _hosts import strip_hosts +from _v0_1_20 import before_0_1_20 ROOT = Path(__file__).resolve().parents[1] @@ -101,13 +102,14 @@ def uses_lines(path: Path) -> list[tuple[str, str, str | None]]: class TemplateConsistencyTests(unittest.TestCase): def test_resources_are_identical_to_the_template_files(self) -> None: + """The resources are frozen (they are what 0.1.18 delivers): equal to the live files without the changes of 0.1.20 (tests/_v0_1_20.py).""" migration = load(MIGRATION_PATH, "migration_0_1_18_resources") for relative in migration.MANAGED_FILES: with self.subTest(file=str(relative)): self.assertEqual( (migration.RESOURCE_ROOT / relative).read_bytes(), - (ROOT / relative).read_bytes(), - "update .template_sync/resources/v0_1_17_to_v0_1_18 when the template file changes", + before_0_1_20(relative, (ROOT / relative).read_bytes()), + "the 0.1.18 resources are frozen: a later template change needs its own migration (and tests/_v0_1_20.py-style undo)", ) def test_the_manifest_lists_the_migration_and_the_new_version(self) -> None: @@ -468,7 +470,10 @@ def test_the_whole_chain_from_0_1_15_reaches_the_live_template_scripts(self) -> replacements = self.helpers.load_replacements(self.repo) newline = "\r\n" if "\r\n" in source else "\n" self.script(name).write_bytes(self.helpers.render_template(source, replacements, newline).encode()) - result = self.sync("0.1.15") + # The chain runs up to the current template version, so the scripts must end equal to the live ones. + latest = re.search(r"template_version: ([0-9.]+)", (ROOT / ".template_sync/manifest.yaml").read_text(encoding="utf-8")) + assert latest is not None + result = self.sync("0.1.15", latest.group(1)) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) replacements = self.helpers.load_replacements(self.repo) for name in SCRIPT_NAMES: diff --git a/tests/test_template_sync_0_1_19.py b/tests/test_template_sync_0_1_19.py index f6ac708..6c322ef 100644 --- a/tests/test_template_sync_0_1_19.py +++ b/tests/test_template_sync_0_1_19.py @@ -41,7 +41,7 @@ def old_text(relative: Path) -> str: class TemplateConsistencyTests(unittest.TestCase): def test_the_manifest_lists_the_migration_and_the_new_version(self) -> None: text = (ROOT / ".template_sync/manifest.yaml").read_text(encoding="utf-8") - self.assertIn("template_version: 0.1.19", text) + self.assertIn("to: 0.1.19", text) # the current template_version is checked by the test of the newest migration self.assertIn("script: migrations/v0_1_18_to_v0_1_19.py", text) def test_chains_from_every_older_version_reach_0_1_19(self) -> None: diff --git a/tests/test_template_sync_0_1_20.py b/tests/test_template_sync_0_1_20.py new file mode 100644 index 0000000..ca6c655 --- /dev/null +++ b/tests/test_template_sync_0_1_20.py @@ -0,0 +1,686 @@ +"""Tests of template 0.1.20: the workflow race fix, the lock files of the pip requirements, the launcher that starts fresh, and their migration.""" +from __future__ import annotations + +import importlib.util +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path +from types import ModuleType +from unittest import mock + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +MIGRATION_PATH = ROOT / ".template_sync/migrations/v0_1_19_to_v0_1_20.py" +SYNC_PATH = ROOT / ".template_sync/sync.py" +LOCK_SCRIPT_PATH = ROOT / ".tools/python/lock_requirements.py" +BUMP_PATH = ROOT / ".tools/python/bump_constructor.py" +LAUNCHER_PATH = ROOT / "app/python_scripts/launch_jupyter.py" +NOTEBOOK_WORKFLOW = ROOT / ".github/workflows/update_on_notebook_change.yml" +SRC_WORKFLOW = ROOT / ".github/workflows/update_on_src_change.yaml" +BASH = shutil.which("bash") +GIT = shutil.which("git") + + +def load(path: Path, name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +MIGRATION = load(MIGRATION_PATH, "migration_0_1_20") +HELPERS = MIGRATION._load_helpers() + + +def lf(path: Path) -> str: + return path.read_bytes().decode("utf-8").replace("\r\n", "\n") + + +def steps(workflow: Path) -> dict[str, dict]: + data = yaml.safe_load(workflow.read_text(encoding="utf-8")) + (job,) = data["jobs"].values() + return {step["name"]: step for step in job["steps"]} + + +# -------------------------------------------------------------------------------------------------------------------------------- +# The template itself +# -------------------------------------------------------------------------------------------------------------------------------- +class TemplateConsistencyTests(unittest.TestCase): + def test_the_manifest_lists_the_migration_and_the_new_version(self) -> None: + text = (ROOT / ".template_sync/manifest.yaml").read_text(encoding="utf-8") + self.assertIn("template_version: 0.1.20", text) + self.assertIn("script: migrations/v0_1_19_to_v0_1_20.py", text) + + def test_chains_from_every_older_version_reach_0_1_20(self) -> None: + sync = load(SYNC_PATH, "sync_for_0_1_20") + manifest = sync.parse_manifest(ROOT / ".template_sync/manifest.yaml") + for start in ("0.0.0", "0.1.9", "0.1.15", "0.1.18", "0.1.19"): + chain = sync.resolve_migration_chain(manifest["migrations"], start, "0.1.20") + self.assertEqual(chain[-1]["to"], "0.1.20", start) + + def test_the_design_note_is_linked(self) -> None: + self.assertIn("(migration_0_1_20.md)", (ROOT / ".tools/docs/README.md").read_text(encoding="utf-8")) + self.assertTrue((ROOT / ".tools/docs/migration_0_1_20.md").is_file()) + + def test_resources_are_identical_to_the_template_files(self) -> None: + for relative in MIGRATION.MANAGED_FILES: + with self.subTest(file=str(relative)): + self.assertEqual( + (MIGRATION.RESOURCE_ROOT / relative).read_bytes(), + (ROOT / relative).read_bytes(), + "update .template_sync/resources/v0_1_19_to_v0_1_20 when the template file changes", + ) + + def test_the_previous_resources_are_the_0_1_19_files(self) -> None: + """The 0.1.19 copies are what the migration compares an app's file with: each must differ from the new file (else nothing is replaced).""" + for relative in MIGRATION.REPLACED_FILES: + with self.subTest(file=str(relative)): + self.assertNotEqual(lf(MIGRATION.PREVIOUS_ROOT / relative), lf(ROOT / relative)) + + def test_the_live_scripts_hold_the_new_blocks_once(self) -> None: + for relative, blocks in ((MIGRATION.POST_INSTALL_SH, MIGRATION.SH_BLOCKS), (MIGRATION.POST_INSTALL_BAT, MIGRATION.BAT_BLOCKS)): + text = lf(ROOT / relative) + for old, new in blocks: + with self.subTest(file=str(relative), block=old.splitlines()[0]): + self.assertEqual(text.count(new), 1) + self.assertNotIn(old, text.replace(new, ""), "the 0.1.19 block is left only inside the new one") + bump = lf(BUMP_PATH) + self.assertEqual(bump.count(MIGRATION.BUMP_SNIPPET + MIGRATION.BUMP_FUNCTION_ANCHOR), 1) + self.assertEqual(bump.count(MIGRATION.BUMP_CALL_NEW), 1) + self.assertIn(MIGRATION.UV_PIN, lf(ROOT / MIGRATION.TOOLS_REQUIREMENTS).splitlines()) + self.assertIn(MIGRATION.USAGE_NEW, lf(ROOT / MIGRATION.USAGE_DOC)) + + def test_the_menu_shortcut_closes_the_windows_terminal_after_enter(self) -> None: + data = json.loads(lf(ROOT / MIGRATION.MENU)) + (item,) = data["menu_items"] + self.assertTrue(item["terminal"]) + self.assertEqual(item["platforms"]["win"]["command"], item["command"] + ["&", "exit"]) + for platform in ("linux", "osx"): + self.assertNotIn("command", item["platforms"][platform]) + + def test_the_workflows_no_longer_share_a_concurrency_group_and_never_use_the_auto_commit_action(self) -> None: + groups = set() + for workflow in (NOTEBOOK_WORKFLOW, SRC_WORKFLOW): + with self.subTest(workflow=workflow.name): + data = yaml.safe_load(workflow.read_text(encoding="utf-8")) + groups.add(data["concurrency"]["group"]) + self.assertFalse(data["concurrency"]["cancel-in-progress"]) + text = workflow.read_text(encoding="utf-8") + self.assertNotIn("git-auto-commit-action", text) + checkout = steps(workflow)["Checkout repository"] + self.assertNotIn("ref", checkout.get("with", {}), "the scripts compare the triggering commit with its parent: keep github.sha") + commit = steps(workflow)["Commit updated files"]["run"] + self.assertIn('git rebase --autostash "origin/$BRANCH"', commit) + self.assertIn('git push origin "HEAD:refs/heads/$BRANCH"', commit) + self.assertEqual(len(groups), 2) + + def test_the_notebook_workflow_locks_after_merging_and_reverts_only_for_the_conda_test(self) -> None: + names = list(steps(NOTEBOOK_WORKFLOW)) + self.assertEqual(names.index("Lock requirements"), names.index("Run merge script") + 1) + self.assertLess(names.index("Lock requirements"), names.index("Check for changes")) + self.assertIn("lock_requirements.py", steps(NOTEBOOK_WORKFLOW)["Lock requirements"]["run"]) + revert = steps(NOTEBOOK_WORKFLOW)["Revert user commit that broke requirements"] + self.assertIn("steps.test-conda-env.outcome == 'failure'", revert["if"]) + self.assertIn("failure()", revert["if"]) + + def test_the_conda_test_installs_the_lock_file(self) -> None: + text = lf(ROOT / MIGRATION.TEST_CONDA) + self.assertIn('REQUIREMENTS_FILE="requirements.lock.txt"', text) + self.assertIn('pip install --prefer-binary -r "$REQUIREMENTS_FILE"', text) + self.assertIn("pip check", text) + + +# -------------------------------------------------------------------------------------------------------------------------------- +# The workflow steps themselves, run with bash and git against a local "origin" that moved while the run worked +# -------------------------------------------------------------------------------------------------------------------------------- +@unittest.skipUnless(BASH and GIT, "bash and git are needed") +class WorkflowScriptTests(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + self.env = { + **os.environ, + "GIT_AUTHOR_NAME": "user", + "GIT_AUTHOR_EMAIL": "user@example.com", + "GIT_COMMITTER_NAME": "user", + "GIT_COMMITTER_EMAIL": "user@example.com", + "GIT_CONFIG_GLOBAL": str(self.tmp / "gitconfig"), + "GIT_CONFIG_NOSYSTEM": "1", + } + (self.tmp / "gitconfig").write_text("[init]\n\tdefaultBranch = main\n[core]\n\tautocrlf = false\n", encoding="utf-8") + # `sleep` of the retry loop: instant in the tests + fake_bin = self.tmp / "bin" + fake_bin.mkdir() + (fake_bin / "sleep").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + (fake_bin / "sleep").chmod(0o755) + self.env["PATH"] = str(fake_bin) + os.pathsep + self.env["PATH"] + self.origin = self.tmp / "origin.git" + self.git(self.tmp, "init", "--bare", "-q", str(self.origin)) + self.seed = self.tmp / "seed" + self.git(self.tmp, "clone", "-q", str(self.origin), str(self.seed)) + self.write(self.seed / "requirements.txt", "pandas==2.2.2\n") + self.write(self.seed / "requirements.lock.txt", "pandas==2.2.2\nnumpy==2.0.0\n") + self.write(self.seed / ".tools/meta/src_change.yaml", 'src_last_modified: "old"\n') + self.write(self.seed / "notebooks/A/A.ipynb", "{}\n") + self.write(self.seed / "src/code.py", "x = 1\n") + self.commit(self.seed, "base") + + def tearDown(self) -> None: + self._tmp.cleanup() + + def git(self, cwd: Path, *args: str) -> str: + result = subprocess.run([GIT or "git", *args], cwd=cwd, env=self.env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, f"git {' '.join(args)}: {result.stdout}{result.stderr}") + return result.stdout.strip() + + def write(self, path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(text.encode("utf-8")) + + def commit(self, repo: Path, message: str) -> str: + self.git(repo, "add", "-A") + self.git(repo, "commit", "-q", "-m", message) + self.git(repo, "push", "-q", "origin", "HEAD:refs/heads/main") + return self.git(repo, "rev-parse", "HEAD") + + def runner_checkout(self, sha: str) -> Path: + """What actions/checkout does for a push: the branch at the commit that triggered the run (fetch-depth 0).""" + work = self.tmp / f"run-{sha[:7]}" + self.git(self.tmp, "clone", "-q", str(self.origin), str(work)) + self.git(work, "checkout", "-q", "-B", "main", sha) + return work + + def run_step(self, workflow: Path, name: str, cwd: Path, **env: str) -> subprocess.CompletedProcess[str]: + step = steps(workflow)[name] + script = self.tmp / "step.sh" + script.write_bytes(step["run"].encode("utf-8")) + merged = {**self.env, **{k: str(v) for k, v in step.get("env", {}).items() if "${{" not in str(v)}, **env} + # GitHub runs `run:` steps with `bash -e {0}` + return subprocess.run([BASH or "bash", "-e", str(script)], cwd=cwd, env=merged, capture_output=True, text=True) + + def origin_log(self) -> list[str]: + self.git(self.seed, "fetch", "-q") + return self.git(self.seed, "log", "--format=%s", "origin/main").splitlines() + + def test_the_src_commit_is_rebased_onto_a_branch_that_moved_while_the_run_waited(self) -> None: + trigger = self.git(self.seed, "rev-parse", "HEAD") + work = self.runner_checkout(trigger) + # the notebook workflow pushed its commit in the meantime (CellMigrationLab/LabChronicle run 41800906840) + self.write(self.seed / "requirements.lock.txt", "pandas==2.2.2\nnumpy==2.1.0\n") + self.commit(self.seed, "GitHub Action - Updated files because of notebook and/or requirements changes") + self.write(work / ".tools/meta/src_change.yaml", 'src_last_modified: "new"\n') + result = self.run_step(SRC_WORKFLOW, "Commit updated files", work, BRANCH="main") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("rebasing onto the latest main", result.stdout) + self.assertEqual(self.origin_log()[:2], ["GitHub Action - Updated src change file", "GitHub Action - Updated files because of notebook and/or requirements changes"]) + self.git(self.seed, "merge", "-q", "--ff-only", "origin/main") + self.assertIn("new", (self.seed / ".tools/meta/src_change.yaml").read_text(encoding="utf-8")) + self.assertIn("numpy==2.1.0", (self.seed / "requirements.lock.txt").read_text(encoding="utf-8")) + + def test_the_notebook_commit_takes_only_its_files_including_a_removed_lock_file(self) -> None: + trigger = self.git(self.seed, "rev-parse", "HEAD") + work = self.runner_checkout(trigger) + self.write(self.seed / ".tools/meta/src_change.yaml", 'src_last_modified: "from the src workflow"\n') + self.commit(self.seed, "GitHub Action - Updated src change file") + (work / "requirements.lock.txt").unlink() # lock_requirements.py removed a stale lock file + self.write(work / "requirements_gpu.lock.txt", "torch==2.7.1\n") + self.write(work / "notebooks/A/A.ipynb", '{"cells": []}\n') + self.write(work / "src/code.py", "x = 2\n") # not a file of this workflow: never committed + result = self.run_step(NOTEBOOK_WORKFLOW, "Commit updated files", work, BRANCH="main") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.git(self.seed, "fetch", "-q") + changed = self.git(self.seed, "show", "--name-status", "--format=", "origin/main").splitlines() + self.assertEqual(sorted(changed), sorted(["D\trequirements.lock.txt", "A\trequirements_gpu.lock.txt", "M\tnotebooks/A/A.ipynb"])) + self.assertEqual(self.git(self.seed, "log", "-1", "--format=%s", "origin/main~1"), "GitHub Action - Updated src change file") + + def test_a_conflicting_commit_fails_with_a_message_and_pushes_nothing(self) -> None: + trigger = self.git(self.seed, "rev-parse", "HEAD") + work = self.runner_checkout(trigger) + self.write(self.seed / "requirements.lock.txt", "pandas==2.2.2\nnumpy==3.0.0\n") + moved = self.commit(self.seed, "someone else") + self.write(work / "requirements.lock.txt", "pandas==2.2.2\nnumpy==2.9.0\n") + result = self.run_step(NOTEBOOK_WORKFLOW, "Commit updated files", work, BRANCH="main") + self.assertNotEqual(result.returncode, 0) + self.assertIn("::error::The commit of this run conflicts with the latest main", result.stdout) + self.git(self.seed, "fetch", "-q") + self.assertEqual(self.git(self.seed, "rev-parse", "origin/main"), moved) + + def test_the_revert_reverts_a_merge_and_a_plain_commit_on_top_of_the_latest_branch(self) -> None: + for kind in ("merge", "plain"): + with self.subTest(kind=kind): + if kind == "merge": + base = self.git(self.seed, "rev-parse", "HEAD") + self.git(self.seed, "checkout", "-q", "-b", "side") + self.write(self.seed / "requirements.txt", "pandas==2.2.2\nbroken==0\n") + self.git(self.seed, "add", "-A") + self.git(self.seed, "commit", "-q", "-m", "break") + self.git(self.seed, "checkout", "-q", "main") + self.git(self.seed, "reset", "-q", "--hard", base) + self.write(self.seed / "notebooks/A/notes.md", "x\n") + self.git(self.seed, "add", "-A") + self.git(self.seed, "commit", "-q", "-m", "other work") + self.git(self.seed, "merge", "-q", "--no-ff", "-m", "Merge branch side", "side") + self.git(self.seed, "push", "-q", "origin", "HEAD:refs/heads/main") + bad = self.git(self.seed, "rev-parse", "HEAD") + else: + self.write(self.seed / "requirements.txt", "pandas==2.2.2\nbroken==1\n") + bad = self.commit(self.seed, "break again") + work = self.runner_checkout(bad) + self.write(work / "requirements.txt", "half written by the merge script\n") # left over by the failed run + # the src workflow committed while the conda test ran + self.write(self.seed / ".tools/meta/src_change.yaml", f'src_last_modified: "{kind}"\n') + self.commit(self.seed, f"src {kind}") + result = self.run_step(NOTEBOOK_WORKFLOW, "Revert user commit that broke requirements", work, BRANCH="main", BAD_COMMIT=bad) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.git(self.seed, "fetch", "-q") + self.assertTrue(self.git(self.seed, "log", "-1", "--format=%s", "origin/main").startswith("Revert ")) + self.assertEqual(self.git(self.seed, "log", "-1", "--format=%s", "origin/main~1"), f"src {kind}") + self.git(self.seed, "checkout", "-q", "main") + self.git(self.seed, "reset", "-q", "--hard", "origin/main") + self.assertEqual((self.seed / "requirements.txt").read_text(encoding="utf-8"), "pandas==2.2.2\n") + + +# -------------------------------------------------------------------------------------------------------------------------------- +# lock_requirements.py (uv is replaced by a fake: the tests need no network) +# -------------------------------------------------------------------------------------------------------------------------------- +class LockScriptTests(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.lock = load(LOCK_SCRIPT_PATH, "lock_requirements_under_test") + (self.root / "environment.yaml").write_text( + "name: demo\ndependencies:\n- python=3.11.1\n- conda\n- git\n- pip>=24.2\n- setuptools\n- certifi\n- jupyterlab=4.4.0\n- pip:\n - ignored\n", + encoding="utf-8", + ) + self.calls: list[dict] = [] + + def tearDown(self) -> None: + self._tmp.cleanup() + + def fake_uv(self, requirements: Path, python_version: str, constraint_files: list[Path], no_emit: list[str]) -> str: + self.calls.append({ + "file": requirements.name, + "python": python_version, + "constraints": [c.read_text(encoding="utf-8") for c in constraint_files], + "no_emit": no_emit, + }) + if "unsolvable" in requirements.read_text(encoding="utf-8"): + raise RuntimeError("No solution found when resolving dependencies") + return "# uv comment\nnumpy==2.4.6 ; python_full_version < '3.12'\npandas==2.2.2\n" + + def run_lock(self) -> str: + buffer = StringIO() + with mock.patch.object(self.lock, "run_uv", self.fake_uv), mock.patch.object(self.lock, "uv_version", lambda: "uv 0.13.3"), redirect_stdout(buffer): + self.assertEqual(self.lock.main(["--root", str(self.root)]), 0) + return buffer.getvalue() + + def test_the_environment_gives_the_python_the_conda_names_and_the_conda_pins(self) -> None: + python, names, constraints = self.lock.read_environment(self.root / "environment.yaml") + self.assertEqual(python, "3.11.1") + self.assertEqual(names, ["certifi", "conda", "git", "jupyterlab", "pip", "setuptools"]) + self.assertEqual(constraints, ["jupyterlab==4.4.0.*"]) + + def test_the_template_placeholders_lock_nothing(self) -> None: + (self.root / "environment.yaml").write_text("dependencies:\n- python=PYTHON_VERSION\n- jupyterlab=JUPYTERLAB_VERSION\n", encoding="utf-8") + (self.root / "requirements.txt").write_text("pandas\n", encoding="utf-8") + self.assertIn("no exact Python version", self.run_lock()) + self.assertFalse((self.root / "requirements.lock.txt").exists()) + + def test_a_lock_file_is_written_once_and_kept_while_the_inputs_do_not_change(self) -> None: + requirements = self.root / "requirements.txt" + requirements.write_text("# Generated by merge_requirements.py on 2026-11-04T08:30:40Z\npandas==2.2.2\n", encoding="utf-8") + self.assertIn("requirements.lock.txt: written", self.run_lock()) + lock = (self.root / "requirements.lock.txt").read_bytes() + text = lock.decode("utf-8") + self.assertNotIn(b"\r\n", lock) + self.assertIn("uv 0.13.3, universal resolution for Python 3.11.1", text) + self.assertIn("numpy==2.4.6 ; python_full_version < '3.12'\npandas==2.2.2\n", text) + self.assertNotIn("# uv comment", text) + self.assertEqual(self.calls[0]["python"], "3.11.1") + self.assertEqual(self.calls[0]["constraints"], ["jupyterlab==4.4.0.*\n"]) + self.assertNotIn("git", self.calls[0]["no_emit"]) + self.assertIn("jupyterlab", self.calls[0]["no_emit"]) + # a new merge only changes the timestamp comment: the lock file is kept and uv is not run again + requirements.write_text("# Generated by merge_requirements.py on 2026-12-01T00:00:00Z\npandas==2.2.2\n", encoding="utf-8") + self.assertIn("requirements.lock.txt: up to date (unchanged)", self.run_lock()) + self.assertEqual((self.root / "requirements.lock.txt").read_bytes(), lock) + self.assertEqual(len(self.calls), 1) + # a new requirement, or another JupyterLab version, writes it again + requirements.write_text("pandas==2.2.2\ntqdm\n", encoding="utf-8") + self.assertIn("requirements.lock.txt: written", self.run_lock()) + env = self.root / "environment.yaml" + env.write_text(env.read_text(encoding="utf-8").replace("jupyterlab=4.4.0", "jupyterlab=4.5.0"), encoding="utf-8") + self.assertIn("requirements.lock.txt: written", self.run_lock()) + self.assertEqual(len(self.calls), 3) + + def test_the_index_options_are_copied_and_the_os_file_is_constrained_by_the_base_lock(self) -> None: + (self.root / "requirements.txt").write_text("pandas==2.2.2\n", encoding="utf-8") + (self.root / "requirements_gpu.txt").write_text("--extra-index-url https://download.pytorch.org/whl/cu128\ntorch\n", encoding="utf-8") + (self.root / "requirements-windows.txt").write_text("pywin32\n", encoding="utf-8") + self.run_lock() + gpu = (self.root / "requirements_gpu.lock.txt").read_text(encoding="utf-8") + self.assertIn("\n--extra-index-url https://download.pytorch.org/whl/cu128\n\nnumpy==", gpu) + windows_call = next(call for call in self.calls if call["file"] == "requirements-windows.txt") + self.assertEqual(len(windows_call["constraints"]), 2) + self.assertIn("pandas==2.2.2", windows_call["constraints"][1]) + self.assertNotIn("# lock-input", windows_call["constraints"][1]) + gpu_call = next(call for call in self.calls if call["file"] == "requirements_gpu.txt") + self.assertEqual(len(gpu_call["constraints"]), 1, "the GPU file is resolved on its own (its torch is not the CPU torch)") + + def test_a_file_that_cannot_be_locked_removes_its_stale_lock_and_warns(self) -> None: + requirements = self.root / "requirements.txt" + requirements.write_text("pandas==2.2.2\n", encoding="utf-8") + self.run_lock() + requirements.write_text("pandas==2.2.2\nunsolvable==1\n", encoding="utf-8") + output = self.run_lock() + self.assertIn("::warning::", output) + self.assertIn("requirements.lock.txt: removed", output) + self.assertFalse((self.root / "requirements.lock.txt").exists()) + self.assertIn("not locked: requirements.txt", output) + + def test_a_lock_file_without_its_requirements_file_is_removed(self) -> None: + (self.root / "requirements.txt").write_text("pandas\n", encoding="utf-8") + (self.root / "requirements-macos.lock.txt").write_text("old\n", encoding="utf-8") + self.assertIn("requirements-macos.lock.txt: removed", self.run_lock()) + self.assertFalse((self.root / "requirements-macos.lock.txt").exists()) + + def test_uv_errors_are_reported_as_runtime_errors(self) -> None: + (self.root / "requirements.txt").write_text("pandas\n", encoding="utf-8") + failed = subprocess.CompletedProcess([], 2, stdout="", stderr="error: boom") + with mock.patch.object(self.lock.subprocess, "run", return_value=failed): + with self.assertRaisesRegex(RuntimeError, "boom"): + self.lock.run_uv(self.root / "requirements.txt", "3.11.1", [], []) + + +# -------------------------------------------------------------------------------------------------------------------------------- +# bump_constructor.py bundles the lock files +# -------------------------------------------------------------------------------------------------------------------------------- +class BumpConstructorTests(unittest.TestCase): + def test_lock_files_are_mapped_when_they_exist_and_unmapped_when_they_are_gone(self) -> None: + bump = load(BUMP_PATH, "bump_constructor_0_1_20") + with tempfile.TemporaryDirectory() as tmp: + cwd = os.getcwd() + os.chdir(tmp) + try: + Path("requirements.lock.txt").write_text("x\n", encoding="utf-8") + data = {"extra_files": [ + {"app/menuinst/Welcome.ipynb": "Demo/notebooks/Welcome.ipynb"}, + {"requirements_gpu.lock.txt": "Demo/requirements_gpu.lock.txt"}, + ]} + with redirect_stdout(StringIO()): + bump.ensure_lock_files_in_extra_files(data) + bump.ensure_lock_files_in_extra_files(data) + finally: + os.chdir(cwd) + self.assertEqual(data["extra_files"], [ + {"app/menuinst/Welcome.ipynb": "Demo/notebooks/Welcome.ipynb"}, + {"requirements.lock.txt": "Demo/requirements.lock.txt"}, + ]) + + +# -------------------------------------------------------------------------------------------------------------------------------- +# launch_jupyter.py: a fresh workspace folder, and Enter before the window closes +# -------------------------------------------------------------------------------------------------------------------------------- +class LauncherTests(unittest.TestCase): + def setUp(self) -> None: + self.launcher = load(LAUNCHER_PATH, "launch_jupyter_0_1_20") + + def test_jupyterlab_gets_an_empty_workspace_folder_that_is_removed_afterwards(self) -> None: + seen: dict[str, object] = {} + + def fake_main() -> int: + argv = list(sys.argv) + (arg,) = [a for a in argv if a.startswith("--LabApp.workspaces_dir=")] + folder = Path(arg.split("=", 1)[1]) + seen.update(argv=argv, folder=folder, empty=folder.is_dir() and not any(folder.iterdir())) + (folder / "auto-0.jupyterlab-workspace").write_text("{}", encoding="utf-8") # JupyterLab saves the session + return 0 + + labapp = ModuleType("jupyterlab.labapp") + labapp.main = fake_main # type: ignore[attr-defined] + with tempfile.TemporaryDirectory() as tmp: + notebook = Path(tmp) / "Welcome.ipynb" + notebook.write_text("{}", encoding="utf-8") + saved_argv = sys.argv + try: + with mock.patch.dict(sys.modules, {"jupyterlab": ModuleType("jupyterlab"), "jupyterlab.labapp": labapp}): + self.assertEqual(self.launcher.run_jupyter(notebook), 0) + finally: + sys.argv = saved_argv + self.assertTrue(seen["empty"]) + self.assertFalse(Path(str(seen["folder"])).exists()) + argv = seen["argv"] + assert isinstance(argv, list) + self.assertIn(str(notebook.resolve()), argv) + + def test_the_window_waits_for_enter_after_a_launch_but_not_for_the_installer_checks(self) -> None: + cases = { + (): True, + ("--no-wait",): False, + ("--self-test",): False, + ("--print-ca-bundle",): False, + } + for args, waits in cases.items(): + with self.subTest(args=args): + with mock.patch.object(self.launcher, "_launch", return_value=7), mock.patch.object(self.launcher, "_wait_for_enter") as wait: + self.assertEqual(self.launcher.main(list(args)), 7) + self.assertEqual(wait.called, waits) + + def test_the_prompt_is_asked_only_in_a_terminal(self) -> None: + tty = mock.Mock() + tty.isatty.return_value = True + pipe = mock.Mock() + pipe.isatty.return_value = False + for stream, asked in ((tty, True), (pipe, False), (None, False)): + with self.subTest(asked=asked): + with mock.patch.object(self.launcher.sys, "stdin", stream), mock.patch("builtins.input") as prompt: + self.launcher._wait_for_enter() + self.assertEqual(prompt.called, asked) + with mock.patch.object(self.launcher.sys, "stdin", tty), mock.patch("builtins.input", side_effect=EOFError): + self.launcher._wait_for_enter() # a closed input never raises + + +# -------------------------------------------------------------------------------------------------------------------------------- +# The migration +# -------------------------------------------------------------------------------------------------------------------------------- +CONSTRUCT = """name: LabChronicle +version: 1.2.0 + +extra_files: +- app/menuinst/Welcome.ipynb: LabChronicle/notebooks/Welcome.ipynb +- requirements.txt: LabChronicle/requirements.txt +""" + + +class MigrationCase(unittest.TestCase): + NEWLINES = { + # line endings of the template files (src workflow and post_install.bat are CRLF in the template) + MIGRATION.SRC_WORKFLOW: "\r\n", + MIGRATION.POST_INSTALL_BAT: "\r\n", + } + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.repo = Path(self._tmp.name) + self.put("construct.yaml", CONSTRUCT) + self.replacements = HELPERS.load_replacements(self.repo) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def put(self, relative: Path | str, text: str, newline: str = "\n") -> Path: + path = self.repo / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(text.replace("\r\n", "\n").replace("\n", newline).encode("utf-8")) + return path + + def rendered(self, text: str) -> str: + return str(HELPERS.render_template(text, self.replacements, "\n")) + + def old(self, relative: Path) -> str: + """The 0.1.19 file of an app (rendered with the app name).""" + if relative in MIGRATION.REPLACED_FILES: + return self.rendered(lf(MIGRATION.PREVIOUS_ROOT / relative)) + text = lf(ROOT / relative) + if relative == MIGRATION.POST_INSTALL_SH: + for old, new in MIGRATION.SH_BLOCKS: + text = text.replace(new, old) + elif relative == MIGRATION.POST_INSTALL_BAT: + for old, new in MIGRATION.BAT_BLOCKS: + text = text.replace(new, old) + elif relative == MIGRATION.BUMP_CONSTRUCTOR: + text = text.replace(MIGRATION.BUMP_SNIPPET, "").replace(MIGRATION.BUMP_CALL_NEW, MIGRATION.BUMP_CALL_OLD) + elif relative == MIGRATION.MENU: + data = json.loads(text) + del data["menu_items"][0]["platforms"]["win"]["command"] + text = json.dumps(data, indent=2, ensure_ascii=False) + "\n" + elif relative == MIGRATION.TOOLS_REQUIREMENTS: + text = text.replace(MIGRATION.UV_LINES, "") + elif relative == MIGRATION.USAGE_DOC: + text = text.replace(MIGRATION.USAGE_NEW, MIGRATION.USAGE_OLD) + return self.rendered(text) + + def new(self, relative: Path) -> str: + return self.rendered(lf(ROOT / relative)) + + ALL = ( + *MIGRATION.REPLACED_FILES, + MIGRATION.TOOLS_REQUIREMENTS, + MIGRATION.POST_INSTALL_SH, + MIGRATION.POST_INSTALL_BAT, + MIGRATION.BUMP_CONSTRUCTOR, + MIGRATION.MENU, + MIGRATION.USAGE_DOC, + ) + + def put_0_1_19_app(self, crlf: bool = False) -> None: + for relative in self.ALL: + self.put(relative, self.old(relative), "\r\n" if crlf else self.NEWLINES.get(relative, "\n")) + + def run_migration(self) -> str: + buffer = StringIO() + with redirect_stdout(buffer): + MIGRATION.migrate(self.repo, {}) + return buffer.getvalue() + + +class MigrationBehaviourTests(MigrationCase): + def test_a_0_1_19_app_reaches_the_0_1_20_files_and_a_second_run_changes_nothing(self) -> None: + self.put_0_1_19_app() + output = self.run_migration() + for relative in (*self.ALL, MIGRATION.LOCK_SCRIPT): + with self.subTest(file=str(relative)): + self.assertRegex(output, rf"{re.escape(str(relative))}: updated") + self.assertEqual(lf(self.repo / relative), self.new(relative)) + self.assertIn("LabChronicle", lf(self.repo / MIGRATION.LAUNCHER)) + before = {p: p.read_bytes() for p in self.repo.rglob("*") if p.is_file()} + again = self.run_migration() + self.assertIn("no repository changes were required", again) + self.assertNotIn("SKIPPED", again) + self.assertEqual({p: p.read_bytes() for p in self.repo.rglob("*") if p.is_file()}, before) + + def test_line_endings_are_kept(self) -> None: + self.put_0_1_19_app(crlf=True) + self.run_migration() + for relative in self.ALL: + with self.subTest(file=str(relative)): + data = (self.repo / relative).read_bytes() + self.assertNotIn(b"\n", data.replace(b"\r\n", b"")) + self.assertNotIn(b"\r\n", (self.repo / MIGRATION.LOCK_SCRIPT).read_bytes(), "a new file is written with LF") + + def test_customised_files_are_skipped_and_left_as_they_are(self) -> None: + self.put_0_1_19_app() + custom = { + MIGRATION.NOTEBOOK_WORKFLOW: self.old(MIGRATION.NOTEBOOK_WORKFLOW).replace("runs-on: ubuntu-latest", "runs-on: ubuntu-22.04"), + MIGRATION.LAUNCHER: self.old(MIGRATION.LAUNCHER) + "\n# our own line\n", + MIGRATION.POST_INSTALL_SH: self.old(MIGRATION.POST_INSTALL_SH).replace('-m pip install -r "$PROJECT_ROOT/requirements-macos.txt"', '-m pip install -U -r "$PROJECT_ROOT/requirements-macos.txt"'), + MIGRATION.BUMP_CONSTRUCTOR: self.old(MIGRATION.BUMP_CONSTRUCTOR).replace(MIGRATION.BUMP_CALL_OLD, " ensure_requirements_in_extra_files(data, strict=True)\n"), + } + for relative, text in custom.items(): + self.put(relative, text, self.NEWLINES.get(relative, "\n")) + output = self.run_migration() + for relative, text in custom.items(): + with self.subTest(file=str(relative)): + self.assertIn(f"{relative}: SKIPPED", output) + self.assertEqual(lf(self.repo / relative), text) + self.assertIn(f"{MIGRATION.SRC_WORKFLOW}: updated", output) + + def test_the_menu_variants(self) -> None: + old = self.old(MIGRATION.MENU) + cases = { + "own windows command": (old.replace('"icon": "', '"command": ["cmd"],\n "icon": "', 1), "SKIPPED: has its own Windows command"), + "other formatting": (json.dumps(json.loads(old)) + "\n", "SKIPPED: formatted differently"), + "no terminal": (old.replace('"terminal": true', '"terminal": false'), "SKIPPED: not the template shape"), + "not json": ("{", "SKIPPED: not valid JSON"), + } + for name, (text, message) in cases.items(): + with self.subTest(case=name): + path = self.put(MIGRATION.MENU, text) + self.assertIn(message, self.run_migration()) + self.assertEqual(lf(path), text) + + def test_uv_and_the_lock_script_of_an_app_are_kept(self) -> None: + self.put(MIGRATION.TOOLS_REQUIREMENTS, "pyyaml==6.0.2\nuv>=0.12\n") + self.put(MIGRATION.LOCK_SCRIPT, "# our own\n") + output = self.run_migration() + self.assertIn(f"{MIGRATION.TOOLS_REQUIREMENTS}: already installs uv: unchanged", output) + self.assertIn(f"{MIGRATION.LOCK_SCRIPT}: SKIPPED: the repository has its own lock_requirements.py", output) + self.assertEqual(lf(self.repo / MIGRATION.LOCK_SCRIPT), "# our own\n") + + def test_a_tools_requirements_file_without_a_final_newline_gets_uv_on_its_own_line(self) -> None: + self.put(MIGRATION.TOOLS_REQUIREMENTS, "pyyaml==6.0.2") + self.run_migration() + self.assertEqual(lf(self.repo / MIGRATION.TOOLS_REQUIREMENTS), "pyyaml==6.0.2\n" + MIGRATION.UV_LINES) + + def test_missing_files_are_not_created(self) -> None: + output = self.run_migration() + for relative in self.ALL: + with self.subTest(file=str(relative)): + self.assertIn(f"{relative}: not present in this repository", output) + self.assertFalse((self.repo / relative).exists()) + self.assertIn(f"{MIGRATION.LOCK_SCRIPT}: SKIPPED: the repository has no .tools/python folder", output) + self.assertFalse((self.repo / MIGRATION.LOCK_SCRIPT).exists()) + + def test_a_labchronicle_like_app_with_its_own_bump_constructor_still_gets_the_lock_function(self) -> None: + """LabChronicle's bump_constructor.py has no ensure_app_manifest_in_extra_files (it differs from the template): the two anchors suffice.""" + text = self.old(MIGRATION.BUMP_CONSTRUCTOR).replace(" ensure_app_manifest_in_extra_files(data)\n", "") + path = self.put(MIGRATION.BUMP_CONSTRUCTOR, text) + self.assertIn(f"{MIGRATION.BUMP_CONSTRUCTOR}: updated", self.run_migration()) + result = lf(path) + self.assertIn(MIGRATION.BUMP_CALL_NEW, result) + compile(result, str(path), "exec") + + +class ChainTests(MigrationCase): + def test_the_sync_script_runs_the_migration_from_0_1_19(self) -> None: + self.put_0_1_19_app() + shutil.copytree(ROOT / ".template_sync", self.repo / ".template_sync") + command = [sys.executable, str(self.repo / ".template_sync/sync.py"), "--from-version", "0.1.19", "--to-version", "0.1.20", "--repository-root", str(self.repo)] + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("[migration 0.1.20]", result.stdout) + self.assertEqual(lf(self.repo / MIGRATION.NOTEBOOK_WORKFLOW), self.new(MIGRATION.NOTEBOOK_WORKFLOW)) + again = subprocess.run(command, capture_output=True, text=True) + self.assertIn("no repository changes were required", again.stdout) + + @unittest.skipUnless(BASH, "bash is not available") + def test_the_migrated_post_install_sh_is_valid_shell(self) -> None: + self.put_0_1_19_app() + self.run_migration() + result = subprocess.run([BASH or "bash", "-n", str(self.repo / MIGRATION.POST_INSTALL_SH)], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + + +if __name__ == "__main__": + unittest.main()