diff --git a/.github/workflows/update_on_notebook_change.yml b/.github/workflows/update_on_notebook_change.yml index c5fae88..4a8c03b 100644 --- a/.github/workflows/update_on_notebook_change.yml +++ b/.github/workflows/update_on_notebook_change.yml @@ -9,7 +9,9 @@ on: - 'notebooks/**/requirements.yaml' workflow_dispatch: -# This enables to run a single workflow at a time +# One run of this workflow at a time per branch. The src workflow has its own group: the two can run at the same time, +# because each one rebases its commit onto the latest branch before it pushes (a shared group kept only one waiting run, so a +# third push cancelled the waiting run of the other workflow). concurrency: group: notebooks-global-lock-${{ github.ref }} cancel-in-progress: false @@ -98,6 +100,12 @@ jobs: .tools/python/merge_requirements.py \ python .tools/python/merge_requirements.py --source-dir notebooks --output requirements.txt + # requirements*.txt -> requirements*.lock.txt: every package the installer installs with pip, with its exact version (uv, valid on + # Windows, macOS and Linux). A lock file is rewritten only when its requirements file, Python or JupyterLab version changed. + - name: Lock requirements + run: | + bash .tools/bash/run_with_context.sh artifacts/error_log_document.log "lock_requirements.py" .tools/python/lock_requirements.py python .tools/python/lock_requirements.py + - name: Check for changes id: check-changes run: | @@ -138,17 +146,66 @@ jobs: - name: Commit updated files id: auto-commit if: steps.check-changes.outputs.has_changes == 'true' && steps.test-conda-env.outcome == 'success' - uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5 - with: - commit_message: GitHub Action - Updated files because of notebook and/or requirements changes - file_pattern: requirements*.txt environment.yaml notebooks/* setup.py + env: + BRANCH: ${{ github.ref_name }} + COMMIT_MESSAGE: GitHub Action - Updated files because of notebook and/or requirements changes + # The run checked out the commit that triggered it (the scripts above compare it with its parent); the branch may have moved + # since (another workflow committed, or a new push). A rejected push is therefore retried after a rebase onto the latest branch. + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git ls-files -z --modified --deleted --others --exclude-standard -- 'requirements*.txt' environment.yaml 'notebooks/*' setup.py | xargs -0 -r git add -A -- + if git diff --cached --quiet; then + echo "No changes to commit." + exit 0 + fi + git commit -m "$COMMIT_MESSAGE" + for attempt in 1 2 3 4 5; do + if git push origin "HEAD:refs/heads/$BRANCH"; then + exit 0 + fi + echo "The push was rejected (attempt $attempt of 5): rebasing onto the latest $BRANCH." + sleep $((attempt * 5)) + git fetch origin "$BRANCH" + if ! git rebase --autostash "origin/$BRANCH"; then + git rebase --abort || true + echo "::error::The commit of this run conflicts with the latest $BRANCH; run this workflow again (Actions > Run workflow)." + exit 1 + fi + done + echo "::error::The push was rejected 5 times; run this workflow again (Actions > Run workflow)." + exit 1 + # Only when the merged requirements could not be installed: a failure of any other step (a script, the push) never reverts a commit. - name: Revert user commit that broke requirements - if: failure() && github.event_name == 'push' + if: failure() && github.event_name == 'push' && steps.test-conda-env.outcome == 'failure' + env: + BRANCH: ${{ github.ref_name }} + BAD_COMMIT: ${{ github.sha }} run: | git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git reset --hard git clean -fd - git revert "${{ github.sha }}" -m 1 --no-edit - git push origin HEAD:${{ github.ref_name }} + for attempt in 1 2 3 4 5; do + git fetch origin "$BRANCH" + git checkout -B "$BRANCH" "origin/$BRANCH" + # A merge commit is reverted against its first parent (-m 1); any other commit is reverted as it is. + if [ "$(git rev-list --parents -n 1 "$BAD_COMMIT" | wc -w)" -gt 2 ]; then + REVERT_ARGS=(-m 1) + else + REVERT_ARGS=() + fi + if ! git revert --no-edit "${REVERT_ARGS[@]}" "$BAD_COMMIT"; then + git revert --abort || true + echo "::error::$BAD_COMMIT broke the requirements but could not be reverted automatically (later commits change the same lines): revert it by hand." + exit 1 + fi + if git push origin "HEAD:refs/heads/$BRANCH"; then + exit 0 + fi + echo "The push of the revert was rejected (attempt $attempt of 5): retrying on the latest $BRANCH." + sleep $((attempt * 5)) + done + echo "::error::The revert of $BAD_COMMIT could not be pushed: revert it by hand." + exit 1 diff --git a/.github/workflows/update_on_src_change.yaml b/.github/workflows/update_on_src_change.yaml index 76b90f0..2ab8870 100644 --- a/.github/workflows/update_on_src_change.yaml +++ b/.github/workflows/update_on_src_change.yaml @@ -7,9 +7,11 @@ on: - 'src/**' workflow_dispatch: -# This enables to run a single workflow at a time +# One run of this workflow at a time per branch. The notebook workflow has its own group: the two can run at the same time, +# because each one rebases its commit onto the latest branch before it pushes (a shared group kept only one waiting run, so a +# third push cancelled the waiting run of the other workflow). concurrency: - group: notebooks-global-lock-${{ github.ref }} + group: src-change-lock-${{ github.ref }} cancel-in-progress: false jobs: @@ -18,13 +20,13 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - + steps: - name: Checkout repository uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 # Fetch all history for all branches and tags - + - name: Update src change file run: | cat > .tools/meta/src_change.yaml < Run workflow)." + exit 1 + fi + done + echo "::error::The push was rejected 5 times; run this workflow again (Actions > Run workflow)." + exit 1 diff --git a/.template_sync/manifest.yaml b/.template_sync/manifest.yaml index ac8d395..43b057a 100644 --- a/.template_sync/manifest.yaml +++ b/.template_sync/manifest.yaml @@ -1,4 +1,4 @@ -template_version: 0.1.19 +template_version: 0.1.20 migrations: - from: 0.0.0 to: 0.0.1 @@ -63,3 +63,6 @@ migrations: - from: 0.1.18 to: 0.1.19 script: migrations/v0_1_18_to_v0_1_19.py + - from: 0.1.19 + to: 0.1.20 + script: migrations/v0_1_19_to_v0_1_20.py diff --git a/.template_sync/migrations/v0_1_19_to_v0_1_20.py b/.template_sync/migrations/v0_1_19_to_v0_1_20.py new file mode 100644 index 0000000..b077854 --- /dev/null +++ b/.template_sync/migrations/v0_1_19_to_v0_1_20.py @@ -0,0 +1,356 @@ +"""Template 0.1.19 -> 0.1.20: no lost pushes between the two workflows, lock files for the pip requirements, a launcher that starts fresh. + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_20.md + +What this changes in an app repository (every step is independent, idempotent and never deletes anything): + +1. Workflows. update_on_notebook_change.yml and update_on_src_change.yaml are replaced by the 0.1.20 files ONLY when the app's file is still + identical to the 0.1.19 template file (line endings are ignored when comparing and kept when writing). The new files rebase their + commit onto the latest branch and retry a rejected push, have one concurrency group each, revert a commit only when the conda test + failed (a merge commit with -m 1, any other commit as it is), and the notebook workflow runs lock_requirements.py. +2. Lock files. .tools/python/lock_requirements.py is created (requirements*.txt -> requirements*.lock.txt with uv), `uv==` is added to + .tools/requirements.txt, .tools/bash/test_conda_environment.sh (replaced when identical to 0.1.19) tests the lock file, + post_install.sh / .bat install the lock file when the installer bundles one (known blocks only), and bump_constructor.py gets the + function that bundles the lock files (inserted at two known lines). +3. Launcher. app/python_scripts/launch_jupyter.py (replaced when identical to the rendered 0.1.19 file) asks to press Enter when JupyterLab + stops and gives JupyterLab an empty workspace folder at every launch (only the Welcome notebook opens). app/menuinst/notebook_launcher.json + gets a Windows command that ends with `& exit`, so the terminal window closes after Enter (cmd /K kept it open). +4. Notebook usage guide. The sentence on stopping JupyterLab in .tools/docs/notebook_usage.md is replaced when it is still the 0.1.19 text. + +Rules this migration keeps: surgical (only known lines or known files), idempotent (a second run reports no changes), never raises for a +customised file (the sync pull request must still be created), the line endings of each file are kept, nothing is deleted. +""" +from __future__ import annotations + +import importlib.util +import json +import re +from pathlib import Path +from typing import Any + + +RESOURCE_ROOT = Path(__file__).resolve().parents[1] / "resources" / "v0_1_19_to_v0_1_20" +PREVIOUS_ROOT = RESOURCE_ROOT / "previous" +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_20.md" +TAG = "[migration 0.1.20]" + +NOTEBOOK_WORKFLOW = Path(".github/workflows/update_on_notebook_change.yml") +SRC_WORKFLOW = Path(".github/workflows/update_on_src_change.yaml") +TEST_CONDA = Path(".tools/bash/test_conda_environment.sh") +LAUNCHER = Path("app/python_scripts/launch_jupyter.py") +LOCK_SCRIPT = Path(".tools/python/lock_requirements.py") +TOOLS_REQUIREMENTS = Path(".tools/requirements.txt") +POST_INSTALL_SH = Path("app/bash_bat_scripts/post_install.sh") +POST_INSTALL_BAT = Path("app/bash_bat_scripts/post_install.bat") +BUMP_CONSTRUCTOR = Path(".tools/python/bump_constructor.py") +MENU = Path("app/menuinst/notebook_launcher.json") +USAGE_DOC = Path(".tools/docs/notebook_usage.md") + +# Files replaced as a whole when the app's file is still the (rendered) 0.1.19 template file. +REPLACED_FILES = (NOTEBOOK_WORKFLOW, SRC_WORKFLOW, TEST_CONDA, LAUNCHER) +# Resource files that are copies of the 0.1.20 template files (a test checks they stay identical to the template). +MANAGED_FILES = (*REPLACED_FILES, LOCK_SCRIPT) + +UV_PIN = "uv==0.13.3" +UV_LINE = re.compile(r"^\s*uv\s*(?:[=<>!~;\[]|$)", re.IGNORECASE) +UV_LINES = "# lock_requirements.py (requirements*.txt -> requirements*.lock.txt)\n" + UV_PIN + "\n" + +# post_install.sh: (0.1.19 text, 0.1.20 text), LF. +SH_BLOCKS = ( + ( + 'echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE"\n' + '"$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1\n', + "# A lock file (requirements*.lock.txt next to the requirements file: every package with its exact version, written by\n" + "# .tools/python/lock_requirements.py) is installed instead of the requirements file when the installer bundles one.\n" + "locked_requirements() {\n" + ' if [ -f "${1%.txt}.lock.txt" ]; then echo "${1%.txt}.lock.txt"; else echo "$1"; fi\n' + "}\n" + 'SELECTED_REQUIREMENTS="$(locked_requirements "$SELECTED_REQUIREMENTS")"\n' + 'echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE"\n' + '"$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1\n', + ), + ( + ' "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-macos.txt" >> "$LOG_FILE" 2>&1\n', + ' "$PYTHON_EXE" -m pip install -r "$(locked_requirements "$PROJECT_ROOT/requirements-macos.txt")" >> "$LOG_FILE" 2>&1\n', + ), + ( + ' "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-linux.txt" >> "$LOG_FILE" 2>&1\n', + ' "$PYTHON_EXE" -m pip install -r "$(locked_requirements "$PROJECT_ROOT/requirements-linux.txt")" >> "$LOG_FILE" 2>&1\n', + ), +) +SH_DONE = "locked_requirements()" + +BAT_BLOCKS = ( + ( + 'echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%"\n' + '"%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1\n', + "REM A lock file (requirements*.lock.txt next to the requirements file: every package with its exact version, written by\n" + "REM .tools/python/lock_requirements.py) is installed instead of the requirements file when the installer bundles one.\n" + 'SET "SELECTED_LOCK=%SELECTED_REQUIREMENTS:~0,-4%.lock.txt"\n' + 'IF EXIST "%SELECTED_LOCK%" SET "SELECTED_REQUIREMENTS=%SELECTED_LOCK%"\n' + 'echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%"\n' + '"%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1\n', + ), + ( + 'IF EXIST "%PROJECT_ROOT%\\requirements-windows.txt" (\n' + ' echo Installing Windows-specific requirements. >> "%LOG_FILE%"\n' + ' "%PYTHON_EXE%" -m pip install -r "%PROJECT_ROOT%\\requirements-windows.txt" >> "%LOG_FILE%" 2>&1\n', + 'SET "WINDOWS_REQUIREMENTS=%PROJECT_ROOT%\\requirements-windows.txt"\n' + 'IF EXIST "%PROJECT_ROOT%\\requirements-windows.lock.txt" SET "WINDOWS_REQUIREMENTS=%PROJECT_ROOT%\\requirements-windows.lock.txt"\n' + 'IF EXIST "%PROJECT_ROOT%\\requirements-windows.txt" (\n' + ' echo Installing Windows-specific requirements from "%WINDOWS_REQUIREMENTS%". >> "%LOG_FILE%"\n' + ' "%PYTHON_EXE%" -m pip install -r "%WINDOWS_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1\n', + ), +) +BAT_DONE = "SELECTED_LOCK=" + +BUMP_FUNCTION_ANCHOR = "def ensure_extra_files(" +BUMP_CALL_OLD = " ensure_requirements_in_extra_files(data)\n" +BUMP_CALL_NEW = " ensure_requirements_in_extra_files(data)\n ensure_lock_files_in_extra_files(data)\n" +BUMP_DONE = "def ensure_lock_files_in_extra_files(" +BUMP_NEEDS = ("def extract_project_folder(", "def debug(", "from pathlib import Path") + +WIN_EXIT = ["&", "exit"] + +USAGE_OLD = ( + "When you have, click on the terminal window and press `Ctrl+C` to stop the Jupyter Lab session and close the terminal." +) +USAGE_NEW = ( + "When you have, stop the Jupyter Lab session with **File > Shut Down** in Jupyter Lab (or click on the terminal window and press `Ctrl+C`). " + "The terminal then asks you to press `Enter`, and closes." +) + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def _load_helpers() -> Any: + """Reuse the helpers of the 0.1.10 migration (token replacement, newline detection, file IO).""" + path = Path(__file__).resolve().parent / "v0_1_9_to_v0_1_10.py" + spec = importlib.util.spec_from_file_location("template_sync_helpers_0_1_20", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _resource(root: Path, relative: Path) -> str: + source = root / relative + if not source.is_file(): + raise FileNotFoundError(f"Template resource not found: {source}") + return source.read_bytes().decode("utf-8").replace("\r\n", "\n") + + +def _render(helpers: Any, text: str, replacements: dict[str, str]) -> str: + return str(helpers.render_template(text, replacements, "\n")) + + +def _write_lf(helpers: Any, path: Path, text_lf: str, newline: str) -> None: + helpers.write_text(path, text_lf.replace("\n", newline)) + + +# -------------------------------------------------------------------------------------------------------------------------------- +# Whole files (workflows, conda test, launcher) +# -------------------------------------------------------------------------------------------------------------------------------- +def _replace_file(helpers: Any, repo_root: Path, relative: Path, replacements: dict[str, str]) -> str: + path = repo_root / relative + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + current_lf = current.replace("\r\n", "\n") + new = _render(helpers, _resource(RESOURCE_ROOT, relative), replacements) + if current_lf == new: + return "already the 0.1.20 template file: unchanged" + if current_lf != _render(helpers, _resource(PREVIOUS_ROOT, relative), replacements): + return f"SKIPPED: customised (not the 0.1.19 template file), so it was left unchanged; apply the 0.1.20 changes by hand, see {DOCS_URL}" + _write_lf(helpers, path, new, helpers.detect_newline(current)) + return "updated: replaced by the 0.1.20 template file" + + +# -------------------------------------------------------------------------------------------------------------------------------- +# Lock files +# -------------------------------------------------------------------------------------------------------------------------------- +def _create_lock_script(helpers: Any, repo_root: Path) -> str: + path = repo_root / LOCK_SCRIPT + new = _resource(RESOURCE_ROOT, LOCK_SCRIPT) + if path.is_file(): + if helpers.read_text(path).replace("\r\n", "\n") == new: + return "already the 0.1.20 template file: unchanged" + return "SKIPPED: the repository has its own lock_requirements.py, left as it is" + if not path.parent.is_dir(): + return "SKIPPED: the repository has no .tools/python folder (nothing was created)" + with path.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(new) + return "updated: created" + + +def _add_uv(helpers: Any, repo_root: Path) -> str: + path = repo_root / TOOLS_REQUIREMENTS + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + if any(UV_LINE.match(line) for line in current.splitlines()): + return "already installs uv: unchanged" + newline = helpers.detect_newline(current) + text = current if not current or current.endswith(("\n", "\r")) else current + newline + helpers.write_text(path, text + UV_LINES.replace("\n", newline)) + return f"updated: added {UV_PIN}" + + +def _replace_blocks(helpers: Any, path: Path, blocks: tuple[tuple[str, str], ...], done: str, what: str) -> str: + current = helpers.read_text(path) + lf = current.replace("\r\n", "\n") + if done in lf: + return f"already {what}: unchanged" + missing = [old.splitlines()[0].strip() for old, _ in blocks if lf.count(old) != 1] + if missing: + shown = "; ".join(missing) + return f"SKIPPED: customised, the known lines were not found once ({shown}); left unchanged, apply the change by hand, see {DOCS_URL}" + for old, new in blocks: + lf = lf.replace(old, new) + _write_lf(helpers, path, lf, helpers.detect_newline(current)) + return f"updated: {what}" + + +def _bump_constructor(helpers: Any, repo_root: Path) -> str: + path = repo_root / BUMP_CONSTRUCTOR + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + lf = current.replace("\r\n", "\n") + if BUMP_DONE in lf: + return "already bundles the lock files: unchanged" + if lf.count(BUMP_FUNCTION_ANCHOR) != 1 or lf.count(BUMP_CALL_OLD) != 1 or not all(need in lf for need in BUMP_NEEDS): + return f"SKIPPED: customised (the known lines were not found once), left unchanged; add ensure_lock_files_in_extra_files by hand, see {DOCS_URL}" + lf = lf.replace(BUMP_FUNCTION_ANCHOR, BUMP_SNIPPET + BUMP_FUNCTION_ANCHOR, 1).replace(BUMP_CALL_OLD, BUMP_CALL_NEW, 1) + _write_lf(helpers, path, lf, helpers.detect_newline(current)) + return "updated: bundles the lock files (ensure_lock_files_in_extra_files)" + + +# The LOCK_FILES tuple and ensure_lock_files_in_extra_files of the 0.1.20 bump_constructor.py, with the blank lines before the next function +# (a test checks that the template script holds this text once). +BUMP_SNIPPET = '''LOCK_FILES = ( + "requirements.lock.txt", + "requirements_gpu.lock.txt", + "requirements-linux.lock.txt", + "requirements-windows.lock.txt", + "requirements-macos.lock.txt", +) + + +def ensure_lock_files_in_extra_files(construct_data: dict) -> None: + """Bundle the lock files (requirements*.lock.txt, written by lock_requirements.py) next to their requirements files. + + post_install installs a lock file instead of its requirements file when the installer bundles one. A mapping whose lock file is gone + (the requirements could not be locked) is removed, because constructor fails on a missing file. + """ + extra_files = construct_data.setdefault("extra_files", []) + project_folder = extract_project_folder(extra_files) + for lock_file in LOCK_FILES: + mapped = [item for item in extra_files if isinstance(item, dict) and lock_file in item] + if Path(lock_file).exists(): + if not mapped: + extra_files.append({lock_file: f"{project_folder}/{lock_file}"}) + debug(f"Added mapping for {lock_file}") + else: + debug(f"{lock_file} mapping already present") + elif mapped: + for item in mapped: + extra_files.remove(item) + debug(f"Removed the mapping of {lock_file}: the file does not exist (the installer installs the requirements file instead)") + + +''' + + +# -------------------------------------------------------------------------------------------------------------------------------- +# Launcher shortcut (Windows: close the terminal after Enter) +# -------------------------------------------------------------------------------------------------------------------------------- +def _menu(helpers: Any, repo_root: Path) -> str: + path = repo_root / MENU + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + lf = current.replace("\r\n", "\n") + try: + data = json.loads(lf) + except json.JSONDecodeError as exc: + return f"SKIPPED: not valid JSON ({exc}), left unchanged" + items = data.get("menu_items") if isinstance(data, dict) else None + if not isinstance(items, list) or len(items) != 1 or not isinstance(items[0], dict): + return f"SKIPPED: not the template shape (one menu item), left unchanged; see {DOCS_URL}" + item = items[0] + command = item.get("command") + win = (item.get("platforms") or {}).get("win") + if not isinstance(command, list) or not isinstance(win, dict) or not item.get("terminal"): + return f"SKIPPED: not the template shape (a terminal command with a win platform), left unchanged; see {DOCS_URL}" + if "command" in win: + if win["command"] == command + WIN_EXIT: + return "already closes the Windows terminal after Enter: unchanged" + return "SKIPPED: has its own Windows command, left unchanged; end it with \"&\", \"exit\" to close the terminal after Enter" + if json.dumps(data, indent=2, ensure_ascii=False) + "\n" != lf: + return f"SKIPPED: formatted differently from the template, left unchanged; add the Windows command by hand, see {DOCS_URL}" + win["command"] = command + WIN_EXIT + _write_lf(helpers, path, json.dumps(data, indent=2, ensure_ascii=False) + "\n", helpers.detect_newline(current)) + return "updated: the Windows command ends with `& exit` (the terminal closes after Enter)" + + +def _usage_doc(helpers: Any, repo_root: Path) -> str: + path = repo_root / USAGE_DOC + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + if USAGE_NEW in current: + return "unchanged" + if current.count(USAGE_OLD) != 1: + return "SKIPPED: the 0.1.19 sentence on stopping Jupyter Lab was not found (customised), left unchanged" + helpers.write_text(path, current.replace(USAGE_OLD, USAGE_NEW)) + return "updated: how to stop Jupyter Lab (File > Shut Down, then Enter)" + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """Replace the two workflows and the launcher, add the lock files (idempotent, surgical, never raises for a customised file).""" + _ = context + helpers = _load_helpers() + try: + replacements = helpers.load_replacements(repo_root) + except (OSError, ValueError) as exc: + log(f"could not read the app name from construct.yaml ({exc}): the launcher is compared without app names.") + replacements = {} + + changed = 0 + + def report(name: Path, step: Any) -> None: + nonlocal changed + try: + status = step() + except (OSError, ValueError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + if status.startswith("updated"): + changed += 1 + log(f"{name}: {status}") + + for relative in REPLACED_FILES: + report(relative, lambda relative=relative: _replace_file(helpers, repo_root, relative, replacements)) + report(LOCK_SCRIPT, lambda: _create_lock_script(helpers, repo_root)) + report(TOOLS_REQUIREMENTS, lambda: _add_uv(helpers, repo_root)) + for relative, blocks, done in ((POST_INSTALL_SH, SH_BLOCKS, SH_DONE), (POST_INSTALL_BAT, BAT_BLOCKS, BAT_DONE)): + if not (repo_root / relative).is_file(): + log(f"{relative}: not present in this repository (nothing was created)") + continue + report(relative, lambda relative=relative, blocks=blocks, done=done: _replace_blocks( + helpers, repo_root / relative, blocks, done, "installs the lock file when the installer bundles one" + )) + report(BUMP_CONSTRUCTOR, lambda: _bump_constructor(helpers, repo_root)) + report(MENU, lambda: _menu(helpers, repo_root)) + report(USAGE_DOC, lambda: _usage_doc(helpers, repo_root)) + + if changed: + log( + f"done: {changed} file(s) changed. Review the pull request before merging. The lock files are written by the next run of " + "'Updates on notebook change' (Actions > Run workflow); the launcher changes reach users with the next release of the app." + ) + else: + log("done: no repository changes were required.") diff --git a/.template_sync/resources/v0_1_17_to_v0_1_18/.tools/docs/install.md b/.template_sync/resources/v0_1_17_to_v0_1_18/.tools/docs/install.md index 4048356..52ae8be 100644 --- a/.template_sync/resources/v0_1_17_to_v0_1_18/.tools/docs/install.md +++ b/.template_sync/resources/v0_1_17_to_v0_1_18/.tools/docs/install.md @@ -33,7 +33,7 @@ Installing takes about 6-8 minutes and you only do it once. You need an internet ## Update an installed app -- **macOS and Windows.** On macOS and Windows, update an app by uninstalling the old version first and then installing the new one: the installer does not install over an existing install. +- **macOS and Windows.** On macOS and Windows, update an app by uninstalling the old version first and then installing the new one: the installer does not install over an existing install. On macOS, if you already ran the new installer over the old version and it stopped or reported a failure at the end, move only the old app shortcut (for example `~/Applications/MyApp.app`) to the Bin, keep the app's folder in your home folder, and run the installer again ([known issue](https://github.com/CellMigrationLab/LabConstrictor/issues/34)). - **Linux.** On Linux, running the installer with `-u` updates in place only if the previous install finished; otherwise delete `/pkgs` or uninstall the app and install it again. ## Where the app is listed diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_notebook_change.yml b/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_notebook_change.yml new file mode 100644 index 0000000..4a8c03b --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_notebook_change.yml @@ -0,0 +1,211 @@ +name: Updates on notebook change + +on: + push: + branches: [ main, master ] + paths: + - 'notebooks/**/*.ipynb' + - 'notebooks/notebook_latest_versions.yaml' + - 'notebooks/**/requirements.yaml' + workflow_dispatch: + +# One run of this workflow at a time per branch. The src workflow has its own group: the two can run at the same time, +# because each one rebases its commit onto the latest branch before it pushes (a shared group kept only one waiting run, so a +# third push cancelled the waiting run of the other workflow). +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon notebook change, update manifests and changelogs + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@2d756ea4c53f7f6b397767d8723b3a10a9f35bf2 # v44 + with: + separator: "," + files: | + notebooks/**/*.ipynb + notebooks/notebook_latest_versions.yaml + notebooks/**/requirements.yaml + + - name: Syncronize notebook versions + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "sync_notebook_versions.py" \ + .tools/python/sync_notebook_versions.py \ + python .tools/python/sync_notebook_versions.py + + - name: Update CHANGELOG + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_changelog.py" \ + .tools/python/update_changelog.py \ + python .tools/python/update_changelog.py notebooks/notebook_latest_versions.yaml + + - name: Update last commit in notebook_latest_versions.yaml + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_version_last_commit.py" \ + .tools/python/update_version_last_commit.py \ + python .tools/python/update_version_last_commit.py notebooks/notebook_latest_versions.yaml + + - name: Hide code cells in notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "hide_code_cells.py" \ + app/python_scripts/hide_code_cells.py \ + python app/python_scripts/hide_code_cells.py . + + - name: Index notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_notebook_README.py" \ + .tools/python/bump_notebook_README.py \ + python .tools/python/bump_notebook_README.py + + - name: Run merge script + if: github.event_name == 'workflow_dispatch' || contains(steps.changed-files.outputs.all_changed_files, 'requirements.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "merge_requirements.py" \ + .tools/python/merge_requirements.py \ + python .tools/python/merge_requirements.py --source-dir notebooks --output requirements.txt + + # requirements*.txt -> requirements*.lock.txt: every package the installer installs with pip, with its exact version (uv, valid on + # Windows, macOS and Linux). A lock file is rewritten only when its requirements file, Python or JupyterLab version changed. + - name: Lock requirements + run: | + bash .tools/bash/run_with_context.sh artifacts/error_log_document.log "lock_requirements.py" .tools/python/lock_requirements.py python .tools/python/lock_requirements.py + + - name: Check for changes + id: check-changes + run: | + if [ -z "$(git status --porcelain)" ]; then + echo "has_changes=false" >> $GITHUB_OUTPUT + else + echo "has_changes=true" >> $GITHUB_OUTPUT + fi + + - name: Set up Miniconda for requirements test + if: steps.check-changes.outputs.has_changes == 'true' + uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + auto-activate: true + miniforge-version: latest + use-mamba: true + + - name: Verify merged requirements inside conda + id: test-conda-env + if: steps.check-changes.outputs.has_changes == 'true' + env: + SKIP_CONDA_ENV_TEST: ${{ vars.SKIP_CONDA_ENV_TEST }} + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "test_conda_environment.sh" \ + .tools/bash/test_conda_environment.sh \ + bash .tools/bash/test_conda_environment.sh + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: notebook-debug-bundle + path: artifacts/error_log_document.log + archive: false + + - name: Commit updated files + id: auto-commit + if: steps.check-changes.outputs.has_changes == 'true' && steps.test-conda-env.outcome == 'success' + env: + BRANCH: ${{ github.ref_name }} + COMMIT_MESSAGE: GitHub Action - Updated files because of notebook and/or requirements changes + # The run checked out the commit that triggered it (the scripts above compare it with its parent); the branch may have moved + # since (another workflow committed, or a new push). A rejected push is therefore retried after a rebase onto the latest branch. + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git ls-files -z --modified --deleted --others --exclude-standard -- 'requirements*.txt' environment.yaml 'notebooks/*' setup.py | xargs -0 -r git add -A -- + if git diff --cached --quiet; then + echo "No changes to commit." + exit 0 + fi + git commit -m "$COMMIT_MESSAGE" + for attempt in 1 2 3 4 5; do + if git push origin "HEAD:refs/heads/$BRANCH"; then + exit 0 + fi + echo "The push was rejected (attempt $attempt of 5): rebasing onto the latest $BRANCH." + sleep $((attempt * 5)) + git fetch origin "$BRANCH" + if ! git rebase --autostash "origin/$BRANCH"; then + git rebase --abort || true + echo "::error::The commit of this run conflicts with the latest $BRANCH; run this workflow again (Actions > Run workflow)." + exit 1 + fi + done + echo "::error::The push was rejected 5 times; run this workflow again (Actions > Run workflow)." + exit 1 + + # Only when the merged requirements could not be installed: a failure of any other step (a script, the push) never reverts a commit. + - name: Revert user commit that broke requirements + if: failure() && github.event_name == 'push' && steps.test-conda-env.outcome == 'failure' + env: + BRANCH: ${{ github.ref_name }} + BAD_COMMIT: ${{ github.sha }} + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git reset --hard + git clean -fd + for attempt in 1 2 3 4 5; do + git fetch origin "$BRANCH" + git checkout -B "$BRANCH" "origin/$BRANCH" + # A merge commit is reverted against its first parent (-m 1); any other commit is reverted as it is. + if [ "$(git rev-list --parents -n 1 "$BAD_COMMIT" | wc -w)" -gt 2 ]; then + REVERT_ARGS=(-m 1) + else + REVERT_ARGS=() + fi + if ! git revert --no-edit "${REVERT_ARGS[@]}" "$BAD_COMMIT"; then + git revert --abort || true + echo "::error::$BAD_COMMIT broke the requirements but could not be reverted automatically (later commits change the same lines): revert it by hand." + exit 1 + fi + if git push origin "HEAD:refs/heads/$BRANCH"; then + exit 0 + fi + echo "The push of the revert was rejected (attempt $attempt of 5): retrying on the latest $BRANCH." + sleep $((attempt * 5)) + done + echo "::error::The revert of $BAD_COMMIT could not be pushed: revert it by hand." + exit 1 diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_src_change.yaml b/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_src_change.yaml new file mode 100644 index 0000000..2ab8870 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_src_change.yaml @@ -0,0 +1,66 @@ +name: Updates on external code change + +on: + push: + branches: [ main, master ] + paths: + - 'src/**' + workflow_dispatch: + +# One run of this workflow at a time per branch. The notebook workflow has its own group: the two can run at the same time, +# because each one rebases its commit onto the latest branch before it pushes (a shared group kept only one waiting run, so a +# third push cancelled the waiting run of the other workflow). +concurrency: + group: src-change-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon external code update the code version + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Update src change file + run: | + cat > .tools/meta/src_change.yaml < Run workflow)." + exit 1 + fi + done + echo "::error::The push was rejected 5 times; run this workflow again (Actions > Run workflow)." + exit 1 diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/bash/test_conda_environment.sh b/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/bash/test_conda_environment.sh new file mode 100644 index 0000000..4742aca --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/bash/test_conda_environment.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Test script to validate conda environment creation and pip requirements installation +# This script creates a test conda environment from environment.yaml and installs pip requirements + +set -e # Exit on any error +set -u # Exit on undefined variables + +is_truthy() { + case "${1:-}" in + 1|true|TRUE|True|yes|YES|Yes|on|ON|On) + return 0 + ;; + *) + return 1 + ;; + esac +} + +if is_truthy "${SKIP_CONDA_ENV_TEST:-}"; then + echo "SKIP_CONDA_ENV_TEST is enabled; skipping conda environment validation." + exit 0 +fi + +echo "==========================================" +echo "Testing Conda Environment Creation" +echo "==========================================" + +# Define test environment name +TEST_ENV_NAME="potential-potato-test-env" + +# Clean up any existing test environment +echo "Cleaning up any existing test environment..." +mamba env remove -n "$TEST_ENV_NAME" -y 2>/dev/null || true + +# Create conda environment from environment.yaml +echo "" +echo "Creating conda environment from environment.yaml..." +if [ ! -f "environment.yaml" ]; then + echo "ERROR: environment.yaml not found!" + exit 1 +fi + +mamba env create -n "$TEST_ENV_NAME" -f environment.yaml -y + +# Activate the environment +echo "" +echo "Activating test environment..." +# Note: In CI, conda/mamba environments need to be activated via conda run +# or by sourcing the activation script + +# Test pip installation of requirements.txt, or of its lock file (requirements.lock.txt, written by +# .tools/python/lock_requirements.py) when there is one: the installer installs the lock file, so that is what is tested. +if [ ! -f "requirements.txt" ]; then + echo "ERROR: requirements.txt not found!" + mamba env remove -n "$TEST_ENV_NAME" -y + exit 1 +fi +REQUIREMENTS_FILE="requirements.txt" +if [ -f "requirements.lock.txt" ]; then + REQUIREMENTS_FILE="requirements.lock.txt" +fi +echo "" +echo "Installing requirements from $REQUIREMENTS_FILE..." + +# Check if the requirements file is not empty +if [ ! -s "$REQUIREMENTS_FILE" ]; then + echo "WARNING: $REQUIREMENTS_FILE is empty, skipping pip install" +else + echo "Installing packages with pip..." + mamba run -n "$TEST_ENV_NAME" pip install --prefer-binary -r "$REQUIREMENTS_FILE" +fi + +# Report dependency conflicts between the pip packages and the conda packages of the environment. Only a warning: a conflict +# among the conda packages themselves must not block the notebooks. +echo "" +echo "Checking the installed packages for conflicts (pip check)..." +if ! mamba run -n "$TEST_ENV_NAME" pip check; then + echo "::warning::pip check found dependency conflicts in the test environment (see the step log); the installer may behave differently than expected." +fi + +# Verify installation by checking Python and key packages +echo "" +echo "Verifying installation..." +echo "Python version:" +mamba run -n "$TEST_ENV_NAME" python --version + +echo "" +echo "Installed pip packages (sample):" +mamba run -n "$TEST_ENV_NAME" pip list | head -n 20 + +# Clean up test environment +echo "" +echo "Cleaning up test environment..." +mamba env remove -n "$TEST_ENV_NAME" -y + +echo "" +echo "==========================================" +echo "✓ Conda environment test completed successfully!" +echo "==========================================" diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/python/lock_requirements.py b/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/python/lock_requirements.py new file mode 100644 index 0000000..c5834a2 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/python/lock_requirements.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Lock the merged requirements files: requirements*.txt -> requirements*.lock.txt. + +merge_requirements.py writes the requirements of the notebooks (requirements.txt, requirements_gpu.txt and the per-OS +requirements-.txt). Those files name the packages the notebooks need, but not the packages these depend on, so two +installations of the same release could get different versions. This script writes, next to each file, a lock file that lists +every package pip installs, each with its exact version, and post_install installs the lock file when the installer bundles one. + +How a lock file is made: +* `uv pip compile --universal`: one lock file that is valid on Windows, macOS and Linux (a package that only one platform needs carries + an environment marker, e.g. `colorama==0.4.6 ; sys_platform == 'win32'`). +* `--python-version` is the Python of environment.yaml. uv reads it as the lowest Python of the resolution, so a few packages can appear + twice with a `python_full_version` marker: pip installs the line that matches the Python of the installer. +* The conda packages of environment.yaml (jupyterlab, pip, setuptools, certifi, ...) are installed by the installer itself, so they are + not written to the lock file (their own dependencies are), and a conda pin such as `jupyterlab=4.4.0` constrains the resolution. +* The per-OS files are resolved with requirements.lock.txt as constraints, so a package that both files need gets the same version. +* Pip options of the requirements file (`--extra-index-url` of the GPU file, ...) are copied to the top of the lock file. + +A lock file is written again only when its requirements file, the Python or JupyterLab version (or this script's lock format) changed: +its header holds a fingerprint of those inputs. Re-running the script without such a change rewrites nothing, so a new release +installs the same versions as the previous one. To move every package to its newest version, delete the lock files and run the +workflow "Updates on notebook change" by hand (Actions > Run workflow). + +If a file cannot be locked (uv is missing, or the requirements cannot be resolved), the script prints a warning, removes the +out-of-date lock file of that file (an installer must never install versions that no longer match the requirements) and goes on: +the installer then installs the plain requirements file. The script itself always exits 0 for such a failure. +""" +from __future__ import annotations + +import argparse +import hashlib +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + +# Bump when the uv arguments or the output change, so that every lock file is written again. +LOCK_FORMAT = "1" +BASE_INPUT = "requirements.txt" +GPU_INPUT = "requirements_gpu.txt" +OS_INPUTS = ("requirements-linux.txt", "requirements-windows.txt", "requirements-macos.txt") +INPUTS = (BASE_INPUT, GPU_INPUT, *OS_INPUTS) +FINGERPRINT_PREFIX = "# lock-input: sha256:" +# Pip options that choose where packages come from: copied to the top of the lock file, so pip finds the locked versions where uv found them. +INDEX_OPTION = re.compile(r"^\s*(--index-url|--extra-index-url|-i|--find-links|-f|--trusted-host)(\s|=)") +CONDA_SPEC = re.compile(r"^\s*([A-Za-z0-9_.-]+)\s*(?:(==?)\s*([0-9][A-Za-z0-9_.+!*-]*))?(?:=\S+)?\s*$") +PLACEHOLDER = re.compile(r"^[A-Z_]+$") + + +def lock_path(requirements: Path) -> Path: + """requirements.txt -> requirements.lock.txt (same folder).""" + return requirements.with_name(requirements.stem + ".lock.txt") + + +def read_environment(path: Path) -> tuple[str | None, list[str], list[str]]: + """(Python version, conda package names, pip constraints from exact conda pins) of environment.yaml.""" + if not path.is_file(): + return None, [], [] + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + python_version = None + names: list[str] = [] + constraints: list[str] = [] + for dependency in data.get("dependencies", []) or []: + if not isinstance(dependency, str): + continue # the `pip:` subsection of an environment file + match = CONDA_SPEC.match(dependency) + if match: + name, operator, version = match.group(1), match.group(2), match.group(3) + else: + # a spec with a range or alternatives (`pip>=24.2`): only its name is used + name, operator, version = re.split(r"[\s<>=!|,\[]", dependency.strip(), maxsplit=1)[0], None, None + if not name: + continue + if version and PLACEHOLDER.match(version): + version = None # the template itself (python=PYTHON_VERSION) + if name.lower() == "python": + python_version = version + continue + names.append(name) + if version: + # conda `name=1.2` means any 1.2.* release, `name==1.2` exactly 1.2 + constraints.append(f"{name}=={version}" if operator == "==" else f"{name}=={version.removesuffix('.*')}.*") + return python_version, sorted(set(names), key=str.lower), constraints + + +def normalized_requirements(text: str) -> str: + """The requirement and option lines of a requirements file (comments, such as the timestamp of merge_requirements.py, are ignored).""" + lines = [] + for line in text.replace("\r\n", "\n").split("\n"): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + lines.append(stripped) + return "\n".join(lines) + + +def fingerprint(requirements_text: str, python_version: str, constraints: list[str], no_emit: list[str], extra_constraints: str) -> str: + payload = "\n---\n".join( + [LOCK_FORMAT, normalized_requirements(requirements_text), python_version, "\n".join(constraints), "\n".join(no_emit), extra_constraints] + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def current_fingerprint(lock: Path) -> str | None: + if not lock.is_file(): + return None + for line in lock.read_text(encoding="utf-8").splitlines(): + if line.startswith(FINGERPRINT_PREFIX): + return line[len(FINGERPRINT_PREFIX):].strip() + return None + + +def run_uv(requirements: Path, python_version: str, constraint_files: list[Path], no_emit: list[str]) -> str: + """The resolution of `requirements` (stdout of uv pip compile); raises RuntimeError with uv's message when it fails.""" + command = [ + sys.executable, "-m", "uv", "pip", "compile", str(requirements), + "--universal", "--python-version", python_version, + "--no-header", "--no-annotate", "--no-config", "--quiet", + ] + for constraint in constraint_files: + command += ["--constraint", str(constraint)] + for name in no_emit: + command += ["--no-emit-package", name] + try: + result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", check=False) + except OSError as exc: + raise RuntimeError(f"uv could not be started: {exc}") from exc + if result.returncode != 0: + message = (result.stderr or result.stdout).strip() or f"exit code {result.returncode}" + raise RuntimeError(message) + return result.stdout + + +def uv_version() -> str: + try: + result = subprocess.run([sys.executable, "-m", "uv", "--version"], capture_output=True, text=True, check=False) + except OSError: + return "uv" + # "uv 0.13.3 (502f93ca6 2026-10-19 x86_64-pc-windows-msvc)" -> "uv 0.13.3": the build details differ between computers + return " ".join(result.stdout.split()[:2]) or "uv" + + +def render_lock(requirements: Path, resolved: str, digest: str, python_version: str, uv: str) -> str: + options = [line.strip() for line in requirements.read_text(encoding="utf-8").splitlines() if INDEX_OPTION.match(line)] + packages = [line.rstrip() for line in resolved.replace("\r\n", "\n").split("\n") if line.strip() and not line.lstrip().startswith("#")] + header = [ + f"# Lock file of {requirements.name}: every package the installer installs with pip, with its exact version.", + f"# Written by .tools/python/lock_requirements.py ({uv}, universal resolution for Python {python_version}); do not edit it by hand.", + f"# It is written again only when {requirements.name}, the Python or JupyterLab version changes (see the fingerprint below).", + f"{FINGERPRINT_PREFIX}{digest}", + ] + body = options + ([""] if options else []) + packages + return "\n".join(header + [""] + body) + "\n" + + +def warn(message: str) -> None: + # `::warning::` shows the message on the summary page of the GitHub Actions run. + print(f"::warning::{message}", flush=True) + + +def lock_file( + requirements: Path, + python_version: str, + constraints: list[str], + no_emit: list[str], + base_lock: Path | None, + work_dir: Path, + uv: str, +) -> str: + """Lock one requirements file; returns what happened ("unchanged", "written", "failed").""" + lock = lock_path(requirements) + extra = base_lock.read_text(encoding="utf-8") if base_lock is not None and base_lock.is_file() else "" + digest = fingerprint(requirements.read_text(encoding="utf-8"), python_version, constraints, no_emit, normalized_requirements(extra)) + if current_fingerprint(lock) == digest: + print(f"{lock}: up to date (unchanged)") + return "unchanged" + constraint_files = [] + if constraints: + conda_constraints = work_dir / "conda_constraints.txt" + conda_constraints.write_text("\n".join(constraints) + "\n", encoding="utf-8") + constraint_files.append(conda_constraints) + if extra: + # Only the package lines: the pip options of the base lock are not constraints. + base_constraints = work_dir / "base_constraints.txt" + base_constraints.write_text( + "\n".join(line for line in extra.splitlines() if line.strip() and not line.startswith(("#", "-"))) + "\n", encoding="utf-8" + ) + constraint_files.append(base_constraints) + try: + resolved = run_uv(requirements, python_version, constraint_files, no_emit) + except RuntimeError as exc: + warn(f"{requirements} could not be locked, so the installer installs it without a lock file: {exc}") + if lock.exists(): + lock.unlink() + print(f"{lock}: removed (it no longer matches {requirements})") + return "failed" + with lock.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(render_lock(requirements, resolved, digest, python_version, uv)) + print(f"{lock}: written") + return "written" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Lock requirements*.txt into requirements*.lock.txt (every package with its exact version).") + parser.add_argument("--root", type=Path, default=Path("."), help="Folder with the requirements files and environment.yaml") + parser.add_argument("--environment", type=Path, default=None, help="Conda environment file (default: /environment.yaml)") + args = parser.parse_args(argv) + root: Path = args.root + environment = args.environment or root / "environment.yaml" + + python_version, conda_names, constraints = read_environment(environment) + if not python_version: + print(f"{environment}: no exact Python version (python=X.Y.Z), so nothing is locked.") + return 0 + no_emit = [name for name in conda_names if name.lower() not in {"git"}] + + # A lock file whose requirements file is gone is removed (the installer must not install it). + for name in INPUTS: + requirements = root / name + if not requirements.is_file() and lock_path(requirements).is_file(): + lock_path(requirements).unlink() + print(f"{lock_path(requirements)}: removed ({requirements} does not exist)") + + present = [root / name for name in INPUTS if (root / name).is_file()] + if not present: + print("No requirements files found: nothing to lock.") + return 0 + uv = uv_version() + results = {} + with tempfile.TemporaryDirectory() as temporary: + work_dir = Path(temporary) + for requirements in present: + base_lock = lock_path(root / BASE_INPUT) if requirements.name in OS_INPUTS else None + results[requirements.name] = lock_file(requirements, python_version, constraints, no_emit, base_lock, work_dir, uv) + failed = [name for name, status in results.items() if status == "failed"] + print(f"Lock files: {len(results) - len(failed)} of {len(results)} requirements files locked" + (f"; not locked: {', '.join(failed)}" if failed else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/app/python_scripts/launch_jupyter.py b/.template_sync/resources/v0_1_19_to_v0_1_20/app/python_scripts/launch_jupyter.py new file mode 100644 index 0000000..f45ec5f --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/app/python_scripts/launch_jupyter.py @@ -0,0 +1,324 @@ +"""TLS-resilient JupyterLab launcher for PROJECT_NAME. + +Certificate handling is configured before JupyterLab imports Tornado. On +Windows this prevents one malformed certificate-store entry from terminating +the application. Certificate and hostname verification remain enabled in every +supported mode. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import re +import shutil +import ssl +import sys +import tempfile +import traceback +from datetime import datetime, timezone +from importlib import metadata +from pathlib import Path + +PROJECT_DISPLAY_NAME = "PROJECT_NAME" +SCRIPT_DIR = Path(__file__).resolve().parent +_INSTALLED_LAYOUT = (SCRIPT_DIR / "notebooks").is_dir() +if _INSTALLED_LAYOUT: + APP_ROOT = SCRIPT_DIR + PREFIX = APP_ROOT.parent +else: + # Repository layout: app/python_scripts/launch_jupyter.py + repository_root = SCRIPT_DIR.parents[1] + APP_ROOT = repository_root if (repository_root / "notebooks").is_dir() else SCRIPT_DIR + PREFIX = APP_ROOT +DEFAULT_NOTEBOOK = APP_ROOT / "notebooks" / "Welcome.ipynb" +LOG_PATH = PREFIX / "launcher_debug.log" + + +def _write_log(message: str, *, console: bool = True) -> None: + timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds") + line = f"[{timestamp}] {message}" + if console: + print(message, flush=True) + try: + with LOG_PATH.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + except OSError: + # Logging must never prevent the application from launching. + pass + + +def _package_version(name: str) -> str: + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return "not installed" + + +def _project_ca_bundle_environment_variable() -> str: + normalized = re.sub(r"[^A-Za-z0-9]+", "_", PROJECT_DISPLAY_NAME).strip("_").upper() + return f"{normalized or 'LABCONSTRICTOR_APP'}_CA_BUNDLE" + + +def _probe_ssl() -> None: + """Create a verified client context without making a network request.""" + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) + if context.verify_mode != ssl.CERT_REQUIRED: + raise RuntimeError("TLS certificate verification is unexpectedly disabled.") + if not context.check_hostname: + raise RuntimeError("TLS hostname verification is unexpectedly disabled.") + + +def _inject_truststore() -> None: + """Route Python SSL contexts through the operating-system trust API.""" + import truststore + + truststore.inject_into_ssl() + + +def _select_ca_bundle() -> Path: + """Choose an explicit CA bundle for verified installer and runtime fallback.""" + for variable in ( + _project_ca_bundle_environment_variable(), + "LABCONSTRICTOR_CA_BUNDLE", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CURL_CA_BUNDLE", + ): + candidate = os.environ.get(variable, "").strip() + if candidate: + path = Path(candidate).expanduser() + if path.is_file(): + return path.resolve() + + import certifi + + return Path(certifi.where()).resolve() + + +def _install_bundle_fallback(bundle: Path) -> None: + """Use a verified PEM bundle without enumerating the Windows cert store.""" + original_context_type = ssl.SSLContext + + def create_context( + purpose: ssl.Purpose = ssl.Purpose.SERVER_AUTH, + *, + cafile: str | os.PathLike[str] | None = None, + capath: str | os.PathLike[str] | None = None, + cadata: str | bytes | None = None, + ) -> ssl.SSLContext: + if purpose == ssl.Purpose.SERVER_AUTH: + context = original_context_type(ssl.PROTOCOL_TLS_CLIENT) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + elif purpose == ssl.Purpose.CLIENT_AUTH: + context = original_context_type(ssl.PROTOCOL_TLS_SERVER) + else: + raise ValueError(purpose) + + if cafile or capath or cadata: + context.load_verify_locations( + cafile=os.fspath(cafile) if cafile else None, + capath=os.fspath(capath) if capath else None, + cadata=cadata, + ) + else: + context.load_verify_locations(cafile=str(bundle)) + + keylogfile = os.environ.get("SSLKEYLOGFILE") + if keylogfile and hasattr(context, "keylog_filename"): + context.keylog_filename = keylogfile + return context + + ssl.create_default_context = create_context # type: ignore[assignment] + ssl._create_default_https_context = create_context # type: ignore[attr-defined] + os.environ["SSL_CERT_FILE"] = str(bundle) + os.environ.setdefault("REQUESTS_CA_BUNDLE", str(bundle)) + os.environ.setdefault("CURL_CA_BUNDLE", str(bundle)) + + +def configure_tls() -> tuple[str, list[str]]: + """Prepare TLS before Jupyter/Tornado imports and return mode plus notes.""" + notes: list[str] = [] + + try: + if "truststore" not in getattr(ssl.SSLContext, "__module__", ""): + _inject_truststore() + _probe_ssl() + return "native truststore", notes + except Exception as exc: + notes.append(f"Native truststore activation failed: {exc}") + + try: + bundle = _select_ca_bundle() + _install_bundle_fallback(bundle) + _probe_ssl() + notes.append(f"Using verified CA bundle fallback: {bundle}") + return "verified CA bundle fallback", notes + except Exception as exc: + notes.append(f"CA bundle fallback failed: {exc}") + raise RuntimeError("Unable to create a verified TLS context. " + " | ".join(notes)) from exc + + +def _diagnostics(tls_mode: str, notes: list[str]) -> dict[str, object]: + return { + "status": "ok", + "application": PROJECT_DISPLAY_NAME, + "tls_mode": tls_mode, + "notes": notes, + "ca_bundle_override_variable": _project_ca_bundle_environment_variable(), + "python": sys.version.split()[0], + "python_executable": sys.executable, + "openssl": ssl.OPENSSL_VERSION, + "platform": platform.platform(), + "jupyterlab": _package_version("jupyterlab"), + "jupyter_server": _package_version("jupyter-server"), + "tornado": _package_version("tornado"), + "truststore": _package_version("truststore"), + "pip_system_certs": _package_version("pip-system-certs"), + "certifi": _package_version("certifi"), + "prefix": str(PREFIX), + "log": str(LOG_PATH), + } + + +def _show_error_dialog(message: str) -> None: + if os.name != "nt": + return + try: + import tkinter as tk + from tkinter import messagebox + + root = tk.Tk() + root.withdraw() + messagebox.showerror(f"{PROJECT_DISPLAY_NAME} could not start", message) + root.destroy() + except Exception: + pass + + +def _wait_for_enter() -> None: + """Keep the terminal window open until the user has read it, then let it close. + + The menu shortcut runs the launcher in a terminal window; without this the window + closes (or stays open, on Windows) as soon as JupyterLab stops. + """ + stream = sys.stdin + if stream is None or not stream.isatty(): + return + try: + input("\nPress Enter to close this window...") + except (EOFError, KeyboardInterrupt): + pass + + +def run_jupyter(notebook: Path) -> int: + if not notebook.is_file(): + raise FileNotFoundError(f"Welcome notebook not found: {notebook}") + + # Import only after configure_tls() has completed. Tornado creates SSL + # contexts during import on some versions. + from jupyterlab.labapp import main + + home = Path.home().resolve() + try: + notebook.resolve().relative_to(home) + root_dir = home + except ValueError: + # All-users installations can live outside the home directory. + root_dir = PREFIX.resolve() + + # A fresh, empty workspace folder for every launch: JupyterLab restores the open + # tabs of its workspaces, and the default folder (~/.jupyter/lab/workspaces) is + # shared by every app, so notebooks of other apps (and of earlier sessions) would + # open next to the Welcome notebook. Settings and themes are kept: they are stored + # in the user settings folder, not in the workspaces. + workspaces_dir = Path(tempfile.mkdtemp(prefix="labconstrictor-workspaces-")) + sys.argv = [ + "jupyter-lab", + str(notebook.resolve()), + "--ServerApp.ip=127.0.0.1", + f"--ServerApp.root_dir={root_dir}", + "--ServerApp.shutdown_no_activity_timeout=30", + "--FileContentsManager.always_delete_dir=True", + f"--LabApp.workspaces_dir={workspaces_dir}", + ] + try: + result = main() + finally: + shutil.rmtree(workspaces_dir, ignore_errors=True) + return int(result or 0) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=f"Launch {PROJECT_DISPLAY_NAME} in its packaged JupyterLab environment safely." + ) + parser.add_argument("notebook", nargs="?", type=Path, default=DEFAULT_NOTEBOOK) + parser.add_argument( + "--self-test", + action="store_true", + help="Validate TLS and Jupyter imports without starting the server.", + ) + parser.add_argument( + "--print-ca-bundle", + action="store_true", + help="Print the selected verified CA bundle path and exit.", + ) + parser.add_argument( + "--no-wait", + action="store_true", + help="Exit when JupyterLab stops, without asking to press Enter.", + ) + args = parser.parse_args(argv) + result = _launch(args) + if not (args.no_wait or args.self_test or args.print_ca_bundle): + _wait_for_enter() + return result + + +def _launch(args: argparse.Namespace) -> int: + try: + if args.print_ca_bundle: + print(_select_ca_bundle(), flush=True) + return 0 + + _write_log("Starting application launcher") + tls_mode, notes = configure_tls() + _write_log(f"TLS mode: {tls_mode}") + for note in notes: + # Recovery details remain available in launcher_debug.log and + # self-test JSON without alarming users during a normal launch. + _write_log(note, console=False) + + import jupyterlab.labapp # noqa: F401 + import tornado.netutil # noqa: F401 + + diagnostics = _diagnostics(tls_mode, notes) + if args.self_test: + print(json.dumps(diagnostics, indent=2), flush=True) + _write_log("Launcher self-test completed successfully") + return 0 + + _write_log(f"Opening notebook: {args.notebook}") + return run_jupyter(args.notebook) + except KeyboardInterrupt: + _write_log("Launcher interrupted by user") + return 130 + except Exception as exc: + details = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) + _write_log(details.rstrip()) + friendly = ( + f"{PROJECT_DISPLAY_NAME} could not start, but no certificate-store changes are required.\n\n" + f"A diagnostic log was written to:\n{LOG_PATH}\n\n" + f"Technical message: {exc}" + ) + print("\n" + friendly, file=sys.stderr, flush=True) + _show_error_dialog(friendly) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_notebook_change.yml b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_notebook_change.yml new file mode 100644 index 0000000..c5fae88 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_notebook_change.yml @@ -0,0 +1,154 @@ +name: Updates on notebook change + +on: + push: + branches: [ main, master ] + paths: + - 'notebooks/**/*.ipynb' + - 'notebooks/notebook_latest_versions.yaml' + - 'notebooks/**/requirements.yaml' + workflow_dispatch: + +# This enables to run a single workflow at a time +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon notebook change, update manifests and changelogs + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@2d756ea4c53f7f6b397767d8723b3a10a9f35bf2 # v44 + with: + separator: "," + files: | + notebooks/**/*.ipynb + notebooks/notebook_latest_versions.yaml + notebooks/**/requirements.yaml + + - name: Syncronize notebook versions + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "sync_notebook_versions.py" \ + .tools/python/sync_notebook_versions.py \ + python .tools/python/sync_notebook_versions.py + + - name: Update CHANGELOG + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_changelog.py" \ + .tools/python/update_changelog.py \ + python .tools/python/update_changelog.py notebooks/notebook_latest_versions.yaml + + - name: Update last commit in notebook_latest_versions.yaml + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_version_last_commit.py" \ + .tools/python/update_version_last_commit.py \ + python .tools/python/update_version_last_commit.py notebooks/notebook_latest_versions.yaml + + - name: Hide code cells in notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "hide_code_cells.py" \ + app/python_scripts/hide_code_cells.py \ + python app/python_scripts/hide_code_cells.py . + + - name: Index notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_notebook_README.py" \ + .tools/python/bump_notebook_README.py \ + python .tools/python/bump_notebook_README.py + + - name: Run merge script + if: github.event_name == 'workflow_dispatch' || contains(steps.changed-files.outputs.all_changed_files, 'requirements.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "merge_requirements.py" \ + .tools/python/merge_requirements.py \ + python .tools/python/merge_requirements.py --source-dir notebooks --output requirements.txt + + - name: Check for changes + id: check-changes + run: | + if [ -z "$(git status --porcelain)" ]; then + echo "has_changes=false" >> $GITHUB_OUTPUT + else + echo "has_changes=true" >> $GITHUB_OUTPUT + fi + + - name: Set up Miniconda for requirements test + if: steps.check-changes.outputs.has_changes == 'true' + uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + auto-activate: true + miniforge-version: latest + use-mamba: true + + - name: Verify merged requirements inside conda + id: test-conda-env + if: steps.check-changes.outputs.has_changes == 'true' + env: + SKIP_CONDA_ENV_TEST: ${{ vars.SKIP_CONDA_ENV_TEST }} + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "test_conda_environment.sh" \ + .tools/bash/test_conda_environment.sh \ + bash .tools/bash/test_conda_environment.sh + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: notebook-debug-bundle + path: artifacts/error_log_document.log + archive: false + + - name: Commit updated files + id: auto-commit + if: steps.check-changes.outputs.has_changes == 'true' && steps.test-conda-env.outcome == 'success' + uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5 + with: + commit_message: GitHub Action - Updated files because of notebook and/or requirements changes + file_pattern: requirements*.txt environment.yaml notebooks/* setup.py + + - name: Revert user commit that broke requirements + if: failure() && github.event_name == 'push' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git reset --hard + git clean -fd + git revert "${{ github.sha }}" -m 1 --no-edit + git push origin HEAD:${{ github.ref_name }} diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_src_change.yaml b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_src_change.yaml new file mode 100644 index 0000000..76b90f0 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_src_change.yaml @@ -0,0 +1,41 @@ +name: Updates on external code change + +on: + push: + branches: [ main, master ] + paths: + - 'src/**' + workflow_dispatch: + +# This enables to run a single workflow at a time +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon external code update the code version + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Update src change file + run: | + cat > .tools/meta/src_change.yaml </dev/null || true + +# Create conda environment from environment.yaml +echo "" +echo "Creating conda environment from environment.yaml..." +if [ ! -f "environment.yaml" ]; then + echo "ERROR: environment.yaml not found!" + exit 1 +fi + +mamba env create -n "$TEST_ENV_NAME" -f environment.yaml -y + +# Activate the environment +echo "" +echo "Activating test environment..." +# Note: In CI, conda/mamba environments need to be activated via conda run +# or by sourcing the activation script + +# Test pip installation of requirements.txt +echo "" +echo "Installing requirements from requirements.txt..." +if [ ! -f "requirements.txt" ]; then + echo "ERROR: requirements.txt not found!" + mamba env remove -n "$TEST_ENV_NAME" -y + exit 1 +fi + +# Check if requirements.txt is not empty +if [ ! -s "requirements.txt" ]; then + echo "WARNING: requirements.txt is empty, skipping pip install" +else + echo "Installing packages with pip..." + mamba run -n "$TEST_ENV_NAME" pip install --prefer-binary -r requirements.txt +fi + +# Verify installation by checking Python and key packages +echo "" +echo "Verifying installation..." +echo "Python version:" +mamba run -n "$TEST_ENV_NAME" python --version + +echo "" +echo "Installed pip packages (sample):" +mamba run -n "$TEST_ENV_NAME" pip list | head -n 20 + +# Clean up test environment +echo "" +echo "Cleaning up test environment..." +mamba env remove -n "$TEST_ENV_NAME" -y + +echo "" +echo "==========================================" +echo "✓ Conda environment test completed successfully!" +echo "==========================================" diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/previous/app/python_scripts/launch_jupyter.py b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/app/python_scripts/launch_jupyter.py new file mode 100644 index 0000000..0f91726 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/app/python_scripts/launch_jupyter.py @@ -0,0 +1,286 @@ +"""TLS-resilient JupyterLab launcher for PROJECT_NAME. + +Certificate handling is configured before JupyterLab imports Tornado. On +Windows this prevents one malformed certificate-store entry from terminating +the application. Certificate and hostname verification remain enabled in every +supported mode. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import re +import ssl +import sys +import traceback +from datetime import datetime, timezone +from importlib import metadata +from pathlib import Path + +PROJECT_DISPLAY_NAME = "PROJECT_NAME" +SCRIPT_DIR = Path(__file__).resolve().parent +_INSTALLED_LAYOUT = (SCRIPT_DIR / "notebooks").is_dir() +if _INSTALLED_LAYOUT: + APP_ROOT = SCRIPT_DIR + PREFIX = APP_ROOT.parent +else: + # Repository layout: app/python_scripts/launch_jupyter.py + repository_root = SCRIPT_DIR.parents[1] + APP_ROOT = repository_root if (repository_root / "notebooks").is_dir() else SCRIPT_DIR + PREFIX = APP_ROOT +DEFAULT_NOTEBOOK = APP_ROOT / "notebooks" / "Welcome.ipynb" +LOG_PATH = PREFIX / "launcher_debug.log" + + +def _write_log(message: str, *, console: bool = True) -> None: + timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds") + line = f"[{timestamp}] {message}" + if console: + print(message, flush=True) + try: + with LOG_PATH.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + except OSError: + # Logging must never prevent the application from launching. + pass + + +def _package_version(name: str) -> str: + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return "not installed" + + +def _project_ca_bundle_environment_variable() -> str: + normalized = re.sub(r"[^A-Za-z0-9]+", "_", PROJECT_DISPLAY_NAME).strip("_").upper() + return f"{normalized or 'LABCONSTRICTOR_APP'}_CA_BUNDLE" + + +def _probe_ssl() -> None: + """Create a verified client context without making a network request.""" + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) + if context.verify_mode != ssl.CERT_REQUIRED: + raise RuntimeError("TLS certificate verification is unexpectedly disabled.") + if not context.check_hostname: + raise RuntimeError("TLS hostname verification is unexpectedly disabled.") + + +def _inject_truststore() -> None: + """Route Python SSL contexts through the operating-system trust API.""" + import truststore + + truststore.inject_into_ssl() + + +def _select_ca_bundle() -> Path: + """Choose an explicit CA bundle for verified installer and runtime fallback.""" + for variable in ( + _project_ca_bundle_environment_variable(), + "LABCONSTRICTOR_CA_BUNDLE", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CURL_CA_BUNDLE", + ): + candidate = os.environ.get(variable, "").strip() + if candidate: + path = Path(candidate).expanduser() + if path.is_file(): + return path.resolve() + + import certifi + + return Path(certifi.where()).resolve() + + +def _install_bundle_fallback(bundle: Path) -> None: + """Use a verified PEM bundle without enumerating the Windows cert store.""" + original_context_type = ssl.SSLContext + + def create_context( + purpose: ssl.Purpose = ssl.Purpose.SERVER_AUTH, + *, + cafile: str | os.PathLike[str] | None = None, + capath: str | os.PathLike[str] | None = None, + cadata: str | bytes | None = None, + ) -> ssl.SSLContext: + if purpose == ssl.Purpose.SERVER_AUTH: + context = original_context_type(ssl.PROTOCOL_TLS_CLIENT) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + elif purpose == ssl.Purpose.CLIENT_AUTH: + context = original_context_type(ssl.PROTOCOL_TLS_SERVER) + else: + raise ValueError(purpose) + + if cafile or capath or cadata: + context.load_verify_locations( + cafile=os.fspath(cafile) if cafile else None, + capath=os.fspath(capath) if capath else None, + cadata=cadata, + ) + else: + context.load_verify_locations(cafile=str(bundle)) + + keylogfile = os.environ.get("SSLKEYLOGFILE") + if keylogfile and hasattr(context, "keylog_filename"): + context.keylog_filename = keylogfile + return context + + ssl.create_default_context = create_context # type: ignore[assignment] + ssl._create_default_https_context = create_context # type: ignore[attr-defined] + os.environ["SSL_CERT_FILE"] = str(bundle) + os.environ.setdefault("REQUESTS_CA_BUNDLE", str(bundle)) + os.environ.setdefault("CURL_CA_BUNDLE", str(bundle)) + + +def configure_tls() -> tuple[str, list[str]]: + """Prepare TLS before Jupyter/Tornado imports and return mode plus notes.""" + notes: list[str] = [] + + try: + if "truststore" not in getattr(ssl.SSLContext, "__module__", ""): + _inject_truststore() + _probe_ssl() + return "native truststore", notes + except Exception as exc: + notes.append(f"Native truststore activation failed: {exc}") + + try: + bundle = _select_ca_bundle() + _install_bundle_fallback(bundle) + _probe_ssl() + notes.append(f"Using verified CA bundle fallback: {bundle}") + return "verified CA bundle fallback", notes + except Exception as exc: + notes.append(f"CA bundle fallback failed: {exc}") + raise RuntimeError("Unable to create a verified TLS context. " + " | ".join(notes)) from exc + + +def _diagnostics(tls_mode: str, notes: list[str]) -> dict[str, object]: + return { + "status": "ok", + "application": PROJECT_DISPLAY_NAME, + "tls_mode": tls_mode, + "notes": notes, + "ca_bundle_override_variable": _project_ca_bundle_environment_variable(), + "python": sys.version.split()[0], + "python_executable": sys.executable, + "openssl": ssl.OPENSSL_VERSION, + "platform": platform.platform(), + "jupyterlab": _package_version("jupyterlab"), + "jupyter_server": _package_version("jupyter-server"), + "tornado": _package_version("tornado"), + "truststore": _package_version("truststore"), + "pip_system_certs": _package_version("pip-system-certs"), + "certifi": _package_version("certifi"), + "prefix": str(PREFIX), + "log": str(LOG_PATH), + } + + +def _show_error_dialog(message: str) -> None: + if os.name != "nt": + return + try: + import tkinter as tk + from tkinter import messagebox + + root = tk.Tk() + root.withdraw() + messagebox.showerror(f"{PROJECT_DISPLAY_NAME} could not start", message) + root.destroy() + except Exception: + pass + + +def run_jupyter(notebook: Path) -> int: + if not notebook.is_file(): + raise FileNotFoundError(f"Welcome notebook not found: {notebook}") + + # Import only after configure_tls() has completed. Tornado creates SSL + # contexts during import on some versions. + from jupyterlab.labapp import main + + home = Path.home().resolve() + try: + notebook.resolve().relative_to(home) + root_dir = home + except ValueError: + # All-users installations can live outside the home directory. + root_dir = PREFIX.resolve() + + sys.argv = [ + "jupyter-lab", + str(notebook.resolve()), + "--ServerApp.ip=127.0.0.1", + f"--ServerApp.root_dir={root_dir}", + "--ServerApp.shutdown_no_activity_timeout=30", + "--FileContentsManager.always_delete_dir=True", + ] + result = main() + return int(result or 0) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=f"Launch {PROJECT_DISPLAY_NAME} in its packaged JupyterLab environment safely." + ) + parser.add_argument("notebook", nargs="?", type=Path, default=DEFAULT_NOTEBOOK) + parser.add_argument( + "--self-test", + action="store_true", + help="Validate TLS and Jupyter imports without starting the server.", + ) + parser.add_argument( + "--print-ca-bundle", + action="store_true", + help="Print the selected verified CA bundle path and exit.", + ) + args = parser.parse_args(argv) + + try: + if args.print_ca_bundle: + print(_select_ca_bundle(), flush=True) + return 0 + + _write_log("Starting application launcher") + tls_mode, notes = configure_tls() + _write_log(f"TLS mode: {tls_mode}") + for note in notes: + # Recovery details remain available in launcher_debug.log and + # self-test JSON without alarming users during a normal launch. + _write_log(note, console=False) + + import jupyterlab.labapp # noqa: F401 + import tornado.netutil # noqa: F401 + + diagnostics = _diagnostics(tls_mode, notes) + if args.self_test: + print(json.dumps(diagnostics, indent=2), flush=True) + _write_log("Launcher self-test completed successfully") + return 0 + + _write_log(f"Opening notebook: {args.notebook}") + return run_jupyter(args.notebook) + except KeyboardInterrupt: + _write_log("Launcher interrupted by user") + return 130 + except Exception as exc: + details = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) + _write_log(details.rstrip()) + friendly = ( + f"{PROJECT_DISPLAY_NAME} could not start, but no certificate-store changes are required.\n\n" + f"A diagnostic log was written to:\n{LOG_PATH}\n\n" + f"Technical message: {exc}" + ) + print("\n" + friendly, file=sys.stderr, flush=True) + _show_error_dialog(friendly) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.tools/bash/test_conda_environment.sh b/.tools/bash/test_conda_environment.sh index 1450f7e..4742aca 100644 --- a/.tools/bash/test_conda_environment.sh +++ b/.tools/bash/test_conda_environment.sh @@ -48,21 +48,34 @@ echo "Activating test environment..." # Note: In CI, conda/mamba environments need to be activated via conda run # or by sourcing the activation script -# Test pip installation of requirements.txt -echo "" -echo "Installing requirements from requirements.txt..." +# Test pip installation of requirements.txt, or of its lock file (requirements.lock.txt, written by +# .tools/python/lock_requirements.py) when there is one: the installer installs the lock file, so that is what is tested. if [ ! -f "requirements.txt" ]; then echo "ERROR: requirements.txt not found!" mamba env remove -n "$TEST_ENV_NAME" -y exit 1 fi +REQUIREMENTS_FILE="requirements.txt" +if [ -f "requirements.lock.txt" ]; then + REQUIREMENTS_FILE="requirements.lock.txt" +fi +echo "" +echo "Installing requirements from $REQUIREMENTS_FILE..." -# Check if requirements.txt is not empty -if [ ! -s "requirements.txt" ]; then - echo "WARNING: requirements.txt is empty, skipping pip install" +# Check if the requirements file is not empty +if [ ! -s "$REQUIREMENTS_FILE" ]; then + echo "WARNING: $REQUIREMENTS_FILE is empty, skipping pip install" else echo "Installing packages with pip..." - mamba run -n "$TEST_ENV_NAME" pip install --prefer-binary -r requirements.txt + mamba run -n "$TEST_ENV_NAME" pip install --prefer-binary -r "$REQUIREMENTS_FILE" +fi + +# Report dependency conflicts between the pip packages and the conda packages of the environment. Only a warning: a conflict +# among the conda packages themselves must not block the notebooks. +echo "" +echo "Checking the installed packages for conflicts (pip check)..." +if ! mamba run -n "$TEST_ENV_NAME" pip check; then + echo "::warning::pip check found dependency conflicts in the test environment (see the step log); the installer may behave differently than expected." fi # Verify installation by checking Python and key packages diff --git a/.tools/python/bump_constructor.py b/.tools/python/bump_constructor.py index b41251c..3c6b769 100644 --- a/.tools/python/bump_constructor.py +++ b/.tools/python/bump_constructor.py @@ -115,6 +115,37 @@ def ensure_requirements_in_extra_files(construct_data: dict): # Update the construct data with the modified extra_files construct_data["extra_files"] = extra_files +LOCK_FILES = ( + "requirements.lock.txt", + "requirements_gpu.lock.txt", + "requirements-linux.lock.txt", + "requirements-windows.lock.txt", + "requirements-macos.lock.txt", +) + + +def ensure_lock_files_in_extra_files(construct_data: dict) -> None: + """Bundle the lock files (requirements*.lock.txt, written by lock_requirements.py) next to their requirements files. + + post_install installs a lock file instead of its requirements file when the installer bundles one. A mapping whose lock file is gone + (the requirements could not be locked) is removed, because constructor fails on a missing file. + """ + extra_files = construct_data.setdefault("extra_files", []) + project_folder = extract_project_folder(extra_files) + for lock_file in LOCK_FILES: + mapped = [item for item in extra_files if isinstance(item, dict) and lock_file in item] + if Path(lock_file).exists(): + if not mapped: + extra_files.append({lock_file: f"{project_folder}/{lock_file}"}) + debug(f"Added mapping for {lock_file}") + else: + debug(f"{lock_file} mapping already present") + elif mapped: + for item in mapped: + extra_files.remove(item) + debug(f"Removed the mapping of {lock_file}: the file does not exist (the installer installs the requirements file instead)") + + def ensure_extra_files(construct_data: dict, notebooks_root: Path, src_root: Path) -> int: extra_files = construct_data.get("extra_files") if extra_files is None: @@ -269,6 +300,7 @@ def main(): data = load_construct(construct_path) # Check if the requirements files are included in extra_files, if not, add them ensure_requirements_in_extra_files(data) + ensure_lock_files_in_extra_files(data) # Update extra_files with notebooks and src files ntbk_added, src_added = ensure_extra_files(data, notebooks_root, src_root) diff --git a/.tools/python/lock_requirements.py b/.tools/python/lock_requirements.py new file mode 100644 index 0000000..c5834a2 --- /dev/null +++ b/.tools/python/lock_requirements.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Lock the merged requirements files: requirements*.txt -> requirements*.lock.txt. + +merge_requirements.py writes the requirements of the notebooks (requirements.txt, requirements_gpu.txt and the per-OS +requirements-.txt). Those files name the packages the notebooks need, but not the packages these depend on, so two +installations of the same release could get different versions. This script writes, next to each file, a lock file that lists +every package pip installs, each with its exact version, and post_install installs the lock file when the installer bundles one. + +How a lock file is made: +* `uv pip compile --universal`: one lock file that is valid on Windows, macOS and Linux (a package that only one platform needs carries + an environment marker, e.g. `colorama==0.4.6 ; sys_platform == 'win32'`). +* `--python-version` is the Python of environment.yaml. uv reads it as the lowest Python of the resolution, so a few packages can appear + twice with a `python_full_version` marker: pip installs the line that matches the Python of the installer. +* The conda packages of environment.yaml (jupyterlab, pip, setuptools, certifi, ...) are installed by the installer itself, so they are + not written to the lock file (their own dependencies are), and a conda pin such as `jupyterlab=4.4.0` constrains the resolution. +* The per-OS files are resolved with requirements.lock.txt as constraints, so a package that both files need gets the same version. +* Pip options of the requirements file (`--extra-index-url` of the GPU file, ...) are copied to the top of the lock file. + +A lock file is written again only when its requirements file, the Python or JupyterLab version (or this script's lock format) changed: +its header holds a fingerprint of those inputs. Re-running the script without such a change rewrites nothing, so a new release +installs the same versions as the previous one. To move every package to its newest version, delete the lock files and run the +workflow "Updates on notebook change" by hand (Actions > Run workflow). + +If a file cannot be locked (uv is missing, or the requirements cannot be resolved), the script prints a warning, removes the +out-of-date lock file of that file (an installer must never install versions that no longer match the requirements) and goes on: +the installer then installs the plain requirements file. The script itself always exits 0 for such a failure. +""" +from __future__ import annotations + +import argparse +import hashlib +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + +# Bump when the uv arguments or the output change, so that every lock file is written again. +LOCK_FORMAT = "1" +BASE_INPUT = "requirements.txt" +GPU_INPUT = "requirements_gpu.txt" +OS_INPUTS = ("requirements-linux.txt", "requirements-windows.txt", "requirements-macos.txt") +INPUTS = (BASE_INPUT, GPU_INPUT, *OS_INPUTS) +FINGERPRINT_PREFIX = "# lock-input: sha256:" +# Pip options that choose where packages come from: copied to the top of the lock file, so pip finds the locked versions where uv found them. +INDEX_OPTION = re.compile(r"^\s*(--index-url|--extra-index-url|-i|--find-links|-f|--trusted-host)(\s|=)") +CONDA_SPEC = re.compile(r"^\s*([A-Za-z0-9_.-]+)\s*(?:(==?)\s*([0-9][A-Za-z0-9_.+!*-]*))?(?:=\S+)?\s*$") +PLACEHOLDER = re.compile(r"^[A-Z_]+$") + + +def lock_path(requirements: Path) -> Path: + """requirements.txt -> requirements.lock.txt (same folder).""" + return requirements.with_name(requirements.stem + ".lock.txt") + + +def read_environment(path: Path) -> tuple[str | None, list[str], list[str]]: + """(Python version, conda package names, pip constraints from exact conda pins) of environment.yaml.""" + if not path.is_file(): + return None, [], [] + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + python_version = None + names: list[str] = [] + constraints: list[str] = [] + for dependency in data.get("dependencies", []) or []: + if not isinstance(dependency, str): + continue # the `pip:` subsection of an environment file + match = CONDA_SPEC.match(dependency) + if match: + name, operator, version = match.group(1), match.group(2), match.group(3) + else: + # a spec with a range or alternatives (`pip>=24.2`): only its name is used + name, operator, version = re.split(r"[\s<>=!|,\[]", dependency.strip(), maxsplit=1)[0], None, None + if not name: + continue + if version and PLACEHOLDER.match(version): + version = None # the template itself (python=PYTHON_VERSION) + if name.lower() == "python": + python_version = version + continue + names.append(name) + if version: + # conda `name=1.2` means any 1.2.* release, `name==1.2` exactly 1.2 + constraints.append(f"{name}=={version}" if operator == "==" else f"{name}=={version.removesuffix('.*')}.*") + return python_version, sorted(set(names), key=str.lower), constraints + + +def normalized_requirements(text: str) -> str: + """The requirement and option lines of a requirements file (comments, such as the timestamp of merge_requirements.py, are ignored).""" + lines = [] + for line in text.replace("\r\n", "\n").split("\n"): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + lines.append(stripped) + return "\n".join(lines) + + +def fingerprint(requirements_text: str, python_version: str, constraints: list[str], no_emit: list[str], extra_constraints: str) -> str: + payload = "\n---\n".join( + [LOCK_FORMAT, normalized_requirements(requirements_text), python_version, "\n".join(constraints), "\n".join(no_emit), extra_constraints] + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def current_fingerprint(lock: Path) -> str | None: + if not lock.is_file(): + return None + for line in lock.read_text(encoding="utf-8").splitlines(): + if line.startswith(FINGERPRINT_PREFIX): + return line[len(FINGERPRINT_PREFIX):].strip() + return None + + +def run_uv(requirements: Path, python_version: str, constraint_files: list[Path], no_emit: list[str]) -> str: + """The resolution of `requirements` (stdout of uv pip compile); raises RuntimeError with uv's message when it fails.""" + command = [ + sys.executable, "-m", "uv", "pip", "compile", str(requirements), + "--universal", "--python-version", python_version, + "--no-header", "--no-annotate", "--no-config", "--quiet", + ] + for constraint in constraint_files: + command += ["--constraint", str(constraint)] + for name in no_emit: + command += ["--no-emit-package", name] + try: + result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", check=False) + except OSError as exc: + raise RuntimeError(f"uv could not be started: {exc}") from exc + if result.returncode != 0: + message = (result.stderr or result.stdout).strip() or f"exit code {result.returncode}" + raise RuntimeError(message) + return result.stdout + + +def uv_version() -> str: + try: + result = subprocess.run([sys.executable, "-m", "uv", "--version"], capture_output=True, text=True, check=False) + except OSError: + return "uv" + # "uv 0.13.3 (502f93ca6 2026-10-19 x86_64-pc-windows-msvc)" -> "uv 0.13.3": the build details differ between computers + return " ".join(result.stdout.split()[:2]) or "uv" + + +def render_lock(requirements: Path, resolved: str, digest: str, python_version: str, uv: str) -> str: + options = [line.strip() for line in requirements.read_text(encoding="utf-8").splitlines() if INDEX_OPTION.match(line)] + packages = [line.rstrip() for line in resolved.replace("\r\n", "\n").split("\n") if line.strip() and not line.lstrip().startswith("#")] + header = [ + f"# Lock file of {requirements.name}: every package the installer installs with pip, with its exact version.", + f"# Written by .tools/python/lock_requirements.py ({uv}, universal resolution for Python {python_version}); do not edit it by hand.", + f"# It is written again only when {requirements.name}, the Python or JupyterLab version changes (see the fingerprint below).", + f"{FINGERPRINT_PREFIX}{digest}", + ] + body = options + ([""] if options else []) + packages + return "\n".join(header + [""] + body) + "\n" + + +def warn(message: str) -> None: + # `::warning::` shows the message on the summary page of the GitHub Actions run. + print(f"::warning::{message}", flush=True) + + +def lock_file( + requirements: Path, + python_version: str, + constraints: list[str], + no_emit: list[str], + base_lock: Path | None, + work_dir: Path, + uv: str, +) -> str: + """Lock one requirements file; returns what happened ("unchanged", "written", "failed").""" + lock = lock_path(requirements) + extra = base_lock.read_text(encoding="utf-8") if base_lock is not None and base_lock.is_file() else "" + digest = fingerprint(requirements.read_text(encoding="utf-8"), python_version, constraints, no_emit, normalized_requirements(extra)) + if current_fingerprint(lock) == digest: + print(f"{lock}: up to date (unchanged)") + return "unchanged" + constraint_files = [] + if constraints: + conda_constraints = work_dir / "conda_constraints.txt" + conda_constraints.write_text("\n".join(constraints) + "\n", encoding="utf-8") + constraint_files.append(conda_constraints) + if extra: + # Only the package lines: the pip options of the base lock are not constraints. + base_constraints = work_dir / "base_constraints.txt" + base_constraints.write_text( + "\n".join(line for line in extra.splitlines() if line.strip() and not line.startswith(("#", "-"))) + "\n", encoding="utf-8" + ) + constraint_files.append(base_constraints) + try: + resolved = run_uv(requirements, python_version, constraint_files, no_emit) + except RuntimeError as exc: + warn(f"{requirements} could not be locked, so the installer installs it without a lock file: {exc}") + if lock.exists(): + lock.unlink() + print(f"{lock}: removed (it no longer matches {requirements})") + return "failed" + with lock.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(render_lock(requirements, resolved, digest, python_version, uv)) + print(f"{lock}: written") + return "written" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Lock requirements*.txt into requirements*.lock.txt (every package with its exact version).") + parser.add_argument("--root", type=Path, default=Path("."), help="Folder with the requirements files and environment.yaml") + parser.add_argument("--environment", type=Path, default=None, help="Conda environment file (default: /environment.yaml)") + args = parser.parse_args(argv) + root: Path = args.root + environment = args.environment or root / "environment.yaml" + + python_version, conda_names, constraints = read_environment(environment) + if not python_version: + print(f"{environment}: no exact Python version (python=X.Y.Z), so nothing is locked.") + return 0 + no_emit = [name for name in conda_names if name.lower() not in {"git"}] + + # A lock file whose requirements file is gone is removed (the installer must not install it). + for name in INPUTS: + requirements = root / name + if not requirements.is_file() and lock_path(requirements).is_file(): + lock_path(requirements).unlink() + print(f"{lock_path(requirements)}: removed ({requirements} does not exist)") + + present = [root / name for name in INPUTS if (root / name).is_file()] + if not present: + print("No requirements files found: nothing to lock.") + return 0 + uv = uv_version() + results = {} + with tempfile.TemporaryDirectory() as temporary: + work_dir = Path(temporary) + for requirements in present: + base_lock = lock_path(root / BASE_INPUT) if requirements.name in OS_INPUTS else None + results[requirements.name] = lock_file(requirements, python_version, constraints, no_emit, base_lock, work_dir, uv) + failed = [name for name, status in results.items() if status == "failed"] + print(f"Lock files: {len(results) - len(failed)} of {len(results)} requirements files locked" + (f"; not locked: {', '.join(failed)}" if failed else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.tools/requirements.txt b/.tools/requirements.txt index d06b04e..8492066 100644 --- a/.tools/requirements.txt +++ b/.tools/requirements.txt @@ -1,3 +1,5 @@ # Python 3.11 pyyaml==6.0.2 -nbformat==5.10.4 \ No newline at end of file +nbformat==5.10.4 +# lock_requirements.py (requirements*.txt -> requirements*.lock.txt) +uv==0.13.3 diff --git a/app/bash_bat_scripts/post_install.bat b/app/bash_bat_scripts/post_install.bat index 376ed5e..1a438ad 100644 --- a/app/bash_bat_scripts/post_install.bat +++ b/app/bash_bat_scripts/post_install.bat @@ -67,6 +67,10 @@ IF EXIST "%GPU_REQUIREMENTS%" ( echo GPU requirements file not found, installing CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" ) +REM A lock file (requirements*.lock.txt next to the requirements file: every package with its exact version, written by +REM .tools/python/lock_requirements.py) is installed instead of the requirements file when the installer bundles one. +SET "SELECTED_LOCK=%SELECTED_REQUIREMENTS:~0,-4%.lock.txt" +IF EXIST "%SELECTED_LOCK%" SET "SELECTED_REQUIREMENTS=%SELECTED_LOCK%" echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" "%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1 IF ERRORLEVEL 1 ( @@ -74,9 +78,11 @@ IF ERRORLEVEL 1 ( GOTO :fail ) +SET "WINDOWS_REQUIREMENTS=%PROJECT_ROOT%\requirements-windows.txt" +IF EXIST "%PROJECT_ROOT%\requirements-windows.lock.txt" SET "WINDOWS_REQUIREMENTS=%PROJECT_ROOT%\requirements-windows.lock.txt" IF EXIST "%PROJECT_ROOT%\requirements-windows.txt" ( - echo Installing Windows-specific requirements. >> "%LOG_FILE%" - "%PYTHON_EXE%" -m pip install -r "%PROJECT_ROOT%\requirements-windows.txt" >> "%LOG_FILE%" 2>&1 + echo Installing Windows-specific requirements from "%WINDOWS_REQUIREMENTS%". >> "%LOG_FILE%" + "%PYTHON_EXE%" -m pip install -r "%WINDOWS_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1 IF ERRORLEVEL 1 ( SET "FAILURE_MESSAGE=Windows-specific requirements installation failed." GOTO :fail diff --git a/app/bash_bat_scripts/post_install.sh b/app/bash_bat_scripts/post_install.sh index c500640..1af362d 100644 --- a/app/bash_bat_scripts/post_install.sh +++ b/app/bash_bat_scripts/post_install.sh @@ -43,18 +43,24 @@ else echo "GPU requirements file not found, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" fi +# A lock file (requirements*.lock.txt next to the requirements file: every package with its exact version, written by +# .tools/python/lock_requirements.py) is installed instead of the requirements file when the installer bundles one. +locked_requirements() { + if [ -f "${1%.txt}.lock.txt" ]; then echo "${1%.txt}.lock.txt"; else echo "$1"; fi +} +SELECTED_REQUIREMENTS="$(locked_requirements "$SELECTED_REQUIREMENTS")" echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE" "$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1 if [[ "${OSTYPE:-}" == "darwin"* ]]; then echo "Detected macOS platform" >> "$LOG_FILE" if [ -f "$PROJECT_ROOT/requirements-macos.txt" ]; then - "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-macos.txt" >> "$LOG_FILE" 2>&1 + "$PYTHON_EXE" -m pip install -r "$(locked_requirements "$PROJECT_ROOT/requirements-macos.txt")" >> "$LOG_FILE" 2>&1 fi elif [[ "${OSTYPE:-}" == "linux-gnu"* ]]; then echo "Detected Linux platform" >> "$LOG_FILE" if [ -f "$PROJECT_ROOT/requirements-linux.txt" ]; then - "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-linux.txt" >> "$LOG_FILE" 2>&1 + "$PYTHON_EXE" -m pip install -r "$(locked_requirements "$PROJECT_ROOT/requirements-linux.txt")" >> "$LOG_FILE" 2>&1 fi else echo "Unknown platform: ${OSTYPE:-unset}" >> "$LOG_FILE" diff --git a/app/menuinst/notebook_launcher.json b/app/menuinst/notebook_launcher.json index 00281f2..401eac5 100644 --- a/app/menuinst/notebook_launcher.json +++ b/app/menuinst/notebook_launcher.json @@ -15,7 +15,14 @@ ], "platforms": { "win": { - "icon": "BASE_PATH_KEYWORD/VLab4Mic/vlab4mic_icon.ico" + "icon": "BASE_PATH_KEYWORD/VLab4Mic/vlab4mic_icon.ico", + "command": [ + "{{ PYTHON }}", + "{{ PREFIX }}/VLab4Mic/launch_jupyter.py", + "{{ PREFIX }}/VLab4Mic/notebooks/Welcome.ipynb", + "&", + "exit" + ] }, "linux": { "icon": "BASE_PATH_KEYWORD/VLab4Mic/vlab4mic_icon.png" diff --git a/app/python_scripts/launch_jupyter.py b/app/python_scripts/launch_jupyter.py index 0eec7aa..78d1031 100644 --- a/app/python_scripts/launch_jupyter.py +++ b/app/python_scripts/launch_jupyter.py @@ -13,8 +13,10 @@ import os import platform import re +import shutil import ssl import sys +import tempfile import traceback from datetime import datetime, timezone from importlib import metadata @@ -197,6 +199,21 @@ def _show_error_dialog(message: str) -> None: pass +def _wait_for_enter() -> None: + """Keep the terminal window open until the user has read it, then let it close. + + The menu shortcut runs the launcher in a terminal window; without this the window + closes (or stays open, on Windows) as soon as JupyterLab stops. + """ + stream = sys.stdin + if stream is None or not stream.isatty(): + return + try: + input("\nPress Enter to close this window...") + except (EOFError, KeyboardInterrupt): + pass + + def run_jupyter(notebook: Path) -> int: if not notebook.is_file(): raise FileNotFoundError(f"Welcome notebook not found: {notebook}") @@ -213,6 +230,12 @@ def run_jupyter(notebook: Path) -> int: # All-users installations can live outside the home directory. root_dir = PREFIX.resolve() + # A fresh, empty workspace folder for every launch: JupyterLab restores the open + # tabs of its workspaces, and the default folder (~/.jupyter/lab/workspaces) is + # shared by every app, so notebooks of other apps (and of earlier sessions) would + # open next to the Welcome notebook. Settings and themes are kept: they are stored + # in the user settings folder, not in the workspaces. + workspaces_dir = Path(tempfile.mkdtemp(prefix="labconstrictor-workspaces-")) sys.argv = [ "jupyter-lab", str(notebook.resolve()), @@ -220,8 +243,12 @@ def run_jupyter(notebook: Path) -> int: f"--ServerApp.root_dir={root_dir}", "--ServerApp.shutdown_no_activity_timeout=30", "--FileContentsManager.always_delete_dir=True", + f"--LabApp.workspaces_dir={workspaces_dir}", ] - result = main() + try: + result = main() + finally: + shutil.rmtree(workspaces_dir, ignore_errors=True) return int(result or 0) @@ -240,8 +267,19 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Print the selected verified CA bundle path and exit.", ) + parser.add_argument( + "--no-wait", + action="store_true", + help="Exit when JupyterLab stops, without asking to press Enter.", + ) args = parser.parse_args(argv) + result = _launch(args) + if not (args.no_wait or args.self_test or args.print_ca_bundle): + _wait_for_enter() + return result + +def _launch(args: argparse.Namespace) -> int: try: if args.print_ca_bundle: print(_select_ca_bundle(), flush=True)