From 987bbcaddc834ff83452224713dad47f15c9cc27 Mon Sep 17 00:00:00 2001 From: guijacquemet <25171305+guijacquemet@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:49:32 +0000 Subject: [PATCH] Sync template to version 0.1.28 --- .github/workflows/publish_pypi.yml | 79 ++++ .github/workflows/release_constructor.yml | 49 ++- .../workflows/update_on_notebook_change.yml | 75 +++- .github/workflows/update_on_src_change.yaml | 45 ++- .template_sync/manifest.yaml | 29 +- .../migrations/v0_1_19_to_v0_1_20.py | 356 ++++++++++++++++++ .../migrations/v0_1_20_to_v0_1_21.py | 86 +++++ .../migrations/v0_1_21_to_v0_1_22.py | 167 ++++++++ .../migrations/v0_1_22_to_v0_1_23.py | 192 ++++++++++ .../migrations/v0_1_23_to_v0_1_24.py | 350 +++++++++++++++++ .../migrations/v0_1_24_to_v0_1_25.py | 270 +++++++++++++ .../migrations/v0_1_25_to_v0_1_26.py | 134 +++++++ .../migrations/v0_1_26_to_v0_1_27.py | 99 +++++ .../migrations/v0_1_27_to_v0_1_28.py | 97 +++++ .../schema/labconstrictor-app.schema.json | 19 + .../v0_1_17_to_v0_1_18/.tools/docs/install.md | 2 +- .../workflows/update_on_notebook_change.yml | 211 +++++++++++ .../workflows/update_on_src_change.yaml | 66 ++++ .../.tools/bash/test_conda_environment.sh | 99 +++++ .../.tools/python/lock_requirements.py | 243 ++++++++++++ .../app/python_scripts/launch_jupyter.py | 324 ++++++++++++++++ .../workflows/update_on_notebook_change.yml | 154 ++++++++ .../workflows/update_on_src_change.yaml | 41 ++ .../.tools/bash/test_conda_environment.sh | 86 +++++ .../app/python_scripts/launch_jupyter.py | 286 ++++++++++++++ .../.tools/python/check_placeholders.py | 109 ++++++ .../.tools/python/release_platforms.py | 116 ++++++ .../.github/workflows/publish_pypi.yml | 79 ++++ .../.tools/python/pypi_package.py | 320 ++++++++++++++++ .../resources/v0_1_24_to_v0_1_25/MANIFEST.in | 3 + .../resources/v0_1_24_to_v0_1_25/setup.py | 57 +++ .../.tools/docs/external_code_upload_pypi.md | 52 +++ .../.tools/docs/troubleshooting_pypi.md | 18 + .../docs/app_manifest_host_requirements.md | 18 + .../schema/labconstrictor-app.schema.json | 91 +++++ .tools/bash/create_constructor_environment.sh | 7 +- .tools/bash/test_conda_environment.sh | 27 +- .tools/docs/app_manifest.md | 19 + .tools/docs/external_code_upload.md | 53 +++ .tools/docs/notebook_upload.md | 4 +- .tools/docs/tools_bridge.md | 6 +- .tools/docs/troubleshooting.md | 19 + .tools/python/bump_constructor.py | 32 ++ .tools/python/bump_version.py | 95 ++++- .tools/python/check_placeholders.py | 109 ++++++ .tools/python/lock_requirements.py | 243 ++++++++++++ .tools/python/merge_requirements.py | 48 ++- .tools/python/pypi_package.py | 320 ++++++++++++++++ .tools/python/release_platforms.py | 116 ++++++ .tools/requirements.txt | 4 +- .tools/schema/labconstrictor-app.schema.json | 19 + app/bash_bat_scripts/post_install.bat | 12 +- app/bash_bat_scripts/post_install.sh | 12 +- app/menuinst/notebook_launcher.json | 9 +- app/python_scripts/launch_jupyter.py | 45 ++- 55 files changed, 5556 insertions(+), 65 deletions(-) create mode 100644 .github/workflows/publish_pypi.yml create mode 100644 .template_sync/migrations/v0_1_19_to_v0_1_20.py create mode 100644 .template_sync/migrations/v0_1_20_to_v0_1_21.py create mode 100644 .template_sync/migrations/v0_1_21_to_v0_1_22.py create mode 100644 .template_sync/migrations/v0_1_22_to_v0_1_23.py create mode 100644 .template_sync/migrations/v0_1_23_to_v0_1_24.py create mode 100644 .template_sync/migrations/v0_1_24_to_v0_1_25.py create mode 100644 .template_sync/migrations/v0_1_25_to_v0_1_26.py create mode 100644 .template_sync/migrations/v0_1_26_to_v0_1_27.py create mode 100644 .template_sync/migrations/v0_1_27_to_v0_1_28.py create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_notebook_change.yml create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_src_change.yaml create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/.tools/bash/test_conda_environment.sh create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/.tools/python/lock_requirements.py create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/app/python_scripts/launch_jupyter.py create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_notebook_change.yml create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_src_change.yaml create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/previous/.tools/bash/test_conda_environment.sh create mode 100644 .template_sync/resources/v0_1_19_to_v0_1_20/previous/app/python_scripts/launch_jupyter.py create mode 100644 .template_sync/resources/v0_1_22_to_v0_1_23/.tools/python/check_placeholders.py create mode 100644 .template_sync/resources/v0_1_23_to_v0_1_24/.tools/python/release_platforms.py create mode 100644 .template_sync/resources/v0_1_24_to_v0_1_25/.github/workflows/publish_pypi.yml create mode 100644 .template_sync/resources/v0_1_24_to_v0_1_25/.tools/python/pypi_package.py create mode 100644 .template_sync/resources/v0_1_24_to_v0_1_25/MANIFEST.in create mode 100644 .template_sync/resources/v0_1_24_to_v0_1_25/setup.py create mode 100644 .template_sync/resources/v0_1_25_to_v0_1_26/.tools/docs/external_code_upload_pypi.md create mode 100644 .template_sync/resources/v0_1_25_to_v0_1_26/.tools/docs/troubleshooting_pypi.md create mode 100644 .template_sync/resources/v0_1_26_to_v0_1_27/.tools/docs/app_manifest_host_requirements.md create mode 100644 .template_sync/resources/v0_1_26_to_v0_1_27/.tools/schema/labconstrictor-app.schema.json create mode 100644 .tools/python/check_placeholders.py create mode 100644 .tools/python/lock_requirements.py create mode 100644 .tools/python/pypi_package.py create mode 100644 .tools/python/release_platforms.py diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml new file mode 100644 index 0000000..cfd03da --- /dev/null +++ b/.github/workflows/publish_pypi.yml @@ -0,0 +1,79 @@ +name: Publish the Python package to PyPI + +# Publishes the app's own Python code (the package in src/) on PyPI at each release, with the version of the release: the same +# moment and the same version as the installers (release_constructor.yml). +# +# OFF BY DEFAULT: both jobs are skipped unless the repository variable PYPI_PUBLISH is +# true -> publish on PyPI (pypi.org) +# testpypi -> publish on TestPyPI (test.pypi.org), a trial +# No secret or token: PyPI trusted publishing. One-time setup (a pending trusted publisher on pypi.org for this workflow file and the +# environment "pypi", then the variable): .tools/docs/external_code_upload.md. To turn it off, delete the variable. + +on: + release: + types: + - published + +permissions: + contents: read + +jobs: + build: + name: Build the package of src/ + if: vars.PYPI_PUBLISH == 'true' || vars.PYPI_PUBLISH == 'testpypi' + runs-on: ubuntu-latest + outputs: + url: ${{ steps.build.outputs.url }} + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt build==1.3.0 + + # The tag (v1.2.3 or 1.2.3) without the v. The checkout is the tagged commit, before the "Update version" commit of + # release_constructor.yml: pypi_package.py writes this version into setup.py itself (not committed). + - name: Build and check the package (nothing is uploaded yet) + id: build + env: + TAG: ${{ github.event.release.tag_name }} + PYPI_PUBLISH: ${{ vars.PYPI_PUBLISH }} + run: python .tools/python/pypi_package.py --version "${TAG#v}" --index "$PYPI_PUBLISH" --outdir dist --github-output "$GITHUB_OUTPUT" + + - name: Store the distributions + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: python-package-distributions + path: dist/ + if-no-files-found: error + + publish: + name: Upload to ${{ vars.PYPI_PUBLISH == 'testpypi' && 'TestPyPI' || 'PyPI' }} + needs: build + if: vars.PYPI_PUBLISH == 'true' || vars.PYPI_PUBLISH == 'testpypi' + runs-on: ubuntu-latest + # The environment named in the trusted publisher on pypi.org (and test.pypi.org). GitHub creates it on the first run; protection + # rules added to it (Settings > Environments > pypi), such as required reviewers, then apply to every upload. + environment: + name: pypi + url: ${{ needs.build.outputs.url }} + permissions: + id-token: write # trusted publishing: PyPI accepts a short-lived token that GitHub signs for this workflow and environment + steps: + - name: Download the distributions + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: python-package-distributions + path: dist/ + + - name: Upload + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + repository-url: ${{ vars.PYPI_PUBLISH == 'testpypi' && 'https://test.pypi.org/legacy/' || 'https://upload.pypi.org/legacy/' }} diff --git a/.github/workflows/release_constructor.yml b/.github/workflows/release_constructor.yml index 4e7939b..fe5ec41 100644 --- a/.github/workflows/release_constructor.yml +++ b/.github/workflows/release_constructor.yml @@ -32,6 +32,8 @@ jobs: run: | set -euxo pipefail RAW="${{ github.event.release.tag_name }}" + # The tag as typed (v1.0.0 or 1.0.0): bump_version.py points the download links to it. + echo "RELEASE_TAG=$RAW" >> "$GITHUB_ENV" RAW="${RAW#v}" echo "release_version=$RAW" >> "$GITHUB_OUTPUT" echo "RELEASE_VERSION=$RAW" >> "$GITHUB_ENV" @@ -61,6 +63,11 @@ jobs: "bump_Welcome_notebook.py" \ .tools/python/bump_Welcome_notebook.py \ python .tools/python/bump_Welcome_notebook.py "${{ github.repository_owner }}" "${{ github.event.repository.name }}" + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "check_placeholders.py" \ + .tools/python/check_placeholders.py \ + python .tools/python/check_placeholders.py - name: Commit updated versioned files @@ -78,22 +85,35 @@ jobs: if-no-files-found: ignore archive: false + platforms: + name: Installers to build + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Read distribution.platforms of labconstrictor-app.yaml + id: matrix + run: python .tools/python/release_platforms.py --github-output "$GITHUB_OUTPUT" + build-installers: name: Build installers (${{ matrix.os_name }}) runs-on: ${{ matrix.runner }} - needs: bump-version + needs: [bump-version, platforms] strategy: fail-fast: false - matrix: - include: - - os_name: linux - runner: ubuntu-latest - - os_name: macos-arm64 - runner: macos-15 - - os_name: macos-intel - runner: macos-15-intel - - os_name: windows - runner: windows-latest + # One build per system of distribution.platforms in labconstrictor-app.yaml (every system without it), from the platforms job. + matrix: ${{ fromJSON(needs.platforms.outputs.matrix) }} defaults: run: shell: bash -l {0} @@ -119,6 +139,8 @@ jobs: exit 1 fi + # The tag as typed (v1.0.0 or 1.0.0): bump_version.py points the download links to it. + echo "RELEASE_TAG=$RAW" >> "$GITHUB_ENV" RAW="${RAW#v}" echo "release_version=$RAW" >> "$GITHUB_OUTPUT" echo "RELEASE_VERSION=$RAW" >> "$GITHUB_ENV" @@ -148,6 +170,11 @@ jobs: "bump_Welcome_notebook.py" \ .tools/python/bump_Welcome_notebook.py \ python .tools/python/bump_Welcome_notebook.py "${{ github.repository_owner }}" "${{ github.event.repository.name }}" + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document.log" \ + "check_placeholders.py" \ + .tools/python/check_placeholders.py \ + python .tools/python/check_placeholders.py - name: Set up Miniconda uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 diff --git a/.github/workflows/update_on_notebook_change.yml b/.github/workflows/update_on_notebook_change.yml index c5fae88..4a8c03b 100644 --- a/.github/workflows/update_on_notebook_change.yml +++ b/.github/workflows/update_on_notebook_change.yml @@ -9,7 +9,9 @@ on: - 'notebooks/**/requirements.yaml' workflow_dispatch: -# This enables to run a single workflow at a time +# One run of this workflow at a time per branch. The src workflow has its own group: the two can run at the same time, +# because each one rebases its commit onto the latest branch before it pushes (a shared group kept only one waiting run, so a +# third push cancelled the waiting run of the other workflow). concurrency: group: notebooks-global-lock-${{ github.ref }} cancel-in-progress: false @@ -98,6 +100,12 @@ jobs: .tools/python/merge_requirements.py \ python .tools/python/merge_requirements.py --source-dir notebooks --output requirements.txt + # requirements*.txt -> requirements*.lock.txt: every package the installer installs with pip, with its exact version (uv, valid on + # Windows, macOS and Linux). A lock file is rewritten only when its requirements file, Python or JupyterLab version changed. + - name: Lock requirements + run: | + bash .tools/bash/run_with_context.sh artifacts/error_log_document.log "lock_requirements.py" .tools/python/lock_requirements.py python .tools/python/lock_requirements.py + - name: Check for changes id: check-changes run: | @@ -138,17 +146,66 @@ jobs: - name: Commit updated files id: auto-commit if: steps.check-changes.outputs.has_changes == 'true' && steps.test-conda-env.outcome == 'success' - uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5 - with: - commit_message: GitHub Action - Updated files because of notebook and/or requirements changes - file_pattern: requirements*.txt environment.yaml notebooks/* setup.py + env: + BRANCH: ${{ github.ref_name }} + COMMIT_MESSAGE: GitHub Action - Updated files because of notebook and/or requirements changes + # The run checked out the commit that triggered it (the scripts above compare it with its parent); the branch may have moved + # since (another workflow committed, or a new push). A rejected push is therefore retried after a rebase onto the latest branch. + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git ls-files -z --modified --deleted --others --exclude-standard -- 'requirements*.txt' environment.yaml 'notebooks/*' setup.py | xargs -0 -r git add -A -- + if git diff --cached --quiet; then + echo "No changes to commit." + exit 0 + fi + git commit -m "$COMMIT_MESSAGE" + for attempt in 1 2 3 4 5; do + if git push origin "HEAD:refs/heads/$BRANCH"; then + exit 0 + fi + echo "The push was rejected (attempt $attempt of 5): rebasing onto the latest $BRANCH." + sleep $((attempt * 5)) + git fetch origin "$BRANCH" + if ! git rebase --autostash "origin/$BRANCH"; then + git rebase --abort || true + echo "::error::The commit of this run conflicts with the latest $BRANCH; run this workflow again (Actions > Run workflow)." + exit 1 + fi + done + echo "::error::The push was rejected 5 times; run this workflow again (Actions > Run workflow)." + exit 1 + # Only when the merged requirements could not be installed: a failure of any other step (a script, the push) never reverts a commit. - name: Revert user commit that broke requirements - if: failure() && github.event_name == 'push' + if: failure() && github.event_name == 'push' && steps.test-conda-env.outcome == 'failure' + env: + BRANCH: ${{ github.ref_name }} + BAD_COMMIT: ${{ github.sha }} run: | git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git reset --hard git clean -fd - git revert "${{ github.sha }}" -m 1 --no-edit - git push origin HEAD:${{ github.ref_name }} + for attempt in 1 2 3 4 5; do + git fetch origin "$BRANCH" + git checkout -B "$BRANCH" "origin/$BRANCH" + # A merge commit is reverted against its first parent (-m 1); any other commit is reverted as it is. + if [ "$(git rev-list --parents -n 1 "$BAD_COMMIT" | wc -w)" -gt 2 ]; then + REVERT_ARGS=(-m 1) + else + REVERT_ARGS=() + fi + if ! git revert --no-edit "${REVERT_ARGS[@]}" "$BAD_COMMIT"; then + git revert --abort || true + echo "::error::$BAD_COMMIT broke the requirements but could not be reverted automatically (later commits change the same lines): revert it by hand." + exit 1 + fi + if git push origin "HEAD:refs/heads/$BRANCH"; then + exit 0 + fi + echo "The push of the revert was rejected (attempt $attempt of 5): retrying on the latest $BRANCH." + sleep $((attempt * 5)) + done + echo "::error::The revert of $BAD_COMMIT could not be pushed: revert it by hand." + exit 1 diff --git a/.github/workflows/update_on_src_change.yaml b/.github/workflows/update_on_src_change.yaml index 76b90f0..2ab8870 100644 --- a/.github/workflows/update_on_src_change.yaml +++ b/.github/workflows/update_on_src_change.yaml @@ -7,9 +7,11 @@ on: - 'src/**' workflow_dispatch: -# This enables to run a single workflow at a time +# One run of this workflow at a time per branch. The notebook workflow has its own group: the two can run at the same time, +# because each one rebases its commit onto the latest branch before it pushes (a shared group kept only one waiting run, so a +# third push cancelled the waiting run of the other workflow). concurrency: - group: notebooks-global-lock-${{ github.ref }} + group: src-change-lock-${{ github.ref }} cancel-in-progress: false jobs: @@ -18,13 +20,13 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - + steps: - name: Checkout repository uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 # Fetch all history for all branches and tags - + - name: Update src change file run: | cat > .tools/meta/src_change.yaml < Run workflow)." + exit 1 + fi + done + echo "::error::The push was rejected 5 times; run this workflow again (Actions > Run workflow)." + exit 1 diff --git a/.template_sync/manifest.yaml b/.template_sync/manifest.yaml index ac8d395..1011ba0 100644 --- a/.template_sync/manifest.yaml +++ b/.template_sync/manifest.yaml @@ -1,4 +1,4 @@ -template_version: 0.1.19 +template_version: 0.1.28 migrations: - from: 0.0.0 to: 0.0.1 @@ -63,3 +63,30 @@ migrations: - from: 0.1.18 to: 0.1.19 script: migrations/v0_1_18_to_v0_1_19.py + - from: 0.1.19 + to: 0.1.20 + script: migrations/v0_1_19_to_v0_1_20.py + - from: 0.1.20 + to: 0.1.21 + script: migrations/v0_1_20_to_v0_1_21.py + - from: 0.1.21 + to: 0.1.22 + script: migrations/v0_1_21_to_v0_1_22.py + - from: 0.1.22 + to: 0.1.23 + script: migrations/v0_1_22_to_v0_1_23.py + - from: 0.1.23 + to: 0.1.24 + script: migrations/v0_1_23_to_v0_1_24.py + - from: 0.1.24 + to: 0.1.25 + script: migrations/v0_1_24_to_v0_1_25.py + - from: 0.1.25 + to: 0.1.26 + script: migrations/v0_1_25_to_v0_1_26.py + - from: 0.1.26 + to: 0.1.27 + script: migrations/v0_1_26_to_v0_1_27.py + - from: 0.1.27 + to: 0.1.28 + script: migrations/v0_1_27_to_v0_1_28.py diff --git a/.template_sync/migrations/v0_1_19_to_v0_1_20.py b/.template_sync/migrations/v0_1_19_to_v0_1_20.py new file mode 100644 index 0000000..b077854 --- /dev/null +++ b/.template_sync/migrations/v0_1_19_to_v0_1_20.py @@ -0,0 +1,356 @@ +"""Template 0.1.19 -> 0.1.20: no lost pushes between the two workflows, lock files for the pip requirements, a launcher that starts fresh. + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_20.md + +What this changes in an app repository (every step is independent, idempotent and never deletes anything): + +1. Workflows. update_on_notebook_change.yml and update_on_src_change.yaml are replaced by the 0.1.20 files ONLY when the app's file is still + identical to the 0.1.19 template file (line endings are ignored when comparing and kept when writing). The new files rebase their + commit onto the latest branch and retry a rejected push, have one concurrency group each, revert a commit only when the conda test + failed (a merge commit with -m 1, any other commit as it is), and the notebook workflow runs lock_requirements.py. +2. Lock files. .tools/python/lock_requirements.py is created (requirements*.txt -> requirements*.lock.txt with uv), `uv==` is added to + .tools/requirements.txt, .tools/bash/test_conda_environment.sh (replaced when identical to 0.1.19) tests the lock file, + post_install.sh / .bat install the lock file when the installer bundles one (known blocks only), and bump_constructor.py gets the + function that bundles the lock files (inserted at two known lines). +3. Launcher. app/python_scripts/launch_jupyter.py (replaced when identical to the rendered 0.1.19 file) asks to press Enter when JupyterLab + stops and gives JupyterLab an empty workspace folder at every launch (only the Welcome notebook opens). app/menuinst/notebook_launcher.json + gets a Windows command that ends with `& exit`, so the terminal window closes after Enter (cmd /K kept it open). +4. Notebook usage guide. The sentence on stopping JupyterLab in .tools/docs/notebook_usage.md is replaced when it is still the 0.1.19 text. + +Rules this migration keeps: surgical (only known lines or known files), idempotent (a second run reports no changes), never raises for a +customised file (the sync pull request must still be created), the line endings of each file are kept, nothing is deleted. +""" +from __future__ import annotations + +import importlib.util +import json +import re +from pathlib import Path +from typing import Any + + +RESOURCE_ROOT = Path(__file__).resolve().parents[1] / "resources" / "v0_1_19_to_v0_1_20" +PREVIOUS_ROOT = RESOURCE_ROOT / "previous" +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_20.md" +TAG = "[migration 0.1.20]" + +NOTEBOOK_WORKFLOW = Path(".github/workflows/update_on_notebook_change.yml") +SRC_WORKFLOW = Path(".github/workflows/update_on_src_change.yaml") +TEST_CONDA = Path(".tools/bash/test_conda_environment.sh") +LAUNCHER = Path("app/python_scripts/launch_jupyter.py") +LOCK_SCRIPT = Path(".tools/python/lock_requirements.py") +TOOLS_REQUIREMENTS = Path(".tools/requirements.txt") +POST_INSTALL_SH = Path("app/bash_bat_scripts/post_install.sh") +POST_INSTALL_BAT = Path("app/bash_bat_scripts/post_install.bat") +BUMP_CONSTRUCTOR = Path(".tools/python/bump_constructor.py") +MENU = Path("app/menuinst/notebook_launcher.json") +USAGE_DOC = Path(".tools/docs/notebook_usage.md") + +# Files replaced as a whole when the app's file is still the (rendered) 0.1.19 template file. +REPLACED_FILES = (NOTEBOOK_WORKFLOW, SRC_WORKFLOW, TEST_CONDA, LAUNCHER) +# Resource files that are copies of the 0.1.20 template files (a test checks they stay identical to the template). +MANAGED_FILES = (*REPLACED_FILES, LOCK_SCRIPT) + +UV_PIN = "uv==0.13.3" +UV_LINE = re.compile(r"^\s*uv\s*(?:[=<>!~;\[]|$)", re.IGNORECASE) +UV_LINES = "# lock_requirements.py (requirements*.txt -> requirements*.lock.txt)\n" + UV_PIN + "\n" + +# post_install.sh: (0.1.19 text, 0.1.20 text), LF. +SH_BLOCKS = ( + ( + 'echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE"\n' + '"$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1\n', + "# A lock file (requirements*.lock.txt next to the requirements file: every package with its exact version, written by\n" + "# .tools/python/lock_requirements.py) is installed instead of the requirements file when the installer bundles one.\n" + "locked_requirements() {\n" + ' if [ -f "${1%.txt}.lock.txt" ]; then echo "${1%.txt}.lock.txt"; else echo "$1"; fi\n' + "}\n" + 'SELECTED_REQUIREMENTS="$(locked_requirements "$SELECTED_REQUIREMENTS")"\n' + 'echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE"\n' + '"$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1\n', + ), + ( + ' "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-macos.txt" >> "$LOG_FILE" 2>&1\n', + ' "$PYTHON_EXE" -m pip install -r "$(locked_requirements "$PROJECT_ROOT/requirements-macos.txt")" >> "$LOG_FILE" 2>&1\n', + ), + ( + ' "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-linux.txt" >> "$LOG_FILE" 2>&1\n', + ' "$PYTHON_EXE" -m pip install -r "$(locked_requirements "$PROJECT_ROOT/requirements-linux.txt")" >> "$LOG_FILE" 2>&1\n', + ), +) +SH_DONE = "locked_requirements()" + +BAT_BLOCKS = ( + ( + 'echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%"\n' + '"%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1\n', + "REM A lock file (requirements*.lock.txt next to the requirements file: every package with its exact version, written by\n" + "REM .tools/python/lock_requirements.py) is installed instead of the requirements file when the installer bundles one.\n" + 'SET "SELECTED_LOCK=%SELECTED_REQUIREMENTS:~0,-4%.lock.txt"\n' + 'IF EXIST "%SELECTED_LOCK%" SET "SELECTED_REQUIREMENTS=%SELECTED_LOCK%"\n' + 'echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%"\n' + '"%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1\n', + ), + ( + 'IF EXIST "%PROJECT_ROOT%\\requirements-windows.txt" (\n' + ' echo Installing Windows-specific requirements. >> "%LOG_FILE%"\n' + ' "%PYTHON_EXE%" -m pip install -r "%PROJECT_ROOT%\\requirements-windows.txt" >> "%LOG_FILE%" 2>&1\n', + 'SET "WINDOWS_REQUIREMENTS=%PROJECT_ROOT%\\requirements-windows.txt"\n' + 'IF EXIST "%PROJECT_ROOT%\\requirements-windows.lock.txt" SET "WINDOWS_REQUIREMENTS=%PROJECT_ROOT%\\requirements-windows.lock.txt"\n' + 'IF EXIST "%PROJECT_ROOT%\\requirements-windows.txt" (\n' + ' echo Installing Windows-specific requirements from "%WINDOWS_REQUIREMENTS%". >> "%LOG_FILE%"\n' + ' "%PYTHON_EXE%" -m pip install -r "%WINDOWS_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1\n', + ), +) +BAT_DONE = "SELECTED_LOCK=" + +BUMP_FUNCTION_ANCHOR = "def ensure_extra_files(" +BUMP_CALL_OLD = " ensure_requirements_in_extra_files(data)\n" +BUMP_CALL_NEW = " ensure_requirements_in_extra_files(data)\n ensure_lock_files_in_extra_files(data)\n" +BUMP_DONE = "def ensure_lock_files_in_extra_files(" +BUMP_NEEDS = ("def extract_project_folder(", "def debug(", "from pathlib import Path") + +WIN_EXIT = ["&", "exit"] + +USAGE_OLD = ( + "When you have, click on the terminal window and press `Ctrl+C` to stop the Jupyter Lab session and close the terminal." +) +USAGE_NEW = ( + "When you have, stop the Jupyter Lab session with **File > Shut Down** in Jupyter Lab (or click on the terminal window and press `Ctrl+C`). " + "The terminal then asks you to press `Enter`, and closes." +) + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def _load_helpers() -> Any: + """Reuse the helpers of the 0.1.10 migration (token replacement, newline detection, file IO).""" + path = Path(__file__).resolve().parent / "v0_1_9_to_v0_1_10.py" + spec = importlib.util.spec_from_file_location("template_sync_helpers_0_1_20", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _resource(root: Path, relative: Path) -> str: + source = root / relative + if not source.is_file(): + raise FileNotFoundError(f"Template resource not found: {source}") + return source.read_bytes().decode("utf-8").replace("\r\n", "\n") + + +def _render(helpers: Any, text: str, replacements: dict[str, str]) -> str: + return str(helpers.render_template(text, replacements, "\n")) + + +def _write_lf(helpers: Any, path: Path, text_lf: str, newline: str) -> None: + helpers.write_text(path, text_lf.replace("\n", newline)) + + +# -------------------------------------------------------------------------------------------------------------------------------- +# Whole files (workflows, conda test, launcher) +# -------------------------------------------------------------------------------------------------------------------------------- +def _replace_file(helpers: Any, repo_root: Path, relative: Path, replacements: dict[str, str]) -> str: + path = repo_root / relative + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + current_lf = current.replace("\r\n", "\n") + new = _render(helpers, _resource(RESOURCE_ROOT, relative), replacements) + if current_lf == new: + return "already the 0.1.20 template file: unchanged" + if current_lf != _render(helpers, _resource(PREVIOUS_ROOT, relative), replacements): + return f"SKIPPED: customised (not the 0.1.19 template file), so it was left unchanged; apply the 0.1.20 changes by hand, see {DOCS_URL}" + _write_lf(helpers, path, new, helpers.detect_newline(current)) + return "updated: replaced by the 0.1.20 template file" + + +# -------------------------------------------------------------------------------------------------------------------------------- +# Lock files +# -------------------------------------------------------------------------------------------------------------------------------- +def _create_lock_script(helpers: Any, repo_root: Path) -> str: + path = repo_root / LOCK_SCRIPT + new = _resource(RESOURCE_ROOT, LOCK_SCRIPT) + if path.is_file(): + if helpers.read_text(path).replace("\r\n", "\n") == new: + return "already the 0.1.20 template file: unchanged" + return "SKIPPED: the repository has its own lock_requirements.py, left as it is" + if not path.parent.is_dir(): + return "SKIPPED: the repository has no .tools/python folder (nothing was created)" + with path.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(new) + return "updated: created" + + +def _add_uv(helpers: Any, repo_root: Path) -> str: + path = repo_root / TOOLS_REQUIREMENTS + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + if any(UV_LINE.match(line) for line in current.splitlines()): + return "already installs uv: unchanged" + newline = helpers.detect_newline(current) + text = current if not current or current.endswith(("\n", "\r")) else current + newline + helpers.write_text(path, text + UV_LINES.replace("\n", newline)) + return f"updated: added {UV_PIN}" + + +def _replace_blocks(helpers: Any, path: Path, blocks: tuple[tuple[str, str], ...], done: str, what: str) -> str: + current = helpers.read_text(path) + lf = current.replace("\r\n", "\n") + if done in lf: + return f"already {what}: unchanged" + missing = [old.splitlines()[0].strip() for old, _ in blocks if lf.count(old) != 1] + if missing: + shown = "; ".join(missing) + return f"SKIPPED: customised, the known lines were not found once ({shown}); left unchanged, apply the change by hand, see {DOCS_URL}" + for old, new in blocks: + lf = lf.replace(old, new) + _write_lf(helpers, path, lf, helpers.detect_newline(current)) + return f"updated: {what}" + + +def _bump_constructor(helpers: Any, repo_root: Path) -> str: + path = repo_root / BUMP_CONSTRUCTOR + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + lf = current.replace("\r\n", "\n") + if BUMP_DONE in lf: + return "already bundles the lock files: unchanged" + if lf.count(BUMP_FUNCTION_ANCHOR) != 1 or lf.count(BUMP_CALL_OLD) != 1 or not all(need in lf for need in BUMP_NEEDS): + return f"SKIPPED: customised (the known lines were not found once), left unchanged; add ensure_lock_files_in_extra_files by hand, see {DOCS_URL}" + lf = lf.replace(BUMP_FUNCTION_ANCHOR, BUMP_SNIPPET + BUMP_FUNCTION_ANCHOR, 1).replace(BUMP_CALL_OLD, BUMP_CALL_NEW, 1) + _write_lf(helpers, path, lf, helpers.detect_newline(current)) + return "updated: bundles the lock files (ensure_lock_files_in_extra_files)" + + +# The LOCK_FILES tuple and ensure_lock_files_in_extra_files of the 0.1.20 bump_constructor.py, with the blank lines before the next function +# (a test checks that the template script holds this text once). +BUMP_SNIPPET = '''LOCK_FILES = ( + "requirements.lock.txt", + "requirements_gpu.lock.txt", + "requirements-linux.lock.txt", + "requirements-windows.lock.txt", + "requirements-macos.lock.txt", +) + + +def ensure_lock_files_in_extra_files(construct_data: dict) -> None: + """Bundle the lock files (requirements*.lock.txt, written by lock_requirements.py) next to their requirements files. + + post_install installs a lock file instead of its requirements file when the installer bundles one. A mapping whose lock file is gone + (the requirements could not be locked) is removed, because constructor fails on a missing file. + """ + extra_files = construct_data.setdefault("extra_files", []) + project_folder = extract_project_folder(extra_files) + for lock_file in LOCK_FILES: + mapped = [item for item in extra_files if isinstance(item, dict) and lock_file in item] + if Path(lock_file).exists(): + if not mapped: + extra_files.append({lock_file: f"{project_folder}/{lock_file}"}) + debug(f"Added mapping for {lock_file}") + else: + debug(f"{lock_file} mapping already present") + elif mapped: + for item in mapped: + extra_files.remove(item) + debug(f"Removed the mapping of {lock_file}: the file does not exist (the installer installs the requirements file instead)") + + +''' + + +# -------------------------------------------------------------------------------------------------------------------------------- +# Launcher shortcut (Windows: close the terminal after Enter) +# -------------------------------------------------------------------------------------------------------------------------------- +def _menu(helpers: Any, repo_root: Path) -> str: + path = repo_root / MENU + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + lf = current.replace("\r\n", "\n") + try: + data = json.loads(lf) + except json.JSONDecodeError as exc: + return f"SKIPPED: not valid JSON ({exc}), left unchanged" + items = data.get("menu_items") if isinstance(data, dict) else None + if not isinstance(items, list) or len(items) != 1 or not isinstance(items[0], dict): + return f"SKIPPED: not the template shape (one menu item), left unchanged; see {DOCS_URL}" + item = items[0] + command = item.get("command") + win = (item.get("platforms") or {}).get("win") + if not isinstance(command, list) or not isinstance(win, dict) or not item.get("terminal"): + return f"SKIPPED: not the template shape (a terminal command with a win platform), left unchanged; see {DOCS_URL}" + if "command" in win: + if win["command"] == command + WIN_EXIT: + return "already closes the Windows terminal after Enter: unchanged" + return "SKIPPED: has its own Windows command, left unchanged; end it with \"&\", \"exit\" to close the terminal after Enter" + if json.dumps(data, indent=2, ensure_ascii=False) + "\n" != lf: + return f"SKIPPED: formatted differently from the template, left unchanged; add the Windows command by hand, see {DOCS_URL}" + win["command"] = command + WIN_EXIT + _write_lf(helpers, path, json.dumps(data, indent=2, ensure_ascii=False) + "\n", helpers.detect_newline(current)) + return "updated: the Windows command ends with `& exit` (the terminal closes after Enter)" + + +def _usage_doc(helpers: Any, repo_root: Path) -> str: + path = repo_root / USAGE_DOC + if not path.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(path) + if USAGE_NEW in current: + return "unchanged" + if current.count(USAGE_OLD) != 1: + return "SKIPPED: the 0.1.19 sentence on stopping Jupyter Lab was not found (customised), left unchanged" + helpers.write_text(path, current.replace(USAGE_OLD, USAGE_NEW)) + return "updated: how to stop Jupyter Lab (File > Shut Down, then Enter)" + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """Replace the two workflows and the launcher, add the lock files (idempotent, surgical, never raises for a customised file).""" + _ = context + helpers = _load_helpers() + try: + replacements = helpers.load_replacements(repo_root) + except (OSError, ValueError) as exc: + log(f"could not read the app name from construct.yaml ({exc}): the launcher is compared without app names.") + replacements = {} + + changed = 0 + + def report(name: Path, step: Any) -> None: + nonlocal changed + try: + status = step() + except (OSError, ValueError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + if status.startswith("updated"): + changed += 1 + log(f"{name}: {status}") + + for relative in REPLACED_FILES: + report(relative, lambda relative=relative: _replace_file(helpers, repo_root, relative, replacements)) + report(LOCK_SCRIPT, lambda: _create_lock_script(helpers, repo_root)) + report(TOOLS_REQUIREMENTS, lambda: _add_uv(helpers, repo_root)) + for relative, blocks, done in ((POST_INSTALL_SH, SH_BLOCKS, SH_DONE), (POST_INSTALL_BAT, BAT_BLOCKS, BAT_DONE)): + if not (repo_root / relative).is_file(): + log(f"{relative}: not present in this repository (nothing was created)") + continue + report(relative, lambda relative=relative, blocks=blocks, done=done: _replace_blocks( + helpers, repo_root / relative, blocks, done, "installs the lock file when the installer bundles one" + )) + report(BUMP_CONSTRUCTOR, lambda: _bump_constructor(helpers, repo_root)) + report(MENU, lambda: _menu(helpers, repo_root)) + report(USAGE_DOC, lambda: _usage_doc(helpers, repo_root)) + + if changed: + log( + f"done: {changed} file(s) changed. Review the pull request before merging. The lock files are written by the next run of " + "'Updates on notebook change' (Actions > Run workflow); the launcher changes reach users with the next release of the app." + ) + else: + log("done: no repository changes were required.") diff --git a/.template_sync/migrations/v0_1_20_to_v0_1_21.py b/.template_sync/migrations/v0_1_20_to_v0_1_21.py new file mode 100644 index 0000000..7f8a212 --- /dev/null +++ b/.template_sync/migrations/v0_1_20_to_v0_1_21.py @@ -0,0 +1,86 @@ +"""Template 0.1.20 -> 0.1.21: build the installer with conda-standalone 26.9.3 or later. + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_21.md + +What this changes in an app repository (idempotent, never deletes anything): + +1. .tools/bash/create_constructor_environment.sh. The constructor environment did not pin conda-standalone, the `_conda.exe` that the + installer uses to install the packages. Versions before 26.9.3 write the package records without `paths_data` when constructor disables + conda's safety checks (conda/conda#16856). conda 26.9 and later in the installed app looks up the Windows launcher stub in that metadata, + cannot find it, and treats the error as a permission problem: the Windows uninstaller then asks twice for administrator rights for + python.exe (`conda init --reverse`). The known `conda create` line is replaced by one that adds `"conda-standalone>=26.9.3"`. + +Rules this migration keeps: surgical (only the one known line), idempotent (a second run reports no changes), never raises for a customised +file (the sync pull request must still be created), the line endings of the file are kept, nothing is deleted. +""" +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any + + +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_21.md" +TAG = "[migration 0.1.21]" +CREATE_ENV = Path(".tools/bash/create_constructor_environment.sh") + +# The 0.1.20 line and its 0.1.21 replacement (LF; the file's own line endings are used when matching and writing). +CREATE_OLD = 'conda create -y -n "${CONSTRUCTOR_ENV:?}" -c conda-forge -c defaults python=3.11 constructor\n' +CREATE_NEW = """# conda-standalone (the installer's _conda.exe) older than 26.9.3 writes package +# records without paths_data when constructor disables conda's safety checks. +# conda 26.9 and later in the installed app then cannot find the Windows +# launcher stub, and `conda init --reverse` in the uninstaller asks for admin +# rights (conda/conda#16856, fixed in conda 26.9.3). +conda create -y -n "${CONSTRUCTOR_ENV:?}" -c conda-forge -c defaults python=3.11 constructor "conda-standalone>=26.9.3" +""" + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def _load_helpers() -> Any: + """Reuse the helpers of the 0.1.10 migration (newline-preserving file IO).""" + path = Path(__file__).resolve().parent / "v0_1_9_to_v0_1_10.py" + spec = importlib.util.spec_from_file_location("template_sync_helpers_0_1_21", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pin_conda_standalone(helpers: Any, path: Path) -> str: + text = helpers.read_text(path) + newline = "\r\n" if "\r\n" in text else "\n" + old_here, new_here = CREATE_OLD.replace("\n", newline), CREATE_NEW.replace("\n", newline) + if new_here in text: + return "unchanged: already pins conda-standalone>=26.9.3" + count = text.count(old_here) + if count == 0: + if "conda-standalone" in text: + return f"SKIPPED: the file already names conda-standalone (customised); check it allows 26.9.3 or later. See {DOCS_URL}" + return f"SKIPPED: the 0.1.20 conda create line was not found (customised file); add \"conda-standalone>=26.9.3\" by hand. See {DOCS_URL}" + if count > 1: + return f"SKIPPED: the 0.1.20 conda create line was found {count} times; add the pin by hand. See {DOCS_URL}" + helpers.write_text(path, text.replace(old_here, new_here, 1)) + return "updated: conda-standalone>=26.9.3" + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """Pin conda-standalone in create_constructor_environment.sh (idempotent, surgical, never raises for a customised file).""" + _ = context + path = repo_root / CREATE_ENV + if not path.is_file(): + log(f"{CREATE_ENV}: SKIPPED, file not found in this repository (nothing was created).") + log("done: no repository changes were required.") + return + try: + status = _pin_conda_standalone(_load_helpers(), path) + except (OSError, ValueError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + log(f"{CREATE_ENV}: {status}") + if status.startswith("updated"): + log("done: 1 file changed. Build and release a new installer: installs made with an older installer keep the admin prompt at uninstall.") + else: + log("done: no repository changes were required.") diff --git a/.template_sync/migrations/v0_1_21_to_v0_1_22.py b/.template_sync/migrations/v0_1_21_to_v0_1_22.py new file mode 100644 index 0000000..14969fb --- /dev/null +++ b/.template_sync/migrations/v0_1_21_to_v0_1_22.py @@ -0,0 +1,167 @@ +"""Template 0.1.21 -> 0.1.22: Windows computers with an NVIDIA driver can install TensorFlow apps; the version cell reads the version file. + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_22.md + +What this changes in an app repository (every step is independent, idempotent and never deletes anything): + +1. .tools/python/merge_requirements.py. The GPU requirements file rewrote `tensorflow` to `tensorflow[and-cuda]` for every system. The + extra pulls nvidia-* wheels that exist only for Linux and carries no platform marker, so the universal lock pinned them for Windows + too, and `pip install` failed on every Windows computer with an NVIDIA driver (post_install.bat picks the GPU file there). The GPU + file now has `tensorflow[and-cuda]==X ; sys_platform == "linux"` and `tensorflow==X ; sys_platform != "linux"`. Three known blocks + are changed (docstring, a new function before build_gpu_lines, the tensorflow branch of build_gpu_lines). +2. .tools/docs/notebook_upload.md. The version cell snippet looked the notebook up at the top level of notebook_latest_versions.yaml, + which sync_notebook_versions.py writes grouped by folder, so Colab users always read "not listed". The one known line is replaced. + Notebooks that already contain the old snippet are not edited (see the design note). + +Rules this migration keeps: surgical (only known blocks), idempotent (a second run reports no changes), never raises for a customised +file (the sync pull request must still be created), the line endings of each file are kept, nothing is deleted. +""" +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any + + +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_22.md" +TAG = "[migration 0.1.22]" +MERGE = Path(".tools/python/merge_requirements.py") +UPLOAD_DOC = Path(".tools/docs/notebook_upload.md") + +# merge_requirements.py: (0.1.21 text, 0.1.22 text), LF. +MERGE_DOC_OLD = ( + "GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` and adds the PyTorch\n" + "CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present.\n" +) +MERGE_DOC_NEW = ( + "GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` on Linux only (TensorFlow's CUDA\n" + "wheels exist only for Linux; other systems keep plain `tensorflow`) and adds the\n" + "PyTorch CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present.\n" +) +MERGE_ANCHOR = "def build_gpu_lines(final_lines: list[str]) -> list[str]:\n" +MERGE_FUNCTION = ( + "# TensorFlow's `and-cuda` extra pulls nvidia-* wheels that exist only for Linux (nvidia-nccl-cu12 has never had a Windows or macOS\n" + '# wheel), and the extra itself carries no platform marker. Without a marker the universal lock of requirements_gpu.txt pins them for\n' + '# every system, so `pip install` fails on a Windows computer with an NVIDIA driver (the installer picks the GPU file there). TensorFlow\n' + '# has had no native Windows GPU build since 2.10, so Windows and macOS keep plain `tensorflow` in the GPU file.\n' + 'GPU_TENSORFLOW_PLATFORM = \'sys_platform == "linux"\'\n' + 'NON_GPU_TENSORFLOW_PLATFORM = \'sys_platform != "linux"\'\n' + '\n' + '\n' + 'def _with_platform(marker: str, platform: str) -> str:\n' + ' """Return ` ; `, combined with the line\'s own marker (`; `) when it has one."""\n' + ' existing = marker.split(";", 1)[1].strip() if ";" in marker else ""\n' + ' return f" ; ({existing}) and {platform}" if existing else f" ; {platform}"\n' + '\n' + '\n' + 'def tensorflow_gpu_lines(line: str) -> list[str]:\n' + ' """Return the GPU-file lines for a tensorflow requirement: `[and-cuda]` on Linux, the plain requirement elsewhere.\n' + '\n' + ' A line that is not a tensorflow requirement is returned unchanged (as a one-item list).\n' + ' """\n' + ' match = pkg_re.match(line)\n' + ' if match:\n' + ' if match.group("name").lower() != "tensorflow":\n' + ' return [line]\n' + ' name, extras = match.group("name"), match.group("extras") or ""\n' + ' spec = f"{match.group(\'op\')}{match.group(\'ver\')}"\n' + ' else:\n' + ' match = PLAIN_PKG_RE.match(line)\n' + ' if not match or match.group("name").lower() != "tensorflow":\n' + ' return [line]\n' + ' name, extras, spec = match.group("name"), match.group("extras") or "", ""\n' + ' marker = match.group("marker") or ""\n' + ' cuda_extras = merge_requirement_extras(extras, "and-cuda")\n' + ' return [\n' + ' f"{name}{cuda_extras}{spec}{_with_platform(marker, GPU_TENSORFLOW_PLATFORM)}",\n' + ' f"{name}{extras}{spec}{_with_platform(marker, NON_GPU_TENSORFLOW_PLATFORM)}",\n' + ' ]\n' + '\n' + '\n' +) +MERGE_BODY_OLD = ( + ' if package_name == "tensorflow":\n' + " line = rewrite_tensorflow_for_gpu(line)\n" + " if package_name in TORCH_GPU_PACKAGES:\n" + " has_torch_package = True\n" +) +MERGE_BODY_NEW = ( + " if package_name in TORCH_GPU_PACKAGES:\n" + " has_torch_package = True\n" + ' if package_name == "tensorflow":\n' + " gpu_lines.extend(tensorflow_gpu_lines(line))\n" + " continue\n" +) +MERGE_DONE = "def tensorflow_gpu_lines(" + +UPLOAD_OLD = ' latest_version = config.get(notebook_name, "")\n' +UPLOAD_NEW = ( + ' # notebook_latest_versions.yaml groups the notebooks by folder: {"": {"": ""}}\n' + " latest_version = next((str(group[notebook_name]) for group in (config or {}).values()\n" + ' if isinstance(group, dict) and notebook_name in group), "")\n' +) +UPLOAD_DONE = "groups the notebooks by folder" + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def _load_helpers() -> Any: + """Reuse the helpers of the 0.1.10 migration (newline-preserving file IO).""" + path = Path(__file__).resolve().parent / "v0_1_9_to_v0_1_10.py" + spec = importlib.util.spec_from_file_location("template_sync_helpers_0_1_22", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _patch_merge(text: str, newline: str) -> tuple[str, str]: + """Return (new text, status) for merge_requirements.py; the code blocks change together or not at all.""" + if MERGE_DONE in text: + return text, "unchanged: the GPU file already limits tensorflow[and-cuda] to Linux" + anchor, body_old, body_new = (s.replace("\n", newline) for s in (MERGE_ANCHOR, MERGE_BODY_OLD, MERGE_BODY_NEW)) + if text.count(anchor) != 1 or text.count(body_old) != 1: + return text, f"SKIPPED: the 0.1.21 build_gpu_lines was not found once (customised file); see {DOCS_URL} for the change to make by hand" + function = MERGE_FUNCTION.replace("\n", newline) + text = text.replace(anchor, function + anchor, 1).replace(body_old, body_new, 1) + doc_old, doc_new = MERGE_DOC_OLD.replace("\n", newline), MERGE_DOC_NEW.replace("\n", newline) + if text.count(doc_old) == 1: + text = text.replace(doc_old, doc_new, 1) + return text, "updated: tensorflow[and-cuda] only on Linux in the GPU file" + + +def _patch_upload(text: str, newline: str) -> tuple[str, str]: + if UPLOAD_DONE in text: + return text, "unchanged: the version cell already reads the grouped version file" + old = UPLOAD_OLD.replace("\n", newline) + if text.count(old) != 1: + return text, f"SKIPPED: the 0.1.21 version lookup line was not found once (customised file). See {DOCS_URL}" + return text.replace(old, UPLOAD_NEW.replace("\n", newline), 1), "updated: the version cell reads the grouped version file" + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """Apply both changes (idempotent, surgical, never raises for a customised or missing file).""" + _ = context + helpers = _load_helpers() + changed = 0 + for relative, patch in ((MERGE, _patch_merge), (UPLOAD_DOC, _patch_upload)): + path = repo_root / relative + if not path.is_file(): + log(f"{relative}: SKIPPED, file not found in this repository (nothing was created).") + continue + try: + text = helpers.read_text(path) + new_text, status = patch(text, "\r\n" if "\r\n" in text else "\n") + if new_text != text: + helpers.write_text(path, new_text) + changed += 1 + except (OSError, ValueError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + log(f"{relative}: {status}") + if changed: + log(f"done: {changed} file(s) changed. Push a notebook change (or run 'Updates on notebook change') to regenerate the GPU lock, then release a new installer.") + else: + log("done: no repository changes were required.") diff --git a/.template_sync/migrations/v0_1_22_to_v0_1_23.py b/.template_sync/migrations/v0_1_22_to_v0_1_23.py new file mode 100644 index 0000000..486fe92 --- /dev/null +++ b/.template_sync/migrations/v0_1_22_to_v0_1_23.py @@ -0,0 +1,192 @@ +"""Template 0.1.22 -> 0.1.23: no template placeholder in the installer (CellMigrationLab/LabConstrictor#40). + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_23.md + +What this changes in an app repository (every step is independent, idempotent and never deletes anything): + +1. Placeholders the repository initialisation missed are replaced with the app's values, only where they are still there: + - app/bash_bat_scripts/post_install.sh / .bat: PYTHON_PROJ_NAME (the import check and the _lc_tools module). An app + initialised on template 0.1.16 or later with Python code in src/ could not be installed: `No module named 'PYTHON_PROJ_NAME'`. + - app/python_scripts/launch_jupyter.py: PROJECT_NAME (PROJECT_DISPLAY_NAME, the launcher's window titles and error messages). + - .tools/templates/Welcome_template.ipynb: PROJECT_NAME and PYTHON_PROJ_NAME ("Welcome to PROJECT_NAME!" in the Playground). + The package is found as in the 0.1.16 migration (the single _lc_tools module or package in src/, then setup.py, then + pyproject.toml); the app name is the menu name of app/menuinst/notebook_launcher.json, which the initialisation wrote. +2. .tools/python/check_placeholders.py (new) and two lines in .github/workflows/release_constructor.yml that run it after the + versioned files are updated: a release whose installer would still ship a placeholder stops with the file, the line and the fix. + +Rules this migration keeps: surgical (only placeholders, one new file and the known workflow lines), idempotent (a second run reports +no changes), never raises for a customised file (the sync pull request must still be created), the line endings of each file are +kept, nothing is deleted. +""" +from __future__ import annotations + +import importlib.util +import json +import re +from functools import partial +from pathlib import Path +from types import ModuleType +from typing import Any, Callable + + +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_23.md" +TAG = "[migration 0.1.23]" +RESOURCE_ROOT = Path(__file__).resolve().parents[1] / "resources" / "v0_1_22_to_v0_1_23" + +POST_INSTALL_SH = Path("app/bash_bat_scripts/post_install.sh") +POST_INSTALL_BAT = Path("app/bash_bat_scripts/post_install.bat") +LAUNCHER = Path("app/python_scripts/launch_jupyter.py") +WELCOME_TEMPLATE = Path(".tools/templates/Welcome_template.ipynb") +NOTEBOOK_LAUNCHER = Path("app/menuinst/notebook_launcher.json") +CHECK_SCRIPT = Path(".tools/python/check_placeholders.py") +RELEASE_WORKFLOW = Path(".github/workflows/release_constructor.yml") + +# Which placeholders each file may still hold, and where their value comes from ("package" or "name"). +PLACEHOLDER_FILES: dict[Path, tuple[tuple[str, str], ...]] = { + POST_INSTALL_SH: (("PYTHON_PROJ_NAME", "package"),), + POST_INSTALL_BAT: (("PYTHON_PROJ_NAME", "package"),), + LAUNCHER: (("PROJECT_NAME", "name"),), + WELCOME_TEMPLATE: (("PYTHON_PROJ_NAME", "package"), ("PROJECT_NAME", "name")), +} + +# The release workflow runs bump_Welcome_notebook.py last in both jobs; the check goes right after it, with the same log file. +WELCOME_STEP = re.compile( + r"(?P[ ]+)bash \.tools/bash/run_with_context\.sh \\\n" + r"(?P=indent) (?P\S+) \\\n" + r"(?P=indent) \"bump_Welcome_notebook\.py\" \\\n" + r"(?P=indent) \.tools/python/bump_Welcome_notebook\.py \\\n" + r"(?P=indent) python \.tools/python/bump_Welcome_notebook\.py [^\n]*\n" +) +CHECK_STEP = ( + "{indent}bash .tools/bash/run_with_context.sh \\\n" + "{indent} {log} \\\n" + '{indent} "check_placeholders.py" \\\n' + "{indent} .tools/python/check_placeholders.py \\\n" + "{indent} python .tools/python/check_placeholders.py\n" +) + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def _load(name: str, alias: str) -> ModuleType: + path = Path(__file__).resolve().parent / name + spec = importlib.util.spec_from_file_location(alias, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def placeholder_pattern(name: str) -> re.Pattern[str]: + """A placeholder on its own or followed by '_' (PYTHON_PROJ_NAME_lc_tools), never inside a longer name (UNDERSCORED_PROJECT_NAME).""" + return re.compile(r"(? str | None: + """The app name the initialisation wrote for PROJECT_NAME: the menu name of notebook_launcher.json, else PROJECT_DISPLAY_NAME.""" + path = repo_root / NOTEBOOK_LAUNCHER + if path.is_file(): + try: + name = json.loads(helpers.read_text(path)).get("menu_name") + except (ValueError, AttributeError): + name = None + if isinstance(name, str) and name.strip() and not placeholder_pattern("PROJECT_NAME").search(name): + return name.strip() + path = repo_root / LAUNCHER + if path.is_file(): + match = re.search(r'^PROJECT_DISPLAY_NAME = "([^"\n]+)"', helpers.read_text(path), re.MULTILINE) + if match and match.group(1) != "PROJECT_NAME": + return match.group(1) + return None + + +def _replace_placeholders(helpers: Any, path: Path, wanted: tuple[tuple[str, str], ...], values: dict[str, str | None]) -> str: + text = helpers.read_text(path) + present = [(name, source) for name, source in wanted if placeholder_pattern(name).search(text)] + if not present: + return "unchanged: no placeholder left" + missing = sorted({source for _, source in present if not values[source]}) + if missing: + what = " and ".join("the Python package (one package in src/, or a name in setup.py)" if m == "package" else "the app name" for m in missing) + names = ", ".join(name for name, _ in present) + return f"SKIPPED: {names} is still there, but {what} could not be found; replace it by hand. See {DOCS_URL}" + done = [] + for name, source in present: + value = values[source] + assert value is not None + text, count = placeholder_pattern(name).subn(value.replace("\\", "\\\\"), text) + done.append(f"{name} -> {value} ({count}x)") + helpers.write_text(path, text) + return "updated: " + ", ".join(done) + + +def _create_check_script(helpers: Any, repo_root: Path) -> str: + path = repo_root / CHECK_SCRIPT + new = (RESOURCE_ROOT / CHECK_SCRIPT).read_text(encoding="utf-8") + if path.is_file(): + if helpers.read_text(path).replace("\r\n", "\n") == new: + return "already the 0.1.23 template file: unchanged" + return "SKIPPED: the repository has its own check_placeholders.py, left as it is" + if not path.parent.is_dir(): + return "SKIPPED: the repository has no .tools/python folder (nothing was created)" + with path.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(new) + return "updated: created" + + +def _add_check_step(helpers: Any, path: Path) -> str: + text = helpers.read_text(path) + newline = "\r\n" if "\r\n" in text else "\n" + lf = text.replace("\r\n", "\n") + if "check_placeholders.py" in lf: + return "unchanged: already runs check_placeholders.py" + steps = list(WELCOME_STEP.finditer(lf)) + if not steps: + return f"SKIPPED: the bump_Welcome_notebook.py command was not found (customised workflow); add the check by hand. See {DOCS_URL}" + out, last = [], 0 + for match in steps: + out.append(lf[last : match.end()]) + out.append(CHECK_STEP.format(indent=match.group("indent"), log=match.group("log"))) + last = match.end() + out.append(lf[last:]) + helpers.write_text(path, "".join(out).replace("\n", newline)) + return f"updated: the check runs after bump_Welcome_notebook.py in {len(steps)} job(s)" + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """Replace the placeholders the initialisation missed and add the release check (idempotent, surgical, never raises for a customised file).""" + _ = context + helpers = _load("v0_1_9_to_v0_1_10.py", "template_sync_helpers_0_1_23") + construct = repo_root / "construct.yaml" + if construct.is_file() and re.search(r"^name:\s*UNDERSCORED_PROJECT_NAME\s*$", helpers.read_text(construct), re.MULTILINE): + log("SKIPPED: this repository is not initialised (construct.yaml still names UNDERSCORED_PROJECT_NAME); nothing was changed.") + return + package = _load("v0_1_15_to_v0_1_16.py", "template_sync_detect_0_1_23").detect_package(helpers, repo_root) + values = {"package": package, "name": app_name(helpers, repo_root)} + changed = 0 + + def report(name: Path, step: Callable[[], str]) -> None: + nonlocal changed + if not (repo_root / name).is_file() and name != CHECK_SCRIPT: + log(f"{name}: not present in this repository (nothing was created)") + return + try: + status = step() + except (OSError, ValueError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + if status.startswith("updated"): + changed += 1 + log(f"{name}: {status}") + + for relative, wanted in PLACEHOLDER_FILES.items(): + report(relative, partial(_replace_placeholders, helpers, repo_root / relative, wanted, values)) + report(CHECK_SCRIPT, lambda: _create_check_script(helpers, repo_root)) + report(RELEASE_WORKFLOW, lambda: _add_check_step(helpers, repo_root / RELEASE_WORKFLOW)) + + if changed: + log(f"done: {changed} file(s) changed. Review the pull request before merging; the fixes reach users with the next release of the app.") + else: + log("done: no repository changes were required.") diff --git a/.template_sync/migrations/v0_1_23_to_v0_1_24.py b/.template_sync/migrations/v0_1_23_to_v0_1_24.py new file mode 100644 index 0000000..c280357 --- /dev/null +++ b/.template_sync/migrations/v0_1_23_to_v0_1_24.py @@ -0,0 +1,350 @@ +"""Template 0.1.23 -> 0.1.24: download links for "v" tags, installers only for the supported systems, the app's own Jupyter kernel. + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_24.md + +What this changes in an app repository (every step is independent, idempotent and never deletes anything): + +1. #41, a release tagged v1.0.0 wrote download links to /releases/download/1.0.0/, which do not exist. The release workflow now exports the + tag as typed (RELEASE_TAG), and .tools/python/bump_version.py points the links to it; the installer names keep the bare version. + bump_version.py reads the tag from the environment, so a workflow without the new line, or an older bump_version.py, still works. +2. #42, every release built the four installers, also for systems an app cannot run on (TensorFlow 2.17+ on Intel Macs). The new + .tools/python/release_platforms.py reads `distribution.platforms` of labconstrictor-app.yaml: the release workflow builds only those + systems (a new `platforms` job writes the build matrix), and the download page has no button for the others. Without the key, + every system is built, as before. A build matrix that is not the template's (an app that removed a system by hand) is left as it is. +3. #43, a user-level "python3" kernelspec (`ipykernel install --user`) won over the app's own kernel, and the notebook ran in another + Python. app/python_scripts/launch_jupyter.py sets JUPYTER_PREFER_ENV_PATH=1 before JupyterLab starts. + +Rules this migration keeps: surgical (only the known blocks, one new file), idempotent (a second run reports no changes), never raises +for a customised file (the sync pull request must still be created), the line endings of each file are kept, nothing is deleted. +""" +from __future__ import annotations + +import importlib.util +import re +from functools import partial +from pathlib import Path +from types import ModuleType +from typing import Any, Callable + + +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_24.md" +TAG = "[migration 0.1.24]" +RESOURCE_ROOT = Path(__file__).resolve().parents[1] / "resources" / "v0_1_23_to_v0_1_24" + +BUMP_VERSION = Path(".tools/python/bump_version.py") +PLATFORMS_SCRIPT = Path(".tools/python/release_platforms.py") +RELEASE_WORKFLOW = Path(".github/workflows/release_constructor.yml") +LAUNCHER = Path("app/python_scripts/launch_jupyter.py") + +# -------------------------------------------------------------------------------------------------------------------------------- +# bump_version.py (#41 the tag in the links, #42 no button for a system that is not built): (0.1.23 text, 0.1.24 text), each once. +# -------------------------------------------------------------------------------------------------------------------------------- +BUMP_BLOCKS: tuple[tuple[str, str], ...] = ( + ("import argparse\nimport pathlib\n", "import argparse\nimport os\nimport pathlib\n"), + ( + '''def replace_version_placeholder(text: str, new_version: str) -> str: + """Render the release version in download documentation. + + Handles both pristine templates containing ``VERSION_NUMBER`` and + initialized repositories where that placeholder was already rendered to an + older explicit version such as ``0.0.0``. + """ + explicit_versions = set( + re.findall(r"/releases/download/(\\d+\\.\\d+\\.\\d+)/", text) + ) + + text = text.replace("VERSION_NUMBER", new_version) + for previous_version in explicit_versions: + text = text.replace(previous_version, new_version) + return text +''', + '''# The folder of a release in a download link: /releases/download// (the tag is 1.2.3 or v1.2.3). +DOWNLOAD_PATH_RE = re.compile(r"/releases/download/v?\\d+\\.\\d+\\.\\d+/") +# A button of the download page: +DOWNLOAD_BUTTON_RE = re.compile(r'\\s*]*>\\s*') +NOT_BUILT_NOTE = "> No installer is built for {label}: this app does not support it." + + +def replace_version_placeholder(text: str, new_version: str, tag: str | None = None) -> str: + """Render the release version in download documentation. + + Handles both pristine templates containing ``VERSION_NUMBER`` and + initialized repositories where that placeholder was already rendered to an + older explicit version such as ``0.0.0``. The links point to the release + tag as it was typed (``v1.2.3`` or ``1.2.3``); the installer names carry + the version without ``v`` (CellMigrationLab/LabConstrictor#41). + """ + explicit_versions = set( + re.findall(r"/releases/download/v?(\\d+\\.\\d+\\.\\d+)/", text) + ) + + text = text.replace("VERSION_NUMBER", new_version) + for previous_version in explicit_versions: + text = text.replace(previous_version, new_version) + return DOWNLOAD_PATH_RE.sub(f"/releases/download/{tag or new_version}/", text) + + +def drop_unbuilt_buttons(text: str, not_built: list[tuple[tuple[str, ...], str]]) -> str: + """Replace the download button of each system the release does not build with a note (CellMigrationLab/LabConstrictor#42). + + ``not_built`` holds (installer name endings, label) pairs, from release_platforms.py. The "or" between + two buttons goes when one of them became a note. + """ + if not not_built: + return text + + def _button(match: re.Match) -> str: + link = match.group(0).split('"', 2)[1] + for endings, label in not_built: + if link.endswith(endings): + return NOT_BUILT_NOTE.format(label=label) + return match.group(0) + + lines = DOWNLOAD_BUTTON_RE.sub(_button, text).split("\\n") + note_start = NOT_BUILT_NOTE.split("{", 1)[0] + kept = [] + for index, line in enumerate(lines): + if line.strip() == "or": + before = next((other for other in reversed(lines[:index]) if other.strip()), "") + after = next((other for other in lines[index + 1 :] if other.strip()), "") + if before.startswith(note_start) or after.startswith(note_start): + continue + kept.append(line) + return "\\n".join(kept) + + +def not_built_systems() -> list[tuple[tuple[str, ...], str]]: + """(installer name endings, label) of each system left out of distribution.platforms in labconstrictor-app.yaml.""" + sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + import release_platforms + + return [(build.files, build.label) for build in release_platforms.skipped_builds(ROOT)] +''', + ), + ( + "def bump_version_in_download_executable_md(new_version: str) -> None:\n", + "def bump_version_in_download_executable_md(\n" + " new_version: str, tag: str | None = None, not_built: list[tuple[tuple[str, ...], str]] | None = None\n" + ") -> None:\n", + ), + ( + " updated_text = replace_version_placeholder(text, new_version)\n", + " updated_text = drop_unbuilt_buttons(replace_version_placeholder(text, new_version, tag), not_built or [])\n", + ), + ( + " args = parser.parse_args()\n", + """ parser.add_argument( + "--tag", + default=os.environ.get("RELEASE_TAG") or None, + help="The release tag the download links point to: 0.0.4 or v0.0.4 (default: $RELEASE_TAG, else the version)", + ) + args = parser.parse_args() +""", + ), + ( + ' sys.exit("Version must look like X.Y.Z")\n', + """ sys.exit("Version must look like X.Y.Z") + if args.tag and args.tag not in (args.new_version, f"v{args.new_version}"): + sys.exit(f"The release tag {args.tag} does not match the version {args.new_version} (expected {args.new_version} or v{args.new_version})") +""", + ), + ( + " # Update download_executable.md\n bump_version_in_download_executable_md(args.new_version)\n", + " # Update download_executable.md: the links point to the release tag, and a system that is not built has no button\n" + " bump_version_in_download_executable_md(args.new_version, args.tag, not_built_systems())\n", + ), +) + +# -------------------------------------------------------------------------------------------------------------------------------- +# release_constructor.yml +# -------------------------------------------------------------------------------------------------------------------------------- +# #41: both "Determine release version" steps strip the "v" of the tag; the tag as typed is exported first. +STRIP_LINE = re.compile(r'^(?P[ ]+)RAW="\$\{RAW#v\}"\n', re.MULTILINE) +TAG_LINES = ( + "{indent}# The tag as typed (v1.0.0 or 1.0.0): bump_version.py points the download links to it.\n" + '{indent}echo "RELEASE_TAG=$RAW" >> "$GITHUB_ENV"\n' +) +# #42: the build matrix of the template, replaced by the one the platforms job writes; the job goes right before the build job. +MATRIX_OLD = """ needs: bump-version + strategy: + fail-fast: false + matrix: + include: + - os_name: linux + runner: ubuntu-latest + - os_name: macos-arm64 + runner: macos-15 + - os_name: macos-intel + runner: macos-15-intel + - os_name: windows + runner: windows-latest +""" +MATRIX_NEW = """ needs: [bump-version, platforms] + strategy: + fail-fast: false + # One build per system of distribution.platforms in labconstrictor-app.yaml (every system without it), from the platforms job. + matrix: ${{ fromJSON(needs.platforms.outputs.matrix) }} +""" +BUILD_JOB = " build-installers:\n" +PLATFORMS_JOB = """ platforms: + name: Installers to build + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Read distribution.platforms of labconstrictor-app.yaml + id: matrix + run: python .tools/python/release_platforms.py --github-output "$GITHUB_OUTPUT" + +""" + +# -------------------------------------------------------------------------------------------------------------------------------- +# launch_jupyter.py (#43): the variable is set right before JupyterLab is imported, in run_jupyter(). +# -------------------------------------------------------------------------------------------------------------------------------- +LAUNCHER_ANCHOR = """ # Import only after configure_tls() has completed. Tornado creates SSL + # contexts during import on some versions. + from jupyterlab.labapp import main +""" +LAUNCHER_LINES = """ # The app's own kernel must win over a user-level "python3" kernelspec (`ipykernel install --user`), which Jupyter + # otherwise prefers: the notebook would run in another Python and fail with "No module named ". Set, not + # setdefault: a JUPYTER_PREFER_ENV_PATH=0 of the user would bring the problem back. + os.environ["JUPYTER_PREFER_ENV_PATH"] = "1" + +""" +LAUNCHER_BLOCK = (LAUNCHER_ANCHOR, LAUNCHER_LINES + LAUNCHER_ANCHOR) + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def _load(name: str, alias: str) -> ModuleType: + path = Path(__file__).resolve().parent / name + spec = importlib.util.spec_from_file_location(alias, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def apply_blocks(text_lf: str, blocks: tuple[tuple[str, str], ...]) -> str | None: + """The text with every block changed, or None when a 0.1.23 block is not there exactly once (a customised file).""" + for old, new in blocks: + if text_lf.count(old) != 1: + return None + text_lf = text_lf.replace(old, new, 1) + return text_lf + + +def add_tag_lines(text_lf: str) -> tuple[str, int]: + """#41: export RELEASE_TAG before each `RAW="${RAW#v}"`; returns the text and how many were added.""" + return STRIP_LINE.subn(lambda m: TAG_LINES.format(indent=m.group("indent")) + m.group(0), text_lf) + + +def build_from_platforms(text_lf: str) -> str | None: + """#42: the platforms job and its matrix, or None when the build job or its matrix is not the template's.""" + if text_lf.count(MATRIX_OLD) != 1 or text_lf.count(BUILD_JOB) != 1: + return None + return text_lf.replace(BUILD_JOB, PLATFORMS_JOB + BUILD_JOB, 1).replace(MATRIX_OLD, MATRIX_NEW, 1) + + +def _blocks_step(helpers: Any, path: Path, blocks: tuple[tuple[str, str], ...], what: str) -> str: + text = helpers.read_text(path) + newline = "\r\n" if "\r\n" in text else "\n" + lf = text.replace("\r\n", "\n") + if all(new in lf for _, new in blocks): + return f"unchanged: already {what}" + new_lf = apply_blocks(lf, blocks) + if new_lf is None: + return f"SKIPPED: customised (a 0.1.23 block was not found), so it was left unchanged; apply the 0.1.24 change by hand, see {DOCS_URL}" + helpers.write_text(path, new_lf.replace("\n", newline)) + return f"updated: {what}" + + +def _create_platforms_script(helpers: Any, repo_root: Path) -> str: + path = repo_root / PLATFORMS_SCRIPT + new = (RESOURCE_ROOT / PLATFORMS_SCRIPT).read_text(encoding="utf-8") + if path.is_file(): + if helpers.read_text(path).replace("\r\n", "\n") == new: + return "already the 0.1.24 template file: unchanged" + return "SKIPPED: the repository has its own release_platforms.py, left as it is" + if not path.parent.is_dir(): + return "SKIPPED: the repository has no .tools/python folder (nothing was created)" + with path.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(new) + return "updated: created" + + +def _update_workflow(helpers: Any, path: Path) -> str: + text = helpers.read_text(path) + newline = "\r\n" if "\r\n" in text else "\n" + lf = text.replace("\r\n", "\n") + done, notes = [], [] + if "RELEASE_TAG=" in lf: + notes.append("already exports RELEASE_TAG") + else: + lf, count = add_tag_lines(lf) + if count: + done.append(f"exports RELEASE_TAG in {count} job(s)") + else: + notes.append(f'SKIPPED the release tag: no RAW="${{RAW#v}}" line (customised); the download links keep the bare version, see {DOCS_URL}') + if MATRIX_NEW in lf: + notes.append("already builds the systems of distribution.platforms") + else: + migrated = build_from_platforms(lf) + if migrated is None: + notes.append( + "SKIPPED the build matrix: it is not the template's four builds (customised), so it was left as it is; to build the systems of " + f"distribution.platforms instead, see {DOCS_URL}" + ) + else: + lf = migrated + done.append("builds the systems of distribution.platforms (new platforms job)") + if done: + helpers.write_text(path, lf.replace("\n", newline)) + return "updated: " + "; ".join(done + notes) + if any(note.startswith("SKIPPED") for note in notes): + return "; ".join(notes) + return "unchanged: " + "; ".join(notes) + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """Release tag in the download links, installers for the systems of distribution.platforms, the app's own kernel (idempotent, surgical).""" + _ = context + helpers = _load("v0_1_9_to_v0_1_10.py", "template_sync_helpers_0_1_24") + changed = 0 + + def report(name: Path, step: Callable[[], str]) -> None: + nonlocal changed + if not (repo_root / name).is_file() and name != PLATFORMS_SCRIPT: + log(f"{name}: not present in this repository (nothing was created)") + return + try: + status = step() + except (OSError, ValueError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + if status.startswith("updated"): + changed += 1 + log(f"{name}: {status}") + + report(PLATFORMS_SCRIPT, partial(_create_platforms_script, helpers, repo_root)) + report(BUMP_VERSION, partial(_blocks_step, helpers, repo_root / BUMP_VERSION, BUMP_BLOCKS, "points the download links to the release tag")) + report(RELEASE_WORKFLOW, partial(_update_workflow, helpers, repo_root / RELEASE_WORKFLOW)) + report(LAUNCHER, partial(_blocks_step, helpers, repo_root / LAUNCHER, (LAUNCHER_BLOCK,), "prefers the app's own Jupyter kernel")) + + if changed: + log(f"done: {changed} file(s) changed. Review the pull request before merging; the fixes reach users with the next release of the app.") + else: + log("done: no repository changes were required.") diff --git a/.template_sync/migrations/v0_1_24_to_v0_1_25.py b/.template_sync/migrations/v0_1_24_to_v0_1_25.py new file mode 100644 index 0000000..cff1608 --- /dev/null +++ b/.template_sync/migrations/v0_1_24_to_v0_1_25.py @@ -0,0 +1,270 @@ +"""Template 0.1.24 -> 0.1.25: publish the app's Python code (src/) on PyPI at each release, off until the developer turns it on. + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_25.md + +What this changes in an app repository (every step is independent, idempotent and never deletes anything): + +1. .github/workflows/publish_pypi.yml (new). At a published release it builds the package of src/ and uploads it to PyPI with trusted + publishing (no secret). Both jobs are skipped unless the repository variable PYPI_PUBLISH is "true" (PyPI) or "testpypi" (TestPyPI), + so nothing changes for an app that does not turn it on. It is NOT added to a repository that already uploads to PyPI with a + workflow of its own (NucleiSky): the package would be uploaded twice. The log says what to do instead. +2. .tools/python/pypi_package.py (new): the checks and the build the workflow runs (version not on the index yet, a package in src/, + the release version written into setup.py, name, version and dependencies of the built files). +3. setup.py, when it is still the template's (any name, version and Python series): the package needs what requirements.txt lists, + README.md is its description on PyPI, and the source repository is linked. MANIFEST.in (new) puts requirements.txt in the source + distribution. A setup.py changed by hand is left as it is (it still builds; the workflow uses it as it is). +4. .tools/python/bump_version.py: the release also writes its version into setup.py (it never did). + +Rules this migration keeps: surgical (only the known blocks, the known setup.py, new files), idempotent (a second run reports no changes), +never raises for a customised file (the sync pull request must still be created), the line endings of each file are kept, nothing is deleted. +""" +from __future__ import annotations + +import importlib.util +import re +from functools import partial +from pathlib import Path +from types import ModuleType +from typing import Any, Callable + + +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_25.md" +TAG = "[migration 0.1.25]" +RESOURCE_ROOT = Path(__file__).resolve().parents[1] / "resources" / "v0_1_24_to_v0_1_25" + +WORKFLOW = Path(".github/workflows/publish_pypi.yml") +PACKAGE_SCRIPT = Path(".tools/python/pypi_package.py") +SETUP = Path("setup.py") +MANIFEST_IN = Path("MANIFEST.in") +BUMP_VERSION = Path(".tools/python/bump_version.py") +NEW_FILES = (WORKFLOW, PACKAGE_SCRIPT, MANIFEST_IN) + +# Text in a workflow that uploads to PyPI (as in .tools/python/pypi_package.py). +PUBLISH_MARKERS = ("pypa/gh-action-pypi-publish", "twine upload") + +# -------------------------------------------------------------------------------------------------------------------------------- +# setup.py of the template up to 0.1.24; the form (and merge_requirements.py) fill in the three placeholders. +# -------------------------------------------------------------------------------------------------------------------------------- +OLD_SETUP = """from setuptools import setup, find_packages + +setup( + name="PYTHON_PROJ_NAME", + version="VERSION_NUMBER", + package_dir={"": "src"}, + packages=find_packages(where="src"), + python_requires=">=PYTHON_VERSION", + install_requires=[ + ], +) +""" +SETUP_FIELDS = (("PYTHON_PROJ_NAME", "name"), ("VERSION_NUMBER", "version"), ("PYTHON_VERSION", "python")) +REQUIRED_LINE = "include requirements.txt" + +# -------------------------------------------------------------------------------------------------------------------------------- +# bump_version.py: (0.1.24 text, 0.1.25 text), each once. +# -------------------------------------------------------------------------------------------------------------------------------- +BUMP_BLOCKS: tuple[tuple[str, str], ...] = ( + ( + 'POST_INSTALL = ROOT / "app" / "bash_bat_scripts" / "post_install.bat"\n', + 'POST_INSTALL = ROOT / "app" / "bash_bat_scripts" / "post_install.bat"\n' + "# version=\"1.2.3\" in setup.py: the version of the app's Python package (src/)\n" + 'SETUP = ROOT / "setup.py"\n' + 'SETUP_VERSION_RE = re.compile(r"^(?P[ \\t]*version\\s*=\\s*)(?P[\\"\'])\\d+\\.\\d+\\.\\d+(?P=quote)", re.MULTILINE)\n', + ), + ( + ' print(f"Updated post_install.bat PKG_VERSION to {new_version}")\n', + ''' print(f"Updated post_install.bat PKG_VERSION to {new_version}") + + +def bump_setup_py(new_version: str) -> None: + """The version of the app's Python package in setup.py, when it is written there as X.Y.Z. + + The package in the installer and the one publish_pypi.yml publishes on PyPI then carry the release version. The line + endings of the file are kept. + """ + if not SETUP.exists(): + return + with SETUP.open(encoding="utf-8", newline="") as handle: + text = handle.read() + updated = SETUP_VERSION_RE.sub(lambda m: f"{m.group('head')}{m.group('quote')}{new_version}{m.group('quote')}", text, count=1) + if updated != text: + with SETUP.open("w", encoding="utf-8", newline="") as handle: + handle.write(updated) + print(f"Updated setup.py version to {new_version}") +''', + ), + ( + " # Also bump PKG_VERSION in post_install.bat\n bump_post_install_bat(args.new_version)\n", + " # Also bump PKG_VERSION in post_install.bat\n bump_post_install_bat(args.new_version)\n\n" + " # The version of the app's Python package (setup.py), also the version publish_pypi.yml publishes on PyPI\n" + " bump_setup_py(args.new_version)\n", + ), +) + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def _load(name: str, alias: str) -> ModuleType: + path = Path(__file__).resolve().parent / name + spec = importlib.util.spec_from_file_location(alias, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def resource(relative: Path) -> str: + return (RESOURCE_ROOT / relative).read_text(encoding="utf-8") + + +def apply_blocks(text_lf: str, blocks: tuple[tuple[str, str], ...]) -> str | None: + """The text with every block changed, or None when a 0.1.24 block is not there exactly once (a customised file).""" + for old, new in blocks: + if text_lf.count(old) != 1: + return None + text_lf = text_lf.replace(old, new, 1) + return text_lf + + +def setup_pattern(template: str) -> re.Pattern[str]: + """A template setup.py as a pattern: each placeholder is the value the form (or merge_requirements.py) wrote, or the placeholder.""" + pattern = re.escape(template.rstrip()) + for placeholder, group in SETUP_FIELDS: + pattern = pattern.replace(re.escape(placeholder), f'(?P<{group}>[^"\\n]*)', 1) + return re.compile(pattern) + + +def setup_values(text_lf: str, template: str) -> dict[str, str] | None: + """The name, version and Python series of a setup.py that is the given template text with its placeholders filled in, else None.""" + match = setup_pattern(template).fullmatch(text_lf.rstrip()) + return match.groupdict() if match else None + + +def render_setup(template: str, values: dict[str, str]) -> str: + for placeholder, group in SETUP_FIELDS: + template = template.replace(placeholder, values[group], 1) + return template + + +def other_publishers(repo_root: Path) -> list[str]: + """The workflows of the repository, other than publish_pypi.yml, that upload to PyPI.""" + folder = repo_root / WORKFLOW.parent + found = [] + for path in sorted(folder.glob("*.y*ml")) if folder.is_dir() else []: + if path.name != WORKFLOW.name and any(marker in path.read_text(encoding="utf-8", errors="replace") for marker in PUBLISH_MARKERS): + found.append(path.relative_to(repo_root).as_posix()) + return found + + +def _create(helpers: Any, repo_root: Path, relative: Path, what: str) -> str: + """Create a new file from the resource (LF); a different file of the repository at that place is kept.""" + path = repo_root / relative + new = resource(relative) + if path.is_file(): + if helpers.read_text(path).replace("\r\n", "\n") == new: + return "unchanged: already the 0.1.25 template file" + return f"SKIPPED: the repository has its own {relative.name}, left as it is; {what}" + if not path.parent.is_dir(): + return f"SKIPPED: the repository has no {relative.parent.as_posix()} folder (nothing was created)" + with path.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(new) + return "updated: created" + + +def _create_workflow(helpers: Any, repo_root: Path) -> str: + others = other_publishers(repo_root) + if others and not (repo_root / WORKFLOW).is_file(): + return ( + f"SKIPPED: {', '.join(others)} already uploads to PyPI, so the template's publish_pypi.yml was not added (the package would be " + "uploaded twice). Nothing to do to keep publishing that way. To publish with the template's workflow instead (at each release, " + f"with the release version): delete your workflow, add publish_pypi.yml by hand and register it on pypi.org, see {DOCS_URL}" + ) + return _create( + helpers, repo_root, WORKFLOW, "to publish the package at each release, compare it with the template's publish_pypi.yml, see " + DOCS_URL + ) + + +def _update_setup(helpers: Any, repo_root: Path) -> str: + path = repo_root / SETUP + text = helpers.read_text(path) + newline = "\r\n" if "\r\n" in text else "\n" + lf = text.replace("\r\n", "\n") + new_template = resource(SETUP) + if setup_values(lf, new_template) is not None: + return "unchanged: already reads its dependencies from requirements.txt and its description from README.md" + values = setup_values(lf, OLD_SETUP) + if values is None: + return ( + "SKIPPED: not the template's setup.py (changed by hand, or the package is described in pyproject.toml), left as it is. It is " + f"published as it is if you turn on PyPI publishing: check its dependencies and description, see {DOCS_URL}" + ) + helpers.write_text(path, render_setup(new_template, values).replace("\n", newline)) + return f"updated: {values['name']} now needs what requirements.txt lists and has README.md as its description" + + +def _create_manifest_in(helpers: Any, repo_root: Path) -> str: + path = repo_root / MANIFEST_IN + setup = repo_root / SETUP + reads_requirements = setup.is_file() and setup_values(helpers.read_text(setup).replace("\r\n", "\n"), resource(SETUP)) is not None + if path.is_file(): + if REQUIRED_LINE in helpers.read_text(path).replace("\r\n", "\n").splitlines(): + return "unchanged: already puts requirements.txt in the source distribution" + if reads_requirements: + return f"SKIPPED: the repository has its own MANIFEST.in, left as it is; add the line `{REQUIRED_LINE}` to it, see {DOCS_URL}" + return "unchanged: the repository's own MANIFEST.in" + if not reads_requirements: + return "unchanged: not needed (setup.py is not the template's, so it does not read requirements.txt)" + return _create(helpers, repo_root, MANIFEST_IN, "") + + +def _update_bump_version(helpers: Any, path: Path) -> str: + text = helpers.read_text(path) + newline = "\r\n" if "\r\n" in text else "\n" + lf = text.replace("\r\n", "\n") + if all(new in lf for _, new in BUMP_BLOCKS): + return "unchanged: already writes the release version into setup.py" + new_lf = apply_blocks(lf, BUMP_BLOCKS) + if new_lf is None: + return ( + "SKIPPED: customised (a 0.1.24 block was not found), so it was left unchanged; the release version does not reach setup.py " + f"(publish_pypi.yml writes it in its own copy anyway), see {DOCS_URL}" + ) + helpers.write_text(path, new_lf.replace("\n", newline)) + return "updated: the release also writes its version into setup.py" + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """Opt-in PyPI publishing of the package in src/: a new workflow and script, setup.py and bump_version.py (idempotent, surgical).""" + _ = context + helpers = _load("v0_1_9_to_v0_1_10.py", "template_sync_helpers_0_1_25") + changed = 0 + + def report(name: Path, step: Callable[[], str]) -> None: + nonlocal changed + if name not in NEW_FILES and not (repo_root / name).is_file(): + log(f"{name}: not present in this repository (nothing was created)") + return + try: + status = step() + except (OSError, ValueError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + if status.startswith("updated"): + changed += 1 + log(f"{name}: {status}") + + report(PACKAGE_SCRIPT, partial(_create, helpers, repo_root, PACKAGE_SCRIPT, "publish_pypi.yml runs the template's, compare them")) + report(WORKFLOW, partial(_create_workflow, helpers, repo_root)) + report(SETUP, partial(_update_setup, helpers, repo_root)) + report(MANIFEST_IN, partial(_create_manifest_in, helpers, repo_root)) + report(BUMP_VERSION, partial(_update_bump_version, helpers, repo_root / BUMP_VERSION)) + + if changed: + log( + f"done: {changed} file(s) changed. Nothing is published until you turn it on (the repository variable PYPI_PUBLISH and a trusted " + f"publisher on pypi.org), see {DOCS_URL}" + ) + else: + log("done: no repository changes were required.") diff --git a/.template_sync/migrations/v0_1_25_to_v0_1_26.py b/.template_sync/migrations/v0_1_25_to_v0_1_26.py new file mode 100644 index 0000000..b4e621b --- /dev/null +++ b/.template_sync/migrations/v0_1_25_to_v0_1_26.py @@ -0,0 +1,134 @@ +"""Template 0.1.25 -> 0.1.26: the app's own documentation explains PyPI publishing (added in 0.1.25). + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_26.md + +The 0.1.25 migration added publish_pypi.yml and pypi_package.py but left the app's documentation as it was, so the one-time setup was +described only in the template's documentation. What this changes in an app repository (each step is independent, idempotent and never +deletes anything): + +1. .tools/docs/external_code_upload.md: the section "4. Publish the code on PyPI (optional)" of the template. A page with a section 4 + of its own is left as it is. +2. .tools/docs/troubleshooting.md: the entry "Publishing to PyPI fails" of the template. + +Each text goes at the end of the page's content, before its Previous / Home / Next links, which stay the last thing on the page (they are +moved there if text followed them). A page without these links gets the text at its end. + +Both texts are the template's, byte for byte (.template_sync/resources/v0_1_25_to_v0_1_26/.tools/docs/): they have no placeholder, +and they link to the design note of 0.1.25 by its address, so every link works in an app. The line endings of each file are kept. +""" +from __future__ import annotations + +import importlib.util +import re +from functools import partial +from pathlib import Path +from types import ModuleType +from typing import Any, Callable + + +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_26.md" +TAG = "[migration 0.1.26]" +RESOURCE_ROOT = Path(__file__).resolve().parents[1] / "resources" / "v0_1_25_to_v0_1_26" + +EXTERNAL_CODE_PAGE = Path(".tools/docs/external_code_upload.md") +TROUBLESHOOTING_PAGE = Path(".tools/docs/troubleshooting.md") +PYPI_SECTION = Path(".tools/docs/external_code_upload_pypi.md") +PYPI_TROUBLESHOOTING = Path(".tools/docs/troubleshooting_pypi.md") + +SECTION_HEADING = "## 4. Publish the code on PyPI (optional)" +TROUBLESHOOTING_HEADING = "## Publishing to PyPI fails" +# A section 4 of the app's own (the template's page has sections 1 to 3 before 0.1.26). +OWN_SECTION_4 = re.compile(r"^##[ \t]+4\.", re.MULTILINE) +# The Previous / Home / Next links at the bottom of a documentation page: a "---" line, then
...
. +FOOTER = re.compile(r'^---[ \t]*\n(?:[ \t]*\n)*
.*?
[ \t]*(?:\n|$)', re.MULTILINE | re.DOTALL) + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def _load(name: str, alias: str) -> ModuleType: + path = Path(__file__).resolve().parent / name + spec = importlib.util.spec_from_file_location(alias, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def resource(relative: Path) -> str: + return (RESOURCE_ROOT / relative).read_text(encoding="utf-8") + + +def has_line(text_lf: str, line: str) -> bool: + return line in (candidate.rstrip() for candidate in text_lf.split("\n")) + + +def add_section(text_lf: str, section: str) -> str: + """The page with the section at the end of its content, then its Previous / Home / Next links (always last), if it has them.""" + footers = list(FOOTER.finditer(text_lf)) + if not footers: + return text_lf.rstrip("\n") + "\n\n" + section + footer = footers[-1] + before, after = text_lf[: footer.start()].rstrip("\n"), text_lf[footer.end():].strip("\n") + content = f"{before}\n\n{after}" if after else before + return content + "\n\n" + section + "\n" + footer.group(0).rstrip("\n") + "\n" + + +def _update(helpers: Any, path: Path, change: Callable[[str], str]) -> None: + text = helpers.read_text(path) + newline = "\r\n" if "\r\n" in text else "\n" + helpers.write_text(path, change(text.replace("\r\n", "\n")).replace("\n", newline)) + + +def _external_code_page(helpers: Any, path: Path) -> str: + lf = helpers.read_text(path).replace("\r\n", "\n") + if has_line(lf, SECTION_HEADING): + return "unchanged: already explains PyPI publishing" + if OWN_SECTION_4.search(lf): + return ( + "SKIPPED: the page has a section 4 of its own, left as it is; the PyPI section is in the template's page " + f"(.tools/docs/external_code_upload.md), see {DOCS_URL}" + ) + section = resource(PYPI_SECTION) + _update(helpers, path, lambda text: add_section(text, section)) + return "updated: added the section 4. Publish the code on PyPI (optional)" + + +def _troubleshooting_page(helpers: Any, path: Path) -> str: + lf = helpers.read_text(path).replace("\r\n", "\n") + if has_line(lf, TROUBLESHOOTING_HEADING): + return "unchanged: already has the entry Publishing to PyPI fails" + entry = resource(PYPI_TROUBLESHOOTING) + _update(helpers, path, lambda text: add_section(text, entry)) + return "updated: added the entry Publishing to PyPI fails" + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """The PyPI section of external_code_upload.md and the PyPI entry of troubleshooting.md (idempotent, surgical).""" + _ = context + helpers = _load("v0_1_9_to_v0_1_10.py", "template_sync_helpers_0_1_26") + changed = 0 + + def report(name: Path, step: Callable[[Path], str]) -> None: + nonlocal changed + path = repo_root / name + if not path.is_file(): + log(f"{name}: not present in this repository (nothing was created)") + return + try: + status = step(path) + except (OSError, ValueError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + if status.startswith("updated"): + changed += 1 + log(f"{name}: {status}") + + report(EXTERNAL_CODE_PAGE, partial(_external_code_page, helpers)) + report(TROUBLESHOOTING_PAGE, partial(_troubleshooting_page, helpers)) + + if changed: + log(f"done: {changed} file(s) changed (documentation only). See {DOCS_URL}") + else: + log("done: no repository changes were required.") diff --git a/.template_sync/migrations/v0_1_26_to_v0_1_27.py b/.template_sync/migrations/v0_1_26_to_v0_1_27.py new file mode 100644 index 0000000..546548e --- /dev/null +++ b/.template_sync/migrations/v0_1_26_to_v0_1_27.py @@ -0,0 +1,99 @@ +"""Template 0.1.26 -> 0.1.27: an app can say which napari, Fiji or QuPath plugin version its tools need (host_requirements). + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_27.md +Proposal: https://github.com/CellMigrationLab/LabConstrictor-Tools/issues/155 + +The App Centre schema gained an optional field, host_requirements. The template's copy of the schema rejects fields it does not +know, so without this migration an app that adds the field fails its "Check App Centre manifest" workflow. What this changes in an +app repository (each step is independent, idempotent and never deletes anything): + +1. .tools/schema/labconstrictor-app.schema.json: replaced by the template's copy (the template owns it; it must equal the App Centre's + schema). A missing schema is not created: the app has no manifest check to update. +2. .tools/docs/app_manifest.md: the section "Plugin versions your tools need (optional)" goes before "## The automatic check". A page + that already has the section is unchanged; a page without that heading gets it before its Previous / Home / Next links, or at + its end. Line endings are kept. + +labconstrictor-app.yaml belongs to the app and is never created or changed. +""" +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + + +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_27.md" +TAG = "[migration 0.1.27]" +RESOURCE_ROOT = Path(__file__).resolve().parents[1] / "resources" / "v0_1_26_to_v0_1_27" + +SCHEMA = Path(".tools/schema/labconstrictor-app.schema.json") +MANIFEST_PAGE = Path(".tools/docs/app_manifest.md") +SECTION = Path(".tools/docs/app_manifest_host_requirements.md") + +SECTION_HEADING = "## Plugin versions your tools need (optional)" +ANCHOR = "## The automatic check" +FOOTER = re.compile(r'^---[ \t]*\n(?:[ \t]*\n)*
.*?
[ \t]*(?:\n|$)', re.MULTILINE | re.DOTALL) + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def resource_bytes(relative: Path) -> bytes: + return (RESOURCE_ROOT / relative).read_bytes() + + +def with_section(text_lf: str, section: str) -> str: + """The page with the section before "## The automatic check", else before its links, else at its end.""" + lines = text_lf.split("\n") + for index, line in enumerate(lines): + if line.rstrip() == ANCHOR: + before = "\n".join(lines[:index]).rstrip("\n") + return before + "\n\n" + section.rstrip("\n") + "\n\n" + "\n".join(lines[index:]) + footers = list(FOOTER.finditer(text_lf)) + if footers: + start = footers[-1].start() + return text_lf[:start].rstrip("\n") + "\n\n" + section.rstrip("\n") + "\n\n" + text_lf[start:] + return text_lf.rstrip("\n") + "\n\n" + section + + +def update_schema(repo_root: Path) -> str: + path = repo_root / SCHEMA + if not path.is_file(): + return "not present in this repository (nothing was created)" + data = resource_bytes(SCHEMA) + if path.read_bytes() == data: + return "unchanged: already the template's copy" + path.write_bytes(data) + return "updated: the template's copy (accepts host_requirements)" + + +def update_page(repo_root: Path) -> str: + path = repo_root / MANIFEST_PAGE + if not path.is_file(): + return "not present in this repository (nothing was created)" + raw = path.read_bytes().decode("utf-8") + newline = "\r\n" if "\r\n" in raw else "\n" + text_lf = raw.replace("\r\n", "\n") + if SECTION_HEADING in (line.rstrip() for line in text_lf.split("\n")): + return "unchanged: already has the section on plugin versions" + section = resource_bytes(SECTION).decode("utf-8").replace("\r\n", "\n") + path.write_bytes(with_section(text_lf, section).replace("\n", newline).encode("utf-8")) + return "updated: added the section on plugin versions" + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """Refresh the schema copy and document host_requirements (idempotent, never deletes).""" + _ = context + changed = 0 + for name, step in ((SCHEMA, update_schema), (MANIFEST_PAGE, update_page)): + try: + status = step(repo_root) + except (OSError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + changed += status.startswith("updated") + log(f"{name}: {status}") + if changed: + log(f"done: {changed} file(s) changed. See {DOCS_URL}") + else: + log("done: no repository changes were required.") diff --git a/.template_sync/migrations/v0_1_27_to_v0_1_28.py b/.template_sync/migrations/v0_1_27_to_v0_1_28.py new file mode 100644 index 0000000..83266b8 --- /dev/null +++ b/.template_sync/migrations/v0_1_27_to_v0_1_28.py @@ -0,0 +1,97 @@ +"""Template 0.1.27 -> 0.1.28: the installers use LabConstrictor-Tools v0.3.0-alpha.2, which registers an app's host_requirements. + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_28.md + +Tools v0.3.0-alpha.2 (feature host_requirements_v1, LabConstrictor-Tools #155) copies the `host_requirements:` block of labconstrictor-app.yaml +into the registry entry, so napari, Fiji and QuPath can tell a person with an older plugin to update it. Older Tools ignore the block. What +this changes in an app repository (each step is independent, idempotent and never deletes anything): + +1. `LC_TOOLS_PIN` in app/bash_bat_scripts/post_install.sh and post_install.bat moves from `v0.3.0-alpha.1` to `v0.3.0-alpha.2`, ONLY when the + line still holds exactly the old template value. An app with its own pin, a script without a pin line or without the Tools hook stays as it is, + with a note. +2. `.tools/docs/tools_bridge.md`: the tag name `v0.3.0-alpha.1` is replaced by `v0.3.0-alpha.2` (the guide names the pinned tag). + +The pin reaches users with the app's next release. +""" +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + + +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_28.md" +TAG = "[migration 0.1.28]" +OLD_PIN = "v0.3.0-alpha.1" +NEW_PIN = "v0.3.0-alpha.2" +POST_INSTALL_SH = Path("app/bash_bat_scripts/post_install.sh") +POST_INSTALL_BAT = Path("app/bash_bat_scripts/post_install.bat") +TOOLS_DOC = Path(".tools/docs/tools_bridge.md") +SH_PIN_LINE = re.compile(r'^([ \t]*LC_TOOLS_PIN=")([^"]*)(".*)$') +BAT_PIN_LINE = re.compile(r'^([ \t]*SET "LC_TOOLS_PIN=)([^"]*)(".*)$', re.IGNORECASE) + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def read(path: Path) -> str: + """The file's text with its line endings as they are (bytes decoded as UTF-8).""" + return path.read_bytes().decode("utf-8") + + +def write(path: Path, text: str) -> None: + path.write_bytes(text.encode("utf-8")) + + +def move_pin(path: Path, pattern: re.Pattern[str]) -> str: + text = read(path) + if "labconstrictor_tools" not in text: + return "has no Tools bridge hook: nothing to move" + lines = text.splitlines(keepends=True) + hits = [(index, match) for index, line in enumerate(lines) if (match := pattern.match(line.rstrip("\r\n")))] + if not hits: + return "has no LC_TOOLS_PIN line (unpinned source or its own Tools source): left as it is" + if len(hits) > 1: + return f"has {len(hits)} LC_TOOLS_PIN lines: left as it is, set the pin by hand to {NEW_PIN}" + index, match = hits[0] + current = match.group(2) + if current == NEW_PIN: + return f"already pins Tools {NEW_PIN}: unchanged" + if current != OLD_PIN: + return f"has its own pin {current}: left as it is (the template pin is now {NEW_PIN})" + ending = lines[index][len(lines[index].rstrip("\r\n")) :] + lines[index] = f"{match.group(1)}{NEW_PIN}{match.group(3)}{ending}" + write(path, "".join(lines)) + return f"updated: Tools pin {OLD_PIN} -> {NEW_PIN}" + + +def refresh_guide(path: Path) -> str: + text = read(path) + if OLD_PIN not in text: + return "does not name the old tag: unchanged" + write(path, text.replace(OLD_PIN, NEW_PIN)) + return f"updated: names {NEW_PIN}" + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """Move the Tools pin to v0.3.0-alpha.2 and refresh the Tools guide (idempotent, surgical, never raises for a customised file).""" + _ = context + changed = 0 + steps = ((POST_INSTALL_SH, lambda p: move_pin(p, SH_PIN_LINE)), (POST_INSTALL_BAT, lambda p: move_pin(p, BAT_PIN_LINE)), + (TOOLS_DOC, refresh_guide)) + for relative, step in steps: + path = repo_root / relative + if not path.is_file(): + log(f"{relative}: not present in this repository (nothing was created)") + continue + try: + status = step(path) + except (OSError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + changed += status.startswith("updated") + log(f"{relative}: {status}") + if changed: + log(f"done: {changed} file(s) changed. The pin reaches users with the next release of the app. See {DOCS_URL}") + else: + log("done: no repository changes were required.") diff --git a/.template_sync/resources/v0_1_15_to_v0_1_16/.tools/schema/labconstrictor-app.schema.json b/.template_sync/resources/v0_1_15_to_v0_1_16/.tools/schema/labconstrictor-app.schema.json index 536d91e..30ae4ae 100644 --- a/.template_sync/resources/v0_1_15_to_v0_1_16/.tools/schema/labconstrictor-app.schema.json +++ b/.template_sync/resources/v0_1_15_to_v0_1_16/.tools/schema/labconstrictor-app.schema.json @@ -21,6 +21,15 @@ "items": {"enum": ["notebook", "napari", "fiji", "qupath"]}, "description": "Where the app's tools can be used besides its own window (the LabConstrictor-Tools bridge)." }, + "host_requirements": { + "type": "object", "additionalProperties": false, + "description": "The oldest LabConstrictor plugin of each host the app needs. labconstrictor-tools register copies it into the app's registry entry; a host whose plugin is older lists the app with a sentence saying which plugin to update (or refuses it with required: true). The notebook needs no plugin.", + "properties": { + "napari": {"$ref": "#/$defs/hostRequirement"}, + "fiji": {"$ref": "#/$defs/hostRequirement"}, + "qupath": {"$ref": "#/$defs/hostRequirement"} + } + }, "distribution": { "type": "object", "additionalProperties": false, "anyOf": [{"required": ["installer"]}, {"required": ["colab"]}, {"required": ["pip"]}], @@ -57,6 +66,16 @@ } }, "$defs": { + "hostRequirement": { + "type": "object", "additionalProperties": false, + "anyOf": [{"required": ["min_version"]}, {"required": ["features"]}], + "properties": { + "min_version": {"type": "string", "pattern": "^v?[0-9]+(\\.[0-9]+)*([-_.]?(a|alpha|b|beta|c|rc|pre|preview)[-_.]?[0-9]*)?([-_.]?(post|rev|r)[-_.]?[0-9]*)?([-_.]?dev[-_.]?[0-9]*)?$", "description": "A release tag of the plugin, with or without v (0.1.0-alpha.6, v0.1.0, 0.2.0rc1). Quote it if YAML could read it as a number (\"0.2\")."}, + "features": {"type": "array", "uniqueItems": true, "minItems": 1, "items": {"type": "string", "pattern": "^[a-z][a-z0-9_]*$"}, "description": "Host feature ids the app needs (LabConstrictor-Tools docs/HOST_FEATURES.md), for example shapes_owner_v1."}, + "why": {"type": "string", "minLength": 3, "maxLength": 160, "pattern": "^[^\\n]+$", "description": "One line shown to the person, saying what the newer plugin brings."}, + "required": {"type": "boolean", "description": "true: an older plugin refuses the app's tools instead of warning. Use it only when the tools really fail."} + } + }, "facts": {"type": "array", "minItems": 1, "maxItems": 6, "items": {"type": "string", "minLength": 5, "maxLength": 200}}, "https": {"type": "string", "pattern": "^https://[^\\s/@]+(/[^\\s]*)?$(?!\\n)"}, "repoPath": {"type": "string", "pattern": "^(?!/)(?!.*\\.\\.)[A-Za-z0-9_./ -]+\\.(png|jpg|jpeg|gif)$(?!\\n)", "description": "Path inside the repository, no leading slash."}, diff --git a/.template_sync/resources/v0_1_17_to_v0_1_18/.tools/docs/install.md b/.template_sync/resources/v0_1_17_to_v0_1_18/.tools/docs/install.md index 4048356..52ae8be 100644 --- a/.template_sync/resources/v0_1_17_to_v0_1_18/.tools/docs/install.md +++ b/.template_sync/resources/v0_1_17_to_v0_1_18/.tools/docs/install.md @@ -33,7 +33,7 @@ Installing takes about 6-8 minutes and you only do it once. You need an internet ## Update an installed app -- **macOS and Windows.** On macOS and Windows, update an app by uninstalling the old version first and then installing the new one: the installer does not install over an existing install. +- **macOS and Windows.** On macOS and Windows, update an app by uninstalling the old version first and then installing the new one: the installer does not install over an existing install. On macOS, if you already ran the new installer over the old version and it stopped or reported a failure at the end, move only the old app shortcut (for example `~/Applications/MyApp.app`) to the Bin, keep the app's folder in your home folder, and run the installer again ([known issue](https://github.com/CellMigrationLab/LabConstrictor/issues/34)). - **Linux.** On Linux, running the installer with `-u` updates in place only if the previous install finished; otherwise delete `/pkgs` or uninstall the app and install it again. ## Where the app is listed diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_notebook_change.yml b/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_notebook_change.yml new file mode 100644 index 0000000..4a8c03b --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_notebook_change.yml @@ -0,0 +1,211 @@ +name: Updates on notebook change + +on: + push: + branches: [ main, master ] + paths: + - 'notebooks/**/*.ipynb' + - 'notebooks/notebook_latest_versions.yaml' + - 'notebooks/**/requirements.yaml' + workflow_dispatch: + +# One run of this workflow at a time per branch. The src workflow has its own group: the two can run at the same time, +# because each one rebases its commit onto the latest branch before it pushes (a shared group kept only one waiting run, so a +# third push cancelled the waiting run of the other workflow). +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon notebook change, update manifests and changelogs + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@2d756ea4c53f7f6b397767d8723b3a10a9f35bf2 # v44 + with: + separator: "," + files: | + notebooks/**/*.ipynb + notebooks/notebook_latest_versions.yaml + notebooks/**/requirements.yaml + + - name: Syncronize notebook versions + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "sync_notebook_versions.py" \ + .tools/python/sync_notebook_versions.py \ + python .tools/python/sync_notebook_versions.py + + - name: Update CHANGELOG + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_changelog.py" \ + .tools/python/update_changelog.py \ + python .tools/python/update_changelog.py notebooks/notebook_latest_versions.yaml + + - name: Update last commit in notebook_latest_versions.yaml + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_version_last_commit.py" \ + .tools/python/update_version_last_commit.py \ + python .tools/python/update_version_last_commit.py notebooks/notebook_latest_versions.yaml + + - name: Hide code cells in notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "hide_code_cells.py" \ + app/python_scripts/hide_code_cells.py \ + python app/python_scripts/hide_code_cells.py . + + - name: Index notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_notebook_README.py" \ + .tools/python/bump_notebook_README.py \ + python .tools/python/bump_notebook_README.py + + - name: Run merge script + if: github.event_name == 'workflow_dispatch' || contains(steps.changed-files.outputs.all_changed_files, 'requirements.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "merge_requirements.py" \ + .tools/python/merge_requirements.py \ + python .tools/python/merge_requirements.py --source-dir notebooks --output requirements.txt + + # requirements*.txt -> requirements*.lock.txt: every package the installer installs with pip, with its exact version (uv, valid on + # Windows, macOS and Linux). A lock file is rewritten only when its requirements file, Python or JupyterLab version changed. + - name: Lock requirements + run: | + bash .tools/bash/run_with_context.sh artifacts/error_log_document.log "lock_requirements.py" .tools/python/lock_requirements.py python .tools/python/lock_requirements.py + + - name: Check for changes + id: check-changes + run: | + if [ -z "$(git status --porcelain)" ]; then + echo "has_changes=false" >> $GITHUB_OUTPUT + else + echo "has_changes=true" >> $GITHUB_OUTPUT + fi + + - name: Set up Miniconda for requirements test + if: steps.check-changes.outputs.has_changes == 'true' + uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + auto-activate: true + miniforge-version: latest + use-mamba: true + + - name: Verify merged requirements inside conda + id: test-conda-env + if: steps.check-changes.outputs.has_changes == 'true' + env: + SKIP_CONDA_ENV_TEST: ${{ vars.SKIP_CONDA_ENV_TEST }} + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "test_conda_environment.sh" \ + .tools/bash/test_conda_environment.sh \ + bash .tools/bash/test_conda_environment.sh + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: notebook-debug-bundle + path: artifacts/error_log_document.log + archive: false + + - name: Commit updated files + id: auto-commit + if: steps.check-changes.outputs.has_changes == 'true' && steps.test-conda-env.outcome == 'success' + env: + BRANCH: ${{ github.ref_name }} + COMMIT_MESSAGE: GitHub Action - Updated files because of notebook and/or requirements changes + # The run checked out the commit that triggered it (the scripts above compare it with its parent); the branch may have moved + # since (another workflow committed, or a new push). A rejected push is therefore retried after a rebase onto the latest branch. + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git ls-files -z --modified --deleted --others --exclude-standard -- 'requirements*.txt' environment.yaml 'notebooks/*' setup.py | xargs -0 -r git add -A -- + if git diff --cached --quiet; then + echo "No changes to commit." + exit 0 + fi + git commit -m "$COMMIT_MESSAGE" + for attempt in 1 2 3 4 5; do + if git push origin "HEAD:refs/heads/$BRANCH"; then + exit 0 + fi + echo "The push was rejected (attempt $attempt of 5): rebasing onto the latest $BRANCH." + sleep $((attempt * 5)) + git fetch origin "$BRANCH" + if ! git rebase --autostash "origin/$BRANCH"; then + git rebase --abort || true + echo "::error::The commit of this run conflicts with the latest $BRANCH; run this workflow again (Actions > Run workflow)." + exit 1 + fi + done + echo "::error::The push was rejected 5 times; run this workflow again (Actions > Run workflow)." + exit 1 + + # Only when the merged requirements could not be installed: a failure of any other step (a script, the push) never reverts a commit. + - name: Revert user commit that broke requirements + if: failure() && github.event_name == 'push' && steps.test-conda-env.outcome == 'failure' + env: + BRANCH: ${{ github.ref_name }} + BAD_COMMIT: ${{ github.sha }} + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git reset --hard + git clean -fd + for attempt in 1 2 3 4 5; do + git fetch origin "$BRANCH" + git checkout -B "$BRANCH" "origin/$BRANCH" + # A merge commit is reverted against its first parent (-m 1); any other commit is reverted as it is. + if [ "$(git rev-list --parents -n 1 "$BAD_COMMIT" | wc -w)" -gt 2 ]; then + REVERT_ARGS=(-m 1) + else + REVERT_ARGS=() + fi + if ! git revert --no-edit "${REVERT_ARGS[@]}" "$BAD_COMMIT"; then + git revert --abort || true + echo "::error::$BAD_COMMIT broke the requirements but could not be reverted automatically (later commits change the same lines): revert it by hand." + exit 1 + fi + if git push origin "HEAD:refs/heads/$BRANCH"; then + exit 0 + fi + echo "The push of the revert was rejected (attempt $attempt of 5): retrying on the latest $BRANCH." + sleep $((attempt * 5)) + done + echo "::error::The revert of $BAD_COMMIT could not be pushed: revert it by hand." + exit 1 diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_src_change.yaml b/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_src_change.yaml new file mode 100644 index 0000000..2ab8870 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/.github/workflows/update_on_src_change.yaml @@ -0,0 +1,66 @@ +name: Updates on external code change + +on: + push: + branches: [ main, master ] + paths: + - 'src/**' + workflow_dispatch: + +# One run of this workflow at a time per branch. The notebook workflow has its own group: the two can run at the same time, +# because each one rebases its commit onto the latest branch before it pushes (a shared group kept only one waiting run, so a +# third push cancelled the waiting run of the other workflow). +concurrency: + group: src-change-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon external code update the code version + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Update src change file + run: | + cat > .tools/meta/src_change.yaml < Run workflow)." + exit 1 + fi + done + echo "::error::The push was rejected 5 times; run this workflow again (Actions > Run workflow)." + exit 1 diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/bash/test_conda_environment.sh b/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/bash/test_conda_environment.sh new file mode 100644 index 0000000..4742aca --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/bash/test_conda_environment.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Test script to validate conda environment creation and pip requirements installation +# This script creates a test conda environment from environment.yaml and installs pip requirements + +set -e # Exit on any error +set -u # Exit on undefined variables + +is_truthy() { + case "${1:-}" in + 1|true|TRUE|True|yes|YES|Yes|on|ON|On) + return 0 + ;; + *) + return 1 + ;; + esac +} + +if is_truthy "${SKIP_CONDA_ENV_TEST:-}"; then + echo "SKIP_CONDA_ENV_TEST is enabled; skipping conda environment validation." + exit 0 +fi + +echo "==========================================" +echo "Testing Conda Environment Creation" +echo "==========================================" + +# Define test environment name +TEST_ENV_NAME="potential-potato-test-env" + +# Clean up any existing test environment +echo "Cleaning up any existing test environment..." +mamba env remove -n "$TEST_ENV_NAME" -y 2>/dev/null || true + +# Create conda environment from environment.yaml +echo "" +echo "Creating conda environment from environment.yaml..." +if [ ! -f "environment.yaml" ]; then + echo "ERROR: environment.yaml not found!" + exit 1 +fi + +mamba env create -n "$TEST_ENV_NAME" -f environment.yaml -y + +# Activate the environment +echo "" +echo "Activating test environment..." +# Note: In CI, conda/mamba environments need to be activated via conda run +# or by sourcing the activation script + +# Test pip installation of requirements.txt, or of its lock file (requirements.lock.txt, written by +# .tools/python/lock_requirements.py) when there is one: the installer installs the lock file, so that is what is tested. +if [ ! -f "requirements.txt" ]; then + echo "ERROR: requirements.txt not found!" + mamba env remove -n "$TEST_ENV_NAME" -y + exit 1 +fi +REQUIREMENTS_FILE="requirements.txt" +if [ -f "requirements.lock.txt" ]; then + REQUIREMENTS_FILE="requirements.lock.txt" +fi +echo "" +echo "Installing requirements from $REQUIREMENTS_FILE..." + +# Check if the requirements file is not empty +if [ ! -s "$REQUIREMENTS_FILE" ]; then + echo "WARNING: $REQUIREMENTS_FILE is empty, skipping pip install" +else + echo "Installing packages with pip..." + mamba run -n "$TEST_ENV_NAME" pip install --prefer-binary -r "$REQUIREMENTS_FILE" +fi + +# Report dependency conflicts between the pip packages and the conda packages of the environment. Only a warning: a conflict +# among the conda packages themselves must not block the notebooks. +echo "" +echo "Checking the installed packages for conflicts (pip check)..." +if ! mamba run -n "$TEST_ENV_NAME" pip check; then + echo "::warning::pip check found dependency conflicts in the test environment (see the step log); the installer may behave differently than expected." +fi + +# Verify installation by checking Python and key packages +echo "" +echo "Verifying installation..." +echo "Python version:" +mamba run -n "$TEST_ENV_NAME" python --version + +echo "" +echo "Installed pip packages (sample):" +mamba run -n "$TEST_ENV_NAME" pip list | head -n 20 + +# Clean up test environment +echo "" +echo "Cleaning up test environment..." +mamba env remove -n "$TEST_ENV_NAME" -y + +echo "" +echo "==========================================" +echo "✓ Conda environment test completed successfully!" +echo "==========================================" diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/python/lock_requirements.py b/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/python/lock_requirements.py new file mode 100644 index 0000000..c5834a2 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/.tools/python/lock_requirements.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Lock the merged requirements files: requirements*.txt -> requirements*.lock.txt. + +merge_requirements.py writes the requirements of the notebooks (requirements.txt, requirements_gpu.txt and the per-OS +requirements-.txt). Those files name the packages the notebooks need, but not the packages these depend on, so two +installations of the same release could get different versions. This script writes, next to each file, a lock file that lists +every package pip installs, each with its exact version, and post_install installs the lock file when the installer bundles one. + +How a lock file is made: +* `uv pip compile --universal`: one lock file that is valid on Windows, macOS and Linux (a package that only one platform needs carries + an environment marker, e.g. `colorama==0.4.6 ; sys_platform == 'win32'`). +* `--python-version` is the Python of environment.yaml. uv reads it as the lowest Python of the resolution, so a few packages can appear + twice with a `python_full_version` marker: pip installs the line that matches the Python of the installer. +* The conda packages of environment.yaml (jupyterlab, pip, setuptools, certifi, ...) are installed by the installer itself, so they are + not written to the lock file (their own dependencies are), and a conda pin such as `jupyterlab=4.4.0` constrains the resolution. +* The per-OS files are resolved with requirements.lock.txt as constraints, so a package that both files need gets the same version. +* Pip options of the requirements file (`--extra-index-url` of the GPU file, ...) are copied to the top of the lock file. + +A lock file is written again only when its requirements file, the Python or JupyterLab version (or this script's lock format) changed: +its header holds a fingerprint of those inputs. Re-running the script without such a change rewrites nothing, so a new release +installs the same versions as the previous one. To move every package to its newest version, delete the lock files and run the +workflow "Updates on notebook change" by hand (Actions > Run workflow). + +If a file cannot be locked (uv is missing, or the requirements cannot be resolved), the script prints a warning, removes the +out-of-date lock file of that file (an installer must never install versions that no longer match the requirements) and goes on: +the installer then installs the plain requirements file. The script itself always exits 0 for such a failure. +""" +from __future__ import annotations + +import argparse +import hashlib +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + +# Bump when the uv arguments or the output change, so that every lock file is written again. +LOCK_FORMAT = "1" +BASE_INPUT = "requirements.txt" +GPU_INPUT = "requirements_gpu.txt" +OS_INPUTS = ("requirements-linux.txt", "requirements-windows.txt", "requirements-macos.txt") +INPUTS = (BASE_INPUT, GPU_INPUT, *OS_INPUTS) +FINGERPRINT_PREFIX = "# lock-input: sha256:" +# Pip options that choose where packages come from: copied to the top of the lock file, so pip finds the locked versions where uv found them. +INDEX_OPTION = re.compile(r"^\s*(--index-url|--extra-index-url|-i|--find-links|-f|--trusted-host)(\s|=)") +CONDA_SPEC = re.compile(r"^\s*([A-Za-z0-9_.-]+)\s*(?:(==?)\s*([0-9][A-Za-z0-9_.+!*-]*))?(?:=\S+)?\s*$") +PLACEHOLDER = re.compile(r"^[A-Z_]+$") + + +def lock_path(requirements: Path) -> Path: + """requirements.txt -> requirements.lock.txt (same folder).""" + return requirements.with_name(requirements.stem + ".lock.txt") + + +def read_environment(path: Path) -> tuple[str | None, list[str], list[str]]: + """(Python version, conda package names, pip constraints from exact conda pins) of environment.yaml.""" + if not path.is_file(): + return None, [], [] + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + python_version = None + names: list[str] = [] + constraints: list[str] = [] + for dependency in data.get("dependencies", []) or []: + if not isinstance(dependency, str): + continue # the `pip:` subsection of an environment file + match = CONDA_SPEC.match(dependency) + if match: + name, operator, version = match.group(1), match.group(2), match.group(3) + else: + # a spec with a range or alternatives (`pip>=24.2`): only its name is used + name, operator, version = re.split(r"[\s<>=!|,\[]", dependency.strip(), maxsplit=1)[0], None, None + if not name: + continue + if version and PLACEHOLDER.match(version): + version = None # the template itself (python=PYTHON_VERSION) + if name.lower() == "python": + python_version = version + continue + names.append(name) + if version: + # conda `name=1.2` means any 1.2.* release, `name==1.2` exactly 1.2 + constraints.append(f"{name}=={version}" if operator == "==" else f"{name}=={version.removesuffix('.*')}.*") + return python_version, sorted(set(names), key=str.lower), constraints + + +def normalized_requirements(text: str) -> str: + """The requirement and option lines of a requirements file (comments, such as the timestamp of merge_requirements.py, are ignored).""" + lines = [] + for line in text.replace("\r\n", "\n").split("\n"): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + lines.append(stripped) + return "\n".join(lines) + + +def fingerprint(requirements_text: str, python_version: str, constraints: list[str], no_emit: list[str], extra_constraints: str) -> str: + payload = "\n---\n".join( + [LOCK_FORMAT, normalized_requirements(requirements_text), python_version, "\n".join(constraints), "\n".join(no_emit), extra_constraints] + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def current_fingerprint(lock: Path) -> str | None: + if not lock.is_file(): + return None + for line in lock.read_text(encoding="utf-8").splitlines(): + if line.startswith(FINGERPRINT_PREFIX): + return line[len(FINGERPRINT_PREFIX):].strip() + return None + + +def run_uv(requirements: Path, python_version: str, constraint_files: list[Path], no_emit: list[str]) -> str: + """The resolution of `requirements` (stdout of uv pip compile); raises RuntimeError with uv's message when it fails.""" + command = [ + sys.executable, "-m", "uv", "pip", "compile", str(requirements), + "--universal", "--python-version", python_version, + "--no-header", "--no-annotate", "--no-config", "--quiet", + ] + for constraint in constraint_files: + command += ["--constraint", str(constraint)] + for name in no_emit: + command += ["--no-emit-package", name] + try: + result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", check=False) + except OSError as exc: + raise RuntimeError(f"uv could not be started: {exc}") from exc + if result.returncode != 0: + message = (result.stderr or result.stdout).strip() or f"exit code {result.returncode}" + raise RuntimeError(message) + return result.stdout + + +def uv_version() -> str: + try: + result = subprocess.run([sys.executable, "-m", "uv", "--version"], capture_output=True, text=True, check=False) + except OSError: + return "uv" + # "uv 0.13.3 (502f93ca6 2026-10-19 x86_64-pc-windows-msvc)" -> "uv 0.13.3": the build details differ between computers + return " ".join(result.stdout.split()[:2]) or "uv" + + +def render_lock(requirements: Path, resolved: str, digest: str, python_version: str, uv: str) -> str: + options = [line.strip() for line in requirements.read_text(encoding="utf-8").splitlines() if INDEX_OPTION.match(line)] + packages = [line.rstrip() for line in resolved.replace("\r\n", "\n").split("\n") if line.strip() and not line.lstrip().startswith("#")] + header = [ + f"# Lock file of {requirements.name}: every package the installer installs with pip, with its exact version.", + f"# Written by .tools/python/lock_requirements.py ({uv}, universal resolution for Python {python_version}); do not edit it by hand.", + f"# It is written again only when {requirements.name}, the Python or JupyterLab version changes (see the fingerprint below).", + f"{FINGERPRINT_PREFIX}{digest}", + ] + body = options + ([""] if options else []) + packages + return "\n".join(header + [""] + body) + "\n" + + +def warn(message: str) -> None: + # `::warning::` shows the message on the summary page of the GitHub Actions run. + print(f"::warning::{message}", flush=True) + + +def lock_file( + requirements: Path, + python_version: str, + constraints: list[str], + no_emit: list[str], + base_lock: Path | None, + work_dir: Path, + uv: str, +) -> str: + """Lock one requirements file; returns what happened ("unchanged", "written", "failed").""" + lock = lock_path(requirements) + extra = base_lock.read_text(encoding="utf-8") if base_lock is not None and base_lock.is_file() else "" + digest = fingerprint(requirements.read_text(encoding="utf-8"), python_version, constraints, no_emit, normalized_requirements(extra)) + if current_fingerprint(lock) == digest: + print(f"{lock}: up to date (unchanged)") + return "unchanged" + constraint_files = [] + if constraints: + conda_constraints = work_dir / "conda_constraints.txt" + conda_constraints.write_text("\n".join(constraints) + "\n", encoding="utf-8") + constraint_files.append(conda_constraints) + if extra: + # Only the package lines: the pip options of the base lock are not constraints. + base_constraints = work_dir / "base_constraints.txt" + base_constraints.write_text( + "\n".join(line for line in extra.splitlines() if line.strip() and not line.startswith(("#", "-"))) + "\n", encoding="utf-8" + ) + constraint_files.append(base_constraints) + try: + resolved = run_uv(requirements, python_version, constraint_files, no_emit) + except RuntimeError as exc: + warn(f"{requirements} could not be locked, so the installer installs it without a lock file: {exc}") + if lock.exists(): + lock.unlink() + print(f"{lock}: removed (it no longer matches {requirements})") + return "failed" + with lock.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(render_lock(requirements, resolved, digest, python_version, uv)) + print(f"{lock}: written") + return "written" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Lock requirements*.txt into requirements*.lock.txt (every package with its exact version).") + parser.add_argument("--root", type=Path, default=Path("."), help="Folder with the requirements files and environment.yaml") + parser.add_argument("--environment", type=Path, default=None, help="Conda environment file (default: /environment.yaml)") + args = parser.parse_args(argv) + root: Path = args.root + environment = args.environment or root / "environment.yaml" + + python_version, conda_names, constraints = read_environment(environment) + if not python_version: + print(f"{environment}: no exact Python version (python=X.Y.Z), so nothing is locked.") + return 0 + no_emit = [name for name in conda_names if name.lower() not in {"git"}] + + # A lock file whose requirements file is gone is removed (the installer must not install it). + for name in INPUTS: + requirements = root / name + if not requirements.is_file() and lock_path(requirements).is_file(): + lock_path(requirements).unlink() + print(f"{lock_path(requirements)}: removed ({requirements} does not exist)") + + present = [root / name for name in INPUTS if (root / name).is_file()] + if not present: + print("No requirements files found: nothing to lock.") + return 0 + uv = uv_version() + results = {} + with tempfile.TemporaryDirectory() as temporary: + work_dir = Path(temporary) + for requirements in present: + base_lock = lock_path(root / BASE_INPUT) if requirements.name in OS_INPUTS else None + results[requirements.name] = lock_file(requirements, python_version, constraints, no_emit, base_lock, work_dir, uv) + failed = [name for name, status in results.items() if status == "failed"] + print(f"Lock files: {len(results) - len(failed)} of {len(results)} requirements files locked" + (f"; not locked: {', '.join(failed)}" if failed else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/app/python_scripts/launch_jupyter.py b/.template_sync/resources/v0_1_19_to_v0_1_20/app/python_scripts/launch_jupyter.py new file mode 100644 index 0000000..f45ec5f --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/app/python_scripts/launch_jupyter.py @@ -0,0 +1,324 @@ +"""TLS-resilient JupyterLab launcher for PROJECT_NAME. + +Certificate handling is configured before JupyterLab imports Tornado. On +Windows this prevents one malformed certificate-store entry from terminating +the application. Certificate and hostname verification remain enabled in every +supported mode. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import re +import shutil +import ssl +import sys +import tempfile +import traceback +from datetime import datetime, timezone +from importlib import metadata +from pathlib import Path + +PROJECT_DISPLAY_NAME = "PROJECT_NAME" +SCRIPT_DIR = Path(__file__).resolve().parent +_INSTALLED_LAYOUT = (SCRIPT_DIR / "notebooks").is_dir() +if _INSTALLED_LAYOUT: + APP_ROOT = SCRIPT_DIR + PREFIX = APP_ROOT.parent +else: + # Repository layout: app/python_scripts/launch_jupyter.py + repository_root = SCRIPT_DIR.parents[1] + APP_ROOT = repository_root if (repository_root / "notebooks").is_dir() else SCRIPT_DIR + PREFIX = APP_ROOT +DEFAULT_NOTEBOOK = APP_ROOT / "notebooks" / "Welcome.ipynb" +LOG_PATH = PREFIX / "launcher_debug.log" + + +def _write_log(message: str, *, console: bool = True) -> None: + timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds") + line = f"[{timestamp}] {message}" + if console: + print(message, flush=True) + try: + with LOG_PATH.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + except OSError: + # Logging must never prevent the application from launching. + pass + + +def _package_version(name: str) -> str: + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return "not installed" + + +def _project_ca_bundle_environment_variable() -> str: + normalized = re.sub(r"[^A-Za-z0-9]+", "_", PROJECT_DISPLAY_NAME).strip("_").upper() + return f"{normalized or 'LABCONSTRICTOR_APP'}_CA_BUNDLE" + + +def _probe_ssl() -> None: + """Create a verified client context without making a network request.""" + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) + if context.verify_mode != ssl.CERT_REQUIRED: + raise RuntimeError("TLS certificate verification is unexpectedly disabled.") + if not context.check_hostname: + raise RuntimeError("TLS hostname verification is unexpectedly disabled.") + + +def _inject_truststore() -> None: + """Route Python SSL contexts through the operating-system trust API.""" + import truststore + + truststore.inject_into_ssl() + + +def _select_ca_bundle() -> Path: + """Choose an explicit CA bundle for verified installer and runtime fallback.""" + for variable in ( + _project_ca_bundle_environment_variable(), + "LABCONSTRICTOR_CA_BUNDLE", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CURL_CA_BUNDLE", + ): + candidate = os.environ.get(variable, "").strip() + if candidate: + path = Path(candidate).expanduser() + if path.is_file(): + return path.resolve() + + import certifi + + return Path(certifi.where()).resolve() + + +def _install_bundle_fallback(bundle: Path) -> None: + """Use a verified PEM bundle without enumerating the Windows cert store.""" + original_context_type = ssl.SSLContext + + def create_context( + purpose: ssl.Purpose = ssl.Purpose.SERVER_AUTH, + *, + cafile: str | os.PathLike[str] | None = None, + capath: str | os.PathLike[str] | None = None, + cadata: str | bytes | None = None, + ) -> ssl.SSLContext: + if purpose == ssl.Purpose.SERVER_AUTH: + context = original_context_type(ssl.PROTOCOL_TLS_CLIENT) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + elif purpose == ssl.Purpose.CLIENT_AUTH: + context = original_context_type(ssl.PROTOCOL_TLS_SERVER) + else: + raise ValueError(purpose) + + if cafile or capath or cadata: + context.load_verify_locations( + cafile=os.fspath(cafile) if cafile else None, + capath=os.fspath(capath) if capath else None, + cadata=cadata, + ) + else: + context.load_verify_locations(cafile=str(bundle)) + + keylogfile = os.environ.get("SSLKEYLOGFILE") + if keylogfile and hasattr(context, "keylog_filename"): + context.keylog_filename = keylogfile + return context + + ssl.create_default_context = create_context # type: ignore[assignment] + ssl._create_default_https_context = create_context # type: ignore[attr-defined] + os.environ["SSL_CERT_FILE"] = str(bundle) + os.environ.setdefault("REQUESTS_CA_BUNDLE", str(bundle)) + os.environ.setdefault("CURL_CA_BUNDLE", str(bundle)) + + +def configure_tls() -> tuple[str, list[str]]: + """Prepare TLS before Jupyter/Tornado imports and return mode plus notes.""" + notes: list[str] = [] + + try: + if "truststore" not in getattr(ssl.SSLContext, "__module__", ""): + _inject_truststore() + _probe_ssl() + return "native truststore", notes + except Exception as exc: + notes.append(f"Native truststore activation failed: {exc}") + + try: + bundle = _select_ca_bundle() + _install_bundle_fallback(bundle) + _probe_ssl() + notes.append(f"Using verified CA bundle fallback: {bundle}") + return "verified CA bundle fallback", notes + except Exception as exc: + notes.append(f"CA bundle fallback failed: {exc}") + raise RuntimeError("Unable to create a verified TLS context. " + " | ".join(notes)) from exc + + +def _diagnostics(tls_mode: str, notes: list[str]) -> dict[str, object]: + return { + "status": "ok", + "application": PROJECT_DISPLAY_NAME, + "tls_mode": tls_mode, + "notes": notes, + "ca_bundle_override_variable": _project_ca_bundle_environment_variable(), + "python": sys.version.split()[0], + "python_executable": sys.executable, + "openssl": ssl.OPENSSL_VERSION, + "platform": platform.platform(), + "jupyterlab": _package_version("jupyterlab"), + "jupyter_server": _package_version("jupyter-server"), + "tornado": _package_version("tornado"), + "truststore": _package_version("truststore"), + "pip_system_certs": _package_version("pip-system-certs"), + "certifi": _package_version("certifi"), + "prefix": str(PREFIX), + "log": str(LOG_PATH), + } + + +def _show_error_dialog(message: str) -> None: + if os.name != "nt": + return + try: + import tkinter as tk + from tkinter import messagebox + + root = tk.Tk() + root.withdraw() + messagebox.showerror(f"{PROJECT_DISPLAY_NAME} could not start", message) + root.destroy() + except Exception: + pass + + +def _wait_for_enter() -> None: + """Keep the terminal window open until the user has read it, then let it close. + + The menu shortcut runs the launcher in a terminal window; without this the window + closes (or stays open, on Windows) as soon as JupyterLab stops. + """ + stream = sys.stdin + if stream is None or not stream.isatty(): + return + try: + input("\nPress Enter to close this window...") + except (EOFError, KeyboardInterrupt): + pass + + +def run_jupyter(notebook: Path) -> int: + if not notebook.is_file(): + raise FileNotFoundError(f"Welcome notebook not found: {notebook}") + + # Import only after configure_tls() has completed. Tornado creates SSL + # contexts during import on some versions. + from jupyterlab.labapp import main + + home = Path.home().resolve() + try: + notebook.resolve().relative_to(home) + root_dir = home + except ValueError: + # All-users installations can live outside the home directory. + root_dir = PREFIX.resolve() + + # A fresh, empty workspace folder for every launch: JupyterLab restores the open + # tabs of its workspaces, and the default folder (~/.jupyter/lab/workspaces) is + # shared by every app, so notebooks of other apps (and of earlier sessions) would + # open next to the Welcome notebook. Settings and themes are kept: they are stored + # in the user settings folder, not in the workspaces. + workspaces_dir = Path(tempfile.mkdtemp(prefix="labconstrictor-workspaces-")) + sys.argv = [ + "jupyter-lab", + str(notebook.resolve()), + "--ServerApp.ip=127.0.0.1", + f"--ServerApp.root_dir={root_dir}", + "--ServerApp.shutdown_no_activity_timeout=30", + "--FileContentsManager.always_delete_dir=True", + f"--LabApp.workspaces_dir={workspaces_dir}", + ] + try: + result = main() + finally: + shutil.rmtree(workspaces_dir, ignore_errors=True) + return int(result or 0) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=f"Launch {PROJECT_DISPLAY_NAME} in its packaged JupyterLab environment safely." + ) + parser.add_argument("notebook", nargs="?", type=Path, default=DEFAULT_NOTEBOOK) + parser.add_argument( + "--self-test", + action="store_true", + help="Validate TLS and Jupyter imports without starting the server.", + ) + parser.add_argument( + "--print-ca-bundle", + action="store_true", + help="Print the selected verified CA bundle path and exit.", + ) + parser.add_argument( + "--no-wait", + action="store_true", + help="Exit when JupyterLab stops, without asking to press Enter.", + ) + args = parser.parse_args(argv) + result = _launch(args) + if not (args.no_wait or args.self_test or args.print_ca_bundle): + _wait_for_enter() + return result + + +def _launch(args: argparse.Namespace) -> int: + try: + if args.print_ca_bundle: + print(_select_ca_bundle(), flush=True) + return 0 + + _write_log("Starting application launcher") + tls_mode, notes = configure_tls() + _write_log(f"TLS mode: {tls_mode}") + for note in notes: + # Recovery details remain available in launcher_debug.log and + # self-test JSON without alarming users during a normal launch. + _write_log(note, console=False) + + import jupyterlab.labapp # noqa: F401 + import tornado.netutil # noqa: F401 + + diagnostics = _diagnostics(tls_mode, notes) + if args.self_test: + print(json.dumps(diagnostics, indent=2), flush=True) + _write_log("Launcher self-test completed successfully") + return 0 + + _write_log(f"Opening notebook: {args.notebook}") + return run_jupyter(args.notebook) + except KeyboardInterrupt: + _write_log("Launcher interrupted by user") + return 130 + except Exception as exc: + details = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) + _write_log(details.rstrip()) + friendly = ( + f"{PROJECT_DISPLAY_NAME} could not start, but no certificate-store changes are required.\n\n" + f"A diagnostic log was written to:\n{LOG_PATH}\n\n" + f"Technical message: {exc}" + ) + print("\n" + friendly, file=sys.stderr, flush=True) + _show_error_dialog(friendly) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_notebook_change.yml b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_notebook_change.yml new file mode 100644 index 0000000..c5fae88 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_notebook_change.yml @@ -0,0 +1,154 @@ +name: Updates on notebook change + +on: + push: + branches: [ main, master ] + paths: + - 'notebooks/**/*.ipynb' + - 'notebooks/notebook_latest_versions.yaml' + - 'notebooks/**/requirements.yaml' + workflow_dispatch: + +# This enables to run a single workflow at a time +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon notebook change, update manifests and changelogs + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@2d756ea4c53f7f6b397767d8723b3a10a9f35bf2 # v44 + with: + separator: "," + files: | + notebooks/**/*.ipynb + notebooks/notebook_latest_versions.yaml + notebooks/**/requirements.yaml + + - name: Syncronize notebook versions + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "sync_notebook_versions.py" \ + .tools/python/sync_notebook_versions.py \ + python .tools/python/sync_notebook_versions.py + + - name: Update CHANGELOG + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_changelog.py" \ + .tools/python/update_changelog.py \ + python .tools/python/update_changelog.py notebooks/notebook_latest_versions.yaml + + - name: Update last commit in notebook_latest_versions.yaml + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_version_last_commit.py" \ + .tools/python/update_version_last_commit.py \ + python .tools/python/update_version_last_commit.py notebooks/notebook_latest_versions.yaml + + - name: Hide code cells in notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "hide_code_cells.py" \ + app/python_scripts/hide_code_cells.py \ + python app/python_scripts/hide_code_cells.py . + + - name: Index notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_notebook_README.py" \ + .tools/python/bump_notebook_README.py \ + python .tools/python/bump_notebook_README.py + + - name: Run merge script + if: github.event_name == 'workflow_dispatch' || contains(steps.changed-files.outputs.all_changed_files, 'requirements.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "merge_requirements.py" \ + .tools/python/merge_requirements.py \ + python .tools/python/merge_requirements.py --source-dir notebooks --output requirements.txt + + - name: Check for changes + id: check-changes + run: | + if [ -z "$(git status --porcelain)" ]; then + echo "has_changes=false" >> $GITHUB_OUTPUT + else + echo "has_changes=true" >> $GITHUB_OUTPUT + fi + + - name: Set up Miniconda for requirements test + if: steps.check-changes.outputs.has_changes == 'true' + uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + auto-activate: true + miniforge-version: latest + use-mamba: true + + - name: Verify merged requirements inside conda + id: test-conda-env + if: steps.check-changes.outputs.has_changes == 'true' + env: + SKIP_CONDA_ENV_TEST: ${{ vars.SKIP_CONDA_ENV_TEST }} + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "test_conda_environment.sh" \ + .tools/bash/test_conda_environment.sh \ + bash .tools/bash/test_conda_environment.sh + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: notebook-debug-bundle + path: artifacts/error_log_document.log + archive: false + + - name: Commit updated files + id: auto-commit + if: steps.check-changes.outputs.has_changes == 'true' && steps.test-conda-env.outcome == 'success' + uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5 + with: + commit_message: GitHub Action - Updated files because of notebook and/or requirements changes + file_pattern: requirements*.txt environment.yaml notebooks/* setup.py + + - name: Revert user commit that broke requirements + if: failure() && github.event_name == 'push' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git reset --hard + git clean -fd + git revert "${{ github.sha }}" -m 1 --no-edit + git push origin HEAD:${{ github.ref_name }} diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_src_change.yaml b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_src_change.yaml new file mode 100644 index 0000000..76b90f0 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/.github/workflows/update_on_src_change.yaml @@ -0,0 +1,41 @@ +name: Updates on external code change + +on: + push: + branches: [ main, master ] + paths: + - 'src/**' + workflow_dispatch: + +# This enables to run a single workflow at a time +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon external code update the code version + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Update src change file + run: | + cat > .tools/meta/src_change.yaml </dev/null || true + +# Create conda environment from environment.yaml +echo "" +echo "Creating conda environment from environment.yaml..." +if [ ! -f "environment.yaml" ]; then + echo "ERROR: environment.yaml not found!" + exit 1 +fi + +mamba env create -n "$TEST_ENV_NAME" -f environment.yaml -y + +# Activate the environment +echo "" +echo "Activating test environment..." +# Note: In CI, conda/mamba environments need to be activated via conda run +# or by sourcing the activation script + +# Test pip installation of requirements.txt +echo "" +echo "Installing requirements from requirements.txt..." +if [ ! -f "requirements.txt" ]; then + echo "ERROR: requirements.txt not found!" + mamba env remove -n "$TEST_ENV_NAME" -y + exit 1 +fi + +# Check if requirements.txt is not empty +if [ ! -s "requirements.txt" ]; then + echo "WARNING: requirements.txt is empty, skipping pip install" +else + echo "Installing packages with pip..." + mamba run -n "$TEST_ENV_NAME" pip install --prefer-binary -r requirements.txt +fi + +# Verify installation by checking Python and key packages +echo "" +echo "Verifying installation..." +echo "Python version:" +mamba run -n "$TEST_ENV_NAME" python --version + +echo "" +echo "Installed pip packages (sample):" +mamba run -n "$TEST_ENV_NAME" pip list | head -n 20 + +# Clean up test environment +echo "" +echo "Cleaning up test environment..." +mamba env remove -n "$TEST_ENV_NAME" -y + +echo "" +echo "==========================================" +echo "✓ Conda environment test completed successfully!" +echo "==========================================" diff --git a/.template_sync/resources/v0_1_19_to_v0_1_20/previous/app/python_scripts/launch_jupyter.py b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/app/python_scripts/launch_jupyter.py new file mode 100644 index 0000000..0f91726 --- /dev/null +++ b/.template_sync/resources/v0_1_19_to_v0_1_20/previous/app/python_scripts/launch_jupyter.py @@ -0,0 +1,286 @@ +"""TLS-resilient JupyterLab launcher for PROJECT_NAME. + +Certificate handling is configured before JupyterLab imports Tornado. On +Windows this prevents one malformed certificate-store entry from terminating +the application. Certificate and hostname verification remain enabled in every +supported mode. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import re +import ssl +import sys +import traceback +from datetime import datetime, timezone +from importlib import metadata +from pathlib import Path + +PROJECT_DISPLAY_NAME = "PROJECT_NAME" +SCRIPT_DIR = Path(__file__).resolve().parent +_INSTALLED_LAYOUT = (SCRIPT_DIR / "notebooks").is_dir() +if _INSTALLED_LAYOUT: + APP_ROOT = SCRIPT_DIR + PREFIX = APP_ROOT.parent +else: + # Repository layout: app/python_scripts/launch_jupyter.py + repository_root = SCRIPT_DIR.parents[1] + APP_ROOT = repository_root if (repository_root / "notebooks").is_dir() else SCRIPT_DIR + PREFIX = APP_ROOT +DEFAULT_NOTEBOOK = APP_ROOT / "notebooks" / "Welcome.ipynb" +LOG_PATH = PREFIX / "launcher_debug.log" + + +def _write_log(message: str, *, console: bool = True) -> None: + timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds") + line = f"[{timestamp}] {message}" + if console: + print(message, flush=True) + try: + with LOG_PATH.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + except OSError: + # Logging must never prevent the application from launching. + pass + + +def _package_version(name: str) -> str: + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return "not installed" + + +def _project_ca_bundle_environment_variable() -> str: + normalized = re.sub(r"[^A-Za-z0-9]+", "_", PROJECT_DISPLAY_NAME).strip("_").upper() + return f"{normalized or 'LABCONSTRICTOR_APP'}_CA_BUNDLE" + + +def _probe_ssl() -> None: + """Create a verified client context without making a network request.""" + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) + if context.verify_mode != ssl.CERT_REQUIRED: + raise RuntimeError("TLS certificate verification is unexpectedly disabled.") + if not context.check_hostname: + raise RuntimeError("TLS hostname verification is unexpectedly disabled.") + + +def _inject_truststore() -> None: + """Route Python SSL contexts through the operating-system trust API.""" + import truststore + + truststore.inject_into_ssl() + + +def _select_ca_bundle() -> Path: + """Choose an explicit CA bundle for verified installer and runtime fallback.""" + for variable in ( + _project_ca_bundle_environment_variable(), + "LABCONSTRICTOR_CA_BUNDLE", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CURL_CA_BUNDLE", + ): + candidate = os.environ.get(variable, "").strip() + if candidate: + path = Path(candidate).expanduser() + if path.is_file(): + return path.resolve() + + import certifi + + return Path(certifi.where()).resolve() + + +def _install_bundle_fallback(bundle: Path) -> None: + """Use a verified PEM bundle without enumerating the Windows cert store.""" + original_context_type = ssl.SSLContext + + def create_context( + purpose: ssl.Purpose = ssl.Purpose.SERVER_AUTH, + *, + cafile: str | os.PathLike[str] | None = None, + capath: str | os.PathLike[str] | None = None, + cadata: str | bytes | None = None, + ) -> ssl.SSLContext: + if purpose == ssl.Purpose.SERVER_AUTH: + context = original_context_type(ssl.PROTOCOL_TLS_CLIENT) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + elif purpose == ssl.Purpose.CLIENT_AUTH: + context = original_context_type(ssl.PROTOCOL_TLS_SERVER) + else: + raise ValueError(purpose) + + if cafile or capath or cadata: + context.load_verify_locations( + cafile=os.fspath(cafile) if cafile else None, + capath=os.fspath(capath) if capath else None, + cadata=cadata, + ) + else: + context.load_verify_locations(cafile=str(bundle)) + + keylogfile = os.environ.get("SSLKEYLOGFILE") + if keylogfile and hasattr(context, "keylog_filename"): + context.keylog_filename = keylogfile + return context + + ssl.create_default_context = create_context # type: ignore[assignment] + ssl._create_default_https_context = create_context # type: ignore[attr-defined] + os.environ["SSL_CERT_FILE"] = str(bundle) + os.environ.setdefault("REQUESTS_CA_BUNDLE", str(bundle)) + os.environ.setdefault("CURL_CA_BUNDLE", str(bundle)) + + +def configure_tls() -> tuple[str, list[str]]: + """Prepare TLS before Jupyter/Tornado imports and return mode plus notes.""" + notes: list[str] = [] + + try: + if "truststore" not in getattr(ssl.SSLContext, "__module__", ""): + _inject_truststore() + _probe_ssl() + return "native truststore", notes + except Exception as exc: + notes.append(f"Native truststore activation failed: {exc}") + + try: + bundle = _select_ca_bundle() + _install_bundle_fallback(bundle) + _probe_ssl() + notes.append(f"Using verified CA bundle fallback: {bundle}") + return "verified CA bundle fallback", notes + except Exception as exc: + notes.append(f"CA bundle fallback failed: {exc}") + raise RuntimeError("Unable to create a verified TLS context. " + " | ".join(notes)) from exc + + +def _diagnostics(tls_mode: str, notes: list[str]) -> dict[str, object]: + return { + "status": "ok", + "application": PROJECT_DISPLAY_NAME, + "tls_mode": tls_mode, + "notes": notes, + "ca_bundle_override_variable": _project_ca_bundle_environment_variable(), + "python": sys.version.split()[0], + "python_executable": sys.executable, + "openssl": ssl.OPENSSL_VERSION, + "platform": platform.platform(), + "jupyterlab": _package_version("jupyterlab"), + "jupyter_server": _package_version("jupyter-server"), + "tornado": _package_version("tornado"), + "truststore": _package_version("truststore"), + "pip_system_certs": _package_version("pip-system-certs"), + "certifi": _package_version("certifi"), + "prefix": str(PREFIX), + "log": str(LOG_PATH), + } + + +def _show_error_dialog(message: str) -> None: + if os.name != "nt": + return + try: + import tkinter as tk + from tkinter import messagebox + + root = tk.Tk() + root.withdraw() + messagebox.showerror(f"{PROJECT_DISPLAY_NAME} could not start", message) + root.destroy() + except Exception: + pass + + +def run_jupyter(notebook: Path) -> int: + if not notebook.is_file(): + raise FileNotFoundError(f"Welcome notebook not found: {notebook}") + + # Import only after configure_tls() has completed. Tornado creates SSL + # contexts during import on some versions. + from jupyterlab.labapp import main + + home = Path.home().resolve() + try: + notebook.resolve().relative_to(home) + root_dir = home + except ValueError: + # All-users installations can live outside the home directory. + root_dir = PREFIX.resolve() + + sys.argv = [ + "jupyter-lab", + str(notebook.resolve()), + "--ServerApp.ip=127.0.0.1", + f"--ServerApp.root_dir={root_dir}", + "--ServerApp.shutdown_no_activity_timeout=30", + "--FileContentsManager.always_delete_dir=True", + ] + result = main() + return int(result or 0) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=f"Launch {PROJECT_DISPLAY_NAME} in its packaged JupyterLab environment safely." + ) + parser.add_argument("notebook", nargs="?", type=Path, default=DEFAULT_NOTEBOOK) + parser.add_argument( + "--self-test", + action="store_true", + help="Validate TLS and Jupyter imports without starting the server.", + ) + parser.add_argument( + "--print-ca-bundle", + action="store_true", + help="Print the selected verified CA bundle path and exit.", + ) + args = parser.parse_args(argv) + + try: + if args.print_ca_bundle: + print(_select_ca_bundle(), flush=True) + return 0 + + _write_log("Starting application launcher") + tls_mode, notes = configure_tls() + _write_log(f"TLS mode: {tls_mode}") + for note in notes: + # Recovery details remain available in launcher_debug.log and + # self-test JSON without alarming users during a normal launch. + _write_log(note, console=False) + + import jupyterlab.labapp # noqa: F401 + import tornado.netutil # noqa: F401 + + diagnostics = _diagnostics(tls_mode, notes) + if args.self_test: + print(json.dumps(diagnostics, indent=2), flush=True) + _write_log("Launcher self-test completed successfully") + return 0 + + _write_log(f"Opening notebook: {args.notebook}") + return run_jupyter(args.notebook) + except KeyboardInterrupt: + _write_log("Launcher interrupted by user") + return 130 + except Exception as exc: + details = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) + _write_log(details.rstrip()) + friendly = ( + f"{PROJECT_DISPLAY_NAME} could not start, but no certificate-store changes are required.\n\n" + f"A diagnostic log was written to:\n{LOG_PATH}\n\n" + f"Technical message: {exc}" + ) + print("\n" + friendly, file=sys.stderr, flush=True) + _show_error_dialog(friendly) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.template_sync/resources/v0_1_22_to_v0_1_23/.tools/python/check_placeholders.py b/.template_sync/resources/v0_1_22_to_v0_1_23/.tools/python/check_placeholders.py new file mode 100644 index 0000000..b06559f --- /dev/null +++ b/.template_sync/resources/v0_1_22_to_v0_1_23/.tools/python/check_placeholders.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Stop a release whose installer would ship a template placeholder (CellMigrationLab/LabConstrictor#40). + +The repository initialisation (the LabConstrictor web form) replaces the template's placeholders with the app's names. A placeholder +it misses survives into the installer: `import PYTHON_PROJ_NAME` in post_install.sh / .bat made every installer of an app with Python +code in src/ fail at its last step. This script runs in the release workflow after the versioned files are updated and before anything +is committed or built, and fails with the file, the line and the fix when an initialisation placeholder is left in a file the +installer ships (app/**, construct.yaml, setup.py, environment.yaml), or when the repository is the raw, uninitialised template. + +Placeholders that are filled in later on purpose are not checked: BASE_PATH_KEYWORD (at install time), PYTHON_VERSION and +JUPYTERLAB_VERSION (by the requirements workflow). + +Usage: python .tools/python/check_placeholders.py [--repository-root PATH] +""" +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_23.md" + +# The placeholders the initialisation replaces (keys of conversion_table in LabConstrictor_streamlit's placeholders.py, +# plus the launcher icon paths it fills in itself). VERSION_NUMBER is also rewritten by bump_version.py, which runs before this check. +INIT_PLACEHOLDERS = ( + "LOWER_PROJ_NAME", + "PYTHON_PROJ_NAME", + "UNDERSCORED_PROJECT_NAME", + "PROJECT_NAME", + "VERSION_NUMBER", + "WELCOME_IMAGE", + "HEADER_IMAGE", + "ICON_IMAGE", + "GITHUB_OWNER", + "GITHUB_REPO_NAME", + "HIDE_CODE_DISABLED", + "ICON_IMAGE_PATH", + "ICON_ICO_IMAGE_PATH", + "ICON_ICNS_IMAGE_PATH", +) +SHIPPED_FILES = ("construct.yaml", "setup.py", "environment.yaml") +SHIPPED_FOLDERS = ("app",) +WELCOME_NOTEBOOK = "app/menuinst/Welcome.ipynb" +WELCOME_TEMPLATE = ".tools/templates/Welcome_template.ipynb" + + +def placeholder_pattern(name: str) -> re.Pattern[str]: + """A placeholder on its own or followed by '_' (PYTHON_PROJ_NAME_lc_tools), never inside a longer name (UNDERSCORED_PROJECT_NAME).""" + return re.compile(r"(? list[Path]: + files = [root / name for name in SHIPPED_FILES if (root / name).is_file()] + for folder in SHIPPED_FOLDERS: + if (root / folder).is_dir(): + files.extend(sorted(p for p in (root / folder).rglob("*") if p.is_file())) + return files + + +def leftover_placeholders(root: Path) -> list[str]: + problems = [] + for path in shipped_files(root): + try: + text = path.read_text(encoding="utf-8") + except (UnicodeDecodeError, OSError): + continue # images, icons and other binary files + rel = path.relative_to(root).as_posix() + # The Welcome notebook is rebuilt from its template at every release: the fix belongs in the template. + where = f" (fix it in {WELCOME_TEMPLATE}, which this notebook is made from)" if rel == WELCOME_NOTEBOOK else "" + for number, line in enumerate(text.splitlines(), start=1): + for name, pattern in PATTERNS.items(): + if pattern.search(line): + problems.append(f"{rel}:{number}: the template placeholder {name} was never replaced{where}: {line.strip()[:160]}") + return problems + + +def check(root: Path) -> list[str]: + construct = root / "construct.yaml" + if construct.is_file() and re.search(r"^name:\s*UNDERSCORED_PROJECT_NAME\s*$", construct.read_text(encoding="utf-8"), re.MULTILINE): + return ["construct.yaml: this is the raw LabConstrictor template; initialise the repository with the LabConstrictor form before releasing it"] + return leftover_placeholders(root) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Fail when a template placeholder is left in a file the installer ships.") + parser.add_argument("--repository-root", type=Path, default=Path.cwd()) + args = parser.parse_args(argv) + problems = check(args.repository_root.resolve()) + if not problems: + print("No template placeholder is left in the files the installer ships.") + return 0 + print("The installer would be broken: these files still hold template placeholders.", file=sys.stderr) + for problem in problems: + print(f" {problem}", file=sys.stderr) + print( + "Replace each placeholder with your app's value (PYTHON_PROJ_NAME: the Python package of the app, usually its folder in src/; " + "PROJECT_NAME: the app name), " + f"commit, and publish the release again. See {DOCS_URL}", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.template_sync/resources/v0_1_23_to_v0_1_24/.tools/python/release_platforms.py b/.template_sync/resources/v0_1_23_to_v0_1_24/.tools/python/release_platforms.py new file mode 100644 index 0000000..4083e6b --- /dev/null +++ b/.template_sync/resources/v0_1_23_to_v0_1_24/.tools/python/release_platforms.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Which installers a release builds: the systems of `distribution.platforms` in labconstrictor-app.yaml (CellMigrationLab/LabConstrictor#42). + +The release workflow builds one installer per system. An app that cannot run on a system (TensorFlow 2.17+ has no wheels for Intel Macs, for +example) leaves that system out of `distribution.platforms`, and then: + * the release workflow does not build it (this script writes the build matrix of release_constructor.yml); + * the download page (.tools/docs/download_executable.md, written by bump_version.py) has no button for it. +Without labconstrictor-app.yaml, or without `distribution.platforms` in it, every system below is built, as before template 0.1.24. + +Usage: python .tools/python/release_platforms.py [--repository-root PATH] [--github-output FILE] +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, NamedTuple + +import yaml + +MANIFEST = "labconstrictor-app.yaml" +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_24.md" + + +class Build(NamedTuple): + platform: str # the name in distribution.platforms + os_name: str # matrix.os_name of release_constructor.yml (names the build job, its log and dist/ folder) + runner: str # the GitHub runner that builds it + files: tuple[str, ...] # how its installers end: - + label: str # for people: the download page and the messages + + +BUILDS = ( + Build("linux-x86_64", "linux", "ubuntu-latest", ("-Linux-x86_64.sh",), "Linux"), + Build("macos-arm64", "macos-arm64", "macos-15", ("-MacOSX-arm64.pkg", "-MacOSX-arm64.sh"), "Macs with Apple silicon (M1, M2, ...)"), + Build("macos-x86_64", "macos-intel", "macos-15-intel", ("-MacOSX-x86_64.pkg", "-MacOSX-x86_64.sh"), "Macs with an Intel processor"), + Build("windows-x86_64", "windows", "windows-latest", ("-Windows-x86_64.exe",), "Windows"), +) + + +def listed_platforms(root: Path) -> list[str] | None: + """`distribution.platforms` of the app manifest; None when there is no manifest or the key is left out (every system).""" + path = root / MANIFEST + if not path.is_file(): + return None + try: + data: Any = yaml.safe_load(path.read_text(encoding="utf-8")) + except yaml.YAMLError as exc: + raise ValueError(f"{MANIFEST} is not valid YAML: {exc}") from exc + distribution = data.get("distribution") if isinstance(data, dict) else None + platforms = distribution.get("platforms") if isinstance(distribution, dict) else None + if platforms is None: + return None + if not isinstance(platforms, list) or not all(isinstance(item, str) for item in platforms): + raise ValueError(f"distribution.platforms in {MANIFEST} must be a list of system names, such as [windows-x86_64, macos-arm64]") + return platforms + + +def builds(root: Path) -> list[Build]: + """The builds of a release, in the order of BUILDS. Raises ValueError when the manifest lists no system this template builds.""" + listed = listed_platforms(root) + if listed is None: + return list(BUILDS) + chosen = [build for build in BUILDS if build.platform in listed] + if not chosen: + known = ", ".join(build.platform for build in BUILDS) + raise ValueError( + f"distribution.platforms in {MANIFEST} lists {', '.join(listed) or 'nothing'}, and the release builds installers only for {known}. " + "Add at least one of them, or remove distribution.installer if the app has no installer." + ) + return chosen + + +def skipped_builds(root: Path) -> list[Build]: + """The systems the release does not build (none without distribution.platforms).""" + chosen = builds(root) + return [build for build in BUILDS if build not in chosen] + + +def matrix(chosen: list[Build]) -> dict[str, list[dict[str, str]]]: + return {"include": [{"os_name": build.os_name, "runner": build.runner} for build in chosen]} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Print the build matrix of the release: the systems of distribution.platforms in labconstrictor-app.yaml.") + parser.add_argument("--repository-root", type=Path, default=Path.cwd()) + parser.add_argument("--github-output", type=Path, help="append matrix= to this file (the GITHUB_OUTPUT of a workflow step)") + args = parser.parse_args(argv) + root = args.repository_root.resolve() + try: + chosen = builds(root) + listed = listed_platforms(root) + except ValueError as exc: + print(f"{exc} See {DOCS_URL}", file=sys.stderr) + return 1 + if listed is None: + print(f"Building every installer: {MANIFEST} does not list distribution.platforms.") + else: + print(f"Building the installers of distribution.platforms in {MANIFEST}: {', '.join(build.platform for build in chosen)}.") + for build in BUILDS: + if build not in chosen: + print(f" Not built: {build.platform} ({build.label}).") + for name in listed: + if name not in {build.platform for build in BUILDS}: + print(f" Not built: {name} (the release has no runner for it).") + value = json.dumps(matrix(chosen), separators=(",", ":")) + print(f"matrix={value}") + if args.github_output: + with args.github_output.open("a", encoding="utf-8") as handle: + handle.write(f"matrix={value}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.template_sync/resources/v0_1_24_to_v0_1_25/.github/workflows/publish_pypi.yml b/.template_sync/resources/v0_1_24_to_v0_1_25/.github/workflows/publish_pypi.yml new file mode 100644 index 0000000..cfd03da --- /dev/null +++ b/.template_sync/resources/v0_1_24_to_v0_1_25/.github/workflows/publish_pypi.yml @@ -0,0 +1,79 @@ +name: Publish the Python package to PyPI + +# Publishes the app's own Python code (the package in src/) on PyPI at each release, with the version of the release: the same +# moment and the same version as the installers (release_constructor.yml). +# +# OFF BY DEFAULT: both jobs are skipped unless the repository variable PYPI_PUBLISH is +# true -> publish on PyPI (pypi.org) +# testpypi -> publish on TestPyPI (test.pypi.org), a trial +# No secret or token: PyPI trusted publishing. One-time setup (a pending trusted publisher on pypi.org for this workflow file and the +# environment "pypi", then the variable): .tools/docs/external_code_upload.md. To turn it off, delete the variable. + +on: + release: + types: + - published + +permissions: + contents: read + +jobs: + build: + name: Build the package of src/ + if: vars.PYPI_PUBLISH == 'true' || vars.PYPI_PUBLISH == 'testpypi' + runs-on: ubuntu-latest + outputs: + url: ${{ steps.build.outputs.url }} + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt build==1.3.0 + + # The tag (v1.2.3 or 1.2.3) without the v. The checkout is the tagged commit, before the "Update version" commit of + # release_constructor.yml: pypi_package.py writes this version into setup.py itself (not committed). + - name: Build and check the package (nothing is uploaded yet) + id: build + env: + TAG: ${{ github.event.release.tag_name }} + PYPI_PUBLISH: ${{ vars.PYPI_PUBLISH }} + run: python .tools/python/pypi_package.py --version "${TAG#v}" --index "$PYPI_PUBLISH" --outdir dist --github-output "$GITHUB_OUTPUT" + + - name: Store the distributions + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: python-package-distributions + path: dist/ + if-no-files-found: error + + publish: + name: Upload to ${{ vars.PYPI_PUBLISH == 'testpypi' && 'TestPyPI' || 'PyPI' }} + needs: build + if: vars.PYPI_PUBLISH == 'true' || vars.PYPI_PUBLISH == 'testpypi' + runs-on: ubuntu-latest + # The environment named in the trusted publisher on pypi.org (and test.pypi.org). GitHub creates it on the first run; protection + # rules added to it (Settings > Environments > pypi), such as required reviewers, then apply to every upload. + environment: + name: pypi + url: ${{ needs.build.outputs.url }} + permissions: + id-token: write # trusted publishing: PyPI accepts a short-lived token that GitHub signs for this workflow and environment + steps: + - name: Download the distributions + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: python-package-distributions + path: dist/ + + - name: Upload + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + repository-url: ${{ vars.PYPI_PUBLISH == 'testpypi' && 'https://test.pypi.org/legacy/' || 'https://upload.pypi.org/legacy/' }} diff --git a/.template_sync/resources/v0_1_24_to_v0_1_25/.tools/python/pypi_package.py b/.template_sync/resources/v0_1_24_to_v0_1_25/.tools/python/pypi_package.py new file mode 100644 index 0000000..d7783b7 --- /dev/null +++ b/.template_sync/resources/v0_1_24_to_v0_1_25/.tools/python/pypi_package.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +"""Build the app's Python package (the code in src/) for PyPI, as .github/workflows/publish_pypi.yml does at each release. + +The workflow runs only when the repository variable PYPI_PUBLISH is "true" (PyPI) or "testpypi" (TestPyPI, a trial). This script: + +1. stops before building, with the reason, when the release cannot be published: no package in src/, a version that is not X.Y.Z, + a version that is already on the index, another workflow of the repository that also publishes to PyPI, or a setup.py that + still has a template placeholder; +2. writes the release version into setup.py of the checkout, without committing it: the release tag points to the commit before + the "Update version" commit of the release workflow, so the committed version cannot be trusted. The name is the one in + setup.py; `distribution.pip` of labconstrictor-app.yaml, when there is one, must be the same name; +3. builds the source distribution and the wheel (`python -m build`: the wheel is built from the source distribution, so a file + missing from it shows), and checks their name, version and dependencies. + +It never uploads: the next job of the workflow does, with PyPI trusted publishing (no token). + +Usage: + python .tools/python/pypi_package.py --version 1.2.3 [--index pypi|testpypi] [--outdir dist] [--github-output FILE] +""" +from __future__ import annotations + +import argparse +import email.parser +import re +import subprocess +import sys +import tarfile +import tomllib +import urllib.error +import urllib.request +import zipfile +from email.message import Message +from pathlib import Path +from typing import Any, Callable + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = "publish_pypi.yml" +DOCS = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/external_code_upload.md#4-publish-the-code-on-pypi-optional" +# --index: the value of PYPI_PUBLISH ("true" is PyPI) -> (label, address) +INDEXES = {"pypi": ("PyPI", "https://pypi.org"), "true": ("PyPI", "https://pypi.org"), "testpypi": ("TestPyPI", "https://test.pypi.org")} +VERSION_RE = re.compile(r"\d+\.\d+\.\d+") +# Text in a workflow that uploads to PyPI (the action of the template, or twine). +PUBLISH_MARKERS = ("pypa/gh-action-pypi-publish", "twine upload") +PLACEHOLDERS = ("PYTHON_PROJ_NAME", "VERSION_NUMBER", "PYTHON_VERSION") + + +class PackageError(Exception): + """A reason not to publish, written to the log of the workflow.""" + + +def normalise(name: str) -> str: + """The name as PyPI compares names (PEP 503): my_App.tools and my-app-tools are the same project.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def packages(root: Path) -> list[str]: + """The packages of src/: its folders with an __init__.py.""" + src = root / "src" + if not src.is_dir(): + return [] + return sorted(path.parent.name for path in src.glob("*/__init__.py")) + + +def other_publishers(root: Path) -> list[str]: + """The workflows of the repository, other than the template's, that upload to PyPI.""" + found = [] + folder = root / ".github" / "workflows" + for path in sorted(folder.glob("*.y*ml")) if folder.is_dir() else []: + if path.name == WORKFLOW: + continue + text = path.read_text(encoding="utf-8", errors="replace") + if any(marker in text for marker in PUBLISH_MARKERS): + found.append(path.relative_to(root).as_posix()) + return found + + +def manifest_pip_name(root: Path) -> str | None: + """`distribution.pip` of labconstrictor-app.yaml: the name of the package on PyPI, when the app gives one.""" + import yaml + + path = root / "labconstrictor-app.yaml" + if not path.is_file(): + return None + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + except yaml.YAMLError as exc: + raise PackageError(f"labconstrictor-app.yaml is not valid YAML ({exc}); run the 'Check app manifest' workflow to see where") from exc + distribution = data.get("distribution") if isinstance(data, dict) else None + name = distribution.get("pip") if isinstance(distribution, dict) else None + return name.strip() if isinstance(name, str) and name.strip() else None + + +def _value_re(key: str) -> re.Pattern[str]: + return re.compile(rf"^(?P[ \t]*{key}\s*=\s*)(?P[\"'])(?P[^\"'\n]*)(?P=quote)", re.MULTILINE) + + +def setup_value(text: str, key: str) -> str | None: + """The first `key="value"` of setup.py (name, version), or None when it is not written there as a string.""" + match = _value_re(key).search(text) + return match.group("value") if match else None + + +def set_setup_value(text: str, key: str, value: str) -> str: + return _value_re(key).sub(lambda m: f"{m.group('head')}{m.group('quote')}{value}{m.group('quote')}", text, count=1) + + +def pyproject_project(root: Path) -> dict[str, Any] | None: + """The [project] table of pyproject.toml, when the package is described there (it then wins over setup.py).""" + path = root / "pyproject.toml" + if not path.is_file(): + return None + try: + project = tomllib.loads(path.read_text(encoding="utf-8")).get("project") + except tomllib.TOMLDecodeError as exc: + raise PackageError(f"pyproject.toml is not valid TOML: {exc}") from exc + return project if isinstance(project, dict) else None + + +def version_on_index(index: str, name: str, version: str, opener: Callable[..., Any] = urllib.request.urlopen) -> bool | None: + """True when the index already has this version of the project, False when not, None when the index could not be asked.""" + url = f"{INDEXES[index][1]}/pypi/{normalise(name)}/{version}/json" + try: + with opener(url, timeout=30) as response: + return bool(response.status == 200) + except urllib.error.HTTPError as exc: + return False if exc.code == 404 else None + except (urllib.error.URLError, OSError): + return None + + +def prepare(root: Path, version: str, index: str, check_index: bool = True) -> str: + """Check that the release can be published and write its version (and name) into setup.py; returns the name on the index.""" + if not VERSION_RE.fullmatch(version): + raise PackageError(f"The release version {version!r} is not X.Y.Z (the tag must be 1.2.3 or v1.2.3, as for the installers).") + found = packages(root) + if not found: + raise PackageError( + "There is no Python package in src/ (a folder with an __init__.py, for example src/my_app/__init__.py), so there is nothing to " + f"publish. Add your code to src/ (see .tools/docs/external_code_upload.md), or delete the repository variable PYPI_PUBLISH. {DOCS}" + ) + others = other_publishers(root) + if others: + raise PackageError( + f"{', '.join(others)} also publishes to PyPI: with PYPI_PUBLISH set, the release would be uploaded twice. Keep one: delete the " + f"repository variable PYPI_PUBLISH (your workflow keeps publishing), or delete your workflow (this one publishes at each release). {DOCS}" + ) + + pip_name = manifest_pip_name(root) + project = pyproject_project(root) + setup_path = root / "setup.py" + if project is not None and "name" in project: + # pyproject.toml describes the package: its name and version are used as they are, and must match. + name = str(project["name"]) + if pip_name and normalise(pip_name) != normalise(name): + raise PackageError(f"distribution.pip of labconstrictor-app.yaml is {pip_name}, but pyproject.toml names the package {name}: make them the same.") + if "version" in project.get("dynamic", []): + return name + if project.get("version") != version: + raise PackageError( + f"pyproject.toml gives the version {project.get('version')}, not the release version {version}: set `version = \"{version}\"` in " + "pyproject.toml, commit, and publish the release again." + ) + return name + if not setup_path.is_file(): + raise PackageError("There is no setup.py (nor a [project] table in pyproject.toml) to build the package from: restore the template's setup.py.") + + text = setup_path.read_bytes().decode("utf-8") # the line endings are kept when the version is written + left = [placeholder for placeholder in PLACEHOLDERS if placeholder in text and placeholder != "VERSION_NUMBER"] + if left: + raise PackageError( + f"setup.py still has the template placeholder {', '.join(left)}: PYTHON_PROJ_NAME is the name of the package (for example my_app), " + "PYTHON_VERSION the oldest Python series it runs on (for example 3.12). Replace it, commit, and publish the release again." + ) + setup_name = setup_value(text, "name") + if pip_name and setup_name and normalise(pip_name) != normalise(setup_name): + raise PackageError( + f"distribution.pip of labconstrictor-app.yaml is {pip_name}, but setup.py names the package {setup_name}: make them the same " + "(distribution.pip is the name the App Centre tells people to pip install; the package of src/ is published under setup.py's name)." + ) + package_name = setup_name or pip_name + if not package_name: + raise PackageError("setup.py has no name=\"...\" for the package: give it the name it should have on PyPI (and in distribution.pip).") + if check_index: + label = INDEXES[index][0] + exists = version_on_index(index, package_name, version) + if exists: + raise PackageError( + f"{label} already has {package_name} {version}, and a version can be uploaded only once (even after it was deleted). " + f"Publish a new release with a higher version (the installers are built for it too). " + f"{INDEXES[index][1]}/project/{normalise(package_name)}/{version}/" + ) + if exists is None: + print(f"WARNING: {label} could not be asked whether {package_name} {version} already exists; the upload fails if it does.") + + updated = set_setup_value(text, "version", version) + if updated != text: + # newline="" keeps the line endings of the file + with setup_path.open("w", encoding="utf-8", newline="") as handle: + handle.write(updated) + return package_name + + +def left_out_requirements(root: Path) -> list[str]: + """Lines of requirements.txt that the template's setup.py does not turn into dependencies (options, paths, URLs).""" + setup_text = (root / "setup.py").read_text(encoding="utf-8") if (root / "setup.py").is_file() else "" + path = root / "requirements.txt" + if "read_requirements()" not in setup_text or not path.is_file(): + return [] + try: + from packaging.requirements import InvalidRequirement, Requirement + except ImportError: + return [] + left = [] + for line in (raw.split("#", 1)[0].strip() for raw in path.read_text(encoding="utf-8").splitlines()): + if not line: + continue + try: + if Requirement(line).url is None: + continue + except InvalidRequirement: + pass + left.append(line) + return left + + +def build(root: Path, outdir: Path, isolation: bool = True) -> None: + """`python -m build`: the source distribution, then the wheel built from it.""" + command = [sys.executable, "-m", "build", "--outdir", str(outdir), *([] if isolation else ["--no-isolation"]), str(root)] + print("Running:", " ".join(command), flush=True) + result = subprocess.run(command) + if result.returncode: + raise PackageError("The package could not be built: see the output of `python -m build` above.") + + +def _headers(data: bytes) -> Message: + return email.parser.BytesParser().parsebytes(data, headersonly=True) + + +def read_metadata(outdir: Path) -> tuple[Message, Message, list[Path]]: + """The metadata of the wheel and of the source distribution in outdir, and the two files.""" + wheels = sorted(outdir.glob("*.whl")) + sdists = sorted(outdir.glob("*.tar.gz")) + if len(wheels) != 1 or len(sdists) != 1: + raise PackageError(f"Expected one wheel and one source distribution in {outdir}, found: {', '.join(p.name for p in sorted(outdir.iterdir())) or 'nothing'}") + with zipfile.ZipFile(wheels[0]) as archive: + member = next((n for n in archive.namelist() if n.count("/") == 1 and n.endswith(".dist-info/METADATA")), None) + if member is None: + raise PackageError(f"{wheels[0].name} has no METADATA") + wheel = _headers(archive.read(member)) + with tarfile.open(sdists[0]) as archive: + info = next((m for m in archive.getmembers() if m.name.count("/") == 1 and m.name.endswith("/PKG-INFO")), None) + handle = archive.extractfile(info) if info is not None else None + if handle is None: + raise PackageError(f"{sdists[0].name} has no PKG-INFO") + sdist = _headers(handle.read()) + return wheel, sdist, [sdists[0], wheels[0]] + + +def check(outdir: Path, name: str, version: str) -> list[str]: + """Check the built files; returns their names.""" + wheel, sdist, files = read_metadata(outdir) + for label, metadata in (("wheel", wheel), ("source distribution", sdist)): + if normalise(str(metadata.get("Name", ""))) != normalise(name): + raise PackageError(f"The {label} is named {metadata.get('Name')}, not {name}: setup.py or pyproject.toml set another name.") + if metadata.get("Version") != version: + raise PackageError(f"The {label} has the version {metadata.get('Version')}, not the release version {version}: setup.py or pyproject.toml set it.") + requirements = wheel.get_all("Requires-Dist") or [] + urls = [requirement for requirement in requirements if " @ " in requirement] + if urls: + raise PackageError( + f"PyPI refuses a package whose dependency is a URL: {'; '.join(urls)}. Publish that dependency on PyPI first and name it with a version, " + "or keep the URL in requirements.txt only (the installer still installs it)." + ) + if sdist.get_all("Requires-Dist") is not None and sorted(sdist.get_all("Requires-Dist") or []) != sorted(requirements): + raise PackageError( + "The wheel built from the source distribution has other dependencies than the source distribution: a file setup.py reads is not in " + "the source distribution (requirements.txt needs `include requirements.txt` in MANIFEST.in)." + ) + return [path.name for path in files] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Build and check the app's Python package (src/) for PyPI. Nothing is uploaded.") + parser.add_argument("--version", required=True, help="The release version, X.Y.Z (the tag without the v)") + parser.add_argument("--index", default="pypi", choices=sorted(INDEXES), help="pypi (or true, the value of PYPI_PUBLISH) or testpypi") + parser.add_argument("--outdir", default="dist", help="Where to write the two files (default: dist)") + parser.add_argument("--repository-root", default=str(ROOT)) + parser.add_argument("--github-output", help="Append name=, version= and url= to this file (GitHub Actions $GITHUB_OUTPUT)") + parser.add_argument("--no-isolation", action="store_true", help="Build with the setuptools already installed (no download)") + parser.add_argument("--offline", action="store_true", help="Do not ask the index whether the version exists") + args = parser.parse_args(argv) + + root = Path(args.repository_root).resolve() + outdir = Path(args.outdir) + if not outdir.is_absolute(): + outdir = root / outdir + label, address = INDEXES[args.index] + try: + name = prepare(root, args.version, args.index, check_index=not args.offline) + for line in left_out_requirements(root): + print(f"WARNING: the line `{line}` of requirements.txt is not a dependency the package can declare on {label} (an option, a path or a URL): left out.") + if outdir.exists() and any(outdir.iterdir()): + raise PackageError(f"{outdir} is not empty: remove it first, so that only this release is uploaded.") + build(root, outdir, isolation=not args.no_isolation) + files = check(outdir, name, args.version) + except PackageError as exc: + print(f"::error title=The package cannot be published::{exc}", file=sys.stderr) + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + url = f"{address}/project/{normalise(name)}/{args.version}/" + print(f"Built {name} {args.version} for {label}: {', '.join(files)}. The next job uploads it: {url}") + if args.github_output: + with open(args.github_output, "a", encoding="utf-8") as handle: + handle.write(f"name={name}\nversion={args.version}\nurl={url}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.template_sync/resources/v0_1_24_to_v0_1_25/MANIFEST.in b/.template_sync/resources/v0_1_24_to_v0_1_25/MANIFEST.in new file mode 100644 index 0000000..757a970 --- /dev/null +++ b/.template_sync/resources/v0_1_24_to_v0_1_25/MANIFEST.in @@ -0,0 +1,3 @@ +# The source distribution of the app's Python package (setup.py, src/, README.md, LICENSE) also carries requirements.txt: +# setup.py reads the dependencies from it when pip builds the package from the source distribution. +include requirements.txt diff --git a/.template_sync/resources/v0_1_24_to_v0_1_25/setup.py b/.template_sync/resources/v0_1_24_to_v0_1_25/setup.py new file mode 100644 index 0000000..ac5f8f4 --- /dev/null +++ b/.template_sync/resources/v0_1_24_to_v0_1_25/setup.py @@ -0,0 +1,57 @@ +"""The Python package of the app: the code in src/ (see .tools/docs/external_code_upload.md). + +The installer installs it into the app. With the repository variable PYPI_PUBLISH, .github/workflows/publish_pypi.yml also +publishes it on PyPI at every release, with the version of the release. The package needs what requirements.txt lists, and +its description on PyPI is README.md. Both files are read when the package is built; they may be missing (the installer). +""" +import os +import re +from pathlib import Path + +from setuptools import find_packages, setup + +HERE = Path(__file__).resolve().parent +# A requirement: a project, optional extras, optional version specifiers, an optional environment marker. +REQUIREMENT_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*\s*(\[[A-Za-z0-9._,\s-]*\])?\s*([<>=!~]=?\s*[^\s,;]+(\s*,\s*[<>=!~]=?\s*[^\s,;]+)*)?\s*(;.+)?") + + +def is_requirement(line): + """True for a requirement setuptools accepts; options (-r, --index-url), paths and URLs are not requirements.""" + try: + from packaging.requirements import InvalidRequirement, Requirement + except ImportError: + return REQUIREMENT_RE.fullmatch(line) is not None + try: + return Requirement(line).url is None + except InvalidRequirement: + return False + + +def read_requirements(): + """The requirements in requirements.txt, one per line (comments and anything else left out); none without the file.""" + path = HERE / "requirements.txt" + if not path.is_file(): + return [] + lines = (line.split("#", 1)[0].strip() for line in path.read_text(encoding="utf-8").splitlines()) + return [line for line in lines if line and is_requirement(line)] + + +def read_readme(): + path = HERE / "README.md" + return path.read_text(encoding="utf-8") if path.is_file() else "" + + +# The repository on GitHub, when the package is built by a workflow (GitHub sets GITHUB_REPOSITORY). +REPOSITORY = os.environ.get("GITHUB_REPOSITORY") + +setup( + name="PYTHON_PROJ_NAME", + version="VERSION_NUMBER", + long_description=read_readme(), + long_description_content_type="text/markdown", + project_urls={"Source": f"/{REPOSITORY}"} if REPOSITORY else {}, + package_dir={"": "src"}, + packages=find_packages(where="src"), + python_requires=">=PYTHON_VERSION", + install_requires=read_requirements(), +) diff --git a/.template_sync/resources/v0_1_25_to_v0_1_26/.tools/docs/external_code_upload_pypi.md b/.template_sync/resources/v0_1_25_to_v0_1_26/.tools/docs/external_code_upload_pypi.md new file mode 100644 index 0000000..15022aa --- /dev/null +++ b/.template_sync/resources/v0_1_25_to_v0_1_26/.tools/docs/external_code_upload_pypi.md @@ -0,0 +1,52 @@ +## 4. Publish the code on PyPI (optional) + +The installer always puts your package into the app. You can also publish it on [PyPI](https://pypi.org), so that anyone can install it with `pip install`, in Google Colab or in their own Python. Each release then does both, with the same version: the GitHub Actions workflow `.github/workflows/publish_pypi.yml` builds the package and uploads it when you publish a release, next to the installers. + +It is **off until you turn it on**, and GitHub stores no password or token for it: PyPI trusts this repository's workflow directly ([trusted publishing](https://docs.pypi.org/trusted-publishers/)). + +**What is published** + +| | | +|---|---| +| Name | The `name=` of `setup.py`: the name of the package, given when the repository was initialised (unless you changed it). If `labconstrictor-app.yaml` has `distribution.pip`, it must be the same name. | +| Version | The release tag without the `v`: release `v1.2.0` publishes 1.2.0. | +| Code | Every package in `src/` (a folder with an `__init__.py`). | +| Dependencies | Each line of `requirements.txt`, as it is written (exact versions included). Options (`-r`, `--extra-index-url`), paths and URLs are left out, with a warning in the log: PyPI refuses dependencies given as URLs. | +| Description | `README.md` of the repository, shown on the PyPI page. | +| License | The `LICENSE` file. | + +### One-time setup + +1. **Check that the name is free.** Open `https://pypi.org/project//`. A "404 Not Found" page means nobody uses it. If it is taken, change `name=` in `setup.py` (and `distribution.pip`) and commit. +2. **Register this workflow on PyPI.** Log in to [pypi.org](https://pypi.org) (create an account if you have none; PyPI asks for two-factor authentication). Open your account menu > **Your projects** > **Publishing** ([pypi.org/manage/account/publishing](https://pypi.org/manage/account/publishing/)). Under **Add a new pending publisher**, choose the **GitHub** tab. The form has five fields; fill them in exactly: + - **PyPI Project Name:** the name of the package. + - **Owner:** the GitHub user or organisation that owns the repository (`CellMigrationLab` in `github.com/CellMigrationLab/MyApp`). + - **Repository name:** the repository (`MyApp`). + - **Workflow name:** `publish_pypi.yml` + - **Environment name:** `pypi` + + Click **Add**. The publisher appears under **Pending publishers** below the form. It does not reserve the name: the project is created by your first release. If the project already exists on PyPI and you own it, add the publisher in the project instead: **Your projects** > the project > **Manage** > **Publishing**. +3. **Turn it on in GitHub.** In the repository, open **Settings** > **Secrets and variables** > **Actions**, then the **Variables** tab (not **Secrets**: a secret cannot turn a workflow on). Click **New repository variable**, type `PYPI_PUBLISH` as the name and `true` as the value, and click **Add variable**. +4. **Publish a release** as usual ([Create executable installers](executable_creation.md)). In the **Actions** tab, the workflow **Publish the Python package to PyPI** runs next to **Build Release Installers**. When both jobs are green, the package is at `https://pypi.org/project//`. The upload job shows the link too. + +### Try it on TestPyPI first (optional) + +[TestPyPI](https://test.pypi.org) is a separate copy of PyPI for trials, with its own account. Add the same pending publisher at [test.pypi.org/manage/account/publishing](https://test.pypi.org/manage/account/publishing/), give the variable `PYPI_PUBLISH` the value `testpypi`, and publish a release. Install the result with: + +```bash +pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ +``` + +Then change the variable to `true`: the next release goes to PyPI. + +### Turn it off + +Delete the variable `PYPI_PUBLISH` (**Settings** > **Secrets and variables** > **Actions** > **Variables**). The workflow stays in the repository and is skipped at each release (a grey icon in the **Actions** tab, not a failure). What was published stays on PyPI. + +### Good to know + +- **A version can be uploaded only once**, even after it was deleted from PyPI. The workflow checks this before it builds anything and stops with a clear message; to publish a fix, make a new release with a higher version. +- **The release writes its version into `setup.py`** (the "Update version" commit of the release workflow), so the file and PyPI agree. +- **To approve each upload by hand**, open **Settings** > **Environments** > **pypi** (GitHub creates it at the first run) and add **Required reviewers**. +- **A repository that already publishes to PyPI with a workflow of its own** does not get `publish_pypi.yml` from the template update, so nothing is uploaded twice. See [the design note of template 0.1.25](https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_25.md). +- If the workflow fails, see [Troubleshooting](troubleshooting.md#publishing-to-pypi-fails). diff --git a/.template_sync/resources/v0_1_25_to_v0_1_26/.tools/docs/troubleshooting_pypi.md b/.template_sync/resources/v0_1_25_to_v0_1_26/.tools/docs/troubleshooting_pypi.md new file mode 100644 index 0000000..a05685f --- /dev/null +++ b/.template_sync/resources/v0_1_25_to_v0_1_26/.tools/docs/troubleshooting_pypi.md @@ -0,0 +1,18 @@ +## Publishing to PyPI fails + +The workflow **Publish the Python package to PyPI** (`publish_pypi.yml`) runs at a release only when the repository variable `PYPI_PUBLISH` is `true` or `testpypi` (see [Upload external code](external_code_upload.md#4-publish-the-code-on-pypi-optional)). Its first job checks and builds the package; its second job uploads it. The installers do not depend on it: a failure here leaves the release and its installers as they are. + +| What you see | What to do | +|---|---| +| Both jobs are grey (skipped) | The variable is missing or has another value. It must be a **variable** (Settings > Secrets and variables > Actions > **Variables**), not a secret, named `PYPI_PUBLISH`, with the value `true` or `testpypi` exactly. | +| `invalid-publisher: valid token, but no corresponding publisher` | The trusted publisher on pypi.org (or test.pypi.org for `testpypi`) does not match this workflow. Check the five fields: owner, repository, workflow `publish_pypi.yml`, environment `pypi`, and the project name. Then re-run the failed job (Actions > the run > **Re-run failed jobs**). | +| `... isn't allowed to upload to project ...` (403) | The name belongs to another PyPI project that this repository is not a publisher of. If it is yours, add the publisher in that project's **Publishing** page; if not, choose another `name=` in `setup.py` (and `distribution.pip`). | +| `PyPI already has ` (or `File already exists`) | That version was uploaded before; a version can be uploaded only once, even after it was deleted. Publish a new release with a higher version. | +| `There is no Python package in src/` | Add your code as a folder with an `__init__.py` in `src/` (see [Upload external code](external_code_upload.md)), or delete the variable `PYPI_PUBLISH`. | +| `... also publishes to PyPI` | Another workflow of the repository uploads to PyPI too. Keep one: delete the variable `PYPI_PUBLISH`, or delete the other workflow. | +| `distribution.pip of labconstrictor-app.yaml is ..., but setup.py names the package ...` | Make the two names the same. | +| `PyPI refuses a package whose dependency is a URL` | `setup.py` lists a dependency as `name @ git+https://...`. Publish that dependency on PyPI and name it with a version, or move the URL to `requirements.txt` (the installer still installs it; the PyPI package leaves it out). | +| `setup.py still has the template placeholder ...` | Replace the placeholder in `setup.py` (the log says which), commit, and publish a new release. | +| The upload job waits | The `pypi` environment has required reviewers (Settings > Environments > pypi): one of them approves the upload in the run. | + +After a fix, re-run the failed jobs of the run if nothing in the repository changed (a publisher fixed on pypi.org); otherwise publish a new release. See [the design note of template 0.1.25](https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_25.md). diff --git a/.template_sync/resources/v0_1_26_to_v0_1_27/.tools/docs/app_manifest_host_requirements.md b/.template_sync/resources/v0_1_26_to_v0_1_27/.tools/docs/app_manifest_host_requirements.md new file mode 100644 index 0000000..4277621 --- /dev/null +++ b/.template_sync/resources/v0_1_26_to_v0_1_27/.tools/docs/app_manifest_host_requirements.md @@ -0,0 +1,18 @@ +## Plugin versions your tools need (optional) + +If your tools rely on a fix or a feature of the napari, Fiji or QuPath plugin that older plugins lack, say so with `host_requirements`. People with an older plugin keep seeing your app, with one sentence saying which plugin to update and where; your app page on the App Centre lists the versions under "Plugin versions". + +```yaml +host_requirements: + qupath: + min_version: 0.1.0-alpha.6 # a release tag of the plugin, with or without v + features: [shapes_owner_v1] # optional: a host feature id + why: Replace keeps your own annotations # optional: one line shown to the person + fiji: + min_version: 0.1.0-alpha.6 +``` + +- Name only `napari`, `fiji` or `qupath`: the notebook ships with your app and needs no plugin. +- Each needs `min_version`, `features`, or both. `required: true` makes an older plugin refuse your tools instead of warning; use it only when they really fail there. +- Write it as a block, as above. The installer passes this file to `labconstrictor-tools register`, which refuses a malformed block (a misspelt host or key, a version like `latest`); `labconstrictor-tools check --module _lc_tools --app-manifest labconstrictor-app.yaml` shows what it read. +- The feature ids and the rule are in the [LabConstrictor-Tools documentation](https://github.com/CellMigrationLab/LabConstrictor-Tools/blob/main/docs/PROTOCOL.md#host-requirements-which-plugin-an-app-needs). diff --git a/.template_sync/resources/v0_1_26_to_v0_1_27/.tools/schema/labconstrictor-app.schema.json b/.template_sync/resources/v0_1_26_to_v0_1_27/.tools/schema/labconstrictor-app.schema.json new file mode 100644 index 0000000..30ae4ae --- /dev/null +++ b/.template_sync/resources/v0_1_26_to_v0_1_27/.tools/schema/labconstrictor-app.schema.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://labconstrictor.cellmig.org/schema/labconstrictor-app.schema.json", + "title": "labconstrictor-app.yaml", + "description": "Manifest that an app keeps in the root of its own repository so that it can be listed in the LabConstrictor App Centre. Edit it with schema support: add `# yaml-language-server: $schema=https://labconstrictor.cellmig.org/schema/labconstrictor-app.schema.json` as the first line.", + "type": "object", + "additionalProperties": false, + "required": ["schema", "name", "summary", "kind", "distribution"], + "properties": { + "schema": {"const": 1, "description": "Manifest format version."}, + "name": {"type": "string", "minLength": 2, "maxLength": 60, "description": "Display name, as users know the app."}, + "summary": {"type": "string", "minLength": 20, "maxLength": 200, "description": "One sentence shown on the app card. Plain text."}, + "description": {"type": "string", "maxLength": 2000, "description": "A few plain-text paragraphs shown on the app page; separate paragraphs with a blank line. No HTML or Markdown."}, + "quickstart": { + "type": "array", "minItems": 1, "maxItems": 6, "items": {"type": "string", "minLength": 5, "maxLength": 200}, + "description": "Up to six short plain-text steps for a first run, shown on the app page. Link to fuller documentation with docs_url." + }, + "kind": {"enum": ["analysis", "segmentation", "simulation", "game", "training", "utility"], "description": "What the app is for; drives the filter on the App Centre."}, + "hosts": { + "type": "array", "uniqueItems": true, + "items": {"enum": ["notebook", "napari", "fiji", "qupath"]}, + "description": "Where the app's tools can be used besides its own window (the LabConstrictor-Tools bridge)." + }, + "host_requirements": { + "type": "object", "additionalProperties": false, + "description": "The oldest LabConstrictor plugin of each host the app needs. labconstrictor-tools register copies it into the app's registry entry; a host whose plugin is older lists the app with a sentence saying which plugin to update (or refuses it with required: true). The notebook needs no plugin.", + "properties": { + "napari": {"$ref": "#/$defs/hostRequirement"}, + "fiji": {"$ref": "#/$defs/hostRequirement"}, + "qupath": {"$ref": "#/$defs/hostRequirement"} + } + }, + "distribution": { + "type": "object", "additionalProperties": false, + "anyOf": [{"required": ["installer"]}, {"required": ["colab"]}, {"required": ["pip"]}], + "dependentRequired": {"platforms": ["installer"]}, + "description": "How people get the app. At least one route is required.", + "properties": { + "installer": {"const": true, "description": "The latest GitHub release carries LabConstrictor installers (Name-version-OS-arch.ext)."}, + "colab": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/link"}, "description": "Google Colab notebooks."}, + "platforms": { + "type": "array", "minItems": 1, "uniqueItems": true, "items": {"enum": ["windows-x86_64", "windows-arm64", "macos-arm64", "macos-x86_64", "linux-x86_64", "linux-arm64"]}, + "description": "The systems you support. Only these installers are offered; leave it out to offer every installer of your release." + }, + "pip": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$(?!\\n)", "description": "Name of the package on PyPI."} + } + }, + "authors": {"type": "array", "items": {"$ref": "#/$defs/author"}}, + "license": {"type": "string", "minLength": 1, "description": "SPDX identifier, for example MIT."}, + "inputs": {"$ref": "#/$defs/facts", "description": "What the app accepts, as a short list (file formats, dimensions, units it needs). A summary, not README copy; use docs_url for the rest."}, + "outputs": {"$ref": "#/$defs/facts", "description": "What the app produces, as a short list (images, masks, tables, plots). A summary, not README copy; use docs_url for the rest."}, + "limitations": {"$ref": "#/$defs/facts", "description": "What the app cannot do or where it may be unreliable, as a short list."}, + "status": {"enum": ["Experimental", "Available", "Deprecated"], "description": "How mature the app is, in the same words the Ecosystem page uses for projects."}, + "categories": {"type": "array", "uniqueItems": true, "maxItems": 6, "items": {"type": "string", "maxLength": 30}, "description": "Words a scientist would search for: tasks, synonyms, methods, tools."}, + "doi": {"type": "string", "pattern": "^10\\.\\d{4,9}/\\S+$(?!\\n)", "description": "DOI of the paper to cite (not a software archive)."}, + "homepage": {"$ref": "#/$defs/https"}, + "docs_url": {"$ref": "#/$defs/https"}, + "icon": {"$ref": "#/$defs/repoPath", "description": "Square PNG in the repository (SVG is not accepted: it can carry scripts)."}, + "screenshots": { + "type": "array", "minItems": 1, "maxItems": 6, + "items": { + "type": "object", "additionalProperties": false, "required": ["path", "caption"], + "properties": {"path": {"$ref": "#/$defs/repoPath"}, "caption": {"type": "string", "minLength": 3, "maxLength": 140}} + }, + "description": "Images in the repository; captions double as alt text." + } + }, + "$defs": { + "hostRequirement": { + "type": "object", "additionalProperties": false, + "anyOf": [{"required": ["min_version"]}, {"required": ["features"]}], + "properties": { + "min_version": {"type": "string", "pattern": "^v?[0-9]+(\\.[0-9]+)*([-_.]?(a|alpha|b|beta|c|rc|pre|preview)[-_.]?[0-9]*)?([-_.]?(post|rev|r)[-_.]?[0-9]*)?([-_.]?dev[-_.]?[0-9]*)?$", "description": "A release tag of the plugin, with or without v (0.1.0-alpha.6, v0.1.0, 0.2.0rc1). Quote it if YAML could read it as a number (\"0.2\")."}, + "features": {"type": "array", "uniqueItems": true, "minItems": 1, "items": {"type": "string", "pattern": "^[a-z][a-z0-9_]*$"}, "description": "Host feature ids the app needs (LabConstrictor-Tools docs/HOST_FEATURES.md), for example shapes_owner_v1."}, + "why": {"type": "string", "minLength": 3, "maxLength": 160, "pattern": "^[^\\n]+$", "description": "One line shown to the person, saying what the newer plugin brings."}, + "required": {"type": "boolean", "description": "true: an older plugin refuses the app's tools instead of warning. Use it only when the tools really fail."} + } + }, + "facts": {"type": "array", "minItems": 1, "maxItems": 6, "items": {"type": "string", "minLength": 5, "maxLength": 200}}, + "https": {"type": "string", "pattern": "^https://[^\\s/@]+(/[^\\s]*)?$(?!\\n)"}, + "repoPath": {"type": "string", "pattern": "^(?!/)(?!.*\\.\\.)[A-Za-z0-9_./ -]+\\.(png|jpg|jpeg|gif)$(?!\\n)", "description": "Path inside the repository, no leading slash."}, + "link": { + "type": "object", "additionalProperties": false, "required": ["label", "url"], + "properties": {"label": {"type": "string", "minLength": 2, "maxLength": 60}, "url": {"type": "string", "pattern": "^https://colab\\.research\\.google\\.com/[^\\s]+$(?!\\n)"}} + }, + "author": { + "type": "object", "additionalProperties": false, "required": ["name"], + "properties": {"name": {"type": "string", "minLength": 2}, "orcid": {"type": "string", "pattern": "^\\d{4}-\\d{4}-\\d{4}-\\d{3}[\\dX]$(?!\\n)"}} + } + } +} diff --git a/.tools/bash/create_constructor_environment.sh b/.tools/bash/create_constructor_environment.sh index 7a1813a..1e8ed9e 100644 --- a/.tools/bash/create_constructor_environment.sh +++ b/.tools/bash/create_constructor_environment.sh @@ -15,4 +15,9 @@ if ! command -v conda >/dev/null 2>&1; then fi fi -conda create -y -n "${CONSTRUCTOR_ENV:?}" -c conda-forge -c defaults python=3.11 constructor +# conda-standalone (the installer's _conda.exe) older than 26.9.3 writes package +# records without paths_data when constructor disables conda's safety checks. +# conda 26.9 and later in the installed app then cannot find the Windows +# launcher stub, and `conda init --reverse` in the uninstaller asks for admin +# rights (conda/conda#16856, fixed in conda 26.9.3). +conda create -y -n "${CONSTRUCTOR_ENV:?}" -c conda-forge -c defaults python=3.11 constructor "conda-standalone>=26.9.3" diff --git a/.tools/bash/test_conda_environment.sh b/.tools/bash/test_conda_environment.sh index 1450f7e..4742aca 100644 --- a/.tools/bash/test_conda_environment.sh +++ b/.tools/bash/test_conda_environment.sh @@ -48,21 +48,34 @@ echo "Activating test environment..." # Note: In CI, conda/mamba environments need to be activated via conda run # or by sourcing the activation script -# Test pip installation of requirements.txt -echo "" -echo "Installing requirements from requirements.txt..." +# Test pip installation of requirements.txt, or of its lock file (requirements.lock.txt, written by +# .tools/python/lock_requirements.py) when there is one: the installer installs the lock file, so that is what is tested. if [ ! -f "requirements.txt" ]; then echo "ERROR: requirements.txt not found!" mamba env remove -n "$TEST_ENV_NAME" -y exit 1 fi +REQUIREMENTS_FILE="requirements.txt" +if [ -f "requirements.lock.txt" ]; then + REQUIREMENTS_FILE="requirements.lock.txt" +fi +echo "" +echo "Installing requirements from $REQUIREMENTS_FILE..." -# Check if requirements.txt is not empty -if [ ! -s "requirements.txt" ]; then - echo "WARNING: requirements.txt is empty, skipping pip install" +# Check if the requirements file is not empty +if [ ! -s "$REQUIREMENTS_FILE" ]; then + echo "WARNING: $REQUIREMENTS_FILE is empty, skipping pip install" else echo "Installing packages with pip..." - mamba run -n "$TEST_ENV_NAME" pip install --prefer-binary -r requirements.txt + mamba run -n "$TEST_ENV_NAME" pip install --prefer-binary -r "$REQUIREMENTS_FILE" +fi + +# Report dependency conflicts between the pip packages and the conda packages of the environment. Only a warning: a conflict +# among the conda packages themselves must not block the notebooks. +echo "" +echo "Checking the installed packages for conflicts (pip check)..." +if ! mamba run -n "$TEST_ENV_NAME" pip check; then + echo "::warning::pip check found dependency conflicts in the test environment (see the step log); the installer may behave differently than expected." fi # Verify installation by checking Python and key packages diff --git a/.tools/docs/app_manifest.md b/.tools/docs/app_manifest.md index a7308f5..4c1def7 100644 --- a/.tools/docs/app_manifest.md +++ b/.tools/docs/app_manifest.md @@ -38,6 +38,25 @@ icon: app/logo/logo.png # optional: a square PNG in your repository The first line makes editors with YAML support (for example VS Code) check the file as you type. `inputs`, `outputs` and `limitations` are short summaries for the app page (a list of one to six sentences, not README copy). More optional fields exist (authors, DOI, license, screenshots, a quick start); the full list is in the [schema](https://labconstrictor.cellmig.org/schema/labconstrictor-app.schema.json). Everything is plain text: no HTML or Markdown. +## Plugin versions your tools need (optional) + +If your tools rely on a fix or a feature of the napari, Fiji or QuPath plugin that older plugins lack, say so with `host_requirements`. People with an older plugin keep seeing your app, with one sentence saying which plugin to update and where; your app page on the App Centre lists the versions under "Plugin versions". + +```yaml +host_requirements: + qupath: + min_version: 0.1.0-alpha.6 # a release tag of the plugin, with or without v + features: [shapes_owner_v1] # optional: a host feature id + why: Replace keeps your own annotations # optional: one line shown to the person + fiji: + min_version: 0.1.0-alpha.6 +``` + +- Name only `napari`, `fiji` or `qupath`: the notebook ships with your app and needs no plugin. +- Each needs `min_version`, `features`, or both. `required: true` makes an older plugin refuse your tools instead of warning; use it only when they really fail there. +- Write it as a block, as above. The installer passes this file to `labconstrictor-tools register`, which refuses a malformed block (a misspelt host or key, a version like `latest`); `labconstrictor-tools check --module _lc_tools --app-manifest labconstrictor-app.yaml` shows what it read. +- The feature ids and the rule are in the [LabConstrictor-Tools documentation](https://github.com/CellMigrationLab/LabConstrictor-Tools/blob/main/docs/PROTOCOL.md#host-requirements-which-plugin-an-app-needs). + ## The automatic check The **Check App Centre manifest** workflow runs when you change the file and when you publish a release: diff --git a/.tools/docs/external_code_upload.md b/.tools/docs/external_code_upload.md index 87448b3..8725d9e 100644 --- a/.tools/docs/external_code_upload.md +++ b/.tools/docs/external_code_upload.md @@ -60,6 +60,59 @@ from vlab4mic import my_script from vlab4mic.subpackage import submodule1 ``` +## 4. Publish the code on PyPI (optional) + +The installer always puts your package into the app. You can also publish it on [PyPI](https://pypi.org), so that anyone can install it with `pip install`, in Google Colab or in their own Python. Each release then does both, with the same version: the GitHub Actions workflow `.github/workflows/publish_pypi.yml` builds the package and uploads it when you publish a release, next to the installers. + +It is **off until you turn it on**, and GitHub stores no password or token for it: PyPI trusts this repository's workflow directly ([trusted publishing](https://docs.pypi.org/trusted-publishers/)). + +**What is published** + +| | | +|---|---| +| Name | The `name=` of `setup.py`: the name of the package, given when the repository was initialised (unless you changed it). If `labconstrictor-app.yaml` has `distribution.pip`, it must be the same name. | +| Version | The release tag without the `v`: release `v1.2.0` publishes 1.2.0. | +| Code | Every package in `src/` (a folder with an `__init__.py`). | +| Dependencies | Each line of `requirements.txt`, as it is written (exact versions included). Options (`-r`, `--extra-index-url`), paths and URLs are left out, with a warning in the log: PyPI refuses dependencies given as URLs. | +| Description | `README.md` of the repository, shown on the PyPI page. | +| License | The `LICENSE` file. | + +### One-time setup + +1. **Check that the name is free.** Open `https://pypi.org/project//`. A "404 Not Found" page means nobody uses it. If it is taken, change `name=` in `setup.py` (and `distribution.pip`) and commit. +2. **Register this workflow on PyPI.** Log in to [pypi.org](https://pypi.org) (create an account if you have none; PyPI asks for two-factor authentication). Open your account menu > **Your projects** > **Publishing** ([pypi.org/manage/account/publishing](https://pypi.org/manage/account/publishing/)). Under **Add a new pending publisher**, choose the **GitHub** tab. The form has five fields; fill them in exactly: + - **PyPI Project Name:** the name of the package. + - **Owner:** the GitHub user or organisation that owns the repository (`CellMigrationLab` in `github.com/CellMigrationLab/MyApp`). + - **Repository name:** the repository (`MyApp`). + - **Workflow name:** `publish_pypi.yml` + - **Environment name:** `pypi` + + Click **Add**. The publisher appears under **Pending publishers** below the form. It does not reserve the name: the project is created by your first release. If the project already exists on PyPI and you own it, add the publisher in the project instead: **Your projects** > the project > **Manage** > **Publishing**. +3. **Turn it on in GitHub.** In the repository, open **Settings** > **Secrets and variables** > **Actions**, then the **Variables** tab (not **Secrets**: a secret cannot turn a workflow on). Click **New repository variable**, type `PYPI_PUBLISH` as the name and `true` as the value, and click **Add variable**. +4. **Publish a release** as usual ([Create executable installers](executable_creation.md)). In the **Actions** tab, the workflow **Publish the Python package to PyPI** runs next to **Build Release Installers**. When both jobs are green, the package is at `https://pypi.org/project//`. The upload job shows the link too. + +### Try it on TestPyPI first (optional) + +[TestPyPI](https://test.pypi.org) is a separate copy of PyPI for trials, with its own account. Add the same pending publisher at [test.pypi.org/manage/account/publishing](https://test.pypi.org/manage/account/publishing/), give the variable `PYPI_PUBLISH` the value `testpypi`, and publish a release. Install the result with: + +```bash +pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ +``` + +Then change the variable to `true`: the next release goes to PyPI. + +### Turn it off + +Delete the variable `PYPI_PUBLISH` (**Settings** > **Secrets and variables** > **Actions** > **Variables**). The workflow stays in the repository and is skipped at each release (a grey icon in the **Actions** tab, not a failure). What was published stays on PyPI. + +### Good to know + +- **A version can be uploaded only once**, even after it was deleted from PyPI. The workflow checks this before it builds anything and stops with a clear message; to publish a fix, make a new release with a higher version. +- **The release writes its version into `setup.py`** (the "Update version" commit of the release workflow), so the file and PyPI agree. +- **To approve each upload by hand**, open **Settings** > **Environments** > **pypi** (GitHub creates it at the first run) and add **Required reviewers**. +- **A repository that already publishes to PyPI with a workflow of its own** does not get `publish_pypi.yml` from the template update, so nothing is uploaded twice. See [the design note of template 0.1.25](https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_25.md). +- If the workflow fails, see [Troubleshooting](troubleshooting.md#publishing-to-pypi-fails). + ---
diff --git a/.tools/docs/notebook_upload.md b/.tools/docs/notebook_upload.md index bd468e2..228f4c8 100644 --- a/.tools/docs/notebook_upload.md +++ b/.tools/docs/notebook_upload.md @@ -109,7 +109,9 @@ try: content = version_response.json()['content'] decoded_content = base64.b64decode(content).decode('utf-8') config = yaml.safe_load(decoded_content) - latest_version = config.get(notebook_name, "") + # notebook_latest_versions.yaml groups the notebooks by folder: {"": {"": ""}} + latest_version = next((str(group[notebook_name]) for group in (config or {}).values() + if isinstance(group, dict) and notebook_name in group), "") output_area.value = (f"Notebook version: `{current_version}`
" f"Latest version available: `{latest_version}`
") diff --git a/.tools/docs/tools_bridge.md b/.tools/docs/tools_bridge.md index 14e69dc..0375ff5 100644 --- a/.tools/docs/tools_bridge.md +++ b/.tools/docs/tools_bridge.md @@ -28,13 +28,13 @@ command line, with forms generated automatically from the function signatures. N Installing Tools before the import check is what lets your package import `labconstrictor_tools` at module level. A failed Tools install is a `WARNING` in `menuinst_debug.log` and the registration is skipped; the installation continues, unless your package needs Tools to be imported (then the import check fails, as it should). **Where `labconstrictor-tools` comes from.** It is not on PyPI yet, so by default the installer downloads the GitHub archive of a **pinned release tag**, - `https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/v0.3.0-alpha.1.zip` (a plain zip: git is not needed on the user's computer). + `https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/v0.3.0-alpha.2.zip` (a plain zip: git is not needed on the user's computer). Set the environment variable `LC_TOOLS_SPEC` when the installer runs to install it from somewhere else (a wheel, a git URL, a mirror, or the name `labconstrictor-tools` once it is published on PyPI); it overrides the pin completely. **Pinned (important).** Two installers built from the same app release get the same version of the tools package, and a change to Tools `main` does not reach new installations until the pin is moved on purpose. - **How to move the pin.** The tag name is written in one place per script: the variable `LC_TOOLS_PIN` (`LC_TOOLS_PIN="v0.3.0-alpha.1"` in - `app/bash_bat_scripts/post_install.sh`, `SET "LC_TOOLS_PIN=v0.3.0-alpha.1"` in `post_install.bat`). To use a new release: create and test the tag in + **How to move the pin.** The tag name is written in one place per script: the variable `LC_TOOLS_PIN` (`LC_TOOLS_PIN="v0.3.0-alpha.2"` in + `app/bash_bat_scripts/post_install.sh`, `SET "LC_TOOLS_PIN=v0.3.0-alpha.2"` in `post_install.bat`). To use a new release: create and test the tag in LabConstrictor-Tools (procedure in its `docs/OPERATIONS.md`), change that string in both scripts and in the copies under `.template_sync/resources/` of the latest migration (the test `test_resources_are_identical_to_the_template_files` fails when they differ), update the tag in this paragraph, and bump `template_version` so apps receive the change. Apps already migrated need a new migration to change the diff --git a/.tools/docs/troubleshooting.md b/.tools/docs/troubleshooting.md index 16bc314..ad1192f 100644 --- a/.tools/docs/troubleshooting.md +++ b/.tools/docs/troubleshooting.md @@ -70,3 +70,22 @@ LabConstrictor-based applications also accept the shared `LABCONSTRICTOR_CA_BUNDLE` variable. Set either variable to an organization PEM bundle before installation or launch when a private certificate authority is required. + +## Publishing to PyPI fails + +The workflow **Publish the Python package to PyPI** (`publish_pypi.yml`) runs at a release only when the repository variable `PYPI_PUBLISH` is `true` or `testpypi` (see [Upload external code](external_code_upload.md#4-publish-the-code-on-pypi-optional)). Its first job checks and builds the package; its second job uploads it. The installers do not depend on it: a failure here leaves the release and its installers as they are. + +| What you see | What to do | +|---|---| +| Both jobs are grey (skipped) | The variable is missing or has another value. It must be a **variable** (Settings > Secrets and variables > Actions > **Variables**), not a secret, named `PYPI_PUBLISH`, with the value `true` or `testpypi` exactly. | +| `invalid-publisher: valid token, but no corresponding publisher` | The trusted publisher on pypi.org (or test.pypi.org for `testpypi`) does not match this workflow. Check the five fields: owner, repository, workflow `publish_pypi.yml`, environment `pypi`, and the project name. Then re-run the failed job (Actions > the run > **Re-run failed jobs**). | +| `... isn't allowed to upload to project ...` (403) | The name belongs to another PyPI project that this repository is not a publisher of. If it is yours, add the publisher in that project's **Publishing** page; if not, choose another `name=` in `setup.py` (and `distribution.pip`). | +| `PyPI already has ` (or `File already exists`) | That version was uploaded before; a version can be uploaded only once, even after it was deleted. Publish a new release with a higher version. | +| `There is no Python package in src/` | Add your code as a folder with an `__init__.py` in `src/` (see [Upload external code](external_code_upload.md)), or delete the variable `PYPI_PUBLISH`. | +| `... also publishes to PyPI` | Another workflow of the repository uploads to PyPI too. Keep one: delete the variable `PYPI_PUBLISH`, or delete the other workflow. | +| `distribution.pip of labconstrictor-app.yaml is ..., but setup.py names the package ...` | Make the two names the same. | +| `PyPI refuses a package whose dependency is a URL` | `setup.py` lists a dependency as `name @ git+https://...`. Publish that dependency on PyPI and name it with a version, or move the URL to `requirements.txt` (the installer still installs it; the PyPI package leaves it out). | +| `setup.py still has the template placeholder ...` | Replace the placeholder in `setup.py` (the log says which), commit, and publish a new release. | +| The upload job waits | The `pypi` environment has required reviewers (Settings > Environments > pypi): one of them approves the upload in the run. | + +After a fix, re-run the failed jobs of the run if nothing in the repository changed (a publisher fixed on pypi.org); otherwise publish a new release. See [the design note of template 0.1.25](https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_25.md). diff --git a/.tools/python/bump_constructor.py b/.tools/python/bump_constructor.py index b41251c..3c6b769 100644 --- a/.tools/python/bump_constructor.py +++ b/.tools/python/bump_constructor.py @@ -115,6 +115,37 @@ def ensure_requirements_in_extra_files(construct_data: dict): # Update the construct data with the modified extra_files construct_data["extra_files"] = extra_files +LOCK_FILES = ( + "requirements.lock.txt", + "requirements_gpu.lock.txt", + "requirements-linux.lock.txt", + "requirements-windows.lock.txt", + "requirements-macos.lock.txt", +) + + +def ensure_lock_files_in_extra_files(construct_data: dict) -> None: + """Bundle the lock files (requirements*.lock.txt, written by lock_requirements.py) next to their requirements files. + + post_install installs a lock file instead of its requirements file when the installer bundles one. A mapping whose lock file is gone + (the requirements could not be locked) is removed, because constructor fails on a missing file. + """ + extra_files = construct_data.setdefault("extra_files", []) + project_folder = extract_project_folder(extra_files) + for lock_file in LOCK_FILES: + mapped = [item for item in extra_files if isinstance(item, dict) and lock_file in item] + if Path(lock_file).exists(): + if not mapped: + extra_files.append({lock_file: f"{project_folder}/{lock_file}"}) + debug(f"Added mapping for {lock_file}") + else: + debug(f"{lock_file} mapping already present") + elif mapped: + for item in mapped: + extra_files.remove(item) + debug(f"Removed the mapping of {lock_file}: the file does not exist (the installer installs the requirements file instead)") + + def ensure_extra_files(construct_data: dict, notebooks_root: Path, src_root: Path) -> int: extra_files = construct_data.get("extra_files") if extra_files is None: @@ -269,6 +300,7 @@ def main(): data = load_construct(construct_path) # Check if the requirements files are included in extra_files, if not, add them ensure_requirements_in_extra_files(data) + ensure_lock_files_in_extra_files(data) # Update extra_files with notebooks and src files ntbk_added, src_added = ensure_extra_files(data, notebooks_root, src_root) diff --git a/.tools/python/bump_version.py b/.tools/python/bump_version.py index 2807615..e8e5f10 100644 --- a/.tools/python/bump_version.py +++ b/.tools/python/bump_version.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import argparse +import os import pathlib import re import shutil @@ -20,22 +21,71 @@ ) PKG_VERSION_RE = re.compile(r'^(\s*SET\s+"PKG_VERSION=)(\d+\.\d+\.\d+)("\s*)$', re.MULTILINE) POST_INSTALL = ROOT / "app" / "bash_bat_scripts" / "post_install.bat" +# version="1.2.3" in setup.py: the version of the app's Python package (src/) +SETUP = ROOT / "setup.py" +SETUP_VERSION_RE = re.compile(r"^(?P[ \t]*version\s*=\s*)(?P[\"'])\d+\.\d+\.\d+(?P=quote)", re.MULTILINE) -def replace_version_placeholder(text: str, new_version: str) -> str: +# The folder of a release in a download link: /releases/download// (the tag is 1.2.3 or v1.2.3). +DOWNLOAD_PATH_RE = re.compile(r"/releases/download/v?\d+\.\d+\.\d+/") +# A button of the download page: +DOWNLOAD_BUTTON_RE = re.compile(r'\s*]*>\s*') +NOT_BUILT_NOTE = "> No installer is built for {label}: this app does not support it." + + +def replace_version_placeholder(text: str, new_version: str, tag: str | None = None) -> str: """Render the release version in download documentation. Handles both pristine templates containing ``VERSION_NUMBER`` and initialized repositories where that placeholder was already rendered to an - older explicit version such as ``0.0.0``. + older explicit version such as ``0.0.0``. The links point to the release + tag as it was typed (``v1.2.3`` or ``1.2.3``); the installer names carry + the version without ``v`` (CellMigrationLab/LabConstrictor#41). """ explicit_versions = set( - re.findall(r"/releases/download/(\d+\.\d+\.\d+)/", text) + re.findall(r"/releases/download/v?(\d+\.\d+\.\d+)/", text) ) text = text.replace("VERSION_NUMBER", new_version) for previous_version in explicit_versions: text = text.replace(previous_version, new_version) - return text + return DOWNLOAD_PATH_RE.sub(f"/releases/download/{tag or new_version}/", text) + + +def drop_unbuilt_buttons(text: str, not_built: list[tuple[tuple[str, ...], str]]) -> str: + """Replace the download button of each system the release does not build with a note (CellMigrationLab/LabConstrictor#42). + + ``not_built`` holds (installer name endings, label) pairs, from release_platforms.py. The "or" between + two buttons goes when one of them became a note. + """ + if not not_built: + return text + + def _button(match: re.Match) -> str: + link = match.group(0).split('"', 2)[1] + for endings, label in not_built: + if link.endswith(endings): + return NOT_BUILT_NOTE.format(label=label) + return match.group(0) + + lines = DOWNLOAD_BUTTON_RE.sub(_button, text).split("\n") + note_start = NOT_BUILT_NOTE.split("{", 1)[0] + kept = [] + for index, line in enumerate(lines): + if line.strip() == "or": + before = next((other for other in reversed(lines[:index]) if other.strip()), "") + after = next((other for other in lines[index + 1 :] if other.strip()), "") + if before.startswith(note_start) or after.startswith(note_start): + continue + kept.append(line) + return "\n".join(kept) + + +def not_built_systems() -> list[tuple[tuple[str, ...], str]]: + """(installer name endings, label) of each system left out of distribution.platforms in labconstrictor-app.yaml.""" + sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + import release_platforms + + return [(build.files, build.label) for build in release_platforms.skipped_builds(ROOT)] def replace_version_in_file(file_path: pathlib.Path, old_version: str, new_version: str) -> bool: """Replace all occurrences of old_version with new_version in a file. @@ -142,7 +192,26 @@ def bump_post_install_bat(new_version: str) -> None: POST_INSTALL.write_text(updated, encoding="utf-8") print(f"Updated post_install.bat PKG_VERSION to {new_version}") -def bump_version_in_download_executable_md(new_version: str) -> None: + +def bump_setup_py(new_version: str) -> None: + """The version of the app's Python package in setup.py, when it is written there as X.Y.Z. + + The package in the installer and the one publish_pypi.yml publishes on PyPI then carry the release version. The line + endings of the file are kept. + """ + if not SETUP.exists(): + return + with SETUP.open(encoding="utf-8", newline="") as handle: + text = handle.read() + updated = SETUP_VERSION_RE.sub(lambda m: f"{m.group('head')}{m.group('quote')}{new_version}{m.group('quote')}", text, count=1) + if updated != text: + with SETUP.open("w", encoding="utf-8", newline="") as handle: + handle.write(updated) + print(f"Updated setup.py version to {new_version}") + +def bump_version_in_download_executable_md( + new_version: str, tag: str | None = None, not_built: list[tuple[tuple[str, ...], str]] | None = None +) -> None: possible_download_md_paths = [ ROOT / "docs" / "download_executable.md", ROOT / ".tools" / "docs" / "download_executable.md" @@ -165,7 +234,7 @@ def bump_version_in_download_executable_md(new_version: str) -> None: # This file contains placeholders, update them with the new version text = download_md.read_text(encoding="utf-8") - updated_text = replace_version_placeholder(text, new_version) + updated_text = drop_unbuilt_buttons(replace_version_placeholder(text, new_version, tag), not_built or []) if updated_text != text: download_md.write_text(updated_text, encoding="utf-8") print(f"Updated download_executable.md to version {new_version}") @@ -190,10 +259,17 @@ def main() -> None: nargs="+", help="Additional files to update (e.g., README.md, __init__.py)" ) + parser.add_argument( + "--tag", + default=os.environ.get("RELEASE_TAG") or None, + help="The release tag the download links point to: 0.0.4 or v0.0.4 (default: $RELEASE_TAG, else the version)", + ) args = parser.parse_args() if not re.fullmatch(r"\d+\.\d+\.\d+", args.new_version): sys.exit("Version must look like X.Y.Z") + if args.tag and args.tag not in (args.new_version, f"v{args.new_version}"): + sys.exit(f"The release tag {args.tag} does not match the version {args.new_version} (expected {args.new_version} or v{args.new_version})") text, current = read_current_version() updated_text = bump_construct_text(text, current, args.new_version) @@ -210,8 +286,11 @@ def main() -> None: # Also bump PKG_VERSION in post_install.bat bump_post_install_bat(args.new_version) - # Update download_executable.md - bump_version_in_download_executable_md(args.new_version) + # The version of the app's Python package (setup.py), also the version publish_pypi.yml publishes on PyPI + bump_setup_py(args.new_version) + + # Update download_executable.md: the links point to the release tag, and a system that is not built has no button + bump_version_in_download_executable_md(args.new_version, args.tag, not_built_systems()) # Update additional files if specified if args.files: diff --git a/.tools/python/check_placeholders.py b/.tools/python/check_placeholders.py new file mode 100644 index 0000000..b06559f --- /dev/null +++ b/.tools/python/check_placeholders.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Stop a release whose installer would ship a template placeholder (CellMigrationLab/LabConstrictor#40). + +The repository initialisation (the LabConstrictor web form) replaces the template's placeholders with the app's names. A placeholder +it misses survives into the installer: `import PYTHON_PROJ_NAME` in post_install.sh / .bat made every installer of an app with Python +code in src/ fail at its last step. This script runs in the release workflow after the versioned files are updated and before anything +is committed or built, and fails with the file, the line and the fix when an initialisation placeholder is left in a file the +installer ships (app/**, construct.yaml, setup.py, environment.yaml), or when the repository is the raw, uninitialised template. + +Placeholders that are filled in later on purpose are not checked: BASE_PATH_KEYWORD (at install time), PYTHON_VERSION and +JUPYTERLAB_VERSION (by the requirements workflow). + +Usage: python .tools/python/check_placeholders.py [--repository-root PATH] +""" +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_23.md" + +# The placeholders the initialisation replaces (keys of conversion_table in LabConstrictor_streamlit's placeholders.py, +# plus the launcher icon paths it fills in itself). VERSION_NUMBER is also rewritten by bump_version.py, which runs before this check. +INIT_PLACEHOLDERS = ( + "LOWER_PROJ_NAME", + "PYTHON_PROJ_NAME", + "UNDERSCORED_PROJECT_NAME", + "PROJECT_NAME", + "VERSION_NUMBER", + "WELCOME_IMAGE", + "HEADER_IMAGE", + "ICON_IMAGE", + "GITHUB_OWNER", + "GITHUB_REPO_NAME", + "HIDE_CODE_DISABLED", + "ICON_IMAGE_PATH", + "ICON_ICO_IMAGE_PATH", + "ICON_ICNS_IMAGE_PATH", +) +SHIPPED_FILES = ("construct.yaml", "setup.py", "environment.yaml") +SHIPPED_FOLDERS = ("app",) +WELCOME_NOTEBOOK = "app/menuinst/Welcome.ipynb" +WELCOME_TEMPLATE = ".tools/templates/Welcome_template.ipynb" + + +def placeholder_pattern(name: str) -> re.Pattern[str]: + """A placeholder on its own or followed by '_' (PYTHON_PROJ_NAME_lc_tools), never inside a longer name (UNDERSCORED_PROJECT_NAME).""" + return re.compile(r"(? list[Path]: + files = [root / name for name in SHIPPED_FILES if (root / name).is_file()] + for folder in SHIPPED_FOLDERS: + if (root / folder).is_dir(): + files.extend(sorted(p for p in (root / folder).rglob("*") if p.is_file())) + return files + + +def leftover_placeholders(root: Path) -> list[str]: + problems = [] + for path in shipped_files(root): + try: + text = path.read_text(encoding="utf-8") + except (UnicodeDecodeError, OSError): + continue # images, icons and other binary files + rel = path.relative_to(root).as_posix() + # The Welcome notebook is rebuilt from its template at every release: the fix belongs in the template. + where = f" (fix it in {WELCOME_TEMPLATE}, which this notebook is made from)" if rel == WELCOME_NOTEBOOK else "" + for number, line in enumerate(text.splitlines(), start=1): + for name, pattern in PATTERNS.items(): + if pattern.search(line): + problems.append(f"{rel}:{number}: the template placeholder {name} was never replaced{where}: {line.strip()[:160]}") + return problems + + +def check(root: Path) -> list[str]: + construct = root / "construct.yaml" + if construct.is_file() and re.search(r"^name:\s*UNDERSCORED_PROJECT_NAME\s*$", construct.read_text(encoding="utf-8"), re.MULTILINE): + return ["construct.yaml: this is the raw LabConstrictor template; initialise the repository with the LabConstrictor form before releasing it"] + return leftover_placeholders(root) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Fail when a template placeholder is left in a file the installer ships.") + parser.add_argument("--repository-root", type=Path, default=Path.cwd()) + args = parser.parse_args(argv) + problems = check(args.repository_root.resolve()) + if not problems: + print("No template placeholder is left in the files the installer ships.") + return 0 + print("The installer would be broken: these files still hold template placeholders.", file=sys.stderr) + for problem in problems: + print(f" {problem}", file=sys.stderr) + print( + "Replace each placeholder with your app's value (PYTHON_PROJ_NAME: the Python package of the app, usually its folder in src/; " + "PROJECT_NAME: the app name), " + f"commit, and publish the release again. See {DOCS_URL}", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.tools/python/lock_requirements.py b/.tools/python/lock_requirements.py new file mode 100644 index 0000000..c5834a2 --- /dev/null +++ b/.tools/python/lock_requirements.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Lock the merged requirements files: requirements*.txt -> requirements*.lock.txt. + +merge_requirements.py writes the requirements of the notebooks (requirements.txt, requirements_gpu.txt and the per-OS +requirements-.txt). Those files name the packages the notebooks need, but not the packages these depend on, so two +installations of the same release could get different versions. This script writes, next to each file, a lock file that lists +every package pip installs, each with its exact version, and post_install installs the lock file when the installer bundles one. + +How a lock file is made: +* `uv pip compile --universal`: one lock file that is valid on Windows, macOS and Linux (a package that only one platform needs carries + an environment marker, e.g. `colorama==0.4.6 ; sys_platform == 'win32'`). +* `--python-version` is the Python of environment.yaml. uv reads it as the lowest Python of the resolution, so a few packages can appear + twice with a `python_full_version` marker: pip installs the line that matches the Python of the installer. +* The conda packages of environment.yaml (jupyterlab, pip, setuptools, certifi, ...) are installed by the installer itself, so they are + not written to the lock file (their own dependencies are), and a conda pin such as `jupyterlab=4.4.0` constrains the resolution. +* The per-OS files are resolved with requirements.lock.txt as constraints, so a package that both files need gets the same version. +* Pip options of the requirements file (`--extra-index-url` of the GPU file, ...) are copied to the top of the lock file. + +A lock file is written again only when its requirements file, the Python or JupyterLab version (or this script's lock format) changed: +its header holds a fingerprint of those inputs. Re-running the script without such a change rewrites nothing, so a new release +installs the same versions as the previous one. To move every package to its newest version, delete the lock files and run the +workflow "Updates on notebook change" by hand (Actions > Run workflow). + +If a file cannot be locked (uv is missing, or the requirements cannot be resolved), the script prints a warning, removes the +out-of-date lock file of that file (an installer must never install versions that no longer match the requirements) and goes on: +the installer then installs the plain requirements file. The script itself always exits 0 for such a failure. +""" +from __future__ import annotations + +import argparse +import hashlib +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + +# Bump when the uv arguments or the output change, so that every lock file is written again. +LOCK_FORMAT = "1" +BASE_INPUT = "requirements.txt" +GPU_INPUT = "requirements_gpu.txt" +OS_INPUTS = ("requirements-linux.txt", "requirements-windows.txt", "requirements-macos.txt") +INPUTS = (BASE_INPUT, GPU_INPUT, *OS_INPUTS) +FINGERPRINT_PREFIX = "# lock-input: sha256:" +# Pip options that choose where packages come from: copied to the top of the lock file, so pip finds the locked versions where uv found them. +INDEX_OPTION = re.compile(r"^\s*(--index-url|--extra-index-url|-i|--find-links|-f|--trusted-host)(\s|=)") +CONDA_SPEC = re.compile(r"^\s*([A-Za-z0-9_.-]+)\s*(?:(==?)\s*([0-9][A-Za-z0-9_.+!*-]*))?(?:=\S+)?\s*$") +PLACEHOLDER = re.compile(r"^[A-Z_]+$") + + +def lock_path(requirements: Path) -> Path: + """requirements.txt -> requirements.lock.txt (same folder).""" + return requirements.with_name(requirements.stem + ".lock.txt") + + +def read_environment(path: Path) -> tuple[str | None, list[str], list[str]]: + """(Python version, conda package names, pip constraints from exact conda pins) of environment.yaml.""" + if not path.is_file(): + return None, [], [] + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + python_version = None + names: list[str] = [] + constraints: list[str] = [] + for dependency in data.get("dependencies", []) or []: + if not isinstance(dependency, str): + continue # the `pip:` subsection of an environment file + match = CONDA_SPEC.match(dependency) + if match: + name, operator, version = match.group(1), match.group(2), match.group(3) + else: + # a spec with a range or alternatives (`pip>=24.2`): only its name is used + name, operator, version = re.split(r"[\s<>=!|,\[]", dependency.strip(), maxsplit=1)[0], None, None + if not name: + continue + if version and PLACEHOLDER.match(version): + version = None # the template itself (python=PYTHON_VERSION) + if name.lower() == "python": + python_version = version + continue + names.append(name) + if version: + # conda `name=1.2` means any 1.2.* release, `name==1.2` exactly 1.2 + constraints.append(f"{name}=={version}" if operator == "==" else f"{name}=={version.removesuffix('.*')}.*") + return python_version, sorted(set(names), key=str.lower), constraints + + +def normalized_requirements(text: str) -> str: + """The requirement and option lines of a requirements file (comments, such as the timestamp of merge_requirements.py, are ignored).""" + lines = [] + for line in text.replace("\r\n", "\n").split("\n"): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + lines.append(stripped) + return "\n".join(lines) + + +def fingerprint(requirements_text: str, python_version: str, constraints: list[str], no_emit: list[str], extra_constraints: str) -> str: + payload = "\n---\n".join( + [LOCK_FORMAT, normalized_requirements(requirements_text), python_version, "\n".join(constraints), "\n".join(no_emit), extra_constraints] + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def current_fingerprint(lock: Path) -> str | None: + if not lock.is_file(): + return None + for line in lock.read_text(encoding="utf-8").splitlines(): + if line.startswith(FINGERPRINT_PREFIX): + return line[len(FINGERPRINT_PREFIX):].strip() + return None + + +def run_uv(requirements: Path, python_version: str, constraint_files: list[Path], no_emit: list[str]) -> str: + """The resolution of `requirements` (stdout of uv pip compile); raises RuntimeError with uv's message when it fails.""" + command = [ + sys.executable, "-m", "uv", "pip", "compile", str(requirements), + "--universal", "--python-version", python_version, + "--no-header", "--no-annotate", "--no-config", "--quiet", + ] + for constraint in constraint_files: + command += ["--constraint", str(constraint)] + for name in no_emit: + command += ["--no-emit-package", name] + try: + result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", check=False) + except OSError as exc: + raise RuntimeError(f"uv could not be started: {exc}") from exc + if result.returncode != 0: + message = (result.stderr or result.stdout).strip() or f"exit code {result.returncode}" + raise RuntimeError(message) + return result.stdout + + +def uv_version() -> str: + try: + result = subprocess.run([sys.executable, "-m", "uv", "--version"], capture_output=True, text=True, check=False) + except OSError: + return "uv" + # "uv 0.13.3 (502f93ca6 2026-10-19 x86_64-pc-windows-msvc)" -> "uv 0.13.3": the build details differ between computers + return " ".join(result.stdout.split()[:2]) or "uv" + + +def render_lock(requirements: Path, resolved: str, digest: str, python_version: str, uv: str) -> str: + options = [line.strip() for line in requirements.read_text(encoding="utf-8").splitlines() if INDEX_OPTION.match(line)] + packages = [line.rstrip() for line in resolved.replace("\r\n", "\n").split("\n") if line.strip() and not line.lstrip().startswith("#")] + header = [ + f"# Lock file of {requirements.name}: every package the installer installs with pip, with its exact version.", + f"# Written by .tools/python/lock_requirements.py ({uv}, universal resolution for Python {python_version}); do not edit it by hand.", + f"# It is written again only when {requirements.name}, the Python or JupyterLab version changes (see the fingerprint below).", + f"{FINGERPRINT_PREFIX}{digest}", + ] + body = options + ([""] if options else []) + packages + return "\n".join(header + [""] + body) + "\n" + + +def warn(message: str) -> None: + # `::warning::` shows the message on the summary page of the GitHub Actions run. + print(f"::warning::{message}", flush=True) + + +def lock_file( + requirements: Path, + python_version: str, + constraints: list[str], + no_emit: list[str], + base_lock: Path | None, + work_dir: Path, + uv: str, +) -> str: + """Lock one requirements file; returns what happened ("unchanged", "written", "failed").""" + lock = lock_path(requirements) + extra = base_lock.read_text(encoding="utf-8") if base_lock is not None and base_lock.is_file() else "" + digest = fingerprint(requirements.read_text(encoding="utf-8"), python_version, constraints, no_emit, normalized_requirements(extra)) + if current_fingerprint(lock) == digest: + print(f"{lock}: up to date (unchanged)") + return "unchanged" + constraint_files = [] + if constraints: + conda_constraints = work_dir / "conda_constraints.txt" + conda_constraints.write_text("\n".join(constraints) + "\n", encoding="utf-8") + constraint_files.append(conda_constraints) + if extra: + # Only the package lines: the pip options of the base lock are not constraints. + base_constraints = work_dir / "base_constraints.txt" + base_constraints.write_text( + "\n".join(line for line in extra.splitlines() if line.strip() and not line.startswith(("#", "-"))) + "\n", encoding="utf-8" + ) + constraint_files.append(base_constraints) + try: + resolved = run_uv(requirements, python_version, constraint_files, no_emit) + except RuntimeError as exc: + warn(f"{requirements} could not be locked, so the installer installs it without a lock file: {exc}") + if lock.exists(): + lock.unlink() + print(f"{lock}: removed (it no longer matches {requirements})") + return "failed" + with lock.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(render_lock(requirements, resolved, digest, python_version, uv)) + print(f"{lock}: written") + return "written" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Lock requirements*.txt into requirements*.lock.txt (every package with its exact version).") + parser.add_argument("--root", type=Path, default=Path("."), help="Folder with the requirements files and environment.yaml") + parser.add_argument("--environment", type=Path, default=None, help="Conda environment file (default: /environment.yaml)") + args = parser.parse_args(argv) + root: Path = args.root + environment = args.environment or root / "environment.yaml" + + python_version, conda_names, constraints = read_environment(environment) + if not python_version: + print(f"{environment}: no exact Python version (python=X.Y.Z), so nothing is locked.") + return 0 + no_emit = [name for name in conda_names if name.lower() not in {"git"}] + + # A lock file whose requirements file is gone is removed (the installer must not install it). + for name in INPUTS: + requirements = root / name + if not requirements.is_file() and lock_path(requirements).is_file(): + lock_path(requirements).unlink() + print(f"{lock_path(requirements)}: removed ({requirements} does not exist)") + + present = [root / name for name in INPUTS if (root / name).is_file()] + if not present: + print("No requirements files found: nothing to lock.") + return 0 + uv = uv_version() + results = {} + with tempfile.TemporaryDirectory() as temporary: + work_dir = Path(temporary) + for requirements in present: + base_lock = lock_path(root / BASE_INPUT) if requirements.name in OS_INPUTS else None + results[requirements.name] = lock_file(requirements, python_version, constraints, no_emit, base_lock, work_dir, uv) + failed = [name for name, status in results.items() if status == "failed"] + print(f"Lock files: {len(results) - len(failed)} of {len(results)} requirements files locked" + (f"; not locked: {', '.join(failed)}" if failed else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.tools/python/merge_requirements.py b/.tools/python/merge_requirements.py index 8b41c68..1fccecb 100644 --- a/.tools/python/merge_requirements.py +++ b/.tools/python/merge_requirements.py @@ -13,8 +13,9 @@ lists, in addition to the common file. Alongside the common requirements file, it emits a GPU-targeted variant. The -GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` and adds the PyTorch -CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present. +GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` on Linux only (TensorFlow's CUDA +wheels exist only for Linux; other systems keep plain `tensorflow`) and adds the +PyTorch CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present. It resolves simple version conflicts by choosing the newest version seen for a given package (and emits a pinned `pkg==`). It still preserves @@ -336,6 +337,44 @@ def rewrite_tensorflow_for_gpu(line: str) -> str: return line +# TensorFlow's `and-cuda` extra pulls nvidia-* wheels that exist only for Linux (nvidia-nccl-cu12 has never had a Windows or macOS +# wheel), and the extra itself carries no platform marker. Without a marker the universal lock of requirements_gpu.txt pins them for +# every system, so `pip install` fails on a Windows computer with an NVIDIA driver (the installer picks the GPU file there). TensorFlow +# has had no native Windows GPU build since 2.10, so Windows and macOS keep plain `tensorflow` in the GPU file. +GPU_TENSORFLOW_PLATFORM = 'sys_platform == "linux"' +NON_GPU_TENSORFLOW_PLATFORM = 'sys_platform != "linux"' + + +def _with_platform(marker: str, platform: str) -> str: + """Return ` ; `, combined with the line's own marker (`; `) when it has one.""" + existing = marker.split(";", 1)[1].strip() if ";" in marker else "" + return f" ; ({existing}) and {platform}" if existing else f" ; {platform}" + + +def tensorflow_gpu_lines(line: str) -> list[str]: + """Return the GPU-file lines for a tensorflow requirement: `[and-cuda]` on Linux, the plain requirement elsewhere. + + A line that is not a tensorflow requirement is returned unchanged (as a one-item list). + """ + match = pkg_re.match(line) + if match: + if match.group("name").lower() != "tensorflow": + return [line] + name, extras = match.group("name"), match.group("extras") or "" + spec = f"{match.group('op')}{match.group('ver')}" + else: + match = PLAIN_PKG_RE.match(line) + if not match or match.group("name").lower() != "tensorflow": + return [line] + name, extras, spec = match.group("name"), match.group("extras") or "", "" + marker = match.group("marker") or "" + cuda_extras = merge_requirement_extras(extras, "and-cuda") + return [ + f"{name}{cuda_extras}{spec}{_with_platform(marker, GPU_TENSORFLOW_PLATFORM)}", + f"{name}{extras}{spec}{_with_platform(marker, NON_GPU_TENSORFLOW_PLATFORM)}", + ] + + def build_gpu_lines(final_lines: list[str]) -> list[str]: gpu_lines: list[str] = [] has_torch_package = False @@ -348,10 +387,11 @@ def build_gpu_lines(final_lines: list[str]) -> list[str]: match = pkg_re.match(line) or PLAIN_PKG_RE.match(line) if match: package_name = match.group("name").lower() - if package_name == "tensorflow": - line = rewrite_tensorflow_for_gpu(line) if package_name in TORCH_GPU_PACKAGES: has_torch_package = True + if package_name == "tensorflow": + gpu_lines.extend(tensorflow_gpu_lines(line)) + continue gpu_lines.append(line) diff --git a/.tools/python/pypi_package.py b/.tools/python/pypi_package.py new file mode 100644 index 0000000..d7783b7 --- /dev/null +++ b/.tools/python/pypi_package.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +"""Build the app's Python package (the code in src/) for PyPI, as .github/workflows/publish_pypi.yml does at each release. + +The workflow runs only when the repository variable PYPI_PUBLISH is "true" (PyPI) or "testpypi" (TestPyPI, a trial). This script: + +1. stops before building, with the reason, when the release cannot be published: no package in src/, a version that is not X.Y.Z, + a version that is already on the index, another workflow of the repository that also publishes to PyPI, or a setup.py that + still has a template placeholder; +2. writes the release version into setup.py of the checkout, without committing it: the release tag points to the commit before + the "Update version" commit of the release workflow, so the committed version cannot be trusted. The name is the one in + setup.py; `distribution.pip` of labconstrictor-app.yaml, when there is one, must be the same name; +3. builds the source distribution and the wheel (`python -m build`: the wheel is built from the source distribution, so a file + missing from it shows), and checks their name, version and dependencies. + +It never uploads: the next job of the workflow does, with PyPI trusted publishing (no token). + +Usage: + python .tools/python/pypi_package.py --version 1.2.3 [--index pypi|testpypi] [--outdir dist] [--github-output FILE] +""" +from __future__ import annotations + +import argparse +import email.parser +import re +import subprocess +import sys +import tarfile +import tomllib +import urllib.error +import urllib.request +import zipfile +from email.message import Message +from pathlib import Path +from typing import Any, Callable + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = "publish_pypi.yml" +DOCS = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/external_code_upload.md#4-publish-the-code-on-pypi-optional" +# --index: the value of PYPI_PUBLISH ("true" is PyPI) -> (label, address) +INDEXES = {"pypi": ("PyPI", "https://pypi.org"), "true": ("PyPI", "https://pypi.org"), "testpypi": ("TestPyPI", "https://test.pypi.org")} +VERSION_RE = re.compile(r"\d+\.\d+\.\d+") +# Text in a workflow that uploads to PyPI (the action of the template, or twine). +PUBLISH_MARKERS = ("pypa/gh-action-pypi-publish", "twine upload") +PLACEHOLDERS = ("PYTHON_PROJ_NAME", "VERSION_NUMBER", "PYTHON_VERSION") + + +class PackageError(Exception): + """A reason not to publish, written to the log of the workflow.""" + + +def normalise(name: str) -> str: + """The name as PyPI compares names (PEP 503): my_App.tools and my-app-tools are the same project.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def packages(root: Path) -> list[str]: + """The packages of src/: its folders with an __init__.py.""" + src = root / "src" + if not src.is_dir(): + return [] + return sorted(path.parent.name for path in src.glob("*/__init__.py")) + + +def other_publishers(root: Path) -> list[str]: + """The workflows of the repository, other than the template's, that upload to PyPI.""" + found = [] + folder = root / ".github" / "workflows" + for path in sorted(folder.glob("*.y*ml")) if folder.is_dir() else []: + if path.name == WORKFLOW: + continue + text = path.read_text(encoding="utf-8", errors="replace") + if any(marker in text for marker in PUBLISH_MARKERS): + found.append(path.relative_to(root).as_posix()) + return found + + +def manifest_pip_name(root: Path) -> str | None: + """`distribution.pip` of labconstrictor-app.yaml: the name of the package on PyPI, when the app gives one.""" + import yaml + + path = root / "labconstrictor-app.yaml" + if not path.is_file(): + return None + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + except yaml.YAMLError as exc: + raise PackageError(f"labconstrictor-app.yaml is not valid YAML ({exc}); run the 'Check app manifest' workflow to see where") from exc + distribution = data.get("distribution") if isinstance(data, dict) else None + name = distribution.get("pip") if isinstance(distribution, dict) else None + return name.strip() if isinstance(name, str) and name.strip() else None + + +def _value_re(key: str) -> re.Pattern[str]: + return re.compile(rf"^(?P[ \t]*{key}\s*=\s*)(?P[\"'])(?P[^\"'\n]*)(?P=quote)", re.MULTILINE) + + +def setup_value(text: str, key: str) -> str | None: + """The first `key="value"` of setup.py (name, version), or None when it is not written there as a string.""" + match = _value_re(key).search(text) + return match.group("value") if match else None + + +def set_setup_value(text: str, key: str, value: str) -> str: + return _value_re(key).sub(lambda m: f"{m.group('head')}{m.group('quote')}{value}{m.group('quote')}", text, count=1) + + +def pyproject_project(root: Path) -> dict[str, Any] | None: + """The [project] table of pyproject.toml, when the package is described there (it then wins over setup.py).""" + path = root / "pyproject.toml" + if not path.is_file(): + return None + try: + project = tomllib.loads(path.read_text(encoding="utf-8")).get("project") + except tomllib.TOMLDecodeError as exc: + raise PackageError(f"pyproject.toml is not valid TOML: {exc}") from exc + return project if isinstance(project, dict) else None + + +def version_on_index(index: str, name: str, version: str, opener: Callable[..., Any] = urllib.request.urlopen) -> bool | None: + """True when the index already has this version of the project, False when not, None when the index could not be asked.""" + url = f"{INDEXES[index][1]}/pypi/{normalise(name)}/{version}/json" + try: + with opener(url, timeout=30) as response: + return bool(response.status == 200) + except urllib.error.HTTPError as exc: + return False if exc.code == 404 else None + except (urllib.error.URLError, OSError): + return None + + +def prepare(root: Path, version: str, index: str, check_index: bool = True) -> str: + """Check that the release can be published and write its version (and name) into setup.py; returns the name on the index.""" + if not VERSION_RE.fullmatch(version): + raise PackageError(f"The release version {version!r} is not X.Y.Z (the tag must be 1.2.3 or v1.2.3, as for the installers).") + found = packages(root) + if not found: + raise PackageError( + "There is no Python package in src/ (a folder with an __init__.py, for example src/my_app/__init__.py), so there is nothing to " + f"publish. Add your code to src/ (see .tools/docs/external_code_upload.md), or delete the repository variable PYPI_PUBLISH. {DOCS}" + ) + others = other_publishers(root) + if others: + raise PackageError( + f"{', '.join(others)} also publishes to PyPI: with PYPI_PUBLISH set, the release would be uploaded twice. Keep one: delete the " + f"repository variable PYPI_PUBLISH (your workflow keeps publishing), or delete your workflow (this one publishes at each release). {DOCS}" + ) + + pip_name = manifest_pip_name(root) + project = pyproject_project(root) + setup_path = root / "setup.py" + if project is not None and "name" in project: + # pyproject.toml describes the package: its name and version are used as they are, and must match. + name = str(project["name"]) + if pip_name and normalise(pip_name) != normalise(name): + raise PackageError(f"distribution.pip of labconstrictor-app.yaml is {pip_name}, but pyproject.toml names the package {name}: make them the same.") + if "version" in project.get("dynamic", []): + return name + if project.get("version") != version: + raise PackageError( + f"pyproject.toml gives the version {project.get('version')}, not the release version {version}: set `version = \"{version}\"` in " + "pyproject.toml, commit, and publish the release again." + ) + return name + if not setup_path.is_file(): + raise PackageError("There is no setup.py (nor a [project] table in pyproject.toml) to build the package from: restore the template's setup.py.") + + text = setup_path.read_bytes().decode("utf-8") # the line endings are kept when the version is written + left = [placeholder for placeholder in PLACEHOLDERS if placeholder in text and placeholder != "VERSION_NUMBER"] + if left: + raise PackageError( + f"setup.py still has the template placeholder {', '.join(left)}: PYTHON_PROJ_NAME is the name of the package (for example my_app), " + "PYTHON_VERSION the oldest Python series it runs on (for example 3.12). Replace it, commit, and publish the release again." + ) + setup_name = setup_value(text, "name") + if pip_name and setup_name and normalise(pip_name) != normalise(setup_name): + raise PackageError( + f"distribution.pip of labconstrictor-app.yaml is {pip_name}, but setup.py names the package {setup_name}: make them the same " + "(distribution.pip is the name the App Centre tells people to pip install; the package of src/ is published under setup.py's name)." + ) + package_name = setup_name or pip_name + if not package_name: + raise PackageError("setup.py has no name=\"...\" for the package: give it the name it should have on PyPI (and in distribution.pip).") + if check_index: + label = INDEXES[index][0] + exists = version_on_index(index, package_name, version) + if exists: + raise PackageError( + f"{label} already has {package_name} {version}, and a version can be uploaded only once (even after it was deleted). " + f"Publish a new release with a higher version (the installers are built for it too). " + f"{INDEXES[index][1]}/project/{normalise(package_name)}/{version}/" + ) + if exists is None: + print(f"WARNING: {label} could not be asked whether {package_name} {version} already exists; the upload fails if it does.") + + updated = set_setup_value(text, "version", version) + if updated != text: + # newline="" keeps the line endings of the file + with setup_path.open("w", encoding="utf-8", newline="") as handle: + handle.write(updated) + return package_name + + +def left_out_requirements(root: Path) -> list[str]: + """Lines of requirements.txt that the template's setup.py does not turn into dependencies (options, paths, URLs).""" + setup_text = (root / "setup.py").read_text(encoding="utf-8") if (root / "setup.py").is_file() else "" + path = root / "requirements.txt" + if "read_requirements()" not in setup_text or not path.is_file(): + return [] + try: + from packaging.requirements import InvalidRequirement, Requirement + except ImportError: + return [] + left = [] + for line in (raw.split("#", 1)[0].strip() for raw in path.read_text(encoding="utf-8").splitlines()): + if not line: + continue + try: + if Requirement(line).url is None: + continue + except InvalidRequirement: + pass + left.append(line) + return left + + +def build(root: Path, outdir: Path, isolation: bool = True) -> None: + """`python -m build`: the source distribution, then the wheel built from it.""" + command = [sys.executable, "-m", "build", "--outdir", str(outdir), *([] if isolation else ["--no-isolation"]), str(root)] + print("Running:", " ".join(command), flush=True) + result = subprocess.run(command) + if result.returncode: + raise PackageError("The package could not be built: see the output of `python -m build` above.") + + +def _headers(data: bytes) -> Message: + return email.parser.BytesParser().parsebytes(data, headersonly=True) + + +def read_metadata(outdir: Path) -> tuple[Message, Message, list[Path]]: + """The metadata of the wheel and of the source distribution in outdir, and the two files.""" + wheels = sorted(outdir.glob("*.whl")) + sdists = sorted(outdir.glob("*.tar.gz")) + if len(wheels) != 1 or len(sdists) != 1: + raise PackageError(f"Expected one wheel and one source distribution in {outdir}, found: {', '.join(p.name for p in sorted(outdir.iterdir())) or 'nothing'}") + with zipfile.ZipFile(wheels[0]) as archive: + member = next((n for n in archive.namelist() if n.count("/") == 1 and n.endswith(".dist-info/METADATA")), None) + if member is None: + raise PackageError(f"{wheels[0].name} has no METADATA") + wheel = _headers(archive.read(member)) + with tarfile.open(sdists[0]) as archive: + info = next((m for m in archive.getmembers() if m.name.count("/") == 1 and m.name.endswith("/PKG-INFO")), None) + handle = archive.extractfile(info) if info is not None else None + if handle is None: + raise PackageError(f"{sdists[0].name} has no PKG-INFO") + sdist = _headers(handle.read()) + return wheel, sdist, [sdists[0], wheels[0]] + + +def check(outdir: Path, name: str, version: str) -> list[str]: + """Check the built files; returns their names.""" + wheel, sdist, files = read_metadata(outdir) + for label, metadata in (("wheel", wheel), ("source distribution", sdist)): + if normalise(str(metadata.get("Name", ""))) != normalise(name): + raise PackageError(f"The {label} is named {metadata.get('Name')}, not {name}: setup.py or pyproject.toml set another name.") + if metadata.get("Version") != version: + raise PackageError(f"The {label} has the version {metadata.get('Version')}, not the release version {version}: setup.py or pyproject.toml set it.") + requirements = wheel.get_all("Requires-Dist") or [] + urls = [requirement for requirement in requirements if " @ " in requirement] + if urls: + raise PackageError( + f"PyPI refuses a package whose dependency is a URL: {'; '.join(urls)}. Publish that dependency on PyPI first and name it with a version, " + "or keep the URL in requirements.txt only (the installer still installs it)." + ) + if sdist.get_all("Requires-Dist") is not None and sorted(sdist.get_all("Requires-Dist") or []) != sorted(requirements): + raise PackageError( + "The wheel built from the source distribution has other dependencies than the source distribution: a file setup.py reads is not in " + "the source distribution (requirements.txt needs `include requirements.txt` in MANIFEST.in)." + ) + return [path.name for path in files] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Build and check the app's Python package (src/) for PyPI. Nothing is uploaded.") + parser.add_argument("--version", required=True, help="The release version, X.Y.Z (the tag without the v)") + parser.add_argument("--index", default="pypi", choices=sorted(INDEXES), help="pypi (or true, the value of PYPI_PUBLISH) or testpypi") + parser.add_argument("--outdir", default="dist", help="Where to write the two files (default: dist)") + parser.add_argument("--repository-root", default=str(ROOT)) + parser.add_argument("--github-output", help="Append name=, version= and url= to this file (GitHub Actions $GITHUB_OUTPUT)") + parser.add_argument("--no-isolation", action="store_true", help="Build with the setuptools already installed (no download)") + parser.add_argument("--offline", action="store_true", help="Do not ask the index whether the version exists") + args = parser.parse_args(argv) + + root = Path(args.repository_root).resolve() + outdir = Path(args.outdir) + if not outdir.is_absolute(): + outdir = root / outdir + label, address = INDEXES[args.index] + try: + name = prepare(root, args.version, args.index, check_index=not args.offline) + for line in left_out_requirements(root): + print(f"WARNING: the line `{line}` of requirements.txt is not a dependency the package can declare on {label} (an option, a path or a URL): left out.") + if outdir.exists() and any(outdir.iterdir()): + raise PackageError(f"{outdir} is not empty: remove it first, so that only this release is uploaded.") + build(root, outdir, isolation=not args.no_isolation) + files = check(outdir, name, args.version) + except PackageError as exc: + print(f"::error title=The package cannot be published::{exc}", file=sys.stderr) + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + url = f"{address}/project/{normalise(name)}/{args.version}/" + print(f"Built {name} {args.version} for {label}: {', '.join(files)}. The next job uploads it: {url}") + if args.github_output: + with open(args.github_output, "a", encoding="utf-8") as handle: + handle.write(f"name={name}\nversion={args.version}\nurl={url}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.tools/python/release_platforms.py b/.tools/python/release_platforms.py new file mode 100644 index 0000000..4083e6b --- /dev/null +++ b/.tools/python/release_platforms.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Which installers a release builds: the systems of `distribution.platforms` in labconstrictor-app.yaml (CellMigrationLab/LabConstrictor#42). + +The release workflow builds one installer per system. An app that cannot run on a system (TensorFlow 2.17+ has no wheels for Intel Macs, for +example) leaves that system out of `distribution.platforms`, and then: + * the release workflow does not build it (this script writes the build matrix of release_constructor.yml); + * the download page (.tools/docs/download_executable.md, written by bump_version.py) has no button for it. +Without labconstrictor-app.yaml, or without `distribution.platforms` in it, every system below is built, as before template 0.1.24. + +Usage: python .tools/python/release_platforms.py [--repository-root PATH] [--github-output FILE] +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, NamedTuple + +import yaml + +MANIFEST = "labconstrictor-app.yaml" +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_24.md" + + +class Build(NamedTuple): + platform: str # the name in distribution.platforms + os_name: str # matrix.os_name of release_constructor.yml (names the build job, its log and dist/ folder) + runner: str # the GitHub runner that builds it + files: tuple[str, ...] # how its installers end: - + label: str # for people: the download page and the messages + + +BUILDS = ( + Build("linux-x86_64", "linux", "ubuntu-latest", ("-Linux-x86_64.sh",), "Linux"), + Build("macos-arm64", "macos-arm64", "macos-15", ("-MacOSX-arm64.pkg", "-MacOSX-arm64.sh"), "Macs with Apple silicon (M1, M2, ...)"), + Build("macos-x86_64", "macos-intel", "macos-15-intel", ("-MacOSX-x86_64.pkg", "-MacOSX-x86_64.sh"), "Macs with an Intel processor"), + Build("windows-x86_64", "windows", "windows-latest", ("-Windows-x86_64.exe",), "Windows"), +) + + +def listed_platforms(root: Path) -> list[str] | None: + """`distribution.platforms` of the app manifest; None when there is no manifest or the key is left out (every system).""" + path = root / MANIFEST + if not path.is_file(): + return None + try: + data: Any = yaml.safe_load(path.read_text(encoding="utf-8")) + except yaml.YAMLError as exc: + raise ValueError(f"{MANIFEST} is not valid YAML: {exc}") from exc + distribution = data.get("distribution") if isinstance(data, dict) else None + platforms = distribution.get("platforms") if isinstance(distribution, dict) else None + if platforms is None: + return None + if not isinstance(platforms, list) or not all(isinstance(item, str) for item in platforms): + raise ValueError(f"distribution.platforms in {MANIFEST} must be a list of system names, such as [windows-x86_64, macos-arm64]") + return platforms + + +def builds(root: Path) -> list[Build]: + """The builds of a release, in the order of BUILDS. Raises ValueError when the manifest lists no system this template builds.""" + listed = listed_platforms(root) + if listed is None: + return list(BUILDS) + chosen = [build for build in BUILDS if build.platform in listed] + if not chosen: + known = ", ".join(build.platform for build in BUILDS) + raise ValueError( + f"distribution.platforms in {MANIFEST} lists {', '.join(listed) or 'nothing'}, and the release builds installers only for {known}. " + "Add at least one of them, or remove distribution.installer if the app has no installer." + ) + return chosen + + +def skipped_builds(root: Path) -> list[Build]: + """The systems the release does not build (none without distribution.platforms).""" + chosen = builds(root) + return [build for build in BUILDS if build not in chosen] + + +def matrix(chosen: list[Build]) -> dict[str, list[dict[str, str]]]: + return {"include": [{"os_name": build.os_name, "runner": build.runner} for build in chosen]} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Print the build matrix of the release: the systems of distribution.platforms in labconstrictor-app.yaml.") + parser.add_argument("--repository-root", type=Path, default=Path.cwd()) + parser.add_argument("--github-output", type=Path, help="append matrix= to this file (the GITHUB_OUTPUT of a workflow step)") + args = parser.parse_args(argv) + root = args.repository_root.resolve() + try: + chosen = builds(root) + listed = listed_platforms(root) + except ValueError as exc: + print(f"{exc} See {DOCS_URL}", file=sys.stderr) + return 1 + if listed is None: + print(f"Building every installer: {MANIFEST} does not list distribution.platforms.") + else: + print(f"Building the installers of distribution.platforms in {MANIFEST}: {', '.join(build.platform for build in chosen)}.") + for build in BUILDS: + if build not in chosen: + print(f" Not built: {build.platform} ({build.label}).") + for name in listed: + if name not in {build.platform for build in BUILDS}: + print(f" Not built: {name} (the release has no runner for it).") + value = json.dumps(matrix(chosen), separators=(",", ":")) + print(f"matrix={value}") + if args.github_output: + with args.github_output.open("a", encoding="utf-8") as handle: + handle.write(f"matrix={value}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.tools/requirements.txt b/.tools/requirements.txt index d06b04e..8492066 100644 --- a/.tools/requirements.txt +++ b/.tools/requirements.txt @@ -1,3 +1,5 @@ # Python 3.11 pyyaml==6.0.2 -nbformat==5.10.4 \ No newline at end of file +nbformat==5.10.4 +# lock_requirements.py (requirements*.txt -> requirements*.lock.txt) +uv==0.13.3 diff --git a/.tools/schema/labconstrictor-app.schema.json b/.tools/schema/labconstrictor-app.schema.json index 536d91e..30ae4ae 100644 --- a/.tools/schema/labconstrictor-app.schema.json +++ b/.tools/schema/labconstrictor-app.schema.json @@ -21,6 +21,15 @@ "items": {"enum": ["notebook", "napari", "fiji", "qupath"]}, "description": "Where the app's tools can be used besides its own window (the LabConstrictor-Tools bridge)." }, + "host_requirements": { + "type": "object", "additionalProperties": false, + "description": "The oldest LabConstrictor plugin of each host the app needs. labconstrictor-tools register copies it into the app's registry entry; a host whose plugin is older lists the app with a sentence saying which plugin to update (or refuses it with required: true). The notebook needs no plugin.", + "properties": { + "napari": {"$ref": "#/$defs/hostRequirement"}, + "fiji": {"$ref": "#/$defs/hostRequirement"}, + "qupath": {"$ref": "#/$defs/hostRequirement"} + } + }, "distribution": { "type": "object", "additionalProperties": false, "anyOf": [{"required": ["installer"]}, {"required": ["colab"]}, {"required": ["pip"]}], @@ -57,6 +66,16 @@ } }, "$defs": { + "hostRequirement": { + "type": "object", "additionalProperties": false, + "anyOf": [{"required": ["min_version"]}, {"required": ["features"]}], + "properties": { + "min_version": {"type": "string", "pattern": "^v?[0-9]+(\\.[0-9]+)*([-_.]?(a|alpha|b|beta|c|rc|pre|preview)[-_.]?[0-9]*)?([-_.]?(post|rev|r)[-_.]?[0-9]*)?([-_.]?dev[-_.]?[0-9]*)?$", "description": "A release tag of the plugin, with or without v (0.1.0-alpha.6, v0.1.0, 0.2.0rc1). Quote it if YAML could read it as a number (\"0.2\")."}, + "features": {"type": "array", "uniqueItems": true, "minItems": 1, "items": {"type": "string", "pattern": "^[a-z][a-z0-9_]*$"}, "description": "Host feature ids the app needs (LabConstrictor-Tools docs/HOST_FEATURES.md), for example shapes_owner_v1."}, + "why": {"type": "string", "minLength": 3, "maxLength": 160, "pattern": "^[^\\n]+$", "description": "One line shown to the person, saying what the newer plugin brings."}, + "required": {"type": "boolean", "description": "true: an older plugin refuses the app's tools instead of warning. Use it only when the tools really fail."} + } + }, "facts": {"type": "array", "minItems": 1, "maxItems": 6, "items": {"type": "string", "minLength": 5, "maxLength": 200}}, "https": {"type": "string", "pattern": "^https://[^\\s/@]+(/[^\\s]*)?$(?!\\n)"}, "repoPath": {"type": "string", "pattern": "^(?!/)(?!.*\\.\\.)[A-Za-z0-9_./ -]+\\.(png|jpg|jpeg|gif)$(?!\\n)", "description": "Path inside the repository, no leading slash."}, diff --git a/app/bash_bat_scripts/post_install.bat b/app/bash_bat_scripts/post_install.bat index 376ed5e..5fcbfd5 100644 --- a/app/bash_bat_scripts/post_install.bat +++ b/app/bash_bat_scripts/post_install.bat @@ -67,6 +67,10 @@ IF EXIST "%GPU_REQUIREMENTS%" ( echo GPU requirements file not found, installing CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" ) +REM A lock file (requirements*.lock.txt next to the requirements file: every package with its exact version, written by +REM .tools/python/lock_requirements.py) is installed instead of the requirements file when the installer bundles one. +SET "SELECTED_LOCK=%SELECTED_REQUIREMENTS:~0,-4%.lock.txt" +IF EXIST "%SELECTED_LOCK%" SET "SELECTED_REQUIREMENTS=%SELECTED_LOCK%" echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" "%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1 IF ERRORLEVEL 1 ( @@ -74,9 +78,11 @@ IF ERRORLEVEL 1 ( GOTO :fail ) +SET "WINDOWS_REQUIREMENTS=%PROJECT_ROOT%\requirements-windows.txt" +IF EXIST "%PROJECT_ROOT%\requirements-windows.lock.txt" SET "WINDOWS_REQUIREMENTS=%PROJECT_ROOT%\requirements-windows.lock.txt" IF EXIST "%PROJECT_ROOT%\requirements-windows.txt" ( - echo Installing Windows-specific requirements. >> "%LOG_FILE%" - "%PYTHON_EXE%" -m pip install -r "%PROJECT_ROOT%\requirements-windows.txt" >> "%LOG_FILE%" 2>&1 + echo Installing Windows-specific requirements from "%WINDOWS_REQUIREMENTS%". >> "%LOG_FILE%" + "%PYTHON_EXE%" -m pip install -r "%WINDOWS_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1 IF ERRORLEVEL 1 ( SET "FAILURE_MESSAGE=Windows-specific requirements installation failed." GOTO :fail @@ -229,7 +235,7 @@ REM This step must never fail the installation: a failure is logged as a WARNING REM LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). REM Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), REM because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. -SET "LC_TOOLS_PIN=v0.3.0-alpha.1" +SET "LC_TOOLS_PIN=v0.3.0-alpha.2" IF NOT DEFINED LC_TOOLS_SPEC SET "LC_TOOLS_SPEC=https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/%LC_TOOLS_PIN%.zip" SET "LC_TOOLS_READY=0" "%PYTHON_EXE%" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('vlab4mic_lc_tools') else 1)" >> "%LOG_FILE%" 2>&1 diff --git a/app/bash_bat_scripts/post_install.sh b/app/bash_bat_scripts/post_install.sh index c500640..51324a9 100644 --- a/app/bash_bat_scripts/post_install.sh +++ b/app/bash_bat_scripts/post_install.sh @@ -43,18 +43,24 @@ else echo "GPU requirements file not found, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" fi +# A lock file (requirements*.lock.txt next to the requirements file: every package with its exact version, written by +# .tools/python/lock_requirements.py) is installed instead of the requirements file when the installer bundles one. +locked_requirements() { + if [ -f "${1%.txt}.lock.txt" ]; then echo "${1%.txt}.lock.txt"; else echo "$1"; fi +} +SELECTED_REQUIREMENTS="$(locked_requirements "$SELECTED_REQUIREMENTS")" echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE" "$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1 if [[ "${OSTYPE:-}" == "darwin"* ]]; then echo "Detected macOS platform" >> "$LOG_FILE" if [ -f "$PROJECT_ROOT/requirements-macos.txt" ]; then - "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-macos.txt" >> "$LOG_FILE" 2>&1 + "$PYTHON_EXE" -m pip install -r "$(locked_requirements "$PROJECT_ROOT/requirements-macos.txt")" >> "$LOG_FILE" 2>&1 fi elif [[ "${OSTYPE:-}" == "linux-gnu"* ]]; then echo "Detected Linux platform" >> "$LOG_FILE" if [ -f "$PROJECT_ROOT/requirements-linux.txt" ]; then - "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-linux.txt" >> "$LOG_FILE" 2>&1 + "$PYTHON_EXE" -m pip install -r "$(locked_requirements "$PROJECT_ROOT/requirements-linux.txt")" >> "$LOG_FILE" 2>&1 fi else echo "Unknown platform: ${OSTYPE:-unset}" >> "$LOG_FILE" @@ -70,7 +76,7 @@ if [ -f "$PROJECT_ROOT/setup.py" ]; then # (it never fails the installation by itself). LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). # Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), # because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. - LC_TOOLS_PIN="v0.3.0-alpha.1" + LC_TOOLS_PIN="v0.3.0-alpha.2" LC_TOOLS_MODULE="vlab4mic_lc_tools" LC_TOOLS_READY=0 if "$PYTHON_EXE" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('$LC_TOOLS_MODULE') else 1)" >> "$LOG_FILE" 2>&1; then diff --git a/app/menuinst/notebook_launcher.json b/app/menuinst/notebook_launcher.json index 00281f2..401eac5 100644 --- a/app/menuinst/notebook_launcher.json +++ b/app/menuinst/notebook_launcher.json @@ -15,7 +15,14 @@ ], "platforms": { "win": { - "icon": "BASE_PATH_KEYWORD/VLab4Mic/vlab4mic_icon.ico" + "icon": "BASE_PATH_KEYWORD/VLab4Mic/vlab4mic_icon.ico", + "command": [ + "{{ PYTHON }}", + "{{ PREFIX }}/VLab4Mic/launch_jupyter.py", + "{{ PREFIX }}/VLab4Mic/notebooks/Welcome.ipynb", + "&", + "exit" + ] }, "linux": { "icon": "BASE_PATH_KEYWORD/VLab4Mic/vlab4mic_icon.png" diff --git a/app/python_scripts/launch_jupyter.py b/app/python_scripts/launch_jupyter.py index 0eec7aa..2d5f38b 100644 --- a/app/python_scripts/launch_jupyter.py +++ b/app/python_scripts/launch_jupyter.py @@ -13,8 +13,10 @@ import os import platform import re +import shutil import ssl import sys +import tempfile import traceback from datetime import datetime, timezone from importlib import metadata @@ -197,10 +199,30 @@ def _show_error_dialog(message: str) -> None: pass +def _wait_for_enter() -> None: + """Keep the terminal window open until the user has read it, then let it close. + + The menu shortcut runs the launcher in a terminal window; without this the window + closes (or stays open, on Windows) as soon as JupyterLab stops. + """ + stream = sys.stdin + if stream is None or not stream.isatty(): + return + try: + input("\nPress Enter to close this window...") + except (EOFError, KeyboardInterrupt): + pass + + def run_jupyter(notebook: Path) -> int: if not notebook.is_file(): raise FileNotFoundError(f"Welcome notebook not found: {notebook}") + # The app's own kernel must win over a user-level "python3" kernelspec (`ipykernel install --user`), which Jupyter + # otherwise prefers: the notebook would run in another Python and fail with "No module named ". Set, not + # setdefault: a JUPYTER_PREFER_ENV_PATH=0 of the user would bring the problem back. + os.environ["JUPYTER_PREFER_ENV_PATH"] = "1" + # Import only after configure_tls() has completed. Tornado creates SSL # contexts during import on some versions. from jupyterlab.labapp import main @@ -213,6 +235,12 @@ def run_jupyter(notebook: Path) -> int: # All-users installations can live outside the home directory. root_dir = PREFIX.resolve() + # A fresh, empty workspace folder for every launch: JupyterLab restores the open + # tabs of its workspaces, and the default folder (~/.jupyter/lab/workspaces) is + # shared by every app, so notebooks of other apps (and of earlier sessions) would + # open next to the Welcome notebook. Settings and themes are kept: they are stored + # in the user settings folder, not in the workspaces. + workspaces_dir = Path(tempfile.mkdtemp(prefix="labconstrictor-workspaces-")) sys.argv = [ "jupyter-lab", str(notebook.resolve()), @@ -220,8 +248,12 @@ def run_jupyter(notebook: Path) -> int: f"--ServerApp.root_dir={root_dir}", "--ServerApp.shutdown_no_activity_timeout=30", "--FileContentsManager.always_delete_dir=True", + f"--LabApp.workspaces_dir={workspaces_dir}", ] - result = main() + try: + result = main() + finally: + shutil.rmtree(workspaces_dir, ignore_errors=True) return int(result or 0) @@ -240,8 +272,19 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Print the selected verified CA bundle path and exit.", ) + parser.add_argument( + "--no-wait", + action="store_true", + help="Exit when JupyterLab stops, without asking to press Enter.", + ) args = parser.parse_args(argv) + result = _launch(args) + if not (args.no_wait or args.self_test or args.print_ca_bundle): + _wait_for_enter() + return result + +def _launch(args: argparse.Namespace) -> int: try: if args.print_ca_bundle: print(_select_ca_bundle(), flush=True)