diff --git a/.github/workflows/check_app_manifest.yml b/.github/workflows/check_app_manifest.yml index 6f5e4b6..113f55f 100644 --- a/.github/workflows/check_app_manifest.yml +++ b/.github/workflows/check_app_manifest.yml @@ -32,10 +32,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 with: python-version: '3.11' diff --git a/.github/workflows/check_schema_drift.yml b/.github/workflows/check_schema_drift.yml index b1edada..3c43c4f 100644 --- a/.github/workflows/check_schema_drift.yml +++ b/.github/workflows/check_schema_drift.yml @@ -22,8 +22,8 @@ jobs: name: Schema copy equals the Site schema runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 with: python-version: '3.11' - run: pip install pytest pyyaml==6.0.2 jsonschema==4.23.0 diff --git a/.github/workflows/release_constructor.yml b/.github/workflows/release_constructor.yml index 6410111..4e7939b 100644 --- a/.github/workflows/release_constructor.yml +++ b/.github/workflows/release_constructor.yml @@ -25,7 +25,7 @@ jobs: if: github.event_name == 'release' steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Determine release version id: version @@ -37,7 +37,7 @@ jobs: echo "RELEASE_VERSION=$RAW" >> "$GITHUB_ENV" - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 with: python-version: '3.11' @@ -64,14 +64,14 @@ jobs: - name: Commit updated versioned files - uses: stefanzweifel/git-auto-commit-action@v5 + uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5 with: branch: main commit_message: GitHub Action - Update version to ${{ github.event.release.tag_name }} - name: Upload diagnostics if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: release-bump-version-logs path: artifacts/error_log_document.log @@ -102,7 +102,7 @@ jobs: OUTPUT_DIR: dist/${{ matrix.os_name }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Determine release version id: version @@ -124,7 +124,7 @@ jobs: echo "RELEASE_VERSION=$RAW" >> "$GITHUB_ENV" - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 with: python-version: '3.11' @@ -150,7 +150,7 @@ jobs: python .tools/python/bump_Welcome_notebook.py "${{ github.repository_owner }}" "${{ github.event.repository.name }}" - name: Set up Miniconda - uses: conda-incubator/setup-miniconda@v3 + uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 with: auto-activate: true miniforge-version: latest @@ -174,7 +174,7 @@ jobs: - name: Upload diagnostics if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: build-installers-${{ matrix.os_name }}-logs path: artifacts/error_log_document_${{ matrix.os_name }}.log @@ -182,7 +182,7 @@ jobs: archive: false - name: Upload installers to release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 with: files: dist/${{ matrix.os_name }}/* fail_on_unmatched_files: true diff --git a/.github/workflows/sync_template.yml b/.github/workflows/sync_template.yml index 22dc7e9..8ba8f42 100644 --- a/.github/workflows/sync_template.yml +++ b/.github/workflows/sync_template.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: ref: main fetch-depth: 0 @@ -157,7 +157,7 @@ jobs: - name: Create pull request if: steps.check-changes.outputs.has_changes == 'true' - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6 with: token: ${{ secrets.LABCONSTRICTOR_SYNC_TOKEN }} branch: template-sync-${{ steps.local-version.outputs.version }}-to-${{ steps.upstream-version.outputs.version }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..01271ed --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,54 @@ +name: Template tests + +# Tests and static checks of the template repository itself (the migrations in .template_sync, the scripts in .tools/python and the tests). +# They run on every pull request and every push to main, without a path filter: a change to any file (a workflow, a doc, a resource copy) can +# break a test that compares files. It does not run in the apps created from the template. +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: template-tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + tests: + if: github.repository == 'CellMigrationLab/LabConstrictor' + name: Tests, ruff and mypy + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + - name: Install the test and check tools + run: pip install pytest==9.1.1 pyyaml==6.0.2 jsonschema==4.23.0 ruff==0.15.20 mypy==1.20.2 types-PyYAML==6.0.12.20260906 + - name: Tests (all tests/test_*.py, including the template_sync migration tests) + run: python -m pytest tests -q + - name: ruff (configuration in ruff.toml) + run: ruff check .template_sync .tools/python tests + - name: mypy (configuration in mypy.ini) + run: python -m mypy .template_sync .tools/python tests + + smoke-install: + if: github.repository == 'CellMigrationLab/LabConstrictor' + name: Smoke install of post_install (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + - name: Run the live post_install script in a fake prefix; the registry entry must carry the hosts of the app's manifest + run: python tests/smoke/post_install_hosts.py diff --git a/.github/workflows/update_on_notebook_change.yml b/.github/workflows/update_on_notebook_change.yml index 1ecd580..c5fae88 100644 --- a/.github/workflows/update_on_notebook_change.yml +++ b/.github/workflows/update_on_notebook_change.yml @@ -22,12 +22,12 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 # Fetch all history for all branches and tags - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 with: python-version: '3.11' @@ -36,7 +36,7 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@v44 + uses: tj-actions/changed-files@2d756ea4c53f7f6b397767d8723b3a10a9f35bf2 # v44 with: separator: "," files: | @@ -109,7 +109,7 @@ jobs: - name: Set up Miniconda for requirements test if: steps.check-changes.outputs.has_changes == 'true' - uses: conda-incubator/setup-miniconda@v3 + uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 with: auto-activate: true miniforge-version: latest @@ -129,7 +129,7 @@ jobs: - name: Upload diagnostics if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: notebook-debug-bundle path: artifacts/error_log_document.log @@ -138,7 +138,7 @@ jobs: - name: Commit updated files id: auto-commit if: steps.check-changes.outputs.has_changes == 'true' && steps.test-conda-env.outcome == 'success' - uses: stefanzweifel/git-auto-commit-action@v5 + uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5 with: commit_message: GitHub Action - Updated files because of notebook and/or requirements changes file_pattern: requirements*.txt environment.yaml notebooks/* setup.py diff --git a/.github/workflows/update_on_src_change.yaml b/.github/workflows/update_on_src_change.yaml index acc7c34..76b90f0 100644 --- a/.github/workflows/update_on_src_change.yaml +++ b/.github/workflows/update_on_src_change.yaml @@ -21,7 +21,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 # Fetch all history for all branches and tags @@ -33,7 +33,7 @@ jobs: - name: Commit updated files id: auto-commit - uses: stefanzweifel/git-auto-commit-action@v5 + uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5 with: commit_message: GitHub Action - Updated src change file diff --git a/.template_sync/manifest.yaml b/.template_sync/manifest.yaml index 2938dc5..d24ad0b 100644 --- a/.template_sync/manifest.yaml +++ b/.template_sync/manifest.yaml @@ -1,4 +1,4 @@ -template_version: 0.1.17 +template_version: 0.1.18 migrations: - from: 0.0.0 to: 0.0.1 @@ -57,3 +57,6 @@ migrations: - from: 0.1.16 to: 0.1.17 script: migrations/v0_1_16_to_v0_1_17.py + - from: 0.1.17 + to: 0.1.18 + script: migrations/v0_1_17_to_v0_1_18.py diff --git a/.template_sync/migrations/v0_1_17_to_v0_1_18.py b/.template_sync/migrations/v0_1_17_to_v0_1_18.py new file mode 100644 index 0000000..02e39f7 --- /dev/null +++ b/.template_sync/migrations/v0_1_17_to_v0_1_18.py @@ -0,0 +1,423 @@ +"""Template 0.1.17 -> 0.1.18: new Tools pin, GitHub Actions pinned by commit SHA, install rules in the install guide, app hosts for Tools. + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_18.md + +What this changes in an app repository (every step is independent, idempotent and never deletes anything): + +1. Tools pin. `LC_TOOLS_PIN` in app/bash_bat_scripts/post_install.sh / .bat moves from the old template value `v0.2.0-rc.1` to the new one + (read from the resource copy of the template script, `v0.3.0-alpha.1`), ONLY when the line still holds exactly the old template value. + An app with its own pin, a script without a pin line (unpinned source, own `LC_TOOLS_SPEC`) or without the hook stays as it is, with a note. +2. Pinned actions. The five template workflows (check_app_manifest.yml, release_constructor.yml, sync_template.yml, + update_on_notebook_change.yml, update_on_src_change.yaml) are replaced by the pinned version ONLY when the app's file is still identical to + the previous template version (the resource with the pins removed; line endings are ignored when comparing and kept when writing). + A file that already is the pinned template file is unchanged; a file that is customised (or pinned in part by the app) is skipped with a + message that lists its still unpinned `uses:` lines. A workflow file that is not in the repository is not created. +3. Install guide. `.tools/docs/install.md`, if it exists, gets the section "Before you install" (path without a space) and the section + "Update an installed app" (macOS/Windows uninstall first, Linux `-u`), inserted before "## Install on Windows" and before "## Next". +4. Tools guide. `.tools/docs/tools_bridge.md` is refreshed (it names the pinned tag) only if it still is the 0.1.17 text. +5. App hosts. The installer passes the app's `hosts:` line to Tools (`register --app-manifest`): `construct.yaml` gets the `extra_files` entry that + bundles `labconstrictor-app.yaml` (only if that file exists in the repository, and only in the known block shape), and the Tools hook of + post_install.sh / .bat gets the lines that pass the bundled file and log when it is missing (only in the known hook shape; the app's own + register arguments stay). A hook that already passes `--app-manifest` or `--hosts` is unchanged; any other shape is SKIPPED with a note. + +Rules this migration keeps: surgical (only known lines or known files), idempotent (a second run reports no changes), never raises for a +customised file (the sync pull request must still be created), the line endings of each file are kept, nothing is deleted. +""" +from __future__ import annotations + +import importlib.util +import re +from pathlib import Path +from typing import Any + + +RESOURCE_ROOT = Path(__file__).resolve().parents[1] / "resources" / "v0_1_17_to_v0_1_18" +SCRIPTS = Path("app/bash_bat_scripts") +POST_INSTALL_SH = SCRIPTS / "post_install.sh" +POST_INSTALL_BAT = SCRIPTS / "post_install.bat" +TOOLS_DOC = Path(".tools/docs/tools_bridge.md") +INSTALL_DOC = Path(".tools/docs/install.md") +WORKFLOWS = ( + Path(".github/workflows/check_app_manifest.yml"), + Path(".github/workflows/release_constructor.yml"), + Path(".github/workflows/sync_template.yml"), + Path(".github/workflows/update_on_notebook_change.yml"), + Path(".github/workflows/update_on_src_change.yaml"), +) +# Resource files: copies of the template files of 0.1.18 (a test checks they stay identical to the template). +MANAGED_FILES = (POST_INSTALL_SH, POST_INSTALL_BAT, TOOLS_DOC, INSTALL_DOC, *WORKFLOWS) + +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_18.md" +TAG = "[migration 0.1.18]" +OLD_PIN = "v0.2.0-rc.1" +NEW_PIN_RE = re.compile(r'LC_TOOLS_PIN="([^"]+)"') + +# The pinned third-party action lines, `uses: owner/repo@<40 hex> # `; removing the pin gives `uses: owner/repo@`. +PINNED_USES = re.compile(r"^(\s*(?:-\s+)?uses:\s*[\w.-]+/[\w./-]+)@[0-9a-f]{40}[ \t]+#[ \t]*(\S+)[ \t]*$") +ANY_USES = re.compile(r"^\s*(?:-\s+)?uses:\s*([\w.-]+/[\w./-]+)@(\S+)") + +INSTALL_BEFORE_HEADING = "## Before you install" +INSTALL_UPDATE_HEADING = "## Update an installed app" +INSTALL_HOSTS_HEADING = "## Where the app is listed" +INSTALL_BEFORE_ANCHOR = "## Install on Windows" +INSTALL_UPDATE_ANCHOR = "## Next" +CONSTRUCT = Path("construct.yaml") +APP_MANIFEST = Path("labconstrictor-app.yaml") + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def _load_helpers() -> Any: + """Reuse the helpers of the 0.1.10 migration (token replacement, newline detection, file IO).""" + path = Path(__file__).resolve().parent / "v0_1_9_to_v0_1_10.py" + spec = importlib.util.spec_from_file_location("template_sync_helpers_0_1_18", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _resource_text(helpers: Any, relative: Path) -> str: + source = RESOURCE_ROOT / relative + if not source.is_file(): + raise FileNotFoundError(f"Template resource not found: {source}") + return str(helpers.read_text(source)).replace("\r\n", "\n") + + +def new_pin(helpers: Any) -> str: + """The pin of the template (the resource copy of post_install.sh is the single source of the value).""" + match = NEW_PIN_RE.search(_resource_text(helpers, POST_INSTALL_SH)) + if not match: + raise ValueError("LC_TOOLS_PIN was not found in the template resource") + return match.group(1) + + +def _unpin(text: str) -> str: + """The text of a workflow before its actions were pinned (the previous template version).""" + lines = [] + for line in text.split("\n"): + match = PINNED_USES.match(line) + lines.append(f"{match.group(1)}@{match.group(2)}" if match else line) + return "\n".join(lines) + + +# -------------------------------------------------------------------------------------------------------------------------------- +# 1. Tools pin +# -------------------------------------------------------------------------------------------------------------------------------- +SH_PIN_LINE = re.compile(r'^([ \t]*LC_TOOLS_PIN=")([^"]*)(".*)$') +BAT_PIN_LINE = re.compile(r'^([ \t]*SET "LC_TOOLS_PIN=)([^"]*)(".*)$', re.IGNORECASE) + + +def _move_pin(helpers: Any, path: Path, pattern: re.Pattern[str], pin: str) -> str: + text = helpers.read_text(path) + if "labconstrictor_tools" not in text: + return "has no Tools bridge hook: nothing to move" + lines = text.splitlines(keepends=True) + hits = [] + for index, line in enumerate(lines): + match = pattern.match(line.rstrip("\r\n")) + if match: + hits.append((index, match)) + if not hits: + return "has no LC_TOOLS_PIN line (unpinned source or its own Tools source): left as it is" + if len(hits) > 1: + return f"has {len(hits)} LC_TOOLS_PIN lines: left as it is, set the pin by hand to {pin}" + index, match = hits[0] + current = match.group(2) + if current == pin: + return f"already pins Tools {pin}: unchanged" + if current != OLD_PIN: + return f"has its own pin {current}: left as it is (the template pin is now {pin})" + ending = lines[index][len(lines[index].rstrip("\r\n")) :] + lines[index] = f"{match.group(1)}{pin}{match.group(3)}{ending}" + helpers.write_text(path, "".join(lines)) + return f"updated: Tools pin {OLD_PIN} -> {pin}" + + +# -------------------------------------------------------------------------------------------------------------------------------- +# 2. Pinned actions +# -------------------------------------------------------------------------------------------------------------------------------- +def _unpinned_uses(text: str) -> list[str]: + """`uses:` lines of third-party actions that are not pinned by SHA (local `./` actions and `docker://` need no pin).""" + found = [] + for line in text.replace("\r\n", "\n").split("\n"): + match = ANY_USES.match(line) + if match and not re.fullmatch(r"[0-9a-f]{40}", match.group(2)) and not match.group(1).startswith(("./", "docker:")): + found.append(line.strip()) + return found + + +def _migrate_workflow(helpers: Any, repo_root: Path, relative: Path) -> str: + path = repo_root / relative + if not path.is_file(): + return "not present in this repository (nothing was created)" + new = _resource_text(helpers, relative) + current = helpers.read_text(path) + current_lf = current.replace("\r\n", "\n") + if current_lf == new: + return "already the pinned template file: unchanged" + if current_lf == _unpin(new): + newline = helpers.detect_newline(current) + helpers.write_text(path, new.replace("\n", newline)) + return "updated: every third-party action is pinned by commit SHA" + left = _unpinned_uses(current) + if not left: + return "customised and already pinned: left as it is" + shown = "; ".join(left[:6]) + ("; ..." if len(left) > 6 else "") + return f"SKIPPED: customised, so it was left unchanged ({len(left)} third-party action(s) not pinned by SHA: {shown}). Pin them by hand, see {DOCS_URL}" + + +# -------------------------------------------------------------------------------------------------------------------------------- +# 3. Install guide +# -------------------------------------------------------------------------------------------------------------------------------- +def _section(text: str, heading: str) -> str: + """The section of `heading` in the (LF) resource text, up to the next `## ` heading, with its trailing blank line.""" + match = re.search(rf"^{re.escape(heading)}\n.*?(?=^## )", text, re.MULTILINE | re.DOTALL) + if not match: + raise ValueError(f"section {heading!r} was not found in the template resource") + return match.group(0) + + +def _migrate_install_doc(helpers: Any, repo_root: Path) -> str: + path = repo_root / INSTALL_DOC + if not path.is_file(): + return "not present in this repository (nothing was created)" + resource = _resource_text(helpers, INSTALL_DOC) + current = helpers.read_text(path) + newline = helpers.detect_newline(current) + lf = current.replace("\r\n", "\n") + done = [] + for heading, anchor in ( + (INSTALL_BEFORE_HEADING, INSTALL_BEFORE_ANCHOR), + (INSTALL_UPDATE_HEADING, INSTALL_UPDATE_ANCHOR), + (INSTALL_HOSTS_HEADING, INSTALL_UPDATE_ANCHOR), + ): + if re.search(rf"^{re.escape(heading)}\s*$", lf, re.MULTILINE): + continue + position = re.search(rf"^{re.escape(anchor)}\s*$", lf, re.MULTILINE) + if not position: + return f"SKIPPED: no line {anchor!r} to insert {heading!r} before; add the sections by hand from {DOCS_URL}" + lf = lf[: position.start()] + _section(resource, heading) + lf[position.start() :] + done.append(heading[3:]) + if not done: + return "already has the install rules: unchanged" + helpers.write_text(path, lf.replace("\n", newline)) + return "updated: added " + " and ".join(repr(d) for d in done) + + +# -------------------------------------------------------------------------------------------------------------------------------- +# 4. Tools guide +# -------------------------------------------------------------------------------------------------------------------------------- +def _migrate_tools_doc(helpers: Any, repo_root: Path, replacements: dict[str, str], pin: str) -> str: + destination = repo_root / TOOLS_DOC + if not destination.is_file(): + return "not present in this repository (nothing was created)" + current = helpers.read_text(destination) + newline = helpers.detect_newline(current) + new = helpers.render_template(_resource_text(helpers, TOOLS_DOC), replacements, newline) + if current == new: + return "unchanged" + old = helpers.render_template(_resource_text(helpers, TOOLS_DOC).replace(pin, OLD_PIN), replacements, newline) + if current != old: + return "SKIPPED: not the 0.1.17 text (customised), so it was left unchanged; it names the pinned Tools tag " + pin + helpers.write_text(destination, new) + return "updated: written" + + +# -------------------------------------------------------------------------------------------------------------------------------- +# 5. App hosts (labconstrictor-app.yaml bundled by the installer, passed to `register --app-manifest`) +# -------------------------------------------------------------------------------------------------------------------------------- +PASSES_HOSTS = re.compile(r"--app-manifest|--hosts") +EXTRA_FILES_HEAD = re.compile(r"^extra_files:\s*$") +EXTRA_FILES_ITEM = re.compile(r"^- [^\s#][^:]*:\s*\S.*$") +MANIFEST_MAPPING = re.compile(r"^\s*-\s*['\"]?labconstrictor-app\.yaml['\"]?\s*:") +WELCOME_MAPPING = re.compile(r"^- [^:]*Welcome\.ipynb:\s*['\"]?([^/'\"\s]+)/") +REGISTER_NAME = re.compile(r'--name "([^"]+)"') +SH_BLOCK_START = " # Hosts of the app:" +SH_BLOCK_END = ' if [ "${LC_TOOLS_READY:-0}" = 1 ]' +BAT_PRE_START = "REM Hosts of the app:" +BAT_LOGS_START = " REM Which hosts list the app" +BAT_LOGS_END = " REM labconstrictor-tools was installed before" +BAT_SPEC_CALL = "importlib.util.find_spec('PYTHON_PROJ_NAME_lc_tools')" +BAT_REGISTER = re.compile(r'^(\s*IF NOT ERRORLEVEL 1 )("%PYTHON_EXE%" -m labconstrictor_tools register .*?)( >> "%LOG_FILE%" 2>&1)\s*$') +SH_LOG_REDIRECT = ' >> "$LOG_FILE" 2>&1' +SH_HOSTS_ARGS = '${LC_HOSTS_ARGS[@]+"${LC_HOSTS_ARGS[@]}"}' + + +def _slice(resource: list[str], start: str, end: str, what: str) -> list[str]: + """The lines of the template resource from the first line that starts with `start` up to (not including) the next line that starts with `end`.""" + begin = next((i for i, line in enumerate(resource) if line.startswith(start)), None) + if begin is None: + raise ValueError(f"{what}: the block was not found in the template resource") + stop = next((i for i in range(begin, len(resource)) if resource[i].startswith(end)), None) + if stop is None: + raise ValueError(f"{what}: the end of the block was not found in the template resource") + return resource[begin:stop] + + +def _only_line(lines: list[str], pattern: str, what: str) -> int: + hits = [i for i, line in enumerate(lines) if pattern in line] + if len(hits) != 1: + raise ValueError(f"{what} was {'not found' if not hits else 'found more than once'}") + return hits[0] + + +def _app_name(line: str) -> str: + match = REGISTER_NAME.search(line) + if not match: + raise ValueError('the register line has no --name "..." argument') + return match.group(1) + + +def _sh_hook_lines(lines: list[str]) -> tuple[int, int, str]: + """(index of the first line of the register statement, index of the line that ends it, the app name) in a known post_install.sh hook.""" + reg = _only_line(lines, "labconstrictor_tools register", "the register line") + if not any(line.lstrip().startswith("lc_note()") for line in lines[:reg]): + raise ValueError("lc_note is not defined before the register line") + first = reg + while first > 0 and lines[first - 1].rstrip("\r\n").rstrip().endswith("\\"): + first -= 1 + if not lines[first].lstrip().startswith("if "): + raise ValueError("the register command is not the condition of an `if` statement") + last = next((i for i in range(reg, min(reg + 4, len(lines))) if SH_LOG_REDIRECT in lines[i]), None) + if last is None or lines[last].count(SH_LOG_REDIRECT) != 1: + raise ValueError(f"the end of the register command (`{SH_LOG_REDIRECT.strip()}`) was not found") + return first, last, _app_name("".join(lines[first : reg + 1])) + + +def _migrate_hosts_sh(helpers: Any, path: Path) -> str: + text = helpers.read_text(path) + if "labconstrictor_tools" not in text: + return "has no Tools bridge hook: nothing to do" + if PASSES_HOSTS.search(text): + return "already passes the hosts of the app to Tools: unchanged" + newline = helpers.detect_newline(text) + lines = text.splitlines(keepends=True) + first, last, name = _sh_hook_lines(lines) + indent = lines[first][: len(lines[first]) - len(lines[first].lstrip())] + block = _slice(_resource_text(helpers, POST_INSTALL_SH).split("\n"), SH_BLOCK_START, SH_BLOCK_END, "post_install.sh") + inserted = [(indent + line[4:] if line.startswith(" ") else line).replace("PROJECT_NAME", name) + newline for line in block] + lines[last] = lines[last].replace(SH_LOG_REDIRECT, f" {SH_HOSTS_ARGS}{SH_LOG_REDIRECT}", 1) + helpers.write_text(path, "".join(lines[:first] + inserted + lines[first:])) + return "updated: the Tools hook passes --app-manifest and logs when labconstrictor-app.yaml is missing" + + +def _bat_hook_lines(lines: list[str]) -> tuple[int, int, int, re.Match[str]]: + """(index of the line before which the manifest lookup goes, index after which the log lines go, register index, its match) in a known post_install.bat hook.""" + reg = _only_line(lines, "labconstrictor_tools register", "the register line") + match = BAT_REGISTER.match(lines[reg].rstrip("\r\n")) + if not match: + raise ValueError("the register line does not have the known shape (IF NOT ERRORLEVEL 1 ... >> LOG_FILE 2>&1)") + found = max((i for i in range(reg) if re.match(r"^\s*echo Found \S+_lc_tools: registering", lines[i])), default=None) + block = max((i for i in range(reg) if lines[i].rstrip("\r\n") == "IF NOT ERRORLEVEL 1 ("), default=None) + if found is None or block is None or block == 0 or "find_spec(" not in lines[block - 1] or block > found: + raise ValueError("the `IF NOT ERRORLEVEL 1 (` block of the hook after the find_spec check was not found") + return block - 1, found, reg, match + + +def _migrate_hosts_bat(helpers: Any, path: Path) -> str: + text = helpers.read_text(path) + if "labconstrictor_tools" not in text: + return "has no Tools bridge hook: nothing to do" + if PASSES_HOSTS.search(text): + return "already passes the hosts of the app to Tools: unchanged" + newline = helpers.detect_newline(text) + lines = text.splitlines(keepends=True) + before, found, reg, match = _bat_hook_lines(lines) + name = _app_name(lines[reg]) + resource = _resource_text(helpers, POST_INSTALL_BAT).split("\n") + pre = _slice(resource, BAT_PRE_START, "\"%PYTHON_EXE%\" -c \"import importlib.util, sys; sys.exit(0 if " + BAT_SPEC_CALL, "post_install.bat") + logs = _slice(resource, BAT_LOGS_START, BAT_LOGS_END, "post_install.bat") + head, command, tail = match.groups() + register = [ + head + "IF DEFINED LC_APP_MANIFEST " + command + ' --app-manifest "%LC_APP_MANIFEST%"' + tail, + head + "IF NOT DEFINED LC_APP_MANIFEST " + command + tail, + ] + + def lines_of(texts: list[str]) -> list[str]: + return [item.replace("PROJECT_NAME", name) + newline for item in texts] + + lines[reg : reg + 1] = lines_of(register) + lines[found + 1 : found + 1] = lines_of(logs) + lines[before:before] = lines_of(pre) + helpers.write_text(path, "".join(lines)) + return "updated: the Tools hook passes --app-manifest and logs when labconstrictor-app.yaml is missing" + + +def _migrate_construct(helpers: Any, repo_root: Path) -> str: + """Add the extra_files entry that bundles labconstrictor-app.yaml (only if the file exists, in the known block shape).""" + path = repo_root / CONSTRUCT + if not path.is_file(): + return "not present in this repository (nothing was created)" + text = helpers.read_text(path) + lines = text.splitlines(keepends=True) + stripped = [line.rstrip("\r\n") for line in lines] + if any(MANIFEST_MAPPING.match(line) for line in stripped): + return "already bundles labconstrictor-app.yaml: unchanged" + if not (repo_root / APP_MANIFEST).is_file(): + return ( + "SKIPPED: the repository has no labconstrictor-app.yaml, so it is not bundled and the app is registered without hosts (listed in every host). " + f"Add the file, then add `- labconstrictor-app.yaml: /labconstrictor-app.yaml` to extra_files. See {DOCS_URL}" + ) + heads = [i for i, line in enumerate(stripped) if EXTRA_FILES_HEAD.match(line)] + if len(heads) != 1: + raise ValueError(f"expected one `extra_files:` block in the usual block shape, found {len(heads)}; add the mapping by hand") + last = heads[0] + while last + 1 < len(stripped) and EXTRA_FILES_ITEM.match(stripped[last + 1]): + last += 1 + folders = {m.group(1) for m in (WELCOME_MAPPING.match(line) for line in stripped[heads[0] + 1 : last + 1]) if m} + if last == heads[0] or len(folders) != 1: + raise ValueError("the project folder could not be read from the Welcome.ipynb entry of extra_files; add the mapping by hand") + newline = helpers.detect_newline(text) + entry = f"- labconstrictor-app.yaml: {folders.pop()}/labconstrictor-app.yaml{newline}" + if not lines[last].endswith(("\n", "\r")): + lines[last] += newline + lines.insert(last + 1, entry) + helpers.write_text(path, "".join(lines)) + return "updated: extra_files bundles labconstrictor-app.yaml" + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """Move the template Tools pin, pin the workflow actions, add the install rules (idempotent, surgical, never raises for a customised file).""" + _ = context + helpers = _load_helpers() + pin = new_pin(helpers) + try: + replacements = helpers.load_replacements(repo_root) + except (OSError, ValueError) as exc: + log(f"could not read the app name from construct.yaml ({exc}): the guides are written without app names.") + replacements = {} + + changed = 0 + + def report(name: Path, step: Any) -> None: + nonlocal changed + try: + status = step() + except (OSError, ValueError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + if status.startswith("updated"): + changed += 1 + log(f"{name}: {status}") + + for relative, pattern in ((POST_INSTALL_SH, SH_PIN_LINE), (POST_INSTALL_BAT, BAT_PIN_LINE)): + if not (repo_root / relative).is_file(): + log(f"{relative}: SKIPPED, file not found in this repository (nothing was created).") + continue + report(relative, lambda relative=relative, pattern=pattern: _move_pin(helpers, repo_root / relative, pattern, pin)) + for relative in WORKFLOWS: + report(relative, lambda relative=relative: _migrate_workflow(helpers, repo_root, relative)) + report(CONSTRUCT, lambda: _migrate_construct(helpers, repo_root)) + for relative, step in ((POST_INSTALL_SH, _migrate_hosts_sh), (POST_INSTALL_BAT, _migrate_hosts_bat)): + if (repo_root / relative).is_file(): + report(relative, lambda relative=relative, step=step: step(helpers, repo_root / relative)) + report(INSTALL_DOC, lambda: _migrate_install_doc(helpers, repo_root)) + report(TOOLS_DOC, lambda: _migrate_tools_doc(helpers, repo_root, replacements, pin)) + + if changed: + log(f"done: {changed} file(s) changed. Review the pull request before merging; the pin reaches users with the next release of the app.") + else: + log("done: no repository changes were required.") diff --git a/.template_sync/resources/v0_1_11_to_v0_1_12/.github/workflows/sync_template.yml b/.template_sync/resources/v0_1_11_to_v0_1_12/.github/workflows/sync_template.yml index 22dc7e9..8ba8f42 100644 --- a/.template_sync/resources/v0_1_11_to_v0_1_12/.github/workflows/sync_template.yml +++ b/.template_sync/resources/v0_1_11_to_v0_1_12/.github/workflows/sync_template.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: ref: main fetch-depth: 0 @@ -157,7 +157,7 @@ jobs: - name: Create pull request if: steps.check-changes.outputs.has_changes == 'true' - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6 with: token: ${{ secrets.LABCONSTRICTOR_SYNC_TOKEN }} branch: template-sync-${{ steps.local-version.outputs.version }}-to-${{ steps.upstream-version.outputs.version }} diff --git a/.template_sync/resources/v0_1_14_to_v0_1_15/.github/workflows/check_app_manifest.yml b/.template_sync/resources/v0_1_14_to_v0_1_15/.github/workflows/check_app_manifest.yml index 6f5e4b6..113f55f 100644 --- a/.template_sync/resources/v0_1_14_to_v0_1_15/.github/workflows/check_app_manifest.yml +++ b/.template_sync/resources/v0_1_14_to_v0_1_15/.github/workflows/check_app_manifest.yml @@ -32,10 +32,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 with: python-version: '3.11' diff --git a/.template_sync/resources/v0_1_15_to_v0_1_16/.github/workflows/check_app_manifest.yml b/.template_sync/resources/v0_1_15_to_v0_1_16/.github/workflows/check_app_manifest.yml index 6f5e4b6..113f55f 100644 --- a/.template_sync/resources/v0_1_15_to_v0_1_16/.github/workflows/check_app_manifest.yml +++ b/.template_sync/resources/v0_1_15_to_v0_1_16/.github/workflows/check_app_manifest.yml @@ -32,10 +32,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 with: python-version: '3.11' diff --git a/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/check_app_manifest.yml b/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/check_app_manifest.yml new file mode 100644 index 0000000..113f55f --- /dev/null +++ b/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/check_app_manifest.yml @@ -0,0 +1,46 @@ +name: Check App Centre manifest + +# Checks labconstrictor-app.yaml (the description of the app for the LabConstrictor App Centre). +# A missing file is a warning, an invalid file fails this check. It never blocks building installers: +# it runs on its own, also when a release is published. +on: + pull_request: + paths: + - labconstrictor-app.yaml + - .tools/schema/** + - .tools/python/check_app_manifest.py + - .github/workflows/check_app_manifest.yml + push: + branches: + - main + paths: + - labconstrictor-app.yaml + - .tools/schema/** + - .tools/python/check_app_manifest.py + - .github/workflows/check_app_manifest.yml + release: + types: + - published + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + name: Check labconstrictor-app.yaml + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install pyyaml==6.0.2 jsonschema==4.23.0 + + - name: Check the manifest + run: python .tools/python/check_app_manifest.py diff --git a/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/release_constructor.yml b/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/release_constructor.yml new file mode 100644 index 0000000..4e7939b --- /dev/null +++ b/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/release_constructor.yml @@ -0,0 +1,188 @@ +name: Build Release Installers + +on: + release: + types: + - published + workflow_dispatch: + inputs: + version: + description: "Version to package (e.g. 0.0.4). Defaults to release tag when empty." + required: false + +# This enables to run a single workflow at a time +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: write + +jobs: + bump-version: + name: Bump version on main branch + runs-on: ubuntu-latest + if: github.event_name == 'release' + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Determine release version + id: version + run: | + set -euxo pipefail + RAW="${{ github.event.release.tag_name }}" + RAW="${RAW#v}" + echo "release_version=$RAW" >> "$GITHUB_OUTPUT" + echo "RELEASE_VERSION=$RAW" >> "$GITHUB_ENV" + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Sync versioned files + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_version.py" \ + .tools/python/bump_version.py \ + python .tools/python/bump_version.py "${RELEASE_VERSION}" + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_constructor.py" \ + .tools/python/bump_constructor.py \ + python .tools/python/bump_constructor.py + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_Welcome_notebook.py" \ + .tools/python/bump_Welcome_notebook.py \ + python .tools/python/bump_Welcome_notebook.py "${{ github.repository_owner }}" "${{ github.event.repository.name }}" + + + - name: Commit updated versioned files + uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5 + with: + branch: main + commit_message: GitHub Action - Update version to ${{ github.event.release.tag_name }} + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: release-bump-version-logs + path: artifacts/error_log_document.log + if-no-files-found: ignore + archive: false + + build-installers: + name: Build installers (${{ matrix.os_name }}) + runs-on: ${{ matrix.runner }} + needs: bump-version + strategy: + fail-fast: false + matrix: + include: + - os_name: linux + runner: ubuntu-latest + - os_name: macos-arm64 + runner: macos-15 + - os_name: macos-intel + runner: macos-15-intel + - os_name: windows + runner: windows-latest + defaults: + run: + shell: bash -l {0} + env: + CONSTRUCTOR_ENV: constructor-env + OUTPUT_DIR: dist/${{ matrix.os_name }} + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Determine release version + id: version + run: | + set -euxo pipefail + if [ "${{ github.event_name }}" = "release" ]; then + RAW="${{ github.event.release.tag_name }}" + else + RAW="${{ inputs.version }}" + fi + + if [ -z "$RAW" ]; then + echo "Version is required when running manually." >&2 + exit 1 + fi + + RAW="${RAW#v}" + echo "release_version=$RAW" >> "$GITHUB_OUTPUT" + echo "RELEASE_VERSION=$RAW" >> "$GITHUB_ENV" + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Sync versioned files and populate Welcome notebook + run: | + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document.log" \ + "bump_version.py" \ + .tools/python/bump_version.py \ + python .tools/python/bump_version.py "${RELEASE_VERSION}" + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document.log" \ + "bump_constructor.py" \ + .tools/python/bump_constructor.py \ + python .tools/python/bump_constructor.py + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document.log" \ + "bump_Welcome_notebook.py" \ + .tools/python/bump_Welcome_notebook.py \ + python .tools/python/bump_Welcome_notebook.py "${{ github.repository_owner }}" "${{ github.event.repository.name }}" + + - name: Set up Miniconda + uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + auto-activate: true + miniforge-version: latest + use-mamba: true + + - name: Create constructor environment + run: | + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document_${{ matrix.os_name }}.log" \ + "create_constructor_environment.sh" \ + .tools/bash/create_constructor_environment.sh \ + bash .tools/bash/create_constructor_environment.sh + + - name: Build installer with constructor + run: | + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document_${{ matrix.os_name }}.log" \ + "build_installer_with_constructor.sh" \ + .tools/bash/build_installer_with_constructor.sh \ + bash .tools/bash/build_installer_with_constructor.sh + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: build-installers-${{ matrix.os_name }}-logs + path: artifacts/error_log_document_${{ matrix.os_name }}.log + if-no-files-found: ignore + archive: false + + - name: Upload installers to release + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + files: dist/${{ matrix.os_name }}/* + fail_on_unmatched_files: true diff --git a/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/sync_template.yml b/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/sync_template.yml new file mode 100644 index 0000000..8ba8f42 --- /dev/null +++ b/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/sync_template.yml @@ -0,0 +1,170 @@ +name: Sync with Template Repository + +on: + schedule: + # Run weekly on Monday at 9 AM UTC + - cron: '0 9 * * 1' + workflow_dispatch: + +# This enables only one template synchronization workflow at a time. +concurrency: + group: template-sync-main + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + +jobs: + sync-template: + name: Sync with template repository + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: main + fetch-depth: 0 + # Do not leave the default GITHUB_TOKEN in the local Git configuration. + # The pull-request step authenticates separately with the repository-scoped + # LABCONSTRICTOR_SYNC_TOKEN secret. + persist-credentials: false + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Add upstream remote + run: | + git remote add upstream https://github.com/CellMigrationLab/LabConstrictor.git + + - name: Fetch upstream changes + run: | + git fetch upstream main + + - name: Read local template version + id: local-version + run: | + local_version=$(python3 - <<'PY' + from pathlib import Path + + manifest = Path(".template_sync/manifest.yaml") + if not manifest.exists(): + print("0.0.0") + raise SystemExit + + for line in manifest.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if stripped.startswith("template_version:"): + print(stripped.split(":", 1)[1].strip()) + break + else: + raise SystemExit("template_version not found in .template_sync/manifest.yaml") + PY + ) + echo "version=$local_version" >> "$GITHUB_OUTPUT" + + - name: Read upstream template version + id: upstream-version + run: | + upstream_version=$(python3 - <<'PY' + import subprocess + + manifest = subprocess.run( + ["git", "show", "upstream/main:.template_sync/manifest.yaml"], + check=True, + capture_output=True, + text=True, + ).stdout + for line in manifest.splitlines(): + stripped = line.strip() + if stripped.startswith("template_version:"): + print(stripped.split(":", 1)[1].strip()) + break + else: + raise SystemExit( + "template_version not found in upstream .template_sync/manifest.yaml" + ) + PY + ) + echo "version=$upstream_version" >> "$GITHUB_OUTPUT" + + - name: Check whether template sync is needed + id: version-check + env: + LOCAL_VERSION: ${{ steps.local-version.outputs.version }} + UPSTREAM_VERSION: ${{ steps.upstream-version.outputs.version }} + run: | + python3 - <<'PY' + import os + from pathlib import Path + + def parse_version(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in version.split(".")) + + local_version = os.environ["LOCAL_VERSION"] + upstream_version = os.environ["UPSTREAM_VERSION"] + sync_required = parse_version(upstream_version) > parse_version(local_version) + + output_path = Path(os.environ["GITHUB_OUTPUT"]) + with output_path.open("a", encoding="utf-8") as fh: + fh.write(f"sync_required={'true' if sync_required else 'false'}\n") + PY + + - name: Check synchronization token setup + if: steps.version-check.outputs.sync_required == 'true' + env: + LABCONSTRICTOR_SYNC_TOKEN: ${{ secrets.LABCONSTRICTOR_SYNC_TOKEN }} + run: | + if [[ -z "$LABCONSTRICTOR_SYNC_TOKEN" ]]; then + { + echo "## One-time LabConstrictor setup required" + echo + echo "A template update is available, but this repository does not have the" + echo '`LABCONSTRICTOR_SYNC_TOKEN` repository secret configured.' + echo + echo "Follow the beginner-friendly guide in:" + echo + echo '`.tools/docs/template_synchronization.md`' + } >> "$GITHUB_STEP_SUMMARY" + + echo "::error title=Synchronization setup required::Configure the LABCONSTRICTOR_SYNC_TOKEN repository secret by following .tools/docs/template_synchronization.md." + exit 1 + fi + + - name: Fetch template sync bootstrap + if: steps.version-check.outputs.sync_required == 'true' + run: | + git checkout upstream/main -- .template_sync + + - name: Run template migrations + if: steps.version-check.outputs.sync_required == 'true' + run: | + python3 .template_sync/sync.py \ + --from-version "${{ steps.local-version.outputs.version }}" \ + --to-version "${{ steps.upstream-version.outputs.version }}" \ + --repository-root "$GITHUB_WORKSPACE" + + - name: Check for changes + id: check-changes + run: | + if [[ -n "$(git status --short)" ]]; then + echo "has_changes=true" >> "$GITHUB_OUTPUT" + else + echo "has_changes=false" >> "$GITHUB_OUTPUT" + fi + + - name: Create pull request + if: steps.check-changes.outputs.has_changes == 'true' + uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6 + with: + token: ${{ secrets.LABCONSTRICTOR_SYNC_TOKEN }} + branch: template-sync-${{ steps.local-version.outputs.version }}-to-${{ steps.upstream-version.outputs.version }} + delete-branch: true + commit-message: Sync template to version ${{ steps.upstream-version.outputs.version }} + title: Sync template to version ${{ steps.upstream-version.outputs.version }} + body: | + This PR updates `.template_sync` from the template repository and applies the migrations needed to move this repository from template version `${{ steps.local-version.outputs.version }}` to `${{ steps.upstream-version.outputs.version }}`. + + The branch is created with the repository-scoped `LABCONSTRICTOR_SYNC_TOKEN` secret so that updates to GitHub workflow files can be included safely. diff --git a/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/update_on_notebook_change.yml b/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/update_on_notebook_change.yml new file mode 100644 index 0000000..c5fae88 --- /dev/null +++ b/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/update_on_notebook_change.yml @@ -0,0 +1,154 @@ +name: Updates on notebook change + +on: + push: + branches: [ main, master ] + paths: + - 'notebooks/**/*.ipynb' + - 'notebooks/notebook_latest_versions.yaml' + - 'notebooks/**/requirements.yaml' + workflow_dispatch: + +# This enables to run a single workflow at a time +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon notebook change, update manifests and changelogs + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@2d756ea4c53f7f6b397767d8723b3a10a9f35bf2 # v44 + with: + separator: "," + files: | + notebooks/**/*.ipynb + notebooks/notebook_latest_versions.yaml + notebooks/**/requirements.yaml + + - name: Syncronize notebook versions + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "sync_notebook_versions.py" \ + .tools/python/sync_notebook_versions.py \ + python .tools/python/sync_notebook_versions.py + + - name: Update CHANGELOG + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_changelog.py" \ + .tools/python/update_changelog.py \ + python .tools/python/update_changelog.py notebooks/notebook_latest_versions.yaml + + - name: Update last commit in notebook_latest_versions.yaml + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_version_last_commit.py" \ + .tools/python/update_version_last_commit.py \ + python .tools/python/update_version_last_commit.py notebooks/notebook_latest_versions.yaml + + - name: Hide code cells in notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "hide_code_cells.py" \ + app/python_scripts/hide_code_cells.py \ + python app/python_scripts/hide_code_cells.py . + + - name: Index notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_notebook_README.py" \ + .tools/python/bump_notebook_README.py \ + python .tools/python/bump_notebook_README.py + + - name: Run merge script + if: github.event_name == 'workflow_dispatch' || contains(steps.changed-files.outputs.all_changed_files, 'requirements.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "merge_requirements.py" \ + .tools/python/merge_requirements.py \ + python .tools/python/merge_requirements.py --source-dir notebooks --output requirements.txt + + - name: Check for changes + id: check-changes + run: | + if [ -z "$(git status --porcelain)" ]; then + echo "has_changes=false" >> $GITHUB_OUTPUT + else + echo "has_changes=true" >> $GITHUB_OUTPUT + fi + + - name: Set up Miniconda for requirements test + if: steps.check-changes.outputs.has_changes == 'true' + uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + auto-activate: true + miniforge-version: latest + use-mamba: true + + - name: Verify merged requirements inside conda + id: test-conda-env + if: steps.check-changes.outputs.has_changes == 'true' + env: + SKIP_CONDA_ENV_TEST: ${{ vars.SKIP_CONDA_ENV_TEST }} + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "test_conda_environment.sh" \ + .tools/bash/test_conda_environment.sh \ + bash .tools/bash/test_conda_environment.sh + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: notebook-debug-bundle + path: artifacts/error_log_document.log + archive: false + + - name: Commit updated files + id: auto-commit + if: steps.check-changes.outputs.has_changes == 'true' && steps.test-conda-env.outcome == 'success' + uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5 + with: + commit_message: GitHub Action - Updated files because of notebook and/or requirements changes + file_pattern: requirements*.txt environment.yaml notebooks/* setup.py + + - name: Revert user commit that broke requirements + if: failure() && github.event_name == 'push' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git reset --hard + git clean -fd + git revert "${{ github.sha }}" -m 1 --no-edit + git push origin HEAD:${{ github.ref_name }} diff --git a/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/update_on_src_change.yaml b/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/update_on_src_change.yaml new file mode 100644 index 0000000..76b90f0 --- /dev/null +++ b/.template_sync/resources/v0_1_17_to_v0_1_18/.github/workflows/update_on_src_change.yaml @@ -0,0 +1,41 @@ +name: Updates on external code change + +on: + push: + branches: [ main, master ] + paths: + - 'src/**' + workflow_dispatch: + +# This enables to run a single workflow at a time +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon external code update the code version + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Update src change file + run: | + cat > .tools/meta/src_change.yaml < **Check where it comes from.** The installers are not code-signed yet, so your system will warn you. Download only from the app's own GitHub release (the address starts with `https://github.com/` followed by the app's owner and repository, as shown on the app's page), and run it only if you trust that owner. A listing in the App Centre checks that the files come from the app's repository; it is not a security audit. + +> A command prompt window opens during installation on Windows; this is normal, do not close it. + +## Before you install + +- **Install path.** The install path must contain no space: the installer refuses a path with a space. Use a path such as `C:\LabConstrictor\` on Windows or `~/labconstrictor/` on macOS and Linux. + +## Install on Windows + +1. Download the `.exe` file and double-click it. +2. If Windows shows "Windows protected your PC", click "More info" and then "Run anyway". The installers are not code-signed yet. + + ![Windows protected your PC: click More info, then Run anyway](https://github.com/CellMigrationLab/LabConstrictor/blob/doc_source/Windows_Protected_your_PC.png) + +3. Follow the on-screen prompts. Launch the app from the Start Menu or your desktop. + +## Install on a Mac + +1. Choose your file: open the Apple menu, then About This Mac. If the Chip or Processor says Intel, download the Intel installer; if it says M1, M2 or similar, download the Apple Silicon one. +2. Double-click the `.pkg` file. If macOS refuses to open it, go to System Settings, Privacy & Security, scroll down to the message about the package and click "Open Anyway" (it appears only after you tried to open the file once). If you do not see it, try opening the file again, then check this page again. +3. Follow the prompts (we recommend "Install only for me"). Launch the app from your Applications folder. + +## Install on Linux + +1. Download the `.sh` file. +2. Open a terminal in the download folder and run `bash` followed by the name of the file you downloaded, for example: `bash MyApp-0.1.0-Linux-x86_64.sh` +3. Follow the prompts, then launch the app from your applications menu. + +## Update an installed app + +- **macOS and Windows.** On macOS and Windows, update an app by uninstalling the old version first and then installing the new one: the installer does not install over an existing install. +- **Linux.** On Linux, running the installer with `-u` updates in place only if the previous install finished; otherwise delete `/pkgs` or uninstall the app and install it again. + +## Where the app is listed + +If the app ships tools for Napari, Fiji or QuPath, the installer registers them with LabConstrictor Tools and passes the `hosts:` line of the app's `labconstrictor-app.yaml` (the installer bundles that file). A host lists the app only if the line names it: with `hosts: [notebook, napari]` the app is listed in Napari, and not in Fiji or QuPath. Without the file, or without a `hosts:` line, the app is registered without hosts, every host lists it, and `menuinst_debug.log` in the install folder says so (`'' declares no hosts, so it is listed in every host; ...`). + +## Next + +Open the app and start working: [Use notebooks after installation](notebook_usage.md). If something goes wrong, see [Troubleshooting an installed app](troubleshooting_installed_app.md). diff --git a/.template_sync/resources/v0_1_17_to_v0_1_18/.tools/docs/tools_bridge.md b/.template_sync/resources/v0_1_17_to_v0_1_18/.tools/docs/tools_bridge.md new file mode 100644 index 0000000..14e69dc --- /dev/null +++ b/.template_sync/resources/v0_1_17_to_v0_1_18/.tools/docs/tools_bridge.md @@ -0,0 +1,58 @@ +# Expose your app's tools to Napari and Fiji (optional, experimental) + +If your repository ships Python code (see [Upload external code](external_code_upload.md)), you can declare some of its functions +as **tools**. After installation they appear in a generic **Napari** widget and a generic **Fiji** command, and work from the +command line, with forms generated automatically from the function signatures. Nothing changes for apps that do not declare tools. + +1. Add a module named `_lc_tools` next to your package under `src/` (so `src/mytool_lc_tools/__init__.py` for + the package `mytool`). Declare tools in it with typed functions, importing your science code **inside** the functions: + + ```python + from typing import Annotated + from labconstrictor_tools import Image, ImageOut, Min, tool + + @tool("Gaussian blur") + def blur(image: Image, sigma: Annotated[float, Min(0)] = 2.0) -> ImageOut: + from mytool.filters import gaussian + return gaussian(image, sigma) + ``` + + Keep the module light: it is imported to read the tool list, so heavy imports at the top slow every start. + **Import `labconstrictor_tools` only in the tools module if you can.** Since template 0.1.17 the installer installs `labconstrictor-tools` *before* it checks that + your package can be imported, so a package that imports `labconstrictor_tools` at module level no longer stops the installation. It is still better practice to import it lazily (inside functions) + or only in `_lc_tools`: the rest of your package then also works where Tools is not installed (a notebook run on Colab, a copy of the package used elsewhere, a failed Tools install). +2. The installer (`post_install`) detects the module, installs `labconstrictor-tools` and registers the app + (`labconstrictor-tools register ...`). `pre_uninstall` unregisters it. A failure in this step is logged to + `menuinst_debug.log` and never fails the installation. + **Order in `post_install`.** (1) your package is installed, (2) if it ships `_lc_tools`, `labconstrictor-tools` is installed, (3) the package import check runs, (4) the app is registered. + Installing Tools before the import check is what lets your package import `labconstrictor_tools` at module level. A failed Tools install is a `WARNING` in `menuinst_debug.log` + and the registration is skipped; the installation continues, unless your package needs Tools to be imported (then the import check fails, as it should). + **Where `labconstrictor-tools` comes from.** It is not on PyPI yet, so by default the installer downloads the GitHub archive of a **pinned release tag**, + `https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/v0.3.0-alpha.1.zip` (a plain zip: git is not needed on the user's computer). + Set the environment variable `LC_TOOLS_SPEC` when the installer runs to install it from somewhere else (a wheel, a git URL, a mirror, or the + name `labconstrictor-tools` once it is published on PyPI); it overrides the pin completely. + **Pinned (important).** Two installers built from the same app release get the same version of the tools package, and a change to Tools `main` + does not reach new installations until the pin is moved on purpose. + **How to move the pin.** The tag name is written in one place per script: the variable `LC_TOOLS_PIN` (`LC_TOOLS_PIN="v0.3.0-alpha.1"` in + `app/bash_bat_scripts/post_install.sh`, `SET "LC_TOOLS_PIN=v0.3.0-alpha.1"` in `post_install.bat`). To use a new release: create and test the tag in + LabConstrictor-Tools (procedure in its `docs/OPERATIONS.md`), change that string in both scripts and in the copies under + `.template_sync/resources/` of the latest migration (the test `test_resources_are_identical_to_the_template_files` fails when they differ), + update the tag in this paragraph, and bump `template_version` so apps receive the change. Apps already migrated need a new migration to change the + string in their own scripts. +3. Test your tools before releasing: `labconstrictor-tools check --module mytool_lc_tools` and + `labconstrictor-tools test --module mytool_lc_tools --cases lc_tests/cases.json`. +4. Install the Napari plugin (`napari-labconstrictor`) or the Fiji plugin (`labconstrictor-fiji`) once per machine. Neither is on PyPI or a Fiji update site yet: see the READMEs of + [napari-labconstrictor](https://github.com/CellMigrationLab/napari-labconstrictor) and [LabConstrictor-Fiji](https://github.com/CellMigrationLab/LabConstrictor-Fiji) for the install commands. +5. Optional: make a long form readable and rich with markers (the authoring guide in the Tools repository lists them all): + - layout: `Group("...")`, `Advanced()`, `Collapsed()`, `EnabledWhen(...)`; + - input widgets: `Widget("slider")` for a bounded number and `Widget("radio")` for a short list of choices; + - `PickChannel()` to let the person choose one channel of a multi-channel image (your function receives that channel as a 2D array); + - `ChoicesFrom("other_tool")` to fill a dropdown from another tool of yours (for example the available devices); + - `RegionOf("image")` (optional labels input) to run on the selection in the host (Napari Shapes layer, Fiji ROI, QuPath annotations); + - outputs: `ImageOut`, `LabelsOut`, `TableOut`, `Scalars`, `MessageOut`, `PointsOut` and `ShapesOut` (polygons: Napari shapes layer, Fiji ROIs, QuPath annotations); + - say "nothing found" with `ToolError("no_match", ...)`. + Napari, Fiji and QuPath do not support every feature in the same way: the table "What each host does with each feature" in `docs/HOST_FEATURES.md` of the Tools repository is the reference. +Documentation and source: https://github.com/CellMigrationLab/LabConstrictor-Tools. +The registered version is read from the `version:` line of `construct.yaml`. The Windows hook (`post_install.bat`) has not yet been run on a real Windows computer; if you can help, follow the +[human test protocol](https://github.com/CellMigrationLab/LabConstrictor-Tools/blob/main/docs/HUMAN_TEST_PROTOCOL.md). +Troubleshooting: `labconstrictor-tools doctor`, `labconstrictor-tools logs`, and `~/.labconstrictor/logs/labconstrictor.log`. diff --git a/.template_sync/resources/v0_1_17_to_v0_1_18/app/bash_bat_scripts/post_install.bat b/.template_sync/resources/v0_1_17_to_v0_1_18/app/bash_bat_scripts/post_install.bat new file mode 100644 index 0000000..4dd92c6 --- /dev/null +++ b/.template_sync/resources/v0_1_17_to_v0_1_18/app/bash_bat_scripts/post_install.bat @@ -0,0 +1,245 @@ +@ECHO OFF +SETLOCAL EnableExtensions DisableDelayedExpansion + +SET "LOG_FILE=%PREFIX%\menuinst_debug.log" +SET "PYTHON_EXE=%PREFIX%\python.exe" +SET "PROJECT_ROOT=%PREFIX%\PROJECT_NAME" +SET "BASE_REQUIREMENTS=%PROJECT_ROOT%\requirements.txt" +SET "GPU_REQUIREMENTS=%PROJECT_ROOT%\requirements_gpu.txt" +SET "SELECTED_REQUIREMENTS=%BASE_REQUIREMENTS%" +SET "NVIDIA_SMI=" +SET "CA_BUNDLE=" +SET "FAILURE_MESSAGE=" +SET "CERTIFI_PATH_FILE=%TEMP%\labconstrictor_certifi_%RANDOM%_%RANDOM%.txt" + +> "%LOG_FILE%" echo Running post_install + +IF NOT EXIST "%PYTHON_EXE%" ( + SET "FAILURE_MESSAGE=Bundled Python executable was not found at %PYTHON_EXE%." + GOTO :fail +) +IF NOT EXIST "%PROJECT_ROOT%\launch_jupyter.py" ( + SET "FAILURE_MESSAGE=TLS-resilient launcher was not found at %PROJECT_ROOT%\launch_jupyter.py." + GOTO :fail +) +IF NOT EXIST "%BASE_REQUIREMENTS%" ( + SET "FAILURE_MESSAGE=Base requirements file was not found at %BASE_REQUIREMENTS%." + GOTO :fail +) + +REM pip 24.2+ uses the operating-system trust store by default. Some Windows +REM certificate stores contain malformed entries that OpenSSL cannot parse. +REM Keep verification enabled while opting out of system trust for installer +REM pip operations and using an explicit verified CA bundle. +SET "PIP_USE_DEPRECATED=legacy-certs" + +"%PYTHON_EXE%" "%PROJECT_ROOT%\launch_jupyter.py" --print-ca-bundle > "%CERTIFI_PATH_FILE%" 2>> "%LOG_FILE%" +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Unable to locate a verified CA bundle." + GOTO :fail +) +SET /P "CA_BUNDLE="<"%CERTIFI_PATH_FILE%" +DEL /Q "%CERTIFI_PATH_FILE%" >NUL 2>&1 + +IF NOT DEFINED CA_BUNDLE ( + SET "FAILURE_MESSAGE=No verified CA bundle could be selected." + GOTO :fail +) +IF NOT EXIST "%CA_BUNDLE%" ( + SET "FAILURE_MESSAGE=Selected CA bundle does not exist: %CA_BUNDLE%" + GOTO :fail +) + +SET "PIP_CERT=%CA_BUNDLE%" +SET "REQUESTS_CA_BUNDLE=%CA_BUNDLE%" +SET "CURL_CA_BUNDLE=%CA_BUNDLE%" +echo Using verified CA bundle for installer network operations: "%CA_BUNDLE%" >> "%LOG_FILE%" + +IF EXIST "%GPU_REQUIREMENTS%" ( + CALL :detect_nvidia_smi + IF DEFINED NVIDIA_SMI ( + echo NVIDIA GPU utility detected, installing GPU requirements from "%GPU_REQUIREMENTS%" >> "%LOG_FILE%" + SET "SELECTED_REQUIREMENTS=%GPU_REQUIREMENTS%" + ) ELSE ( + echo NVIDIA GPU not detected, installing CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" + ) +) ELSE ( + echo GPU requirements file not found, installing CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" +) + +echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" +"%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Application requirements installation failed." + GOTO :fail +) + +IF EXIST "%PROJECT_ROOT%\requirements-windows.txt" ( + echo Installing Windows-specific requirements. >> "%LOG_FILE%" + "%PYTHON_EXE%" -m pip install -r "%PROJECT_ROOT%\requirements-windows.txt" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Windows-specific requirements installation failed." + GOTO :fail + ) +) + +REM External Python code is optional in LabConstrictor projects. If setup.py is +REM bundled, install it without a temporary build environment and verify that +REM the generated import package is usable. Otherwise continue without it. +IF EXIST "%PROJECT_ROOT%\setup.py" ( + echo Found setup.py, installing PROJECT_NAME package locally without build isolation. >> "%LOG_FILE%" + "%PYTHON_EXE%" -m pip install --no-deps --no-build-isolation "%PROJECT_ROOT%" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=PROJECT_NAME package installation failed." + GOTO :fail + ) + + REM Tools first, only for an app whose package ships a tools module: the package may import labconstrictor_tools at module level, + REM so labconstrictor-tools is installed BEFORE the import check below. A failed install is only a WARNING, see :install_lc_tools at the end of this file. + CALL :install_lc_tools + + "%PYTHON_EXE%" -c "import PYTHON_PROJ_NAME; print('PROJECT_NAME import successful:', PYTHON_PROJ_NAME.__file__)" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Installed PROJECT_NAME package could not be imported as PYTHON_PROJ_NAME." + GOTO :fail + ) +) ELSE ( + echo No setup.py detected; this project does not bundle an optional Python package. >> "%LOG_FILE%" +) + +REM Optional: expose the app's tools to Napari, Fiji and the command line (LabConstrictor tools bridge). +REM If the app's package ships a module named _lc_tools, install labconstrictor-tools and register that module. +REM This step must never fail the installation. LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +REM labconstrictor-tools is installed by :install_lc_tools, called above before the import check of the package (it sets LC_TOOLS_PIN, LC_TOOLS_SPEC and LC_TOOLS_READY). +SET "LC_APP_VERSION=" +IF EXIST "%PROJECT_ROOT%\construct.yaml" FOR /F "usebackq tokens=1,* delims=: " %%A IN (`findstr /B /C:"version:" "%PROJECT_ROOT%\construct.yaml"`) DO SET "LC_APP_VERSION=%%~B" +IF NOT DEFINED LC_APP_VERSION ( + SET "LC_APP_VERSION=0" + echo WARNING: no version: line at the start of a line in construct.yaml - registering the tools with version 0. >> "%LOG_FILE%" +) +REM Hosts of the app: the app's labconstrictor-app.yaml (bundled by construct.yaml extra_files) goes to Tools with --app-manifest, so that +REM Napari, Fiji and QuPath list the app only in the hosts its hosts: line names. Without the file the app is +REM registered without hosts (every host lists it) and the log says so (see the lines after "Found ... registering" below). +SET "LC_APP_MANIFEST=" +IF EXIST "%PROJECT_ROOT%\labconstrictor-app.yaml" SET "LC_APP_MANIFEST=%PROJECT_ROOT%\labconstrictor-app.yaml" +"%PYTHON_EXE%" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('PYTHON_PROJ_NAME_lc_tools') else 1)" >> "%LOG_FILE%" 2>&1 +IF NOT ERRORLEVEL 1 ( + echo Found PYTHON_PROJ_NAME_lc_tools: registering the tools of PROJECT_NAME for Napari and Fiji. >> "%LOG_FILE%" + REM Which hosts list the app (the sentence is the one Tools uses for an app without hosts). Logged before the READY check, which sets the errorlevel. + IF NOT DEFINED LC_APP_MANIFEST echo WARNING: labconstrictor-app.yaml was not found in "%PROJECT_ROOT%" ^(add it to extra_files in construct.yaml^), so PROJECT_NAME is registered without hosts. 'PROJECT_NAME' declares no hosts, so it is listed in every host; register it with --app-manifest ^(or --hosts^) to choose >> "%LOG_FILE%" + IF DEFINED LC_APP_MANIFEST echo Passing the hosts of "%LC_APP_MANIFEST%" to Tools ^(--app-manifest^); Tools logs the same sentence if the file declares no hosts. >> "%LOG_FILE%" + REM labconstrictor-tools was installed before the import check (:install_lc_tools): register only if that worked (errorlevel 1 otherwise). + "%PYTHON_EXE%" -c "import sys; sys.exit(0 if sys.argv[1] == '1' else 1)" "%LC_TOOLS_READY%" >> "%LOG_FILE%" 2>&1 + IF NOT ERRORLEVEL 1 IF DEFINED LC_APP_MANIFEST "%PYTHON_EXE%" -m labconstrictor_tools register --name "PROJECT_NAME" --prefix "%PREFIX%" --module PYTHON_PROJ_NAME_lc_tools --version "%LC_APP_VERSION%" --display-name "PROJECT_NAME" --app-manifest "%LC_APP_MANIFEST%" >> "%LOG_FILE%" 2>&1 + IF NOT ERRORLEVEL 1 IF NOT DEFINED LC_APP_MANIFEST "%PYTHON_EXE%" -m labconstrictor_tools register --name "PROJECT_NAME" --prefix "%PREFIX%" --module PYTHON_PROJ_NAME_lc_tools --version "%LC_APP_VERSION%" --display-name "PROJECT_NAME" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 echo WARNING: tool registration failed - see the pip and register output above in this file; PROJECT_NAME itself is installed. >> "%LOG_FILE%" +) + +"%PYTHON_EXE%" "%PROJECT_ROOT%\include_path.py" --path "%PREFIX%" --files "%PROJECT_ROOT%\notebook_launcher.json" --keyword "BASE_PATH_KEYWORD" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Launcher path preparation failed." + GOTO :fail +) + +"%PYTHON_EXE%" "%PROJECT_ROOT%\hide_code_cells.py" "%PROJECT_ROOT%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Notebook metadata preparation failed." + GOTO :fail +) + +REM Keep conda verification enabled, but avoid the Windows trust store that +REM failed during installation. Conda accepts a CA bundle path for ssl_verify. +echo Configuring conda to use verified CA bundle: "%CA_BUNDLE%" >> "%LOG_FILE%" +"%PYTHON_EXE%" -m conda config --system --set ssl_verify "%CA_BUNDLE%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + echo WARNING: Could not configure conda with the selected CA bundle. The application launcher still has its own verified fallback. >> "%LOG_FILE%" +) + +REM Validate TLS and Jupyter imports before creating shortcuts or registering +REM the application. +"%PYTHON_EXE%" "%PROJECT_ROOT%\launch_jupyter.py" --self-test >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Launcher preflight failed. See menuinst_debug.log and launcher_debug.log." + GOTO :fail +) +echo Launcher preflight completed successfully. >> "%LOG_FILE%" + +"%PYTHON_EXE%" -c "import os, sys; print('Python:', sys.executable); print('Prefix:', os.environ.get('PREFIX'))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Python installation diagnostics failed." + GOTO :fail +) + +"%PYTHON_EXE%" -c "from menuinst.api import install; import os; print(install(os.path.join(r'%PREFIX%', 'PROJECT_NAME', 'notebook_launcher.json')))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Application shortcut creation failed." + GOTO :fail +) + +SET "ARP_KEY=HKCU\Software\Microsoft\Windows\CurrentVersion\Uninstall\UNDERSCORED_PROJECT_NAME" +SET "UNINSTALL_EXE=%PREFIX%\Uninstall-UNDERSCORED_PROJECT_NAME.exe" +SET "DISPLAY_ICON=%PROJECT_ROOT%\logo.ico" +SET "DISPLAY_VERSION=VERSION_NUMBER" +SET "PUBLISHER=GITHUB_OWNER" +echo Registering PROJECT_NAME in Windows Apps list >> "%LOG_FILE%" +reg add "%ARP_KEY%" /v DisplayName /d "PROJECT_NAME" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v DisplayVersion /d "%DISPLAY_VERSION%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v Publisher /d "%PUBLISHER%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v InstallLocation /d "%PREFIX%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v DisplayIcon /d "%DISPLAY_ICON%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v UninstallString /d "\"%UNINSTALL_EXE%\"" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v QuietUninstallString /d "\"%UNINSTALL_EXE%\" /S" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v NoModify /t REG_DWORD /d 1 /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v NoRepair /t REG_DWORD /d 1 /f >> "%LOG_FILE%" 2>&1 + +echo Post-install completed successfully. >> "%LOG_FILE%" +echo Post-install completed successfully. +ENDLOCAL +EXIT /B 0 + +:fail +IF EXIST "%CERTIFI_PATH_FILE%" DEL /Q "%CERTIFI_PATH_FILE%" >NUL 2>&1 +IF NOT DEFINED FAILURE_MESSAGE SET "FAILURE_MESSAGE=Post-install failed for an unknown reason." +echo ERROR: %FAILURE_MESSAGE% >> "%LOG_FILE%" +echo ERROR: %FAILURE_MESSAGE% +echo See "%LOG_FILE%" for details. +ENDLOCAL +EXIT /B 1 + +:detect_nvidia_smi +FOR %%P IN ( + "%ProgramFiles%\NVIDIA Corporation\NVSMI\nvidia-smi.exe" + "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvidia-smi.exe" + "%SystemRoot%\System32\nvidia-smi.exe" + "%SystemRoot%\Sysnative\nvidia-smi.exe" +) DO ( + IF NOT DEFINED NVIDIA_SMI IF EXIST "%%~P" SET "NVIDIA_SMI=%%~P" +) +IF DEFINED NVIDIA_SMI GOTO :EOF + +FOR /F "delims=" %%I IN ('where.exe nvidia-smi.exe 2^>NUL') DO ( + IF NOT DEFINED NVIDIA_SMI SET "NVIDIA_SMI=%%~fI" +) +GOTO :EOF + +:install_lc_tools +REM Optional (LabConstrictor tools bridge): install labconstrictor-tools when the app's package ships a module named _lc_tools. +REM Called before the import check of the package, which may import labconstrictor_tools at module level. +REM This step must never fail the installation: a failure is logged as a WARNING and LC_TOOLS_READY stays 0 (no registration later). +REM LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +REM Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), +REM because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. +SET "LC_TOOLS_PIN=v0.3.0-alpha.1" +IF NOT DEFINED LC_TOOLS_SPEC SET "LC_TOOLS_SPEC=https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/%LC_TOOLS_PIN%.zip" +SET "LC_TOOLS_READY=0" +"%PYTHON_EXE%" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('PYTHON_PROJ_NAME_lc_tools') else 1)" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 GOTO :EOF +echo Found PYTHON_PROJ_NAME_lc_tools: installing labconstrictor-tools before the import check of PROJECT_NAME. >> "%LOG_FILE%" +REM LC_TOOLS_SPEC goes to pip as one argument through Python: cmd.exe never re-parses it (a quote or & in it cannot run anything). +"%PYTHON_EXE%" -c "import os, subprocess, sys; sys.exit(subprocess.call([sys.executable, '-m', 'pip', 'install', os.environ['LC_TOOLS_SPEC']]))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + echo WARNING: labconstrictor-tools could not be installed - see the pip output above in this file; the tools of PROJECT_NAME will not be registered ^(the import check fails only if the package needs labconstrictor-tools at import time^). >> "%LOG_FILE%" + GOTO :EOF +) +SET "LC_TOOLS_READY=1" +GOTO :EOF diff --git a/.template_sync/resources/v0_1_17_to_v0_1_18/app/bash_bat_scripts/post_install.sh b/.template_sync/resources/v0_1_17_to_v0_1_18/app/bash_bat_scripts/post_install.sh new file mode 100644 index 0000000..ebcaf0b --- /dev/null +++ b/.template_sync/resources/v0_1_17_to_v0_1_18/app/bash_bat_scripts/post_install.sh @@ -0,0 +1,147 @@ +#!/bin/bash +set -euo pipefail + +LOG_FILE="$PREFIX/menuinst_debug.log" +PYTHON_EXE="$PREFIX/bin/python" +PROJECT_ROOT="$PREFIX/PROJECT_NAME" +BASE_REQUIREMENTS="$PROJECT_ROOT/requirements.txt" +GPU_REQUIREMENTS="$PROJECT_ROOT/requirements_gpu.txt" +SELECTED_REQUIREMENTS="$BASE_REQUIREMENTS" + +echo "Running post_install" > "$LOG_FILE" + +fail() { + echo "ERROR: $1" | tee -a "$LOG_FILE" >&2 + exit 1 +} + +[ -x "$PYTHON_EXE" ] || fail "Bundled Python executable was not found at $PYTHON_EXE." +[ -f "$PROJECT_ROOT/launch_jupyter.py" ] || fail "TLS-resilient launcher was not found at $PROJECT_ROOT/launch_jupyter.py." +[ -f "$BASE_REQUIREMENTS" ] || fail "Base requirements file was not found at $BASE_REQUIREMENTS." + +# pip 24.2+ uses system certificates by default. Opt out for installer pip +# operations and use an explicit verified CA bundle instead. +export PIP_USE_DEPRECATED=legacy-certs +CA_BUNDLE="$("$PYTHON_EXE" "$PROJECT_ROOT/launch_jupyter.py" --print-ca-bundle 2>> "$LOG_FILE")" +[ -s "$CA_BUNDLE" ] || fail "No valid verified CA bundle could be selected: $CA_BUNDLE" + +export PIP_CERT="$CA_BUNDLE" +export REQUESTS_CA_BUNDLE="$CA_BUNDLE" +export CURL_CA_BUNDLE="$CA_BUNDLE" +echo "Using verified CA bundle for installer network operations: $CA_BUNDLE" >> "$LOG_FILE" + +if [ -f "$GPU_REQUIREMENTS" ]; then + if [[ "${OSTYPE:-}" == "darwin"* ]]; then + echo "macOS detected, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" + elif command -v nvidia-smi >/dev/null 2>&1; then + echo "NVIDIA GPU detected, installing GPU requirements from $GPU_REQUIREMENTS" >> "$LOG_FILE" + SELECTED_REQUIREMENTS="$GPU_REQUIREMENTS" + else + echo "NVIDIA GPU not detected, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" + fi +else + echo "GPU requirements file not found, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" +fi + +echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE" +"$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1 + +if [[ "${OSTYPE:-}" == "darwin"* ]]; then + echo "Detected macOS platform" >> "$LOG_FILE" + if [ -f "$PROJECT_ROOT/requirements-macos.txt" ]; then + "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-macos.txt" >> "$LOG_FILE" 2>&1 + fi +elif [[ "${OSTYPE:-}" == "linux-gnu"* ]]; then + echo "Detected Linux platform" >> "$LOG_FILE" + if [ -f "$PROJECT_ROOT/requirements-linux.txt" ]; then + "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-linux.txt" >> "$LOG_FILE" 2>&1 + fi +else + echo "Unknown platform: ${OSTYPE:-unset}" >> "$LOG_FILE" +fi + +# External Python code is optional. Verify the generated package only when the +# constructor bundled setup.py and src/. +if [ -f "$PROJECT_ROOT/setup.py" ]; then + echo "Found setup.py, installing PROJECT_NAME package locally without build isolation" >> "$LOG_FILE" + "$PYTHON_EXE" -m pip install --no-deps --no-build-isolation "$PROJECT_ROOT" >> "$LOG_FILE" 2>&1 + # Tools first, only for an app whose package ships a _lc_tools module: the app package may import labconstrictor_tools + # at module level, so labconstrictor-tools must be installed BEFORE the import check below. A failed install is only a WARNING + # (it never fails the installation by itself). LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). + # Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), + # because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. + LC_TOOLS_PIN="v0.3.0-alpha.1" + LC_TOOLS_MODULE="PYTHON_PROJ_NAME_lc_tools" + LC_TOOLS_READY=0 + if "$PYTHON_EXE" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('$LC_TOOLS_MODULE') else 1)" >> "$LOG_FILE" 2>&1; then + echo "Found $LC_TOOLS_MODULE: installing labconstrictor-tools before the import check of PROJECT_NAME" >> "$LOG_FILE" || : + if "$PYTHON_EXE" -m pip install "${LC_TOOLS_SPEC:-https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/${LC_TOOLS_PIN}.zip}" >> "$LOG_FILE" 2>&1; then + LC_TOOLS_READY=1 + else + echo "WARNING: labconstrictor-tools could not be installed - see the pip output above in this file; the tools of PROJECT_NAME will not be registered (the import check below fails only if the package needs labconstrictor-tools at import time)." >> "$LOG_FILE" || : + fi + fi + "$PYTHON_EXE" -c "import PYTHON_PROJ_NAME; print('PROJECT_NAME import successful:', PYTHON_PROJ_NAME.__file__)" >> "$LOG_FILE" 2>&1 +else + echo "No setup.py detected; this project does not bundle an optional Python package." >> "$LOG_FILE" +fi + + +# --- Optional: expose the app's tools to Napari, Fiji and the command line (LabConstrictor tools bridge) ------------------- +# If the app's package ships a module named _lc_tools, install labconstrictor-tools and register that module. +# This step must never fail the installation. LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +# labconstrictor-tools is installed above, before the import check of the package (LC_TOOLS_PIN, LC_TOOLS_MODULE and LC_TOOLS_READY are set there). +if [ -f "$PROJECT_ROOT/setup.py" ] && "$PYTHON_EXE" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('$LC_TOOLS_MODULE') else 1)" >> "$LOG_FILE" 2>&1; then + lc_note() { echo "$*" >> "$LOG_FILE" || :; } # a full or unwritable log must not fail the installation + lc_note "Found $LC_TOOLS_MODULE: registering the tools of PROJECT_NAME for Napari and Fiji" + LC_APP_VERSION="$(sed -n 's/^version: *//p' "$PROJECT_ROOT/construct.yaml" 2>/dev/null | head -1 | tr -d "\"'\r" || true)" + if [ -z "$LC_APP_VERSION" ]; then + lc_note "WARNING: no version: line at the start of a line in construct.yaml - registering the tools with version 0." + LC_APP_VERSION=0 + fi + # Hosts of the app: the app's labconstrictor-app.yaml (bundled by construct.yaml extra_files) goes to Tools with --app-manifest, so that + # Napari, Fiji and QuPath list the app only in the hosts its hosts: line names. Without the file the app is registered + # without hosts (every host lists it) and the log says so, with the sentence Tools uses for it. Tools itself reads the file: a manifest + # without a hosts: line (or with an invalid one) is answered by Tools, whose message goes to this log. + LC_APP_MANIFEST="$PROJECT_ROOT/labconstrictor-app.yaml" + LC_HOSTS_ARGS=() + LC_NO_HOSTS="'PROJECT_NAME' declares no hosts, so it is listed in every host; register it with --app-manifest (or --hosts) to choose" + if [ ! -f "$LC_APP_MANIFEST" ]; then + lc_note "WARNING: $LC_APP_MANIFEST was not found (add labconstrictor-app.yaml to extra_files in construct.yaml), so PROJECT_NAME is registered without hosts. $LC_NO_HOSTS" + else + LC_HOSTS_ARGS=(--app-manifest "$LC_APP_MANIFEST") + lc_note "Passing the hosts of $LC_APP_MANIFEST to Tools (--app-manifest); Tools logs the same sentence if the file declares no hosts." + fi + if [ "${LC_TOOLS_READY:-0}" = 1 ] \ + && "$PYTHON_EXE" -m labconstrictor_tools register --name "PROJECT_NAME" --prefix "$PREFIX" \ + --module "$LC_TOOLS_MODULE" --version "$LC_APP_VERSION" --display-name "PROJECT_NAME" ${LC_HOSTS_ARGS[@]+"${LC_HOSTS_ARGS[@]}"} >> "$LOG_FILE" 2>&1; then + lc_note "Tools registered (labconstrictor-tools list shows them)." + else + lc_note "WARNING: tool registration failed - see the pip and register output above in this file; PROJECT_NAME itself is installed." + fi +fi + +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PROJECT_ROOT/notebook_launcher.json" --keyword "BASE_PATH_KEYWORD" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PREFIX/pre_uninstall.sh" --keyword "BASE_PATH" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PREFIX/uninstall.sh" --keyword "BASE_PATH" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/hide_code_cells.py" "$PROJECT_ROOT" >> "$LOG_FILE" 2>&1 + +# Keep conda verification enabled while avoiding the system certificate store +# that failed. Conda accepts a CA bundle path for ssl_verify. +if ! "$PYTHON_EXE" -m conda config --system --set ssl_verify "$CA_BUNDLE" >> "$LOG_FILE" 2>&1; then + echo "WARNING: Could not configure conda with the selected CA bundle. The launcher still has its own verified fallback." >> "$LOG_FILE" +fi + +# Do not create the application shortcut unless launcher preflight succeeds. +"$PYTHON_EXE" "$PROJECT_ROOT/launch_jupyter.py" --self-test >> "$LOG_FILE" 2>&1 +echo "Launcher preflight completed successfully." >> "$LOG_FILE" + +"$PYTHON_EXE" -c "import os, sys; print('Python:', sys.executable); print('Prefix:', os.environ.get('PREFIX'))" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" -c "from menuinst.api import install; import os; print(install(os.path.join('$PREFIX', 'PROJECT_NAME', 'notebook_launcher.json')))" >> "$LOG_FILE" 2>&1 + +echo "Post-install completed successfully." >> "$LOG_FILE" + +if [ -t 0 ]; then + echo + read -rp "Press Enter to close the installer..." _ +fi diff --git a/.tools/docs/README.md b/.tools/docs/README.md index d89cdc4..ece3106 100644 --- a/.tools/docs/README.md +++ b/.tools/docs/README.md @@ -29,6 +29,7 @@ This file temporarily lists all internal LabConstrictor documentation. After you - [Enable automatic template updates](template_synchronization.md) - [Template 0.1.16: how the Tools bridge hooks reach an existing app](migration_0_1_16.md) - [Template 0.1.17: install Tools before the package import check](migration_0_1_17.md) +- [Template 0.1.18: new Tools pin, pinned GitHub Actions, install rules](migration_0_1_18.md) ## Notebook authoring tips diff --git a/.tools/docs/install.md b/.tools/docs/install.md index 247e071..4048356 100644 --- a/.tools/docs/install.md +++ b/.tools/docs/install.md @@ -6,6 +6,10 @@ Installing takes about 6-8 minutes and you only do it once. You need an internet > A command prompt window opens during installation on Windows; this is normal, do not close it. +## Before you install + +- **Install path.** The install path must contain no space: the installer refuses a path with a space. Use a path such as `C:\LabConstrictor\` on Windows or `~/labconstrictor/` on macOS and Linux. + ## Install on Windows 1. Download the `.exe` file and double-click it. @@ -27,6 +31,15 @@ Installing takes about 6-8 minutes and you only do it once. You need an internet 2. Open a terminal in the download folder and run `bash` followed by the name of the file you downloaded, for example: `bash MyApp-0.1.0-Linux-x86_64.sh` 3. Follow the prompts, then launch the app from your applications menu. +## Update an installed app + +- **macOS and Windows.** On macOS and Windows, update an app by uninstalling the old version first and then installing the new one: the installer does not install over an existing install. +- **Linux.** On Linux, running the installer with `-u` updates in place only if the previous install finished; otherwise delete `/pkgs` or uninstall the app and install it again. + +## Where the app is listed + +If the app ships tools for Napari, Fiji or QuPath, the installer registers them with LabConstrictor Tools and passes the `hosts:` line of the app's `labconstrictor-app.yaml` (the installer bundles that file). A host lists the app only if the line names it: with `hosts: [notebook, napari]` the app is listed in Napari, and not in Fiji or QuPath. Without the file, or without a `hosts:` line, the app is registered without hosts, every host lists it, and `menuinst_debug.log` in the install folder says so (`'' declares no hosts, so it is listed in every host; ...`). + ## Next Open the app and start working: [Use notebooks after installation](notebook_usage.md). If something goes wrong, see [Troubleshooting an installed app](troubleshooting_installed_app.md). diff --git a/.tools/docs/migration_0_1_18.md b/.tools/docs/migration_0_1_18.md new file mode 100644 index 0000000..c67185d --- /dev/null +++ b/.tools/docs/migration_0_1_18.md @@ -0,0 +1,79 @@ +# Template 0.1.18: new Tools pin, pinned GitHub Actions, install rules + +Design note for the 0.1.18 migration (`.template_sync/migrations/v0_1_17_to_v0_1_18.py`). How a change reaches an installed copy is described in [migration_0_1_16.md](migration_0_1_16.md): a new installer is needed, nothing edits an existing install. + +## 1. What changes + +| Part | Change | +|---|---| +| Tools pin | `LC_TOOLS_PIN` in `app/bash_bat_scripts/post_install.sh` and `post_install.bat` moves from `v0.2.0-rc.1` to `v0.3.0-alpha.1`. | +| GitHub Actions | Every third-party `uses:` in the workflows is pinned by the commit SHA of the tag it named (the tag stays as a trailing comment). No version is upgraded. | +| Install guide | `.tools/docs/install.md` states three install rules (section 4). | +| Tools guide | `.tools/docs/tools_bridge.md` names the new pinned tag. | +| App hosts | The installer bundles the app's `labconstrictor-app.yaml` and `post_install` passes it to `labconstrictor-tools register --app-manifest`, so the `hosts:` line of the app decides which hosts list it (section 7). | +| Template CI | Only in the template repository itself: the workflow `.github/workflows/tests.yml` runs the template's tests, `black`, `ruff` and `mypy`. It does not run in apps (it checks `github.repository`) and is not delivered by the migration. | + +The pinned tag `v0.3.0-alpha.1` of LabConstrictor-Tools must exist before an installer built from this template is used: the installer downloads the archive of that tag. + +## 2. What the migration does + +Every step is independent, idempotent, never deletes, never raises for a customised file (the sync pull request is still created) and keeps the line endings of each file. Each file gets one line in the output: `updated`, `unchanged` or `SKIPPED` with the reason. + +* **Pin.** In each of the two scripts the line `LC_TOOLS_PIN="v0.2.0-rc.1"` (`SET "LC_TOOLS_PIN=v0.2.0-rc.1"` in the `.bat`) is changed to the new tag **only if it still holds exactly the old template value**. An app with its own pin, an unpinned hook (the source is `refs/heads/main.zip` or its own `LC_TOOLS_SPEC`) or no hook stays as it is and a note says so. Only the value inside the quotes changes. +* **Workflows.** Five files: `check_app_manifest.yml`, `release_constructor.yml`, `sync_template.yml`, `update_on_notebook_change.yml`, `update_on_src_change.yaml`. A file is replaced by the pinned template file **only if it is still identical to the previous template version** (line endings are ignored when comparing). A file that is already the pinned template file is unchanged. A file that the app customised (or pinned in part) is skipped, and the message lists the third-party actions that are still pinned by tag only, so the maintainer can pin them by hand with the table in section 3. A workflow that is not in the repository is not created. +* **Install guide.** If `.tools/docs/install.md` exists, the sections "Before you install" and "Update an installed app" are inserted before `## Install on Windows` and before `## Next` (each only if its heading is missing); if an anchor line is missing the step is skipped with a message. +* **Tools guide.** `.tools/docs/tools_bridge.md` is rewritten only if it is still the 0.1.17 text. +* **App hosts.** See section 7. + +Changing files under `.github/workflows/` needs the synchronization token to have the `workflows=write` permission, as for every earlier workflow change ([template_synchronization.md](template_synchronization.md)). + +## 3. Pinned actions + +| Action | Tag | Commit SHA | +|---|---|---| +| actions/checkout | v4 | `11d5960a326750d5838078e36cf38b85af677262` | +| actions/setup-python | v4 | `7f4fc3e22c37d6ff65e88745f38bd3157c663f7c` | +| actions/upload-artifact | v7 | `cf430e030ddbb5b0abf93d22962f4752f3646cd9` | +| tj-actions/changed-files | v44 | `2d756ea4c53f7f6b397767d8723b3a10a9f35bf2` | +| conda-incubator/setup-miniconda | v3 | `fc2d68f6413eb2d87b895e92f8584b5b94a10167` | +| stefanzweifel/git-auto-commit-action | v5 | `b863ae1933cb653a53c021fe36dbb774e1fb9403` | +| softprops/action-gh-release | v2 | `3bb12739c298aeb8a4eeaf626c5b8d85266b0e65` | +| peter-evans/create-pull-request | v6 | `c5a7806660adbe173f04e3e038b0ccdcd758773c` | + +Mutable tags can be moved to other code (the March 2025 `tj-actions/changed-files` incident), and these workflows hold write tokens. The SHAs are the ones apps already carry after their own pinning pull requests (for example NucleiSky pull request 37). To move to a newer version, change the SHA and the tag comment together, in the live workflow and in the copy under `.template_sync/resources/` of the latest migration. + +## 4. The three install rules + +Stated in `.tools/docs/install.md` with this wording (a test checks that every doc that states them uses the same sentences): + +* The install path must contain no space: the installer refuses a path with a space. Use a path such as `C:\LabConstrictor\` on Windows or `~/labconstrictor/` on macOS and Linux. +* On macOS and Windows, update an app by uninstalling the old version first and then installing the new one: the installer does not install over an existing install. +* On Linux, running the installer with `-u` updates in place only if the previous install finished; otherwise delete `/pkgs` or uninstall the app and install it again. + +## 5. Template tests in CI + +`.github/workflows/tests.yml` runs on every pull request and on every push to `main`, with read-only permissions: all `tests/test_*.py`, then `black --check`, `ruff check` and `mypy` on `.template_sync`, `.tools/python` and `tests`. The linters are configured in `pyproject.toml` to the state of the code at 0.1.18 (no behaviour change was made to satisfy them); tighten the configuration when code is cleaned up. + +## 6. What is not verified here + +* Installed copies are not changed: a new installer must be built and run. +* The Windows script is not run by the template tests; the end-to-end check is `smoke_migration.yml` of LabConstrictor-Tools on a Windows runner. + +## 7. App hosts: the installer passes the `hosts:` line to Tools + +Problem found by the Tools readiness check: Tools lists an app in a host (Napari, Fiji, QuPath) only for the hosts its registry entry names, but the installer never told Tools. `labconstrictor-app.yaml` was not in the installer and `post_install` called `register` without `--app-manifest`, so every registered app had no `hosts` and every host listed it. + +What the template does now: + +* **Bundle.** `bump_constructor.py` adds `labconstrictor-app.yaml: /labconstrictor-app.yaml` to `extra_files` in `construct.yaml` when the file exists in the repository (a mapping to a missing file would fail the installer build, so it is not added otherwise; an existing mapping whose file is gone is reported, not removed). +* **Register.** In `post_install.sh` / `.bat` the Tools hook passes `--app-manifest /labconstrictor-app.yaml` to `register`. If the file is not in the installation, the app is registered without hosts and `menuinst_debug.log` says so with the sentence Tools uses: `'' declares no hosts, so it is listed in every host; register it with --app-manifest (or --hosts) to choose`. A manifest that is there but has no `hosts:` line is read by Tools, which registers the app without hosts and prints the same sentence (it reaches the same log); the hook does not parse the file itself. There is no silent fallback; a manifest with an invalid `hosts:` line makes `register` refuse (Tools exit code 2), which the hook logs as `WARNING: tool registration failed`. + +What the migration does in an app repository (same rules as above: surgical, idempotent, never deletes, never raises, line endings kept): + +* `construct.yaml`: adds the `extra_files` entry **only if** the repository has a `labconstrictor-app.yaml` and the entry is not there yet, with the project folder read from the `Welcome.ipynb` entry. No manifest: `SKIPPED` with the instruction (add the file, then the entry). An `extra_files` that is not a block list, or no readable project folder: `SKIPPED`. +* `post_install.sh` / `.bat`: inserts the manifest lookup and the log lines in the Tools hook and adds the argument to the app's own `register` line (the app's own arguments stay). A hook that already has `--app-manifest` or `--hosts` is unchanged. A hook of another shape (no `lc_note` before the register line in the `.sh`; a register line that is not `IF NOT ERRORLEVEL 1 ... >> LOG 2>&1` in the `.bat`) is `SKIPPED` and nothing is written. +* `bump_constructor.py` of an app is not touched: the entry the migration writes into `construct.yaml` survives it. An app that adds its manifest later adds the mapping by hand (or takes the new `bump_constructor.py` of the template). + +Adopting it, per app: have a `labconstrictor-app.yaml` with a `hosts:` line at the repository root, take this template version (the synchronization pull request), build and release a new installer. Installed copies do not change: reinstall. + +The template CI runs the live `post_install` in a fake prefix on Linux, macOS and Windows (`tests/smoke/post_install_hosts.py`) with the real `labconstrictor-tools` and checks that the registry entry's `hosts` equals the `hosts:` line of each real manifest in `tests/fixtures/manifests`. The job installs Tools through `post_install` from the pinned tag `v0.3.0-alpha.1` (`LC_TOOLS_PIN`), so it also tests that URL: if the tag does not install, the job fails. diff --git a/.tools/docs/tools_bridge.md b/.tools/docs/tools_bridge.md index 236e827..14e69dc 100644 --- a/.tools/docs/tools_bridge.md +++ b/.tools/docs/tools_bridge.md @@ -28,13 +28,13 @@ command line, with forms generated automatically from the function signatures. N Installing Tools before the import check is what lets your package import `labconstrictor_tools` at module level. A failed Tools install is a `WARNING` in `menuinst_debug.log` and the registration is skipped; the installation continues, unless your package needs Tools to be imported (then the import check fails, as it should). **Where `labconstrictor-tools` comes from.** It is not on PyPI yet, so by default the installer downloads the GitHub archive of a **pinned release tag**, - `https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/v0.2.0-rc.1.zip` (a plain zip: git is not needed on the user's computer). + `https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/v0.3.0-alpha.1.zip` (a plain zip: git is not needed on the user's computer). Set the environment variable `LC_TOOLS_SPEC` when the installer runs to install it from somewhere else (a wheel, a git URL, a mirror, or the name `labconstrictor-tools` once it is published on PyPI); it overrides the pin completely. **Pinned (important).** Two installers built from the same app release get the same version of the tools package, and a change to Tools `main` does not reach new installations until the pin is moved on purpose. - **How to move the pin.** The tag name is written in one place per script: the variable `LC_TOOLS_PIN` (`LC_TOOLS_PIN="v0.2.0-rc.1"` in - `app/bash_bat_scripts/post_install.sh`, `SET "LC_TOOLS_PIN=v0.2.0-rc.1"` in `post_install.bat`). To use a new release: create and test the tag in + **How to move the pin.** The tag name is written in one place per script: the variable `LC_TOOLS_PIN` (`LC_TOOLS_PIN="v0.3.0-alpha.1"` in + `app/bash_bat_scripts/post_install.sh`, `SET "LC_TOOLS_PIN=v0.3.0-alpha.1"` in `post_install.bat`). To use a new release: create and test the tag in LabConstrictor-Tools (procedure in its `docs/OPERATIONS.md`), change that string in both scripts and in the copies under `.template_sync/resources/` of the latest migration (the test `test_resources_are_identical_to_the_template_files` fails when they differ), update the tag in this paragraph, and bump `template_version` so apps receive the change. Apps already migrated need a new migration to change the diff --git a/.tools/python/bump_constructor.py b/.tools/python/bump_constructor.py index 12e93bb..c1a3f14 100644 --- a/.tools/python/bump_constructor.py +++ b/.tools/python/bump_constructor.py @@ -116,6 +116,28 @@ def ensure_requirements_in_extra_files(construct_data: dict): # Update the construct data with the modified extra_files construct_data["extra_files"] = extra_files + +APP_MANIFEST = "labconstrictor-app.yaml" + + +def ensure_app_manifest_in_extra_files(construct_data: dict, manifest_path: Path = Path(APP_MANIFEST)) -> bool: + """Bundle labconstrictor-app.yaml in the installer, so post_install can pass its hosts: line to Tools (--app-manifest). + + The mapping is added only when the file exists (it is optional, and a mapping to a missing file would fail the installer build). + Returns True when the mapping was added. A mapping whose file is gone is reported, never removed. + """ + extra_files = construct_data.setdefault("extra_files", []) + mapped = any(isinstance(item, dict) and APP_MANIFEST in item for item in extra_files) + if manifest_path.exists() and not mapped: + extra_files.append({APP_MANIFEST: f"{extract_project_folder(extra_files)}/{APP_MANIFEST}"}) + debug(f"Added mapping for {APP_MANIFEST} (its hosts: line goes to Tools at install time)") + return True + if mapped and not manifest_path.exists(): + debug(f"WARNING: extra_files maps {APP_MANIFEST} but the file does not exist: the installer build will fail until it is added or the mapping is removed") + elif not mapped: + debug(f"{APP_MANIFEST} not found: the app will be registered without hosts (listed in every host)") + return False + def ensure_extra_files(construct_data: dict, notebooks_root: Path, src_root: Path) -> int: extra_files = construct_data.get("extra_files") if extra_files is None: @@ -274,6 +296,7 @@ def main(): data = load_construct(construct_path) # Check if the requirements files are included in extra_files, if not, add them ensure_requirements_in_extra_files(data) + ensure_app_manifest_in_extra_files(data) # Update extra_files with notebooks and src files ntbk_added, src_added = ensure_extra_files(data, notebooks_root, src_root) diff --git a/app/bash_bat_scripts/post_install.bat b/app/bash_bat_scripts/post_install.bat index 55d9761..4dd92c6 100644 --- a/app/bash_bat_scripts/post_install.bat +++ b/app/bash_bat_scripts/post_install.bat @@ -117,12 +117,21 @@ IF NOT DEFINED LC_APP_VERSION ( SET "LC_APP_VERSION=0" echo WARNING: no version: line at the start of a line in construct.yaml - registering the tools with version 0. >> "%LOG_FILE%" ) +REM Hosts of the app: the app's labconstrictor-app.yaml (bundled by construct.yaml extra_files) goes to Tools with --app-manifest, so that +REM Napari, Fiji and QuPath list the app only in the hosts its hosts: line names. Without the file the app is +REM registered without hosts (every host lists it) and the log says so (see the lines after "Found ... registering" below). +SET "LC_APP_MANIFEST=" +IF EXIST "%PROJECT_ROOT%\labconstrictor-app.yaml" SET "LC_APP_MANIFEST=%PROJECT_ROOT%\labconstrictor-app.yaml" "%PYTHON_EXE%" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('PYTHON_PROJ_NAME_lc_tools') else 1)" >> "%LOG_FILE%" 2>&1 IF NOT ERRORLEVEL 1 ( echo Found PYTHON_PROJ_NAME_lc_tools: registering the tools of PROJECT_NAME for Napari and Fiji. >> "%LOG_FILE%" + REM Which hosts list the app (the sentence is the one Tools uses for an app without hosts). Logged before the READY check, which sets the errorlevel. + IF NOT DEFINED LC_APP_MANIFEST echo WARNING: labconstrictor-app.yaml was not found in "%PROJECT_ROOT%" ^(add it to extra_files in construct.yaml^), so PROJECT_NAME is registered without hosts. 'PROJECT_NAME' declares no hosts, so it is listed in every host; register it with --app-manifest ^(or --hosts^) to choose >> "%LOG_FILE%" + IF DEFINED LC_APP_MANIFEST echo Passing the hosts of "%LC_APP_MANIFEST%" to Tools ^(--app-manifest^); Tools logs the same sentence if the file declares no hosts. >> "%LOG_FILE%" REM labconstrictor-tools was installed before the import check (:install_lc_tools): register only if that worked (errorlevel 1 otherwise). "%PYTHON_EXE%" -c "import sys; sys.exit(0 if sys.argv[1] == '1' else 1)" "%LC_TOOLS_READY%" >> "%LOG_FILE%" 2>&1 - IF NOT ERRORLEVEL 1 "%PYTHON_EXE%" -m labconstrictor_tools register --name "PROJECT_NAME" --prefix "%PREFIX%" --module PYTHON_PROJ_NAME_lc_tools --version "%LC_APP_VERSION%" --display-name "PROJECT_NAME" >> "%LOG_FILE%" 2>&1 + IF NOT ERRORLEVEL 1 IF DEFINED LC_APP_MANIFEST "%PYTHON_EXE%" -m labconstrictor_tools register --name "PROJECT_NAME" --prefix "%PREFIX%" --module PYTHON_PROJ_NAME_lc_tools --version "%LC_APP_VERSION%" --display-name "PROJECT_NAME" --app-manifest "%LC_APP_MANIFEST%" >> "%LOG_FILE%" 2>&1 + IF NOT ERRORLEVEL 1 IF NOT DEFINED LC_APP_MANIFEST "%PYTHON_EXE%" -m labconstrictor_tools register --name "PROJECT_NAME" --prefix "%PREFIX%" --module PYTHON_PROJ_NAME_lc_tools --version "%LC_APP_VERSION%" --display-name "PROJECT_NAME" >> "%LOG_FILE%" 2>&1 IF ERRORLEVEL 1 echo WARNING: tool registration failed - see the pip and register output above in this file; PROJECT_NAME itself is installed. >> "%LOG_FILE%" ) @@ -220,7 +229,7 @@ REM This step must never fail the installation: a failure is logged as a WARNING REM LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). REM Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), REM because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. -SET "LC_TOOLS_PIN=v0.2.0-rc.1" +SET "LC_TOOLS_PIN=v0.3.0-alpha.1" IF NOT DEFINED LC_TOOLS_SPEC SET "LC_TOOLS_SPEC=https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/%LC_TOOLS_PIN%.zip" SET "LC_TOOLS_READY=0" "%PYTHON_EXE%" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('PYTHON_PROJ_NAME_lc_tools') else 1)" >> "%LOG_FILE%" 2>&1 diff --git a/app/bash_bat_scripts/post_install.sh b/app/bash_bat_scripts/post_install.sh index 67d91a2..ebcaf0b 100644 --- a/app/bash_bat_scripts/post_install.sh +++ b/app/bash_bat_scripts/post_install.sh @@ -70,7 +70,7 @@ if [ -f "$PROJECT_ROOT/setup.py" ]; then # (it never fails the installation by itself). LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). # Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), # because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. - LC_TOOLS_PIN="v0.2.0-rc.1" + LC_TOOLS_PIN="v0.3.0-alpha.1" LC_TOOLS_MODULE="PYTHON_PROJ_NAME_lc_tools" LC_TOOLS_READY=0 if "$PYTHON_EXE" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('$LC_TOOLS_MODULE') else 1)" >> "$LOG_FILE" 2>&1; then @@ -99,9 +99,22 @@ if [ -f "$PROJECT_ROOT/setup.py" ] && "$PYTHON_EXE" -c "import importlib.util, s lc_note "WARNING: no version: line at the start of a line in construct.yaml - registering the tools with version 0." LC_APP_VERSION=0 fi + # Hosts of the app: the app's labconstrictor-app.yaml (bundled by construct.yaml extra_files) goes to Tools with --app-manifest, so that + # Napari, Fiji and QuPath list the app only in the hosts its hosts: line names. Without the file the app is registered + # without hosts (every host lists it) and the log says so, with the sentence Tools uses for it. Tools itself reads the file: a manifest + # without a hosts: line (or with an invalid one) is answered by Tools, whose message goes to this log. + LC_APP_MANIFEST="$PROJECT_ROOT/labconstrictor-app.yaml" + LC_HOSTS_ARGS=() + LC_NO_HOSTS="'PROJECT_NAME' declares no hosts, so it is listed in every host; register it with --app-manifest (or --hosts) to choose" + if [ ! -f "$LC_APP_MANIFEST" ]; then + lc_note "WARNING: $LC_APP_MANIFEST was not found (add labconstrictor-app.yaml to extra_files in construct.yaml), so PROJECT_NAME is registered without hosts. $LC_NO_HOSTS" + else + LC_HOSTS_ARGS=(--app-manifest "$LC_APP_MANIFEST") + lc_note "Passing the hosts of $LC_APP_MANIFEST to Tools (--app-manifest); Tools logs the same sentence if the file declares no hosts." + fi if [ "${LC_TOOLS_READY:-0}" = 1 ] \ && "$PYTHON_EXE" -m labconstrictor_tools register --name "PROJECT_NAME" --prefix "$PREFIX" \ - --module "$LC_TOOLS_MODULE" --version "$LC_APP_VERSION" --display-name "PROJECT_NAME" >> "$LOG_FILE" 2>&1; then + --module "$LC_TOOLS_MODULE" --version "$LC_APP_VERSION" --display-name "PROJECT_NAME" ${LC_HOSTS_ARGS[@]+"${LC_HOSTS_ARGS[@]}"} >> "$LOG_FILE" 2>&1; then lc_note "Tools registered (labconstrictor-tools list shows them)." else lc_note "WARNING: tool registration failed - see the pip and register output above in this file; PROJECT_NAME itself is installed." diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..8b65979 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,12 @@ +# Configuration of mypy for the template's own Python (.template_sync, .tools/python, tests), run by .github/workflows/tests.yml. +# A separate file (not pyproject.toml) so that nothing changes for `pip install` of an app package built from this repository. +[mypy] +python_version = 3.11 +ignore_missing_imports = True +exclude = ^(\.template_sync/resources|tests/fixtures)/ +warn_unused_configs = True + +# Modules that do not pass mypy yet (missing annotations or optional values used without a check, found when the check was added at 0.1.18). +# They are listed one by one, not excluded, so a new error in any other module fails the check; remove an entry when its module is cleaned up. +[mypy-update_changelog,update_version_last_commit,merge_requirements,bump_constructor,v0_1_16_to_v0_1_17,test_template_sync_0_1_16,test_template_sync_0_1_17] +ignore_errors = True diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..0387686 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,18 @@ +# Configuration of ruff for the template's own Python (.template_sync, .tools/python, tests), run by .github/workflows/tests.yml. +# A separate file (not pyproject.toml) so that nothing changes for `pip install` of an app package built from this repository. +# The configuration describes the state of the code at template 0.1.18: nothing was reformatted or rewritten to satisfy a tool, because the +# migrations are frozen history and the files under .template_sync/resources are byte copies that tests compare with the live files. +line-length = 160 +target-version = "py311" +extend-exclude = ["tests/fixtures", ".template_sync/resources"] + +[lint] +# ruff's default rule set (pycodestyle E4/E7/E9, pyflakes F) except: +# * E741 (ambiguous variable name `l`): used throughout the migrations and tests as the loop variable over lines. +ignore = ["E741"] + +[lint.per-file-ignores] +# unused imports in files that are frozen history or shipped to apps unchanged +".template_sync/migrations/v0_1_5_to_v0_1_6.py" = ["F401"] +".template_sync/sync.py" = ["F401"] +".tools/python/update_changelog.py" = ["F401"] diff --git a/tests/_hosts.py b/tests/_hosts.py new file mode 100644 index 0000000..2bdde26 --- /dev/null +++ b/tests/_hosts.py @@ -0,0 +1,49 @@ +"""Helper of the tests: the template post_install scripts without the app-hosts lines (the step added in 0.1.18 after the pin step). + +Tests of the pin step, and of the frozen 0.1.17 resources, compare scripts with the older shape; `strip_hosts` removes exactly the lines +the hosts step adds, so those tests keep checking what they checked, and the hosts step has its own tests (test_template_sync_0_1_18_hosts.py). +""" +from __future__ import annotations + +import re + + +SH_START = "# Hosts of the app:" +SH_ARGS = ' ${LC_HOSTS_ARGS[@]+"${LC_HOSTS_ARGS[@]}"}' +BAT_PRE_START = "REM Hosts of the app:" +BAT_PRE_LAST = 'IF EXIST "%PROJECT_ROOT%\\labconstrictor-app.yaml"' +BAT_LOGS_START = " REM Which hosts list the app" +BAT_LOG_LINE = re.compile(r"^ IF (NOT )?DEFINED LC_APP_MANIFEST echo ") + + +def _drop_range(lines: list[str], start: str, last: str) -> list[str]: + """Remove the lines from the first one that starts with `start` up to and including the next one that starts with `last`.""" + begin = next((i for i, line in enumerate(lines) if line.startswith(start)), None) + if begin is None: + return lines + stop = next(i for i in range(begin, len(lines)) if lines[i].startswith(last)) + return lines[:begin] + lines[stop + 1 :] + + +def strip_hosts_sh(text: str) -> str: + """Remove the inserted block: from its first comment line to the `fi` that closes it (same indent as the comment), and the register argument.""" + lines = text.split("\n") + begin = next((i for i, line in enumerate(lines) if line.lstrip().startswith(SH_START)), None) + if begin is not None: + indent = lines[begin][: len(lines[begin]) - len(lines[begin].lstrip())] + stop = next(i for i in range(begin, len(lines)) if lines[i].rstrip("\r") == indent + "fi") + lines = lines[:begin] + lines[stop + 1 :] + return "\n".join(lines).replace(SH_ARGS, "") + + +def strip_hosts_bat(text: str) -> str: + lines = _drop_range(text.split("\n"), BAT_PRE_START, BAT_PRE_LAST) + lines = [line for line in lines if not line.startswith(BAT_LOGS_START) and not BAT_LOG_LINE.match(line.rstrip("\r"))] + lines = [line for line in lines if " IF NOT DEFINED LC_APP_MANIFEST " not in line] + return "\n".join(line.replace(" IF DEFINED LC_APP_MANIFEST ", " ").replace(' --app-manifest "%LC_APP_MANIFEST%"', "") for line in lines) + + +def strip_hosts(name: str, data: bytes) -> bytes: + """The script `name` (post_install.sh or post_install.bat) as bytes, without the lines of the hosts step (line endings kept).""" + strip = strip_hosts_sh if name.endswith(".sh") else strip_hosts_bat + return strip(data.decode()).encode() diff --git a/tests/fixtures/manifests/celltrackscolab_labconstrictor-app.yaml b/tests/fixtures/manifests/celltrackscolab_labconstrictor-app.yaml new file mode 100644 index 0000000..7b27072 --- /dev/null +++ b/tests/fixtures/manifests/celltrackscolab_labconstrictor-app.yaml @@ -0,0 +1,64 @@ +# yaml-language-server: $schema=https://labconstrictor.cellmig.org/schema/labconstrictor-app.schema.json +schema: 1 +name: CellTracksColab +summary: Compile, visualise and analyse cell tracking data across fields of view, biological repeats and conditions. +description: |- + Import and compile cell tracking data from TrackMate (CSV or XML), multiwell plate exports, or other compatible CSV files. Explore trajectories and track metrics, filter tracks, and inspect variability across fields of view, biological repeats and conditions. + + Separate notebooks visualise compiled data, explore movement phenotypes with UMAP, t-SNE and HDBSCAN, analyse spatial clustering with Ripley's L, and measure distances to regions of interest. Use the notebooks in Google Colab or through the desktop app. +kind: analysis +hosts: [notebook] +distribution: + installer: true + colab: + - label: TrackMate (CSV or XML) + url: https://colab.research.google.com/github/CellMigrationLab/CellTracksColab/blob/main/notebooks/CellTracksColab_TrackMate/CellTracksColab_TrackMate.ipynb + - label: TrackMate Plate (multiwell CSV) + url: https://colab.research.google.com/github/CellMigrationLab/CellTracksColab/blob/main/notebooks/CellTracksColab_TrackMate_Plate/CellTracksColab_TrackMate_Plate.ipynb + - label: Custom (other tracking CSV files) + url: https://colab.research.google.com/github/CellMigrationLab/CellTracksColab/blob/main/notebooks/CellTracksColab_Custom/CellTracksColab_Custom.ipynb + - label: Viewer (compiled tracks) + url: https://colab.research.google.com/github/CellMigrationLab/CellTracksColab/blob/main/notebooks/CellTracksColab_Viewer/CellTracksColab_Viewer.ipynb + - label: Dimensionality Reduction (UMAP, t-SNE) + url: https://colab.research.google.com/github/CellMigrationLab/CellTracksColab/blob/main/notebooks/CellTracksColab_Dimensionality_Reduction/CellTracksColab_Dimensionality_Reduction.ipynb + - label: Track Clustering (Ripley's L) + url: https://colab.research.google.com/github/CellMigrationLab/CellTracksColab/blob/main/notebooks/CellTracksColab_Track_Clustering/CellTracksColab_Track_Clustering.ipynb + - label: Distance to ROI (requires masks) + url: https://colab.research.google.com/github/CellMigrationLab/CellTracksColab/blob/main/notebooks/CellTracksColab_Distance_to_ROI/CellTracksColab_Distance_to_ROI.ipynb +license: MIT +categories: [cell tracking, cell migration, trajectory analysis, TrackMate] +doi: 10.1371/journal.pbio.3002740 +icon: app/logo/CellTracksColab_logo.png +docs_url: https://github.com/CellMigrationLab/CellTracksColab/wiki +inputs: + - "TrackMate CSV (spots and tracks) or XML exports, organised by condition and biological repeat (TrackMate)" + - "TrackMate CSV exports from multiwell or plate-based experiments (TrackMate Plate)" + - "Compatible tracking CSV files from CellProfiler, Icy, ilastik or Fiji Manual Tracker; column mapping may be required (Custom)" + - "Compiled CellTracksColab spots and tracks tables (CSV or CSV.gz) for Viewer and advanced notebooks" + - "TIFF ROI masks matched to tracked fields of view for Distance to ROI" +outputs: + - "Compiled spot and track tables (CSV/CSV.gz), identified by field of view, biological repeat and condition" + - "Trajectory plots, track metrics (speed, directionality and more), quality-control plots and condition comparisons" + - "UMAP and t-SNE embeddings, HDBSCAN clusters and representative trajectories (Dimensionality Reduction)" + - "Ripley's L spatial clustering curves and Monte Carlo comparisons (Track Clustering)" + - "Distances from tracks to ROI masks, derived metrics and visual checks (Distance to ROI)" +limitations: + - "Does not yet support track merging or splitting" + - "Viewer, Dimensionality Reduction, Track Clustering and Distance to ROI require compiled CellTracksColab tables" + - "Custom imports need correct column mapping and spatial/time calibration" + - "ROI analysis needs masks matched to each field of view and consistent coordinate calibration" + - "Dimensionality reduction and clustering are exploratory; results depend on the selected features and parameters" +quickstart: + - "Choose TrackMate, TrackMate Plate or Custom to import data; Viewer and advanced analyses need compiled CellTracksColab tables" + - "Desktop: install CellTracksColab, launch it and choose your notebook from the Welcome dashboard" + - "Google Colab: open a notebook below, run Load key dependencies and connect Google Drive when prompted" + - "Select the input folder or compiled spot/track files and choose a results folder" + - "For Distance to ROI, check the matching TIFF masks and the track-to-image calibration" + - "Inspect plots and field-of-view/repeat quality control before comparing conditions" +screenshots: + - path: Wiki/Screenshot1.png + caption: Compile your data + - path: Wiki/Screenshot2.png + caption: Visualise your tracks + - path: Wiki/Screenshot4.png + caption: Plot track parameters diff --git a/tests/fixtures/manifests/guessthecondition_labconstrictor-app.yaml b/tests/fixtures/manifests/guessthecondition_labconstrictor-app.yaml new file mode 100644 index 0000000..581ee16 --- /dev/null +++ b/tests/fixtures/manifests/guessthecondition_labconstrictor-app.yaml @@ -0,0 +1,35 @@ +# yaml-language-server: $schema=https://labconstrictor.cellmig.org/schema/labconstrictor-app.schema.json +schema: 1 +name: Guess the Condition +summary: A blinded test of whether experimental conditions can be told apart by eye, with statistics across biological repeats. +description: |- + You see randomised images without their condition and guess it. The app records your guesses and decision times, then reports whether the conditions can be told apart across biological repeats, with accuracy against chance and a confusion matrix. +kind: utility +hosts: [notebook, napari, fiji] +distribution: + installer: true + colab: + - label: Interactive game + url: https://colab.research.google.com/github/CellMigrationLab/GuessTheCondition/blob/main/notebooks/GuessTheCondition/GuessTheCondition.ipynb + - label: Python API example (synthetic data) + url: https://colab.research.google.com/github/CellMigrationLab/GuessTheCondition/blob/main/notebooks/GuessTheCondition_API_Example/GuessTheCondition_API_Example.ipynb +license: MIT +icon: app/logo/guessthecondition.png +categories: [experimental design, blinding] +inputs: + - "TIFF images (.tif or .tiff): one folder per condition, inside it one folder per biological repeat" + - "2D images, multi-channel images (shown side by side), and z-stacks or time series (shown as a maximum projection)" +outputs: + - "Whether the conditions can be told apart across biological repeats, with n and the 95% interval" + - "Accuracy of every repeat and of every condition against chance, the confusion matrix and the decision times (a SuperPlot)" + - "A PDF report with the text, the figures and the tables" + - "game_results.csv and session.json, in a folder with your name inside the results folder" +limitations: + - "Every channel is contrast-stretched on its own, so conditions must be told apart by what the images show, not by overall brightness" + - "With one or two repeats there is no verdict about the experiment; a randomization test is exploratory and does not generalise to new repeats" + - "The test on all images pooled is exploratory: it counts every image as independent, so its false-positive chance is higher than stated" +quickstart: + - "Prepare your images: one folder per condition, inside it one folder per biological repeat" + - "Step 1: give your name, the folder with your experiment, a folder for the results and the share of the images you want to guess" + - "Step 2: press Start / continue, choose the condition you think each image comes from and press Submit guess" + - "Step 3: run the analysis cell when you have guessed enough images" diff --git a/tests/fixtures/manifests/vlab4mic_labconstrictor-app.yaml b/tests/fixtures/manifests/vlab4mic_labconstrictor-app.yaml new file mode 100644 index 0000000..a0303ac --- /dev/null +++ b/tests/fixtures/manifests/vlab4mic_labconstrictor-app.yaml @@ -0,0 +1,37 @@ +# yaml-language-server: $schema=https://labconstrictor.cellmig.org/schema/labconstrictor-app.schema.json +schema: 1 +name: VLab4Mic +summary: A virtual lab to simulate fluorescence microscopy images, from molecular structures to acquisition, with no coding. +description: |- + Build virtual samples from molecular structures, apply fluorescent labelling and simulate image acquisition across microscopy modalities, before stepping into the microscope room. + + This is the desktop installer distribution of VLab4Mic. Two of its tools also run in napari and Fiji. +kind: simulation +hosts: [notebook, napari, fiji] +distribution: + installer: true + colab: + - label: Build and image a virtual sample + url: https://colab.research.google.com/github/CellMigrationLab/LabConstrictor-VLab4Mic/blob/main/notebooks/VLab4Mic_main/VLab4Mic_main.ipynb + - label: Parameter sweeps + url: https://colab.research.google.com/github/CellMigrationLab/LabConstrictor-VLab4Mic/blob/main/notebooks/VLab4Mic_parameter_sweeps/VLab4Mic_parameter_sweeps.ipynb +license: MIT +icon: app/logo/vlab4mic_icon.png +categories: [microscopy simulation, experiment design] +docs_url: https://github.com/CellMigrationLab/LabConstrictor-VLab4Mic#readme +homepage: https://vlab4mic.henriqueslab.org/ +doi: 10.64898/2026.06.02.729521 +inputs: + - "Molecular structures: choose a PDB ID or provide a .cif/.pdb file; configure probe, labelling and acquisition" + - "Image comparison: a reference and simulated 2D image with their pixel sizes (napari/Fiji tools)" +outputs: + - "Simulated and noiseless fluorescence microscopy images, with pixel size" + - "Image comparison scores (SSIM and Pearson correlation) when using the comparison tool" +limitations: + - The napari and Fiji tools return one frame; multi-frame runs, parameter sweeps and advanced probe design require notebooks + - In the default 1000 nm field a widefield image at 100 nm per pixel has only 10 x 10 pixels; raise Sample size XY +quickstart: + - "Install and launch VLab4Mic, then open the Welcome dashboard in JupyterLab" + - "Open VLab4Mic_main to select a molecular structure, fluorescent probe and imaging modality" + - "Run the acquisition and inspect the simulated and noiseless images" + - "Use VLab4Mic_parameter_sweeps when you need to compare multiple parameter combinations" diff --git a/tests/fixtures/template_0_1_17_apps/README.md b/tests/fixtures/template_0_1_17_apps/README.md new file mode 100644 index 0000000..6de2d1e --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/README.md @@ -0,0 +1,15 @@ +Real files of apps at template 0.1.17, as stored in their repositories on main (13 October 2026; the .bat files with their line endings), used by +tests/test_template_sync_0_1_18.py: + +* `playground/`: CellMigrationLab/LabConstrictor-Playground (CRLF `.bat`). `release_constructor.yml` is its workflow as it is: customised (no Intel Mac + build) and already pinned by SHA. +* `nucleisky/`: CellMigrationLab/NucleiSky +* `vlab4mic/`: CellMigrationLab/LabConstrictor-VLab4Mic +* `guessthecondition/`: CellMigrationLab/GuessTheCondition + +All four hold the template pin `v0.2.0-rc.1` in both scripts. Their other workflows are byte-identical to the resources of the 0.1.18 migration +(the apps pinned the actions by their own pull requests), so the tests use the resources for that case. + +`../template_0_1_17_workflows/` holds the five workflows of template 0.1.17 (unpinned), taken from the template at commit 2dbc5ef. + +Each app folder also holds its `construct.yaml` as on main (14 October 2026), used by `tests/test_template_sync_0_1_18_hosts.py` (the `extra_files` block the hosts step edits). The matching real `labconstrictor-app.yaml` files are in `../manifests/`. diff --git a/tests/fixtures/template_0_1_17_apps/guessthecondition/construct.yaml b/tests/fixtures/template_0_1_17_apps/guessthecondition/construct.yaml new file mode 100644 index 0000000..86cbd76 --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/guessthecondition/construct.yaml @@ -0,0 +1,43 @@ +name: GuessTheCondition +version: 0.0.0 +environment_file: environment.yaml +installer_type: all +license_file: LICENSE +welcome_image: app/logo/logo.png +header_image: app/logo/logo.png +icon_image: app/logo/guessthecondition.png +conclusion_text: Thank you! You have successfully installed GuessTheCondition 0.0.0. +keep_pkgs: false +register_python: false +initialize_conda: false +extra_files: +- .tools/meta/src_change.yaml: GuessTheCondition/src_change.yaml +- app/bash_bat_scripts/pre_uninstall.sh: pre_uninstall.sh +- app/bash_bat_scripts/uninstall.sh: uninstall.sh +- app/logo/guessthecondition.icns: GuessTheCondition/guessthecondition.icns +- app/logo/guessthecondition.ico: GuessTheCondition/guessthecondition.ico +- app/logo/guessthecondition.png: GuessTheCondition/guessthecondition.png +- app/menuinst/Welcome.ipynb: GuessTheCondition/notebooks/Welcome.ipynb +- app/menuinst/notebook_launcher.json: GuessTheCondition/notebook_launcher.json +- app/python_scripts/hide_code_cells.py: GuessTheCondition/hide_code_cells.py +- app/python_scripts/include_path.py: GuessTheCondition/include_path.py +- app/python_scripts/launch_jupyter.py: GuessTheCondition/launch_jupyter.py +- construct.yaml: GuessTheCondition/construct.yaml +- notebooks/GuessTheCondition/GuessTheCondition.ipynb: GuessTheCondition/notebooks/GuessTheCondition/GuessTheCondition.ipynb +- notebooks/GuessTheCondition_API_Example/GuessTheCondition_API_Example.ipynb: GuessTheCondition/notebooks/GuessTheCondition_API_Example/GuessTheCondition_API_Example.ipynb +- notebooks/notebook_latest_versions.yaml: GuessTheCondition/notebook_latest_versions.yaml +- requirements.txt: GuessTheCondition/requirements.txt +- requirements_gpu.txt: GuessTheCondition/requirements_gpu.txt +- setup.py: GuessTheCondition/setup.py +- src/guessthecondition/__init__.py: GuessTheCondition/src/guessthecondition/__init__.py +- src/guessthecondition/analysis.py: GuessTheCondition/src/guessthecondition/analysis.py +- src/guessthecondition/dataset.py: GuessTheCondition/src/guessthecondition/dataset.py +- src/guessthecondition/demo.py: GuessTheCondition/src/guessthecondition/demo.py +- src/guessthecondition/images.py: GuessTheCondition/src/guessthecondition/images.py +- src/guessthecondition/report.py: GuessTheCondition/src/guessthecondition/report.py +- src/guessthecondition/session.py: GuessTheCondition/src/guessthecondition/session.py +- src/guessthecondition_lc_tools/__init__.py: GuessTheCondition/src/guessthecondition_lc_tools/__init__.py +post_install: app/bash_bat_scripts/post_install.bat # [win] +post_install: app/bash_bat_scripts/post_install.sh # [unix] +pre_uninstall: app/bash_bat_scripts/pre_uninstall.bat # [win] +pre_uninstall: app/bash_bat_scripts/pre_uninstall.sh # [unix] diff --git a/tests/fixtures/template_0_1_17_apps/guessthecondition/post_install.bat b/tests/fixtures/template_0_1_17_apps/guessthecondition/post_install.bat new file mode 100644 index 0000000..efda826 --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/guessthecondition/post_install.bat @@ -0,0 +1,236 @@ +@ECHO OFF +SETLOCAL EnableExtensions DisableDelayedExpansion + +SET "LOG_FILE=%PREFIX%\menuinst_debug.log" +SET "PYTHON_EXE=%PREFIX%\python.exe" +SET "PROJECT_ROOT=%PREFIX%\GuessTheCondition" +SET "BASE_REQUIREMENTS=%PROJECT_ROOT%\requirements.txt" +SET "GPU_REQUIREMENTS=%PROJECT_ROOT%\requirements_gpu.txt" +SET "SELECTED_REQUIREMENTS=%BASE_REQUIREMENTS%" +SET "NVIDIA_SMI=" +SET "CA_BUNDLE=" +SET "FAILURE_MESSAGE=" +SET "CERTIFI_PATH_FILE=%TEMP%\labconstrictor_certifi_%RANDOM%_%RANDOM%.txt" + +> "%LOG_FILE%" echo Running post_install + +IF NOT EXIST "%PYTHON_EXE%" ( + SET "FAILURE_MESSAGE=Bundled Python executable was not found at %PYTHON_EXE%." + GOTO :fail +) +IF NOT EXIST "%PROJECT_ROOT%\launch_jupyter.py" ( + SET "FAILURE_MESSAGE=TLS-resilient launcher was not found at %PROJECT_ROOT%\launch_jupyter.py." + GOTO :fail +) +IF NOT EXIST "%BASE_REQUIREMENTS%" ( + SET "FAILURE_MESSAGE=Base requirements file was not found at %BASE_REQUIREMENTS%." + GOTO :fail +) + +REM pip 24.2+ uses the operating-system trust store by default. Some Windows +REM certificate stores contain malformed entries that OpenSSL cannot parse. +REM Keep verification enabled while opting out of system trust for installer +REM pip operations and using an explicit verified CA bundle. +SET "PIP_USE_DEPRECATED=legacy-certs" + +"%PYTHON_EXE%" "%PROJECT_ROOT%\launch_jupyter.py" --print-ca-bundle > "%CERTIFI_PATH_FILE%" 2>> "%LOG_FILE%" +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Unable to locate a verified CA bundle." + GOTO :fail +) +SET /P "CA_BUNDLE="<"%CERTIFI_PATH_FILE%" +DEL /Q "%CERTIFI_PATH_FILE%" >NUL 2>&1 + +IF NOT DEFINED CA_BUNDLE ( + SET "FAILURE_MESSAGE=No verified CA bundle could be selected." + GOTO :fail +) +IF NOT EXIST "%CA_BUNDLE%" ( + SET "FAILURE_MESSAGE=Selected CA bundle does not exist: %CA_BUNDLE%" + GOTO :fail +) + +SET "PIP_CERT=%CA_BUNDLE%" +SET "REQUESTS_CA_BUNDLE=%CA_BUNDLE%" +SET "CURL_CA_BUNDLE=%CA_BUNDLE%" +echo Using verified CA bundle for installer network operations: "%CA_BUNDLE%" >> "%LOG_FILE%" + +IF EXIST "%GPU_REQUIREMENTS%" ( + CALL :detect_nvidia_smi + IF DEFINED NVIDIA_SMI ( + echo NVIDIA GPU utility detected, installing GPU requirements from "%GPU_REQUIREMENTS%" >> "%LOG_FILE%" + SET "SELECTED_REQUIREMENTS=%GPU_REQUIREMENTS%" + ) ELSE ( + echo NVIDIA GPU not detected, installing CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" + ) +) ELSE ( + echo GPU requirements file not found, installing CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" +) + +echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" +"%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Application requirements installation failed." + GOTO :fail +) + +IF EXIST "%PROJECT_ROOT%\requirements-windows.txt" ( + echo Installing Windows-specific requirements. >> "%LOG_FILE%" + "%PYTHON_EXE%" -m pip install -r "%PROJECT_ROOT%\requirements-windows.txt" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Windows-specific requirements installation failed." + GOTO :fail + ) +) + +REM External Python code is optional in LabConstrictor projects. If setup.py is +REM bundled, install it without a temporary build environment and verify that +REM the generated import package is usable. Otherwise continue without it. +IF EXIST "%PROJECT_ROOT%\setup.py" ( + echo Found setup.py, installing GuessTheCondition package locally without build isolation. >> "%LOG_FILE%" + "%PYTHON_EXE%" -m pip install --no-deps --no-build-isolation "%PROJECT_ROOT%" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=GuessTheCondition package installation failed." + GOTO :fail + ) + + REM Tools first, only for an app whose package ships a tools module: the package may import labconstrictor_tools at module level, + REM so labconstrictor-tools is installed BEFORE the import check below. A failed install is only a WARNING, see :install_lc_tools at the end of this file. + CALL :install_lc_tools + + "%PYTHON_EXE%" -c "import guessthecondition; print('GuessTheCondition import successful:', guessthecondition.__file__)" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Installed GuessTheCondition package could not be imported as guessthecondition." + GOTO :fail + ) +) ELSE ( + echo No setup.py detected; this project does not bundle an optional Python package. >> "%LOG_FILE%" +) + +REM Optional: expose the app's tools to Napari, Fiji and the command line (LabConstrictor tools bridge). +REM If the app's package ships a module named _lc_tools, install labconstrictor-tools and register that module. +REM This step must never fail the installation. LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +REM labconstrictor-tools is installed by :install_lc_tools, called above before the import check of the package (it sets LC_TOOLS_PIN, LC_TOOLS_SPEC and LC_TOOLS_READY). +SET "LC_APP_VERSION=" +IF EXIST "%PROJECT_ROOT%\construct.yaml" FOR /F "usebackq tokens=1,* delims=: " %%A IN (`findstr /B /C:"version:" "%PROJECT_ROOT%\construct.yaml"`) DO SET "LC_APP_VERSION=%%~B" +IF NOT DEFINED LC_APP_VERSION ( + SET "LC_APP_VERSION=0" + echo WARNING: no version: line at the start of a line in construct.yaml - registering the tools with version 0. >> "%LOG_FILE%" +) +"%PYTHON_EXE%" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('guessthecondition_lc_tools') else 1)" >> "%LOG_FILE%" 2>&1 +IF NOT ERRORLEVEL 1 ( + echo Found guessthecondition_lc_tools: registering the tools of GuessTheCondition for Napari and Fiji. >> "%LOG_FILE%" + REM labconstrictor-tools was installed before the import check (:install_lc_tools): register only if that worked (errorlevel 1 otherwise). + "%PYTHON_EXE%" -c "import sys; sys.exit(0 if sys.argv[1] == '1' else 1)" "%LC_TOOLS_READY%" >> "%LOG_FILE%" 2>&1 + IF NOT ERRORLEVEL 1 "%PYTHON_EXE%" -m labconstrictor_tools register --name "GuessTheCondition" --prefix "%PREFIX%" --module guessthecondition_lc_tools --version "%LC_APP_VERSION%" --display-name "GuessTheCondition" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 echo WARNING: tool registration failed - see the pip and register output above in this file; GuessTheCondition itself is installed. >> "%LOG_FILE%" +) + +"%PYTHON_EXE%" "%PROJECT_ROOT%\include_path.py" --path "%PREFIX%" --files "%PROJECT_ROOT%\notebook_launcher.json" --keyword "BASE_PATH_KEYWORD" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Launcher path preparation failed." + GOTO :fail +) + +"%PYTHON_EXE%" "%PROJECT_ROOT%\hide_code_cells.py" "%PROJECT_ROOT%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Notebook metadata preparation failed." + GOTO :fail +) + +REM Keep conda verification enabled, but avoid the Windows trust store that +REM failed during installation. Conda accepts a CA bundle path for ssl_verify. +echo Configuring conda to use verified CA bundle: "%CA_BUNDLE%" >> "%LOG_FILE%" +"%PYTHON_EXE%" -m conda config --system --set ssl_verify "%CA_BUNDLE%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + echo WARNING: Could not configure conda with the selected CA bundle. The application launcher still has its own verified fallback. >> "%LOG_FILE%" +) + +REM Validate TLS and Jupyter imports before creating shortcuts or registering +REM the application. +"%PYTHON_EXE%" "%PROJECT_ROOT%\launch_jupyter.py" --self-test >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Launcher preflight failed. See menuinst_debug.log and launcher_debug.log." + GOTO :fail +) +echo Launcher preflight completed successfully. >> "%LOG_FILE%" + +"%PYTHON_EXE%" -c "import os, sys; print('Python:', sys.executable); print('Prefix:', os.environ.get('PREFIX'))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Python installation diagnostics failed." + GOTO :fail +) + +"%PYTHON_EXE%" -c "from menuinst.api import install; import os; print(install(os.path.join(r'%PREFIX%', 'GuessTheCondition', 'notebook_launcher.json')))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Application shortcut creation failed." + GOTO :fail +) + +SET "ARP_KEY=HKCU\Software\Microsoft\Windows\CurrentVersion\Uninstall\GuessTheCondition" +SET "UNINSTALL_EXE=%PREFIX%\Uninstall-GuessTheCondition.exe" +SET "DISPLAY_ICON=%PROJECT_ROOT%\logo.ico" +SET "DISPLAY_VERSION=0.1.3" +SET "PUBLISHER=CellMigrationLab" +echo Registering GuessTheCondition in Windows Apps list >> "%LOG_FILE%" +reg add "%ARP_KEY%" /v DisplayName /d "GuessTheCondition" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v DisplayVersion /d "%DISPLAY_VERSION%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v Publisher /d "%PUBLISHER%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v InstallLocation /d "%PREFIX%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v DisplayIcon /d "%DISPLAY_ICON%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v UninstallString /d "\"%UNINSTALL_EXE%\"" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v QuietUninstallString /d "\"%UNINSTALL_EXE%\" /S" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v NoModify /t REG_DWORD /d 1 /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v NoRepair /t REG_DWORD /d 1 /f >> "%LOG_FILE%" 2>&1 + +echo Post-install completed successfully. >> "%LOG_FILE%" +echo Post-install completed successfully. +ENDLOCAL +EXIT /B 0 + +:fail +IF EXIST "%CERTIFI_PATH_FILE%" DEL /Q "%CERTIFI_PATH_FILE%" >NUL 2>&1 +IF NOT DEFINED FAILURE_MESSAGE SET "FAILURE_MESSAGE=Post-install failed for an unknown reason." +echo ERROR: %FAILURE_MESSAGE% >> "%LOG_FILE%" +echo ERROR: %FAILURE_MESSAGE% +echo See "%LOG_FILE%" for details. +ENDLOCAL +EXIT /B 1 + +:detect_nvidia_smi +FOR %%P IN ( + "%ProgramFiles%\NVIDIA Corporation\NVSMI\nvidia-smi.exe" + "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvidia-smi.exe" + "%SystemRoot%\System32\nvidia-smi.exe" + "%SystemRoot%\Sysnative\nvidia-smi.exe" +) DO ( + IF NOT DEFINED NVIDIA_SMI IF EXIST "%%~P" SET "NVIDIA_SMI=%%~P" +) +IF DEFINED NVIDIA_SMI GOTO :EOF + +FOR /F "delims=" %%I IN ('where.exe nvidia-smi.exe 2^>NUL') DO ( + IF NOT DEFINED NVIDIA_SMI SET "NVIDIA_SMI=%%~fI" +) +GOTO :EOF + +:install_lc_tools +REM Optional (LabConstrictor tools bridge): install labconstrictor-tools when the app's package ships a module named _lc_tools. +REM Called before the import check of the package, which may import labconstrictor_tools at module level. +REM This step must never fail the installation: a failure is logged as a WARNING and LC_TOOLS_READY stays 0 (no registration later). +REM LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +REM Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), +REM because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. +SET "LC_TOOLS_PIN=v0.2.0-rc.1" +IF NOT DEFINED LC_TOOLS_SPEC SET "LC_TOOLS_SPEC=https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/%LC_TOOLS_PIN%.zip" +SET "LC_TOOLS_READY=0" +"%PYTHON_EXE%" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('guessthecondition_lc_tools') else 1)" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 GOTO :EOF +echo Found guessthecondition_lc_tools: installing labconstrictor-tools before the import check of GuessTheCondition. >> "%LOG_FILE%" +REM LC_TOOLS_SPEC goes to pip as one argument through Python: cmd.exe never re-parses it (a quote or & in it cannot run anything). +"%PYTHON_EXE%" -c "import os, subprocess, sys; sys.exit(subprocess.call([sys.executable, '-m', 'pip', 'install', os.environ['LC_TOOLS_SPEC']]))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + echo WARNING: labconstrictor-tools could not be installed - see the pip output above in this file; the tools of GuessTheCondition will not be registered ^(the import check fails only if the package needs labconstrictor-tools at import time^). >> "%LOG_FILE%" + GOTO :EOF +) +SET "LC_TOOLS_READY=1" +GOTO :EOF diff --git a/tests/fixtures/template_0_1_17_apps/guessthecondition/post_install.sh b/tests/fixtures/template_0_1_17_apps/guessthecondition/post_install.sh new file mode 100644 index 0000000..3aad4ca --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/guessthecondition/post_install.sh @@ -0,0 +1,134 @@ +#!/bin/bash +set -euo pipefail + +LOG_FILE="$PREFIX/menuinst_debug.log" +PYTHON_EXE="$PREFIX/bin/python" +PROJECT_ROOT="$PREFIX/GuessTheCondition" +BASE_REQUIREMENTS="$PROJECT_ROOT/requirements.txt" +GPU_REQUIREMENTS="$PROJECT_ROOT/requirements_gpu.txt" +SELECTED_REQUIREMENTS="$BASE_REQUIREMENTS" + +echo "Running post_install" > "$LOG_FILE" + +fail() { + echo "ERROR: $1" | tee -a "$LOG_FILE" >&2 + exit 1 +} + +[ -x "$PYTHON_EXE" ] || fail "Bundled Python executable was not found at $PYTHON_EXE." +[ -f "$PROJECT_ROOT/launch_jupyter.py" ] || fail "TLS-resilient launcher was not found at $PROJECT_ROOT/launch_jupyter.py." +[ -f "$BASE_REQUIREMENTS" ] || fail "Base requirements file was not found at $BASE_REQUIREMENTS." + +# pip 24.2+ uses system certificates by default. Opt out for installer pip +# operations and use an explicit verified CA bundle instead. +export PIP_USE_DEPRECATED=legacy-certs +CA_BUNDLE="$("$PYTHON_EXE" "$PROJECT_ROOT/launch_jupyter.py" --print-ca-bundle 2>> "$LOG_FILE")" +[ -s "$CA_BUNDLE" ] || fail "No valid verified CA bundle could be selected: $CA_BUNDLE" + +export PIP_CERT="$CA_BUNDLE" +export REQUESTS_CA_BUNDLE="$CA_BUNDLE" +export CURL_CA_BUNDLE="$CA_BUNDLE" +echo "Using verified CA bundle for installer network operations: $CA_BUNDLE" >> "$LOG_FILE" + +if [ -f "$GPU_REQUIREMENTS" ]; then + if [[ "${OSTYPE:-}" == "darwin"* ]]; then + echo "macOS detected, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" + elif command -v nvidia-smi >/dev/null 2>&1; then + echo "NVIDIA GPU detected, installing GPU requirements from $GPU_REQUIREMENTS" >> "$LOG_FILE" + SELECTED_REQUIREMENTS="$GPU_REQUIREMENTS" + else + echo "NVIDIA GPU not detected, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" + fi +else + echo "GPU requirements file not found, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" +fi + +echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE" +"$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1 + +if [[ "${OSTYPE:-}" == "darwin"* ]]; then + echo "Detected macOS platform" >> "$LOG_FILE" + if [ -f "$PROJECT_ROOT/requirements-macos.txt" ]; then + "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-macos.txt" >> "$LOG_FILE" 2>&1 + fi +elif [[ "${OSTYPE:-}" == "linux-gnu"* ]]; then + echo "Detected Linux platform" >> "$LOG_FILE" + if [ -f "$PROJECT_ROOT/requirements-linux.txt" ]; then + "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-linux.txt" >> "$LOG_FILE" 2>&1 + fi +else + echo "Unknown platform: ${OSTYPE:-unset}" >> "$LOG_FILE" +fi + +# External Python code is optional. Verify the generated package only when the +# constructor bundled setup.py and src/. +if [ -f "$PROJECT_ROOT/setup.py" ]; then + echo "Found setup.py, installing GuessTheCondition package locally without build isolation" >> "$LOG_FILE" + "$PYTHON_EXE" -m pip install --no-deps --no-build-isolation "$PROJECT_ROOT" >> "$LOG_FILE" 2>&1 + # Tools first, only for an app whose package ships a _lc_tools module: the app package may import labconstrictor_tools + # at module level, so labconstrictor-tools must be installed BEFORE the import check below. A failed install is only a WARNING + # (it never fails the installation by itself). LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). + # Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), + # because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. + LC_TOOLS_PIN="v0.2.0-rc.1" + LC_TOOLS_MODULE="guessthecondition_lc_tools" + LC_TOOLS_READY=0 + if "$PYTHON_EXE" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('$LC_TOOLS_MODULE') else 1)" >> "$LOG_FILE" 2>&1; then + echo "Found $LC_TOOLS_MODULE: installing labconstrictor-tools before the import check of GuessTheCondition" >> "$LOG_FILE" || : + if "$PYTHON_EXE" -m pip install "${LC_TOOLS_SPEC:-https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/${LC_TOOLS_PIN}.zip}" >> "$LOG_FILE" 2>&1; then + LC_TOOLS_READY=1 + else + echo "WARNING: labconstrictor-tools could not be installed - see the pip output above in this file; the tools of GuessTheCondition will not be registered (the import check below fails only if the package needs labconstrictor-tools at import time)." >> "$LOG_FILE" || : + fi + fi + "$PYTHON_EXE" -c "import guessthecondition; print('GuessTheCondition import successful:', guessthecondition.__file__)" >> "$LOG_FILE" 2>&1 +else + echo "No setup.py detected; this project does not bundle an optional Python package." >> "$LOG_FILE" +fi + + +# --- Optional: expose the app's tools to Napari, Fiji and the command line (LabConstrictor tools bridge) ------------------- +# If the app's package ships a module named _lc_tools, install labconstrictor-tools and register that module. +# This step must never fail the installation. LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +# labconstrictor-tools is installed above, before the import check of the package (LC_TOOLS_PIN, LC_TOOLS_MODULE and LC_TOOLS_READY are set there). +if [ -f "$PROJECT_ROOT/setup.py" ] && "$PYTHON_EXE" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('$LC_TOOLS_MODULE') else 1)" >> "$LOG_FILE" 2>&1; then + lc_note() { echo "$*" >> "$LOG_FILE" || :; } # a full or unwritable log must not fail the installation + lc_note "Found $LC_TOOLS_MODULE: registering the tools of GuessTheCondition for Napari and Fiji" + LC_APP_VERSION="$(sed -n 's/^version: *//p' "$PROJECT_ROOT/construct.yaml" 2>/dev/null | head -1 | tr -d "\"'\r" || true)" + if [ -z "$LC_APP_VERSION" ]; then + lc_note "WARNING: no version: line at the start of a line in construct.yaml - registering the tools with version 0." + LC_APP_VERSION=0 + fi + if [ "${LC_TOOLS_READY:-0}" = 1 ] \ + && "$PYTHON_EXE" -m labconstrictor_tools register --name "GuessTheCondition" --prefix "$PREFIX" \ + --module "$LC_TOOLS_MODULE" --version "$LC_APP_VERSION" --display-name "GuessTheCondition" >> "$LOG_FILE" 2>&1; then + lc_note "Tools registered (labconstrictor-tools list shows them)." + else + lc_note "WARNING: tool registration failed - see the pip and register output above in this file; GuessTheCondition itself is installed." + fi +fi + +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PROJECT_ROOT/notebook_launcher.json" --keyword "BASE_PATH_KEYWORD" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PREFIX/pre_uninstall.sh" --keyword "BASE_PATH" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PREFIX/uninstall.sh" --keyword "BASE_PATH" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/hide_code_cells.py" "$PROJECT_ROOT" >> "$LOG_FILE" 2>&1 + +# Keep conda verification enabled while avoiding the system certificate store +# that failed. Conda accepts a CA bundle path for ssl_verify. +if ! "$PYTHON_EXE" -m conda config --system --set ssl_verify "$CA_BUNDLE" >> "$LOG_FILE" 2>&1; then + echo "WARNING: Could not configure conda with the selected CA bundle. The launcher still has its own verified fallback." >> "$LOG_FILE" +fi + +# Do not create the application shortcut unless launcher preflight succeeds. +"$PYTHON_EXE" "$PROJECT_ROOT/launch_jupyter.py" --self-test >> "$LOG_FILE" 2>&1 +echo "Launcher preflight completed successfully." >> "$LOG_FILE" + +"$PYTHON_EXE" -c "import os, sys; print('Python:', sys.executable); print('Prefix:', os.environ.get('PREFIX'))" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" -c "from menuinst.api import install; import os; print(install(os.path.join('$PREFIX', 'GuessTheCondition', 'notebook_launcher.json')))" >> "$LOG_FILE" 2>&1 + +echo "Post-install completed successfully." >> "$LOG_FILE" + +if [ -t 0 ]; then + echo + read -rp "Press Enter to close the installer..." _ +fi diff --git a/tests/fixtures/template_0_1_17_apps/nucleisky/construct.yaml b/tests/fixtures/template_0_1_17_apps/nucleisky/construct.yaml new file mode 100644 index 0000000..5c1096a --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/nucleisky/construct.yaml @@ -0,0 +1,110 @@ +name: NucleiSky +version: 0.1.3 +environment_file: environment.yaml +installer_type: all +license_file: LICENSE +welcome_image: app/logo/Gemini_Generated_Image_fga0qrfga0qrfga0.png +header_image: app/logo/Gemini_Generated_Image_27i68r27i68r27i6.png +icon_image: app/logo/Gemini_Generated_Image_kc5ezfkc5ezfkc5e.png +conclusion_text: Thank you! You have successfully installed NucleiSky 0.1.3. +keep_pkgs: false +register_python: false +initialize_conda: false +extra_files: +- app/bash_bat_scripts/pre_uninstall.sh: pre_uninstall.sh +- app/bash_bat_scripts/uninstall.sh: uninstall.sh +- app/logo/Gemini_Generated_Image_kc5ezfkc5ezfkc5e.icns: NucleiSky/Gemini_Generated_Image_kc5ezfkc5ezfkc5e.icns +- app/logo/Gemini_Generated_Image_kc5ezfkc5ezfkc5e.ico: NucleiSky/Gemini_Generated_Image_kc5ezfkc5ezfkc5e.ico +- app/logo/Gemini_Generated_Image_kc5ezfkc5ezfkc5e.png: NucleiSky/Gemini_Generated_Image_kc5ezfkc5ezfkc5e.png +- app/menuinst/Welcome.ipynb: NucleiSky/notebooks/Welcome.ipynb +- app/menuinst/notebook_launcher.json: NucleiSky/notebook_launcher.json +- app/python_scripts/hide_code_cells.py: NucleiSky/hide_code_cells.py +- app/python_scripts/include_path.py: NucleiSky/include_path.py +- app/python_scripts/launch_jupyter.py: NucleiSky/launch_jupyter.py +- construct.yaml: NucleiSky/construct.yaml +- notebooks/NucleiSky2DApp/NucleiSky2DApp.ipynb: NucleiSky/notebooks/NucleiSky2DApp/NucleiSky2DApp.ipynb +- notebooks/NucleiSky2DBenchmarking/NucleiSky2DBenchmarking.ipynb: NucleiSky/notebooks/NucleiSky2DBenchmarking/NucleiSky2DBenchmarking.ipynb +- notebooks/NucleiSky2D_API_Workflow_Example/NucleiSky2D_API_Workflow_Example.ipynb: NucleiSky/notebooks/NucleiSky2D_API_Workflow_Example/NucleiSky2D_API_Workflow_Example.ipynb +- notebooks/NucleiSky3DApp/NucleiSky3DApp.ipynb: NucleiSky/notebooks/NucleiSky3DApp/NucleiSky3DApp.ipynb +- notebooks/NucleiSky3DBenchmarking/NucleiSky3DBenchmarking.ipynb: NucleiSky/notebooks/NucleiSky3DBenchmarking/NucleiSky3DBenchmarking.ipynb +- notebooks/NucleiSky3D_API_Workflow_Example/NucleiSky3D_API_Workflow_Example.ipynb: NucleiSky/notebooks/NucleiSky3D_API_Workflow_Example/NucleiSky3D_API_Workflow_Example.ipynb +- notebooks/notebook_latest_versions.yaml: NucleiSky/notebook_latest_versions.yaml +- requirements.txt: NucleiSky/requirements.txt +- requirements_gpu.txt: NucleiSky/requirements_gpu.txt +- setup.py: NucleiSky/setup.py +- src/nucleisky/__init__.py: NucleiSky/src/nucleisky/__init__.py +- src/nucleisky/nucleisky2d/__init__.py: NucleiSky/src/nucleisky/nucleisky2d/__init__.py +- src/nucleisky/nucleisky2d/config.py: NucleiSky/src/nucleisky/nucleisky2d/config.py +- src/nucleisky/nucleisky2d/demo_utils.py: NucleiSky/src/nucleisky/nucleisky2d/demo_utils.py +- src/nucleisky/nucleisky2d/export.py: NucleiSky/src/nucleisky/nucleisky2d/export.py +- src/nucleisky/nucleisky2d/features.py: NucleiSky/src/nucleisky/nucleisky2d/features.py +- src/nucleisky/nucleisky2d/io.py: NucleiSky/src/nucleisky/nucleisky2d/io.py +- src/nucleisky/nucleisky2d/matching/__init__.py: NucleiSky/src/nucleisky/nucleisky2d/matching/__init__.py +- src/nucleisky/nucleisky2d/matching/geometry.py: NucleiSky/src/nucleisky/nucleisky2d/matching/geometry.py +- src/nucleisky/nucleisky2d/matching/graph.py: NucleiSky/src/nucleisky/nucleisky2d/matching/graph.py +- src/nucleisky/nucleisky2d/matching/hashing.py: NucleiSky/src/nucleisky/nucleisky2d/matching/hashing.py +- src/nucleisky/nucleisky2d/matching/quad.py: NucleiSky/src/nucleisky/nucleisky2d/matching/quad.py +- src/nucleisky/nucleisky2d/matching/triangle.py: NucleiSky/src/nucleisky/nucleisky2d/matching/triangle.py +- src/nucleisky/nucleisky2d/pipeline.py: NucleiSky/src/nucleisky/nucleisky2d/pipeline.py +- src/nucleisky/nucleisky2d/preprocess.py: NucleiSky/src/nucleisky/nucleisky2d/preprocess.py +- src/nucleisky/nucleisky2d/segmentation.py: NucleiSky/src/nucleisky/nucleisky2d/segmentation.py +- src/nucleisky/nucleisky2d/types.py: NucleiSky/src/nucleisky/nucleisky2d/types.py +- src/nucleisky/nucleisky2d/utils.py: NucleiSky/src/nucleisky/nucleisky2d/utils.py +- src/nucleisky/nucleisky2d/visualization.py: NucleiSky/src/nucleisky/nucleisky2d/visualization.py +- src/nucleisky/nucleisky3d/__init__.py: NucleiSky/src/nucleisky/nucleisky3d/__init__.py +- src/nucleisky/nucleisky3d/config.py: NucleiSky/src/nucleisky/nucleisky3d/config.py +- src/nucleisky/nucleisky3d/demo_utils.py: NucleiSky/src/nucleisky/nucleisky3d/demo_utils.py +- src/nucleisky/nucleisky3d/export.py: NucleiSky/src/nucleisky/nucleisky3d/export.py +- src/nucleisky/nucleisky3d/features.py: NucleiSky/src/nucleisky/nucleisky3d/features.py +- src/nucleisky/nucleisky3d/io.py: NucleiSky/src/nucleisky/nucleisky3d/io.py +- src/nucleisky/nucleisky3d/matching/__init__.py: NucleiSky/src/nucleisky/nucleisky3d/matching/__init__.py +- src/nucleisky/nucleisky3d/matching/geometry.py: NucleiSky/src/nucleisky/nucleisky3d/matching/geometry.py +- src/nucleisky/nucleisky3d/matching/hashing3d.py: NucleiSky/src/nucleisky/nucleisky3d/matching/hashing3d.py +- src/nucleisky/nucleisky3d/matching/pyramid.py: NucleiSky/src/nucleisky/nucleisky3d/matching/pyramid.py +- src/nucleisky/nucleisky3d/pipeline.py: NucleiSky/src/nucleisky/nucleisky3d/pipeline.py +- src/nucleisky/nucleisky3d/preprocess.py: NucleiSky/src/nucleisky/nucleisky3d/preprocess.py +- src/nucleisky/nucleisky3d/segmentation.py: NucleiSky/src/nucleisky/nucleisky3d/segmentation.py +- src/nucleisky/nucleisky3d/types.py: NucleiSky/src/nucleisky/nucleisky3d/types.py +- src/nucleisky/nucleisky3d/utils.py: NucleiSky/src/nucleisky/nucleisky3d/utils.py +- src/nucleisky/nucleisky3d/visualization.py: NucleiSky/src/nucleisky/nucleisky3d/visualization.py +- src/nucleisky2d/__init__.py: NucleiSky/src/nucleisky2d/__init__.py +- src/nucleisky2d/segmentation.py: NucleiSky/src/nucleisky2d/segmentation.py +- src/nucleisky3d/__init__.py: NucleiSky/src/nucleisky3d/__init__.py +- src/nucleisky_lc_tools/__init__.py: NucleiSky/src/nucleisky_lc_tools/__init__.py + +- src/nucleisky/nucleisky2d/config.py: NucleiSky/src/nucleisky/nucleisky2d/config.py +- src/nucleisky/nucleisky2d/demo_utils.py: NucleiSky/src/nucleisky/nucleisky2d/demo_utils.py +- src/nucleisky/nucleisky2d/export.py: NucleiSky/src/nucleisky/nucleisky2d/export.py +- src/nucleisky/nucleisky2d/features.py: NucleiSky/src/nucleisky/nucleisky2d/features.py +- src/nucleisky/nucleisky2d/io.py: NucleiSky/src/nucleisky/nucleisky2d/io.py +- src/nucleisky/nucleisky2d/matching/geometry.py: NucleiSky/src/nucleisky/nucleisky2d/matching/geometry.py +- src/nucleisky/nucleisky2d/matching/graph.py: NucleiSky/src/nucleisky/nucleisky2d/matching/graph.py +- src/nucleisky/nucleisky2d/matching/hashing.py: NucleiSky/src/nucleisky/nucleisky2d/matching/hashing.py +- src/nucleisky/nucleisky2d/matching/quad.py: NucleiSky/src/nucleisky/nucleisky2d/matching/quad.py +- src/nucleisky/nucleisky2d/matching/triangle.py: NucleiSky/src/nucleisky/nucleisky2d/matching/triangle.py +- src/nucleisky/nucleisky2d/pipeline.py: NucleiSky/src/nucleisky/nucleisky2d/pipeline.py +- src/nucleisky/nucleisky2d/preprocess.py: NucleiSky/src/nucleisky/nucleisky2d/preprocess.py +- src/nucleisky/nucleisky2d/segmentation.py: NucleiSky/src/nucleisky/nucleisky2d/segmentation.py +- src/nucleisky/nucleisky2d/types.py: NucleiSky/src/nucleisky/nucleisky2d/types.py +- src/nucleisky/nucleisky2d/utils.py: NucleiSky/src/nucleisky/nucleisky2d/utils.py +- src/nucleisky/nucleisky2d/visualization.py: NucleiSky/src/nucleisky/nucleisky2d/visualization.py +- src/nucleisky/nucleisky3d/config.py: NucleiSky/src/nucleisky/nucleisky3d/config.py +- src/nucleisky/nucleisky3d/demo_utils.py: NucleiSky/src/nucleisky/nucleisky3d/demo_utils.py +- src/nucleisky/nucleisky3d/export.py: NucleiSky/src/nucleisky/nucleisky3d/export.py +- src/nucleisky/nucleisky3d/features.py: NucleiSky/src/nucleisky/nucleisky3d/features.py +- src/nucleisky/nucleisky3d/io.py: NucleiSky/src/nucleisky/nucleisky3d/io.py +- src/nucleisky/nucleisky3d/matching/geometry.py: NucleiSky/src/nucleisky/nucleisky3d/matching/geometry.py +- src/nucleisky/nucleisky3d/matching/hashing3d.py: NucleiSky/src/nucleisky/nucleisky3d/matching/hashing3d.py +- src/nucleisky/nucleisky3d/matching/pyramid.py: NucleiSky/src/nucleisky/nucleisky3d/matching/pyramid.py +- src/nucleisky/nucleisky3d/pipeline.py: NucleiSky/src/nucleisky/nucleisky3d/pipeline.py +- src/nucleisky/nucleisky3d/preprocess.py: NucleiSky/src/nucleisky/nucleisky3d/preprocess.py +- src/nucleisky/nucleisky3d/segmentation.py: NucleiSky/src/nucleisky/nucleisky3d/segmentation.py +- src/nucleisky/nucleisky3d/types.py: NucleiSky/src/nucleisky/nucleisky3d/types.py +- src/nucleisky/nucleisky3d/utils.py: NucleiSky/src/nucleisky/nucleisky3d/utils.py +- src/nucleisky/nucleisky3d/visualization.py: NucleiSky/src/nucleisky/nucleisky3d/visualization.py +- src/nucleisky2d/segmentation.py: NucleiSky/src/nucleisky2d/segmentation.py +- src/nucleisky_lc_tools/__init__.py: NucleiSky/src/nucleisky_lc_tools/__init__.py +post_install: app/bash_bat_scripts/post_install.bat # [win] +post_install: app/bash_bat_scripts/post_install.sh # [unix] +pre_uninstall: app/bash_bat_scripts/pre_uninstall.bat # [win] +pre_uninstall: app/bash_bat_scripts/pre_uninstall.sh # [unix] diff --git a/tests/fixtures/template_0_1_17_apps/nucleisky/post_install.bat b/tests/fixtures/template_0_1_17_apps/nucleisky/post_install.bat new file mode 100644 index 0000000..dd920f3 --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/nucleisky/post_install.bat @@ -0,0 +1,236 @@ +@ECHO OFF +SETLOCAL EnableExtensions DisableDelayedExpansion + +SET "LOG_FILE=%PREFIX%\menuinst_debug.log" +SET "PYTHON_EXE=%PREFIX%\python.exe" +SET "PROJECT_ROOT=%PREFIX%\NucleiSky" +SET "BASE_REQUIREMENTS=%PROJECT_ROOT%\requirements.txt" +SET "GPU_REQUIREMENTS=%PROJECT_ROOT%\requirements_gpu.txt" +SET "SELECTED_REQUIREMENTS=%BASE_REQUIREMENTS%" +SET "NVIDIA_SMI=" +SET "CA_BUNDLE=" +SET "FAILURE_MESSAGE=" +SET "CERTIFI_PATH_FILE=%TEMP%\labconstrictor_certifi_%RANDOM%_%RANDOM%.txt" + +> "%LOG_FILE%" echo Running post_install + +IF NOT EXIST "%PYTHON_EXE%" ( + SET "FAILURE_MESSAGE=Bundled Python executable was not found at %PYTHON_EXE%." + GOTO :fail +) +IF NOT EXIST "%PROJECT_ROOT%\launch_jupyter.py" ( + SET "FAILURE_MESSAGE=TLS-resilient launcher was not found at %PROJECT_ROOT%\launch_jupyter.py." + GOTO :fail +) +IF NOT EXIST "%BASE_REQUIREMENTS%" ( + SET "FAILURE_MESSAGE=Base requirements file was not found at %BASE_REQUIREMENTS%." + GOTO :fail +) + +REM pip 24.2+ uses the operating-system trust store by default. Some Windows +REM certificate stores contain malformed entries that OpenSSL cannot parse. +REM Keep verification enabled while opting out of system trust for installer +REM pip operations and using an explicit verified CA bundle. +SET "PIP_USE_DEPRECATED=legacy-certs" + +"%PYTHON_EXE%" "%PROJECT_ROOT%\launch_jupyter.py" --print-ca-bundle > "%CERTIFI_PATH_FILE%" 2>> "%LOG_FILE%" +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Unable to locate a verified CA bundle." + GOTO :fail +) +SET /P "CA_BUNDLE="<"%CERTIFI_PATH_FILE%" +DEL /Q "%CERTIFI_PATH_FILE%" >NUL 2>&1 + +IF NOT DEFINED CA_BUNDLE ( + SET "FAILURE_MESSAGE=No verified CA bundle could be selected." + GOTO :fail +) +IF NOT EXIST "%CA_BUNDLE%" ( + SET "FAILURE_MESSAGE=Selected CA bundle does not exist: %CA_BUNDLE%" + GOTO :fail +) + +SET "PIP_CERT=%CA_BUNDLE%" +SET "REQUESTS_CA_BUNDLE=%CA_BUNDLE%" +SET "CURL_CA_BUNDLE=%CA_BUNDLE%" +echo Using verified CA bundle for installer network operations: "%CA_BUNDLE%" >> "%LOG_FILE%" + +IF EXIST "%GPU_REQUIREMENTS%" ( + CALL :detect_nvidia_smi + IF DEFINED NVIDIA_SMI ( + echo NVIDIA GPU utility detected, installing GPU requirements from "%GPU_REQUIREMENTS%" >> "%LOG_FILE%" + SET "SELECTED_REQUIREMENTS=%GPU_REQUIREMENTS%" + ) ELSE ( + echo NVIDIA GPU not detected, installing CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" + ) +) ELSE ( + echo GPU requirements file not found, installing CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" +) + +echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" +"%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Application requirements installation failed." + GOTO :fail +) + +IF EXIST "%PROJECT_ROOT%\requirements-windows.txt" ( + echo Installing Windows-specific requirements. >> "%LOG_FILE%" + "%PYTHON_EXE%" -m pip install -r "%PROJECT_ROOT%\requirements-windows.txt" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Windows-specific requirements installation failed." + GOTO :fail + ) +) + +REM External Python code is optional in LabConstrictor projects. If setup.py is +REM bundled, install it without a temporary build environment and verify that +REM the generated import package is usable. Otherwise continue without it. +IF EXIST "%PROJECT_ROOT%\setup.py" ( + echo Found setup.py, installing NucleiSky package locally without build isolation. >> "%LOG_FILE%" + "%PYTHON_EXE%" -m pip install --no-deps --no-build-isolation "%PROJECT_ROOT%" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=NucleiSky package installation failed." + GOTO :fail + ) + + REM Tools first, only for an app whose package ships a tools module: the package may import labconstrictor_tools at module level, + REM so labconstrictor-tools is installed BEFORE the import check below. A failed install is only a WARNING, see :install_lc_tools at the end of this file. + CALL :install_lc_tools + + "%PYTHON_EXE%" -c "import nucleisky; print('NucleiSky import successful:', nucleisky.__file__)" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Installed NucleiSky package could not be imported as nucleisky." + GOTO :fail + ) +) ELSE ( + echo No setup.py detected; this project does not bundle an optional Python package. >> "%LOG_FILE%" +) + +REM Optional: expose the app's tools to Napari, Fiji and the command line (LabConstrictor tools bridge). +REM If the app's package ships a module named _lc_tools, install labconstrictor-tools and register that module. +REM This step must never fail the installation. LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +REM labconstrictor-tools is installed by :install_lc_tools, called above before the import check of the package (it sets LC_TOOLS_PIN, LC_TOOLS_SPEC and LC_TOOLS_READY). +SET "LC_APP_VERSION=" +IF EXIST "%PROJECT_ROOT%\construct.yaml" FOR /F "usebackq tokens=1,* delims=: " %%A IN (`findstr /B /C:"version:" "%PROJECT_ROOT%\construct.yaml"`) DO SET "LC_APP_VERSION=%%~B" +IF NOT DEFINED LC_APP_VERSION ( + SET "LC_APP_VERSION=0" + echo WARNING: no version: line at the start of a line in construct.yaml - registering the tools with version 0. >> "%LOG_FILE%" +) +"%PYTHON_EXE%" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('nucleisky_lc_tools') else 1)" >> "%LOG_FILE%" 2>&1 +IF NOT ERRORLEVEL 1 ( + echo Found nucleisky_lc_tools: registering the tools of NucleiSky for Napari and Fiji. >> "%LOG_FILE%" + REM labconstrictor-tools was installed before the import check (:install_lc_tools): register only if that worked (errorlevel 1 otherwise). + "%PYTHON_EXE%" -c "import sys; sys.exit(0 if sys.argv[1] == '1' else 1)" "%LC_TOOLS_READY%" >> "%LOG_FILE%" 2>&1 + IF NOT ERRORLEVEL 1 "%PYTHON_EXE%" -m labconstrictor_tools register --name "NucleiSky" --prefix "%PREFIX%" --module nucleisky_lc_tools --version "%LC_APP_VERSION%" --display-name "NucleiSky" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 echo WARNING: tool registration failed - see the pip and register output above in this file; NucleiSky itself is installed. >> "%LOG_FILE%" +) + +"%PYTHON_EXE%" "%PROJECT_ROOT%\include_path.py" --path "%PREFIX%" --files "%PROJECT_ROOT%\notebook_launcher.json" --keyword "BASE_PATH_KEYWORD" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Launcher path preparation failed." + GOTO :fail +) + +"%PYTHON_EXE%" "%PROJECT_ROOT%\hide_code_cells.py" "%PROJECT_ROOT%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Notebook metadata preparation failed." + GOTO :fail +) + +REM Keep conda verification enabled, but avoid the Windows trust store that +REM failed during installation. Conda accepts a CA bundle path for ssl_verify. +echo Configuring conda to use verified CA bundle: "%CA_BUNDLE%" >> "%LOG_FILE%" +"%PYTHON_EXE%" -m conda config --system --set ssl_verify "%CA_BUNDLE%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + echo WARNING: Could not configure conda with the selected CA bundle. The application launcher still has its own verified fallback. >> "%LOG_FILE%" +) + +REM Validate TLS and Jupyter imports before creating shortcuts or registering +REM the application. +"%PYTHON_EXE%" "%PROJECT_ROOT%\launch_jupyter.py" --self-test >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Launcher preflight failed. See menuinst_debug.log and launcher_debug.log." + GOTO :fail +) +echo Launcher preflight completed successfully. >> "%LOG_FILE%" + +"%PYTHON_EXE%" -c "import os, sys; print('Python:', sys.executable); print('Prefix:', os.environ.get('PREFIX'))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Python installation diagnostics failed." + GOTO :fail +) + +"%PYTHON_EXE%" -c "from menuinst.api import install; import os; print(install(os.path.join(r'%PREFIX%', 'NucleiSky', 'notebook_launcher.json')))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Application shortcut creation failed." + GOTO :fail +) + +SET "ARP_KEY=HKCU\Software\Microsoft\Windows\CurrentVersion\Uninstall\NucleiSky" +SET "UNINSTALL_EXE=%PREFIX%\Uninstall-NucleiSky.exe" +SET "DISPLAY_ICON=%PROJECT_ROOT%\logo.ico" +SET "DISPLAY_VERSION=0.1.1" +SET "PUBLISHER=CellMigrationLab" +echo Registering NucleiSky in Windows Apps list >> "%LOG_FILE%" +reg add "%ARP_KEY%" /v DisplayName /d "NucleiSky" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v DisplayVersion /d "%DISPLAY_VERSION%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v Publisher /d "%PUBLISHER%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v InstallLocation /d "%PREFIX%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v DisplayIcon /d "%DISPLAY_ICON%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v UninstallString /d "\"%UNINSTALL_EXE%\"" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v QuietUninstallString /d "\"%UNINSTALL_EXE%\" /S" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v NoModify /t REG_DWORD /d 1 /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v NoRepair /t REG_DWORD /d 1 /f >> "%LOG_FILE%" 2>&1 + +echo Post-install completed successfully. >> "%LOG_FILE%" +echo Post-install completed successfully. +ENDLOCAL +EXIT /B 0 + +:fail +IF EXIST "%CERTIFI_PATH_FILE%" DEL /Q "%CERTIFI_PATH_FILE%" >NUL 2>&1 +IF NOT DEFINED FAILURE_MESSAGE SET "FAILURE_MESSAGE=Post-install failed for an unknown reason." +echo ERROR: %FAILURE_MESSAGE% >> "%LOG_FILE%" +echo ERROR: %FAILURE_MESSAGE% +echo See "%LOG_FILE%" for details. +ENDLOCAL +EXIT /B 1 + +:detect_nvidia_smi +FOR %%P IN ( + "%ProgramFiles%\NVIDIA Corporation\NVSMI\nvidia-smi.exe" + "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvidia-smi.exe" + "%SystemRoot%\System32\nvidia-smi.exe" + "%SystemRoot%\Sysnative\nvidia-smi.exe" +) DO ( + IF NOT DEFINED NVIDIA_SMI IF EXIST "%%~P" SET "NVIDIA_SMI=%%~P" +) +IF DEFINED NVIDIA_SMI GOTO :EOF + +FOR /F "delims=" %%I IN ('where.exe nvidia-smi.exe 2^>NUL') DO ( + IF NOT DEFINED NVIDIA_SMI SET "NVIDIA_SMI=%%~fI" +) +GOTO :EOF + +:install_lc_tools +REM Optional (LabConstrictor tools bridge): install labconstrictor-tools when the app's package ships a module named _lc_tools. +REM Called before the import check of the package, which may import labconstrictor_tools at module level. +REM This step must never fail the installation: a failure is logged as a WARNING and LC_TOOLS_READY stays 0 (no registration later). +REM LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +REM Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), +REM because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. +SET "LC_TOOLS_PIN=v0.2.0-rc.1" +IF NOT DEFINED LC_TOOLS_SPEC SET "LC_TOOLS_SPEC=https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/%LC_TOOLS_PIN%.zip" +SET "LC_TOOLS_READY=0" +"%PYTHON_EXE%" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('nucleisky_lc_tools') else 1)" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 GOTO :EOF +echo Found nucleisky_lc_tools: installing labconstrictor-tools before the import check of NucleiSky. >> "%LOG_FILE%" +REM LC_TOOLS_SPEC goes to pip as one argument through Python: cmd.exe never re-parses it (a quote or & in it cannot run anything). +"%PYTHON_EXE%" -c "import os, subprocess, sys; sys.exit(subprocess.call([sys.executable, '-m', 'pip', 'install', os.environ['LC_TOOLS_SPEC']]))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + echo WARNING: labconstrictor-tools could not be installed - see the pip output above in this file; the tools of NucleiSky will not be registered ^(the import check fails only if the package needs labconstrictor-tools at import time^). >> "%LOG_FILE%" + GOTO :EOF +) +SET "LC_TOOLS_READY=1" +GOTO :EOF diff --git a/tests/fixtures/template_0_1_17_apps/nucleisky/post_install.sh b/tests/fixtures/template_0_1_17_apps/nucleisky/post_install.sh new file mode 100644 index 0000000..3034812 --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/nucleisky/post_install.sh @@ -0,0 +1,134 @@ +#!/bin/bash +set -euo pipefail + +LOG_FILE="$PREFIX/menuinst_debug.log" +PYTHON_EXE="$PREFIX/bin/python" +PROJECT_ROOT="$PREFIX/NucleiSky" +BASE_REQUIREMENTS="$PROJECT_ROOT/requirements.txt" +GPU_REQUIREMENTS="$PROJECT_ROOT/requirements_gpu.txt" +SELECTED_REQUIREMENTS="$BASE_REQUIREMENTS" + +echo "Running post_install" > "$LOG_FILE" + +fail() { + echo "ERROR: $1" | tee -a "$LOG_FILE" >&2 + exit 1 +} + +[ -x "$PYTHON_EXE" ] || fail "Bundled Python executable was not found at $PYTHON_EXE." +[ -f "$PROJECT_ROOT/launch_jupyter.py" ] || fail "TLS-resilient launcher was not found at $PROJECT_ROOT/launch_jupyter.py." +[ -f "$BASE_REQUIREMENTS" ] || fail "Base requirements file was not found at $BASE_REQUIREMENTS." + +# pip 24.2+ uses system certificates by default. Opt out for installer pip +# operations and use an explicit verified CA bundle instead. +export PIP_USE_DEPRECATED=legacy-certs +CA_BUNDLE="$("$PYTHON_EXE" "$PROJECT_ROOT/launch_jupyter.py" --print-ca-bundle 2>> "$LOG_FILE")" +[ -s "$CA_BUNDLE" ] || fail "No valid verified CA bundle could be selected: $CA_BUNDLE" + +export PIP_CERT="$CA_BUNDLE" +export REQUESTS_CA_BUNDLE="$CA_BUNDLE" +export CURL_CA_BUNDLE="$CA_BUNDLE" +echo "Using verified CA bundle for installer network operations: $CA_BUNDLE" >> "$LOG_FILE" + +if [ -f "$GPU_REQUIREMENTS" ]; then + if [[ "${OSTYPE:-}" == "darwin"* ]]; then + echo "macOS detected, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" + elif command -v nvidia-smi >/dev/null 2>&1; then + echo "NVIDIA GPU detected, installing GPU requirements from $GPU_REQUIREMENTS" >> "$LOG_FILE" + SELECTED_REQUIREMENTS="$GPU_REQUIREMENTS" + else + echo "NVIDIA GPU not detected, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" + fi +else + echo "GPU requirements file not found, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" +fi + +echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE" +"$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1 + +if [[ "${OSTYPE:-}" == "darwin"* ]]; then + echo "Detected macOS platform" >> "$LOG_FILE" + if [ -f "$PROJECT_ROOT/requirements-macos.txt" ]; then + "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-macos.txt" >> "$LOG_FILE" 2>&1 + fi +elif [[ "${OSTYPE:-}" == "linux-gnu"* ]]; then + echo "Detected Linux platform" >> "$LOG_FILE" + if [ -f "$PROJECT_ROOT/requirements-linux.txt" ]; then + "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-linux.txt" >> "$LOG_FILE" 2>&1 + fi +else + echo "Unknown platform: ${OSTYPE:-unset}" >> "$LOG_FILE" +fi + +# External Python code is optional. Verify the generated package only when the +# constructor bundled setup.py and src/. +if [ -f "$PROJECT_ROOT/setup.py" ]; then + echo "Found setup.py, installing NucleiSky package locally without build isolation" >> "$LOG_FILE" + "$PYTHON_EXE" -m pip install --no-deps --no-build-isolation "$PROJECT_ROOT" >> "$LOG_FILE" 2>&1 + # Tools first, only for an app whose package ships a _lc_tools module: the app package may import labconstrictor_tools + # at module level, so labconstrictor-tools must be installed BEFORE the import check below. A failed install is only a WARNING + # (it never fails the installation by itself). LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). + # Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), + # because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. + LC_TOOLS_PIN="v0.2.0-rc.1" + LC_TOOLS_MODULE="nucleisky_lc_tools" + LC_TOOLS_READY=0 + if "$PYTHON_EXE" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('$LC_TOOLS_MODULE') else 1)" >> "$LOG_FILE" 2>&1; then + echo "Found $LC_TOOLS_MODULE: installing labconstrictor-tools before the import check of NucleiSky" >> "$LOG_FILE" || : + if "$PYTHON_EXE" -m pip install "${LC_TOOLS_SPEC:-https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/${LC_TOOLS_PIN}.zip}" >> "$LOG_FILE" 2>&1; then + LC_TOOLS_READY=1 + else + echo "WARNING: labconstrictor-tools could not be installed - see the pip output above in this file; the tools of NucleiSky will not be registered (the import check below fails only if the package needs labconstrictor-tools at import time)." >> "$LOG_FILE" || : + fi + fi + "$PYTHON_EXE" -c "import nucleisky; print('NucleiSky import successful:', nucleisky.__file__)" >> "$LOG_FILE" 2>&1 +else + echo "No setup.py detected; this project does not bundle an optional Python package." >> "$LOG_FILE" +fi + + +# --- Optional: expose the app's tools to Napari, Fiji and the command line (LabConstrictor tools bridge) ------------------- +# If the app's package ships a module named _lc_tools, install labconstrictor-tools and register that module. +# This step must never fail the installation. LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +# labconstrictor-tools is installed above, before the import check of the package (LC_TOOLS_PIN, LC_TOOLS_MODULE and LC_TOOLS_READY are set there). +if [ -f "$PROJECT_ROOT/setup.py" ] && "$PYTHON_EXE" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('$LC_TOOLS_MODULE') else 1)" >> "$LOG_FILE" 2>&1; then + lc_note() { echo "$*" >> "$LOG_FILE" || :; } # a full or unwritable log must not fail the installation + lc_note "Found $LC_TOOLS_MODULE: registering the tools of NucleiSky for Napari and Fiji" + LC_APP_VERSION="$(sed -n 's/^version: *//p' "$PROJECT_ROOT/construct.yaml" 2>/dev/null | head -1 | tr -d "\"'\r" || true)" + if [ -z "$LC_APP_VERSION" ]; then + lc_note "WARNING: no version: line at the start of a line in construct.yaml - registering the tools with version 0." + LC_APP_VERSION=0 + fi + if [ "${LC_TOOLS_READY:-0}" = 1 ] \ + && "$PYTHON_EXE" -m labconstrictor_tools register --name "NucleiSky" --prefix "$PREFIX" \ + --module "$LC_TOOLS_MODULE" --version "$LC_APP_VERSION" --display-name "NucleiSky" >> "$LOG_FILE" 2>&1; then + lc_note "Tools registered (labconstrictor-tools list shows them)." + else + lc_note "WARNING: tool registration failed - see the pip and register output above in this file; NucleiSky itself is installed." + fi +fi + +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PROJECT_ROOT/notebook_launcher.json" --keyword "BASE_PATH_KEYWORD" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PREFIX/pre_uninstall.sh" --keyword "BASE_PATH" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PREFIX/uninstall.sh" --keyword "BASE_PATH" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/hide_code_cells.py" "$PROJECT_ROOT" >> "$LOG_FILE" 2>&1 + +# Keep conda verification enabled while avoiding the system certificate store +# that failed. Conda accepts a CA bundle path for ssl_verify. +if ! "$PYTHON_EXE" -m conda config --system --set ssl_verify "$CA_BUNDLE" >> "$LOG_FILE" 2>&1; then + echo "WARNING: Could not configure conda with the selected CA bundle. The launcher still has its own verified fallback." >> "$LOG_FILE" +fi + +# Do not create the application shortcut unless launcher preflight succeeds. +"$PYTHON_EXE" "$PROJECT_ROOT/launch_jupyter.py" --self-test >> "$LOG_FILE" 2>&1 +echo "Launcher preflight completed successfully." >> "$LOG_FILE" + +"$PYTHON_EXE" -c "import os, sys; print('Python:', sys.executable); print('Prefix:', os.environ.get('PREFIX'))" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" -c "from menuinst.api import install; import os; print(install(os.path.join('$PREFIX', 'NucleiSky', 'notebook_launcher.json')))" >> "$LOG_FILE" 2>&1 + +echo "Post-install completed successfully." >> "$LOG_FILE" + +if [ -t 0 ]; then + echo + read -rp "Press Enter to close the installer..." _ +fi diff --git a/tests/fixtures/template_0_1_17_apps/playground/construct.yaml b/tests/fixtures/template_0_1_17_apps/playground/construct.yaml new file mode 100644 index 0000000..cd1ec68 --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/playground/construct.yaml @@ -0,0 +1,49 @@ +name: LabConstrictorPlayground +version: 0.0.3 + +environment_file: environment.yaml +installer_type: all +license_file: LICENSE + +welcome_image: app/logo/logo.png +header_image: app/logo/logo.png +icon_image: app/logo/labconstrictor_playground.png + +conclusion_text: 'Thank you! You have successfully installed LabConstrictorPlayground 0.0.3.' + +keep_pkgs: false +register_python: false +initialize_conda: false + +extra_files: +- .tools/meta/src_change.yaml: LabConstrictorPlayground/src_change.yaml +- app/bash_bat_scripts/pre_uninstall.sh: pre_uninstall.sh +- app/bash_bat_scripts/uninstall.sh: uninstall.sh +- app/logo/labconstrictor_playground.icns: LabConstrictorPlayground/labconstrictor_playground.icns +- app/logo/labconstrictor_playground.ico: LabConstrictorPlayground/labconstrictor_playground.ico +- app/logo/labconstrictor_playground.png: LabConstrictorPlayground/labconstrictor_playground.png +- app/menuinst/Welcome.ipynb: LabConstrictorPlayground/notebooks/Welcome.ipynb +- app/menuinst/notebook_launcher.json: LabConstrictorPlayground/notebook_launcher.json +- app/python_scripts/hide_code_cells.py: LabConstrictorPlayground/hide_code_cells.py +- app/python_scripts/include_path.py: LabConstrictorPlayground/include_path.py +- app/python_scripts/launch_jupyter.py: LabConstrictorPlayground/launch_jupyter.py +- construct.yaml: LabConstrictorPlayground/construct.yaml +- notebooks/LabConstrictorPlayground/LabConstrictorPlayground.ipynb: LabConstrictorPlayground/notebooks/LabConstrictorPlayground/LabConstrictorPlayground.ipynb +- notebooks/notebook_latest_versions.yaml: LabConstrictorPlayground/notebook_latest_versions.yaml +- requirements.txt: LabConstrictorPlayground/requirements.txt +- requirements_gpu.txt: LabConstrictorPlayground/requirements_gpu.txt +- setup.py: LabConstrictorPlayground/setup.py +- src/labconstrictor_playground/__init__.py: LabConstrictorPlayground/src/labconstrictor_playground/__init__.py +- src/labconstrictor_playground/checks.py: LabConstrictorPlayground/src/labconstrictor_playground/checks.py +- src/labconstrictor_playground/data.py: LabConstrictorPlayground/src/labconstrictor_playground/data.py +- src/labconstrictor_playground/stress.py: LabConstrictorPlayground/src/labconstrictor_playground/stress.py +- src/labconstrictor_playground/timing.py: LabConstrictorPlayground/src/labconstrictor_playground/timing.py +- src/labconstrictor_playground/tour.py: LabConstrictorPlayground/src/labconstrictor_playground/tour.py +- src/labconstrictor_playground_lc_tools/__init__.py: LabConstrictorPlayground/src/labconstrictor_playground_lc_tools/__init__.py + +post_install: app/bash_bat_scripts/post_install.bat # [win] +post_install: app/bash_bat_scripts/post_install.sh # [unix] + +pre_uninstall: app/bash_bat_scripts/pre_uninstall.bat # [win] +pre_uninstall: app/bash_bat_scripts/pre_uninstall.sh # [unix] + diff --git a/tests/fixtures/template_0_1_17_apps/playground/post_install.bat b/tests/fixtures/template_0_1_17_apps/playground/post_install.bat new file mode 100644 index 0000000..6672ad4 --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/playground/post_install.bat @@ -0,0 +1,245 @@ +@ECHO OFF +SETLOCAL EnableExtensions DisableDelayedExpansion + +SET "LOG_FILE=%PREFIX%\menuinst_debug.log" +SET "PYTHON_EXE=%PREFIX%\python.exe" +SET "PROJECT_ROOT=%PREFIX%\LabConstrictorPlayground" +SET "BASE_REQUIREMENTS=%PROJECT_ROOT%\requirements.txt" +SET "GPU_REQUIREMENTS=%PROJECT_ROOT%\requirements_gpu.txt" +SET "SELECTED_REQUIREMENTS=%BASE_REQUIREMENTS%" +SET "NVIDIA_SMI=" +SET "CA_BUNDLE=" +SET "FAILURE_MESSAGE=" +SET "CERTIFI_PATH_FILE=%TEMP%\labconstrictor_certifi_%RANDOM%_%RANDOM%.txt" + +> "%LOG_FILE%" echo Running post_install + +IF NOT EXIST "%PYTHON_EXE%" ( + SET "FAILURE_MESSAGE=Bundled Python executable was not found at %PYTHON_EXE%." + GOTO :fail +) +IF NOT EXIST "%PROJECT_ROOT%\launch_jupyter.py" ( + SET "FAILURE_MESSAGE=TLS-resilient launcher was not found at %PROJECT_ROOT%\launch_jupyter.py." + GOTO :fail +) +IF NOT EXIST "%BASE_REQUIREMENTS%" ( + SET "FAILURE_MESSAGE=Base requirements file was not found at %BASE_REQUIREMENTS%." + GOTO :fail +) + +REM pip 24.2+ uses the operating-system trust store by default. Some Windows +REM certificate stores contain malformed entries that OpenSSL cannot parse. +REM Keep verification enabled while opting out of system trust for installer +REM pip operations and using an explicit verified CA bundle. +SET "PIP_USE_DEPRECATED=legacy-certs" + +"%PYTHON_EXE%" "%PROJECT_ROOT%\launch_jupyter.py" --print-ca-bundle > "%CERTIFI_PATH_FILE%" 2>> "%LOG_FILE%" +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Unable to locate a verified CA bundle." + GOTO :fail +) +SET /P "CA_BUNDLE="<"%CERTIFI_PATH_FILE%" +DEL /Q "%CERTIFI_PATH_FILE%" >NUL 2>&1 + +IF NOT DEFINED CA_BUNDLE ( + SET "FAILURE_MESSAGE=No verified CA bundle could be selected." + GOTO :fail +) +IF NOT EXIST "%CA_BUNDLE%" ( + SET "FAILURE_MESSAGE=Selected CA bundle does not exist: %CA_BUNDLE%" + GOTO :fail +) + +SET "PIP_CERT=%CA_BUNDLE%" +SET "REQUESTS_CA_BUNDLE=%CA_BUNDLE%" +SET "CURL_CA_BUNDLE=%CA_BUNDLE%" +echo Using verified CA bundle for installer network operations: "%CA_BUNDLE%" >> "%LOG_FILE%" + +IF EXIST "%GPU_REQUIREMENTS%" ( + CALL :detect_nvidia_smi + IF DEFINED NVIDIA_SMI ( + echo NVIDIA GPU utility detected, installing GPU requirements from "%GPU_REQUIREMENTS%" >> "%LOG_FILE%" + SET "SELECTED_REQUIREMENTS=%GPU_REQUIREMENTS%" + ) ELSE ( + echo NVIDIA GPU not detected, installing CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" + ) +) ELSE ( + echo GPU requirements file not found, installing CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" +) + +echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" +"%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + IF /I NOT "%SELECTED_REQUIREMENTS%"=="%BASE_REQUIREMENTS%" ( + REM The GPU build could not be installed (network, proxy, an index that is down): the application must still install. + echo WARNING: installing the GPU requirements failed; falling back to the CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" + SET "SELECTED_REQUIREMENTS=%BASE_REQUIREMENTS%" + "%PYTHON_EXE%" -m pip install -r "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1 + ) +) +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Application requirements installation failed." + GOTO :fail +) +echo Requirements installed from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" + +IF EXIST "%PROJECT_ROOT%\requirements-windows.txt" ( + echo Installing Windows-specific requirements. >> "%LOG_FILE%" + "%PYTHON_EXE%" -m pip install -r "%PROJECT_ROOT%\requirements-windows.txt" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Windows-specific requirements installation failed." + GOTO :fail + ) +) + +REM External Python code is optional in LabConstrictor projects. If setup.py is +REM bundled, install it without a temporary build environment and verify that +REM the generated import package is usable. Otherwise continue without it. +IF EXIST "%PROJECT_ROOT%\setup.py" ( + echo Found setup.py, installing LabConstrictorPlayground package locally without build isolation. >> "%LOG_FILE%" + "%PYTHON_EXE%" -m pip install --no-deps --no-build-isolation "%PROJECT_ROOT%" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=LabConstrictorPlayground package installation failed." + GOTO :fail + ) + + REM Tools first, only for an app whose package ships a tools module: the package may import labconstrictor_tools at module level, + REM so labconstrictor-tools is installed BEFORE the import check below. A failed install is only a WARNING, see :install_lc_tools at the end of this file. + CALL :install_lc_tools + + "%PYTHON_EXE%" -c "import labconstrictor_playground; print('LabConstrictorPlayground import successful:', labconstrictor_playground.__file__)" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Installed LabConstrictorPlayground package could not be imported as labconstrictor_playground." + GOTO :fail + ) +) ELSE ( + echo No setup.py detected; this project does not bundle an optional Python package. >> "%LOG_FILE%" +) + +REM Optional: expose the app's tools to Napari, Fiji and the command line (LabConstrictor tools bridge). +REM If the app's package ships a module named _lc_tools, install labconstrictor-tools and register that module. +REM This step must never fail the installation. LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +REM labconstrictor-tools is installed by :install_lc_tools, called above before the import check of the package (it sets LC_TOOLS_PIN, LC_TOOLS_SPEC and LC_TOOLS_READY). +SET "LC_APP_VERSION=" +IF EXIST "%PROJECT_ROOT%\construct.yaml" FOR /F "usebackq tokens=1,* delims=: " %%A IN (`findstr /B /C:"version:" "%PROJECT_ROOT%\construct.yaml"`) DO SET "LC_APP_VERSION=%%~B" +IF NOT DEFINED LC_APP_VERSION ( + SET "LC_APP_VERSION=0" + echo WARNING: no version: line at the start of a line in construct.yaml - registering the tools with version 0. >> "%LOG_FILE%" +) +"%PYTHON_EXE%" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('labconstrictor_playground_lc_tools') else 1)" >> "%LOG_FILE%" 2>&1 +IF NOT ERRORLEVEL 1 ( + echo Found labconstrictor_playground_lc_tools: registering the tools of LabConstrictorPlayground for Napari and Fiji. >> "%LOG_FILE%" + REM labconstrictor-tools was installed before the import check (:install_lc_tools): register only if that worked (errorlevel 1 otherwise). + "%PYTHON_EXE%" -c "import sys; sys.exit(0 if sys.argv[1] == '1' else 1)" "%LC_TOOLS_READY%" >> "%LOG_FILE%" 2>&1 + IF NOT ERRORLEVEL 1 "%PYTHON_EXE%" -m labconstrictor_tools register --name "LabConstrictorPlayground" --prefix "%PREFIX%" --module labconstrictor_playground_lc_tools --version "%LC_APP_VERSION%" --display-name "LabConstrictorPlayground" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 echo WARNING: tool registration failed - see the pip and register output above in this file; LabConstrictorPlayground itself is installed. >> "%LOG_FILE%" +) + +"%PYTHON_EXE%" "%PROJECT_ROOT%\include_path.py" --path "%PREFIX%" --files "%PROJECT_ROOT%\notebook_launcher.json" --keyword "BASE_PATH_KEYWORD" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Launcher path preparation failed." + GOTO :fail +) + +"%PYTHON_EXE%" "%PROJECT_ROOT%\hide_code_cells.py" "%PROJECT_ROOT%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Notebook metadata preparation failed." + GOTO :fail +) + +REM Keep conda verification enabled, but avoid the Windows trust store that +REM failed during installation. Conda accepts a CA bundle path for ssl_verify. +echo Configuring conda to use verified CA bundle: "%CA_BUNDLE%" >> "%LOG_FILE%" +"%PYTHON_EXE%" -m conda config --system --set ssl_verify "%CA_BUNDLE%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + echo WARNING: Could not configure conda with the selected CA bundle. The application launcher still has its own verified fallback. >> "%LOG_FILE%" +) + +REM Validate TLS and Jupyter imports before creating shortcuts or registering +REM the application. +"%PYTHON_EXE%" "%PROJECT_ROOT%\launch_jupyter.py" --self-test >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Launcher preflight failed. See menuinst_debug.log and launcher_debug.log." + GOTO :fail +) +echo Launcher preflight completed successfully. >> "%LOG_FILE%" + +"%PYTHON_EXE%" -c "import os, sys; print('Python:', sys.executable); print('Prefix:', os.environ.get('PREFIX'))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Python installation diagnostics failed." + GOTO :fail +) + +"%PYTHON_EXE%" -c "from menuinst.api import install; import os; print(install(os.path.join(r'%PREFIX%', 'LabConstrictorPlayground', 'notebook_launcher.json')))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Application shortcut creation failed." + GOTO :fail +) + +SET "ARP_KEY=HKCU\Software\Microsoft\Windows\CurrentVersion\Uninstall\LabConstrictorPlayground" +SET "UNINSTALL_EXE=%PREFIX%\Uninstall-LabConstrictorPlayground.exe" +SET "DISPLAY_ICON=%PROJECT_ROOT%\logo.ico" +SET "DISPLAY_VERSION=0.1.0" +SET "PUBLISHER=CellMigrationLab" +echo Registering LabConstrictorPlayground in Windows Apps list >> "%LOG_FILE%" +reg add "%ARP_KEY%" /v DisplayName /d "LabConstrictorPlayground" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v DisplayVersion /d "%DISPLAY_VERSION%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v Publisher /d "%PUBLISHER%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v InstallLocation /d "%PREFIX%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v DisplayIcon /d "%DISPLAY_ICON%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v UninstallString /d "\"%UNINSTALL_EXE%\"" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v QuietUninstallString /d "\"%UNINSTALL_EXE%\" /S" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v NoModify /t REG_DWORD /d 1 /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v NoRepair /t REG_DWORD /d 1 /f >> "%LOG_FILE%" 2>&1 + +echo Post-install completed successfully. >> "%LOG_FILE%" +echo Post-install completed successfully. +ENDLOCAL +EXIT /B 0 + +:fail +IF EXIST "%CERTIFI_PATH_FILE%" DEL /Q "%CERTIFI_PATH_FILE%" >NUL 2>&1 +IF NOT DEFINED FAILURE_MESSAGE SET "FAILURE_MESSAGE=Post-install failed for an unknown reason." +echo ERROR: %FAILURE_MESSAGE% >> "%LOG_FILE%" +echo ERROR: %FAILURE_MESSAGE% +echo See "%LOG_FILE%" for details. +ENDLOCAL +EXIT /B 1 + +:detect_nvidia_smi +FOR %%P IN ( + "%ProgramFiles%\NVIDIA Corporation\NVSMI\nvidia-smi.exe" + "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvidia-smi.exe" + "%SystemRoot%\System32\nvidia-smi.exe" + "%SystemRoot%\Sysnative\nvidia-smi.exe" +) DO ( + IF NOT DEFINED NVIDIA_SMI IF EXIST "%%~P" SET "NVIDIA_SMI=%%~P" +) +IF DEFINED NVIDIA_SMI GOTO :EOF + +FOR /F "delims=" %%I IN ('where.exe nvidia-smi.exe 2^>NUL') DO ( + IF NOT DEFINED NVIDIA_SMI SET "NVIDIA_SMI=%%~fI" +) +GOTO :EOF + +:install_lc_tools +REM Optional (LabConstrictor tools bridge): install labconstrictor-tools when the app's package ships a module named _lc_tools. +REM Called before the import check of the package, which may import labconstrictor_tools at module level. +REM This step must never fail the installation: a failure is logged as a WARNING and LC_TOOLS_READY stays 0 (no registration later). +REM LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +REM Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), +REM because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. +SET "LC_TOOLS_PIN=v0.2.0-rc.1" +IF NOT DEFINED LC_TOOLS_SPEC SET "LC_TOOLS_SPEC=https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/%LC_TOOLS_PIN%.zip" +SET "LC_TOOLS_READY=0" +"%PYTHON_EXE%" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('labconstrictor_playground_lc_tools') else 1)" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 GOTO :EOF +echo Found labconstrictor_playground_lc_tools: installing labconstrictor-tools before the import check of LabConstrictorPlayground. >> "%LOG_FILE%" +REM LC_TOOLS_SPEC goes to pip as one argument through Python: cmd.exe never re-parses it (a quote or & in it cannot run anything). +"%PYTHON_EXE%" -c "import os, subprocess, sys; sys.exit(subprocess.call([sys.executable, '-m', 'pip', 'install', os.environ['LC_TOOLS_SPEC']]))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + echo WARNING: labconstrictor-tools could not be installed - see the pip output above in this file; the tools of LabConstrictorPlayground will not be registered ^(the import check fails only if the package needs labconstrictor-tools at import time^). >> "%LOG_FILE%" + GOTO :EOF +) +SET "LC_TOOLS_READY=1" +GOTO :EOF diff --git a/tests/fixtures/template_0_1_17_apps/playground/post_install.sh b/tests/fixtures/template_0_1_17_apps/playground/post_install.sh new file mode 100644 index 0000000..d27ae36 --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/playground/post_install.sh @@ -0,0 +1,144 @@ +#!/bin/bash +set -euo pipefail + +LOG_FILE="$PREFIX/menuinst_debug.log" +PYTHON_EXE="$PREFIX/bin/python" +PROJECT_ROOT="$PREFIX/LabConstrictorPlayground" +BASE_REQUIREMENTS="$PROJECT_ROOT/requirements.txt" +GPU_REQUIREMENTS="$PROJECT_ROOT/requirements_gpu.txt" +SELECTED_REQUIREMENTS="$BASE_REQUIREMENTS" + +echo "Running post_install" > "$LOG_FILE" + +fail() { + echo "ERROR: $1" | tee -a "$LOG_FILE" >&2 + exit 1 +} + +[ -x "$PYTHON_EXE" ] || fail "Bundled Python executable was not found at $PYTHON_EXE." +[ -f "$PROJECT_ROOT/launch_jupyter.py" ] || fail "TLS-resilient launcher was not found at $PROJECT_ROOT/launch_jupyter.py." +[ -f "$BASE_REQUIREMENTS" ] || fail "Base requirements file was not found at $BASE_REQUIREMENTS." + +# pip 24.2+ uses system certificates by default. Opt out for installer pip +# operations and use an explicit verified CA bundle instead. +export PIP_USE_DEPRECATED=legacy-certs +CA_BUNDLE="$("$PYTHON_EXE" "$PROJECT_ROOT/launch_jupyter.py" --print-ca-bundle 2>> "$LOG_FILE")" +[ -s "$CA_BUNDLE" ] || fail "No valid verified CA bundle could be selected: $CA_BUNDLE" + +export PIP_CERT="$CA_BUNDLE" +export REQUESTS_CA_BUNDLE="$CA_BUNDLE" +export CURL_CA_BUNDLE="$CA_BUNDLE" +echo "Using verified CA bundle for installer network operations: $CA_BUNDLE" >> "$LOG_FILE" + +if [ -f "$GPU_REQUIREMENTS" ]; then + if [[ "${OSTYPE:-}" == "darwin"* ]]; then + echo "macOS detected, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" + elif command -v nvidia-smi >/dev/null 2>&1; then + echo "NVIDIA GPU detected, installing GPU requirements from $GPU_REQUIREMENTS" >> "$LOG_FILE" + SELECTED_REQUIREMENTS="$GPU_REQUIREMENTS" + else + echo "NVIDIA GPU not detected, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" + fi +else + echo "GPU requirements file not found, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" +fi + +echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE" +if ! "$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1; then + if [ "$SELECTED_REQUIREMENTS" != "$BASE_REQUIREMENTS" ]; then + # The GPU build could not be installed (network, proxy, an index that is down): the application must still install. + echo "WARNING: installing the GPU requirements failed; falling back to the CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" + SELECTED_REQUIREMENTS="$BASE_REQUIREMENTS" + "$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1 || fail "Application requirements installation failed." + else + fail "Application requirements installation failed." + fi +fi +echo "Requirements installed from $SELECTED_REQUIREMENTS" >> "$LOG_FILE" + +if [[ "${OSTYPE:-}" == "darwin"* ]]; then + echo "Detected macOS platform" >> "$LOG_FILE" + if [ -f "$PROJECT_ROOT/requirements-macos.txt" ]; then + "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-macos.txt" >> "$LOG_FILE" 2>&1 + fi +elif [[ "${OSTYPE:-}" == "linux-gnu"* ]]; then + echo "Detected Linux platform" >> "$LOG_FILE" + if [ -f "$PROJECT_ROOT/requirements-linux.txt" ]; then + "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-linux.txt" >> "$LOG_FILE" 2>&1 + fi +else + echo "Unknown platform: ${OSTYPE:-unset}" >> "$LOG_FILE" +fi + +# External Python code is optional. Verify the generated package only when the +# constructor bundled setup.py and src/. +if [ -f "$PROJECT_ROOT/setup.py" ]; then + echo "Found setup.py, installing LabConstrictorPlayground package locally without build isolation" >> "$LOG_FILE" + "$PYTHON_EXE" -m pip install --no-deps --no-build-isolation "$PROJECT_ROOT" >> "$LOG_FILE" 2>&1 + # Tools first, only for an app whose package ships a _lc_tools module: the app package may import labconstrictor_tools + # at module level, so labconstrictor-tools must be installed BEFORE the import check below. A failed install is only a WARNING + # (it never fails the installation by itself). LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). + # Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), + # because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. + LC_TOOLS_PIN="v0.2.0-rc.1" + LC_TOOLS_MODULE="labconstrictor_playground_lc_tools" + LC_TOOLS_READY=0 + if "$PYTHON_EXE" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('$LC_TOOLS_MODULE') else 1)" >> "$LOG_FILE" 2>&1; then + echo "Found $LC_TOOLS_MODULE: installing labconstrictor-tools before the import check of LabConstrictorPlayground" >> "$LOG_FILE" || : + if "$PYTHON_EXE" -m pip install "${LC_TOOLS_SPEC:-https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/${LC_TOOLS_PIN}.zip}" >> "$LOG_FILE" 2>&1; then + LC_TOOLS_READY=1 + else + echo "WARNING: labconstrictor-tools could not be installed - see the pip output above in this file; the tools of LabConstrictorPlayground will not be registered (the import check below fails only if the package needs labconstrictor-tools at import time)." >> "$LOG_FILE" || : + fi + fi + "$PYTHON_EXE" -c "import labconstrictor_playground; print('LabConstrictorPlayground import successful:', labconstrictor_playground.__file__)" >> "$LOG_FILE" 2>&1 +else + echo "No setup.py detected; this project does not bundle an optional Python package." >> "$LOG_FILE" +fi + + +# --- Optional: expose the app's tools to Napari, Fiji and the command line (LabConstrictor tools bridge) ------------------- +# If the app's package ships a module named _lc_tools, install labconstrictor-tools and register that module. +# This step must never fail the installation. LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +# labconstrictor-tools is installed above, before the import check of the package (LC_TOOLS_PIN, LC_TOOLS_MODULE and LC_TOOLS_READY are set there). +if [ -f "$PROJECT_ROOT/setup.py" ] && "$PYTHON_EXE" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('$LC_TOOLS_MODULE') else 1)" >> "$LOG_FILE" 2>&1; then + lc_note() { echo "$*" >> "$LOG_FILE" || :; } # a full or unwritable log must not fail the installation + lc_note "Found $LC_TOOLS_MODULE: registering the tools of LabConstrictorPlayground for Napari and Fiji" + LC_APP_VERSION="$(sed -n 's/^version: *//p' "$PROJECT_ROOT/construct.yaml" 2>/dev/null | head -1 | tr -d "\"'\r" || true)" + if [ -z "$LC_APP_VERSION" ]; then + lc_note "WARNING: no version: line at the start of a line in construct.yaml - registering the tools with version 0." + LC_APP_VERSION=0 + fi + if [ "${LC_TOOLS_READY:-0}" = 1 ] \ + && "$PYTHON_EXE" -m labconstrictor_tools register --name "LabConstrictorPlayground" --prefix "$PREFIX" \ + --module "$LC_TOOLS_MODULE" --version "$LC_APP_VERSION" --display-name "LabConstrictorPlayground" >> "$LOG_FILE" 2>&1; then + lc_note "Tools registered (labconstrictor-tools list shows them)." + else + lc_note "WARNING: tool registration failed - see the pip and register output above in this file; LabConstrictorPlayground itself is installed." + fi +fi + +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PROJECT_ROOT/notebook_launcher.json" --keyword "BASE_PATH_KEYWORD" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PREFIX/pre_uninstall.sh" --keyword "BASE_PATH" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PREFIX/uninstall.sh" --keyword "BASE_PATH" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/hide_code_cells.py" "$PROJECT_ROOT" >> "$LOG_FILE" 2>&1 + +# Keep conda verification enabled while avoiding the system certificate store +# that failed. Conda accepts a CA bundle path for ssl_verify. +if ! "$PYTHON_EXE" -m conda config --system --set ssl_verify "$CA_BUNDLE" >> "$LOG_FILE" 2>&1; then + echo "WARNING: Could not configure conda with the selected CA bundle. The launcher still has its own verified fallback." >> "$LOG_FILE" +fi + +# Do not create the application shortcut unless launcher preflight succeeds. +"$PYTHON_EXE" "$PROJECT_ROOT/launch_jupyter.py" --self-test >> "$LOG_FILE" 2>&1 +echo "Launcher preflight completed successfully." >> "$LOG_FILE" + +"$PYTHON_EXE" -c "import os, sys; print('Python:', sys.executable); print('Prefix:', os.environ.get('PREFIX'))" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" -c "from menuinst.api import install; import os; print(install(os.path.join('$PREFIX', 'LabConstrictorPlayground', 'notebook_launcher.json')))" >> "$LOG_FILE" 2>&1 + +echo "Post-install completed successfully." >> "$LOG_FILE" + +if [ -t 0 ]; then + echo + read -rp "Press Enter to close the installer..." _ +fi diff --git a/tests/fixtures/template_0_1_17_apps/playground/release_constructor.yml b/tests/fixtures/template_0_1_17_apps/playground/release_constructor.yml new file mode 100644 index 0000000..ff5e100 --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/playground/release_constructor.yml @@ -0,0 +1,186 @@ +name: Build Release Installers + +on: + release: + types: + - published + workflow_dispatch: + inputs: + version: + description: "Version to package (e.g. 0.0.4). Defaults to release tag when empty." + required: false + +# This enables to run a single workflow at a time +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: write + +jobs: + bump-version: + name: Bump version on main branch + runs-on: ubuntu-latest + if: github.event_name == 'release' + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Determine release version + id: version + run: | + set -euxo pipefail + RAW="${{ github.event.release.tag_name }}" + RAW="${RAW#v}" + echo "release_version=$RAW" >> "$GITHUB_OUTPUT" + echo "RELEASE_VERSION=$RAW" >> "$GITHUB_ENV" + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Sync versioned files + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_version.py" \ + .tools/python/bump_version.py \ + python .tools/python/bump_version.py "${RELEASE_VERSION}" + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_constructor.py" \ + .tools/python/bump_constructor.py \ + python .tools/python/bump_constructor.py + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_Welcome_notebook.py" \ + .tools/python/bump_Welcome_notebook.py \ + python .tools/python/bump_Welcome_notebook.py "${{ github.repository_owner }}" "${{ github.event.repository.name }}" + + + - name: Commit updated versioned files + uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5 + with: + branch: main + commit_message: GitHub Action - Update version to ${{ github.event.release.tag_name }} + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: release-bump-version-logs + path: artifacts/error_log_document.log + if-no-files-found: ignore + archive: false + + build-installers: + name: Build installers (${{ matrix.os_name }}) + runs-on: ${{ matrix.runner }} + needs: bump-version + strategy: + fail-fast: false + matrix: + include: + - os_name: linux + runner: ubuntu-latest + - os_name: macos-arm64 + runner: macos-15 + - os_name: windows + runner: windows-latest + defaults: + run: + shell: bash -l {0} + env: + CONSTRUCTOR_ENV: constructor-env + OUTPUT_DIR: dist/${{ matrix.os_name }} + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Determine release version + id: version + run: | + set -euxo pipefail + if [ "${{ github.event_name }}" = "release" ]; then + RAW="${{ github.event.release.tag_name }}" + else + RAW="${{ inputs.version }}" + fi + + if [ -z "$RAW" ]; then + echo "Version is required when running manually." >&2 + exit 1 + fi + + RAW="${RAW#v}" + echo "release_version=$RAW" >> "$GITHUB_OUTPUT" + echo "RELEASE_VERSION=$RAW" >> "$GITHUB_ENV" + + - name: Set up Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Sync versioned files and populate Welcome notebook + run: | + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document.log" \ + "bump_version.py" \ + .tools/python/bump_version.py \ + python .tools/python/bump_version.py "${RELEASE_VERSION}" + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document.log" \ + "bump_constructor.py" \ + .tools/python/bump_constructor.py \ + python .tools/python/bump_constructor.py + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document.log" \ + "bump_Welcome_notebook.py" \ + .tools/python/bump_Welcome_notebook.py \ + python .tools/python/bump_Welcome_notebook.py "${{ github.repository_owner }}" "${{ github.event.repository.name }}" + + - name: Set up Miniconda + uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + auto-activate: true + miniforge-version: latest + use-mamba: true + + - name: Create constructor environment + run: | + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document_${{ matrix.os_name }}.log" \ + "create_constructor_environment.sh" \ + .tools/bash/create_constructor_environment.sh \ + bash .tools/bash/create_constructor_environment.sh + + - name: Build installer with constructor + run: | + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document_${{ matrix.os_name }}.log" \ + "build_installer_with_constructor.sh" \ + .tools/bash/build_installer_with_constructor.sh \ + bash .tools/bash/build_installer_with_constructor.sh + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: build-installers-${{ matrix.os_name }}-logs + path: artifacts/error_log_document_${{ matrix.os_name }}.log + if-no-files-found: ignore + archive: false + + - name: Upload installers to release + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + files: dist/${{ matrix.os_name }}/* + fail_on_unmatched_files: true diff --git a/tests/fixtures/template_0_1_17_apps/vlab4mic/construct.yaml b/tests/fixtures/template_0_1_17_apps/vlab4mic/construct.yaml new file mode 100644 index 0000000..20a2abf --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/vlab4mic/construct.yaml @@ -0,0 +1,37 @@ +name: VLab4Mic +version: 0.1.5 +environment_file: environment.yaml +installer_type: all +license_file: LICENSE +welcome_image: app/logo/logo.png +header_image: app/logo/logo.png +icon_image: app/logo/vlab4mic_icon.png +conclusion_text: Thank you! You have successfully installed VLab4Mic 0.1.5. +keep_pkgs: false +register_python: false +initialize_conda: false +extra_files: +- .tools/meta/src_change.yaml: VLab4Mic/src_change.yaml +- app/bash_bat_scripts/pre_uninstall.sh: pre_uninstall.sh +- app/bash_bat_scripts/uninstall.sh: uninstall.sh +- app/logo/vlab4mic_icon.icns: VLab4Mic/vlab4mic_icon.icns +- app/logo/vlab4mic_icon.ico: VLab4Mic/vlab4mic_icon.ico +- app/logo/vlab4mic_icon.png: VLab4Mic/vlab4mic_icon.png +- app/menuinst/Welcome.ipynb: VLab4Mic/notebooks/Welcome.ipynb +- app/menuinst/notebook_launcher.json: VLab4Mic/notebook_launcher.json +- app/python_scripts/hide_code_cells.py: VLab4Mic/hide_code_cells.py +- app/python_scripts/include_path.py: VLab4Mic/include_path.py +- app/python_scripts/launch_jupyter.py: VLab4Mic/launch_jupyter.py +- construct.yaml: VLab4Mic/construct.yaml +- notebooks/VLab4Mic_main/VLab4Mic_main.ipynb: VLab4Mic/notebooks/VLab4Mic_main/VLab4Mic_main.ipynb +- notebooks/VLab4Mic_parameter_sweeps/VLab4Mic_parameter_sweeps.ipynb: VLab4Mic/notebooks/VLab4Mic_parameter_sweeps/VLab4Mic_parameter_sweeps.ipynb +- notebooks/notebook_latest_versions.yaml: VLab4Mic/notebook_latest_versions.yaml +- requirements.txt: VLab4Mic/requirements.txt +- requirements_gpu.txt: VLab4Mic/requirements_gpu.txt +- setup.py: VLab4Mic/setup.py +- src/vlab4mic_lc_tools/__init__.py: VLab4Mic/src/vlab4mic_lc_tools/__init__.py +- src/vlab4mic_lc_tools/_labelling.py: VLab4Mic/src/vlab4mic_lc_tools/_labelling.py +post_install: app/bash_bat_scripts/post_install.bat # [win] +post_install: app/bash_bat_scripts/post_install.sh # [unix] +pre_uninstall: app/bash_bat_scripts/pre_uninstall.bat # [win] +pre_uninstall: app/bash_bat_scripts/pre_uninstall.sh # [unix] diff --git a/tests/fixtures/template_0_1_17_apps/vlab4mic/post_install.bat b/tests/fixtures/template_0_1_17_apps/vlab4mic/post_install.bat new file mode 100644 index 0000000..68b2973 --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/vlab4mic/post_install.bat @@ -0,0 +1,236 @@ +@ECHO OFF +SETLOCAL EnableExtensions DisableDelayedExpansion + +SET "LOG_FILE=%PREFIX%\menuinst_debug.log" +SET "PYTHON_EXE=%PREFIX%\python.exe" +SET "PROJECT_ROOT=%PREFIX%\VLab4Mic" +SET "BASE_REQUIREMENTS=%PROJECT_ROOT%\requirements.txt" +SET "GPU_REQUIREMENTS=%PROJECT_ROOT%\requirements_gpu.txt" +SET "SELECTED_REQUIREMENTS=%BASE_REQUIREMENTS%" +SET "NVIDIA_SMI=" +SET "CA_BUNDLE=" +SET "FAILURE_MESSAGE=" +SET "CERTIFI_PATH_FILE=%TEMP%\labconstrictor_certifi_%RANDOM%_%RANDOM%.txt" + +> "%LOG_FILE%" echo Running post_install + +IF NOT EXIST "%PYTHON_EXE%" ( + SET "FAILURE_MESSAGE=Bundled Python executable was not found at %PYTHON_EXE%." + GOTO :fail +) +IF NOT EXIST "%PROJECT_ROOT%\launch_jupyter.py" ( + SET "FAILURE_MESSAGE=TLS-resilient launcher was not found at %PROJECT_ROOT%\launch_jupyter.py." + GOTO :fail +) +IF NOT EXIST "%BASE_REQUIREMENTS%" ( + SET "FAILURE_MESSAGE=Base requirements file was not found at %BASE_REQUIREMENTS%." + GOTO :fail +) + +REM pip 24.2+ uses the operating-system trust store by default. Some Windows +REM certificate stores contain malformed entries that OpenSSL cannot parse. +REM Keep verification enabled while opting out of system trust for installer +REM pip operations and using an explicit verified CA bundle. +SET "PIP_USE_DEPRECATED=legacy-certs" + +"%PYTHON_EXE%" "%PROJECT_ROOT%\launch_jupyter.py" --print-ca-bundle > "%CERTIFI_PATH_FILE%" 2>> "%LOG_FILE%" +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Unable to locate a verified CA bundle." + GOTO :fail +) +SET /P "CA_BUNDLE="<"%CERTIFI_PATH_FILE%" +DEL /Q "%CERTIFI_PATH_FILE%" >NUL 2>&1 + +IF NOT DEFINED CA_BUNDLE ( + SET "FAILURE_MESSAGE=No verified CA bundle could be selected." + GOTO :fail +) +IF NOT EXIST "%CA_BUNDLE%" ( + SET "FAILURE_MESSAGE=Selected CA bundle does not exist: %CA_BUNDLE%" + GOTO :fail +) + +SET "PIP_CERT=%CA_BUNDLE%" +SET "REQUESTS_CA_BUNDLE=%CA_BUNDLE%" +SET "CURL_CA_BUNDLE=%CA_BUNDLE%" +echo Using verified CA bundle for installer network operations: "%CA_BUNDLE%" >> "%LOG_FILE%" + +IF EXIST "%GPU_REQUIREMENTS%" ( + CALL :detect_nvidia_smi + IF DEFINED NVIDIA_SMI ( + echo NVIDIA GPU utility detected, installing GPU requirements from "%GPU_REQUIREMENTS%" >> "%LOG_FILE%" + SET "SELECTED_REQUIREMENTS=%GPU_REQUIREMENTS%" + ) ELSE ( + echo NVIDIA GPU not detected, installing CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" + ) +) ELSE ( + echo GPU requirements file not found, installing CPU requirements from "%BASE_REQUIREMENTS%" >> "%LOG_FILE%" +) + +echo Installing requirements from "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" +"%PYTHON_EXE%" -m pip install -r "%SELECTED_REQUIREMENTS%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Application requirements installation failed." + GOTO :fail +) + +IF EXIST "%PROJECT_ROOT%\requirements-windows.txt" ( + echo Installing Windows-specific requirements. >> "%LOG_FILE%" + "%PYTHON_EXE%" -m pip install -r "%PROJECT_ROOT%\requirements-windows.txt" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Windows-specific requirements installation failed." + GOTO :fail + ) +) + +REM External Python code is optional in LabConstrictor projects. If setup.py is +REM bundled, install it without a temporary build environment and verify that +REM the generated import package is usable. Otherwise continue without it. +IF EXIST "%PROJECT_ROOT%\setup.py" ( + echo Found setup.py, installing VLab4Mic package locally without build isolation. >> "%LOG_FILE%" + "%PYTHON_EXE%" -m pip install --no-deps --no-build-isolation "%PROJECT_ROOT%" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=VLab4Mic package installation failed." + GOTO :fail + ) + + REM Tools first, only for an app whose package ships a tools module: the package may import labconstrictor_tools at module level, + REM so labconstrictor-tools is installed BEFORE the import check below. A failed install is only a WARNING, see :install_lc_tools at the end of this file. + CALL :install_lc_tools + + "%PYTHON_EXE%" -c "import vlab4mic; print('VLab4Mic import successful:', vlab4mic.__file__)" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Installed VLab4Mic package could not be imported as vlab4mic." + GOTO :fail + ) +) ELSE ( + echo No setup.py detected; this project does not bundle an optional Python package. >> "%LOG_FILE%" +) + +REM Optional: expose the app's tools to Napari, Fiji and the command line (LabConstrictor tools bridge). +REM If the app's package ships a module named _lc_tools, install labconstrictor-tools and register that module. +REM This step must never fail the installation. LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +REM labconstrictor-tools is installed by :install_lc_tools, called above before the import check of the package (it sets LC_TOOLS_PIN, LC_TOOLS_SPEC and LC_TOOLS_READY). +SET "LC_APP_VERSION=" +IF EXIST "%PROJECT_ROOT%\construct.yaml" FOR /F "usebackq tokens=1,* delims=: " %%A IN (`findstr /B /C:"version:" "%PROJECT_ROOT%\construct.yaml"`) DO SET "LC_APP_VERSION=%%~B" +IF NOT DEFINED LC_APP_VERSION ( + SET "LC_APP_VERSION=0" + echo WARNING: no version: line at the start of a line in construct.yaml - registering the tools with version 0. >> "%LOG_FILE%" +) +"%PYTHON_EXE%" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('vlab4mic_lc_tools') else 1)" >> "%LOG_FILE%" 2>&1 +IF NOT ERRORLEVEL 1 ( + echo Found vlab4mic_lc_tools: registering the tools of VLab4Mic for Napari and Fiji. >> "%LOG_FILE%" + REM labconstrictor-tools was installed before the import check (:install_lc_tools): register only if that worked (errorlevel 1 otherwise). + "%PYTHON_EXE%" -c "import sys; sys.exit(0 if sys.argv[1] == '1' else 1)" "%LC_TOOLS_READY%" >> "%LOG_FILE%" 2>&1 + IF NOT ERRORLEVEL 1 "%PYTHON_EXE%" -m labconstrictor_tools register --name "VLab4Mic" --prefix "%PREFIX%" --module vlab4mic_lc_tools --version "%LC_APP_VERSION%" --display-name "VLab4Mic" >> "%LOG_FILE%" 2>&1 + IF ERRORLEVEL 1 echo WARNING: tool registration failed - see the pip and register output above in this file; VLab4Mic itself is installed. >> "%LOG_FILE%" +) + +"%PYTHON_EXE%" "%PROJECT_ROOT%\include_path.py" --path "%PREFIX%" --files "%PROJECT_ROOT%\notebook_launcher.json" --keyword "BASE_PATH_KEYWORD" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Launcher path preparation failed." + GOTO :fail +) + +"%PYTHON_EXE%" "%PROJECT_ROOT%\hide_code_cells.py" "%PROJECT_ROOT%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Notebook metadata preparation failed." + GOTO :fail +) + +REM Keep conda verification enabled, but avoid the Windows trust store that +REM failed during installation. Conda accepts a CA bundle path for ssl_verify. +echo Configuring conda to use verified CA bundle: "%CA_BUNDLE%" >> "%LOG_FILE%" +"%PYTHON_EXE%" -m conda config --system --set ssl_verify "%CA_BUNDLE%" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + echo WARNING: Could not configure conda with the selected CA bundle. The application launcher still has its own verified fallback. >> "%LOG_FILE%" +) + +REM Validate TLS and Jupyter imports before creating shortcuts or registering +REM the application. +"%PYTHON_EXE%" "%PROJECT_ROOT%\launch_jupyter.py" --self-test >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Launcher preflight failed. See menuinst_debug.log and launcher_debug.log." + GOTO :fail +) +echo Launcher preflight completed successfully. >> "%LOG_FILE%" + +"%PYTHON_EXE%" -c "import os, sys; print('Python:', sys.executable); print('Prefix:', os.environ.get('PREFIX'))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Python installation diagnostics failed." + GOTO :fail +) + +"%PYTHON_EXE%" -c "from menuinst.api import install; import os; print(install(os.path.join(r'%PREFIX%', 'VLab4Mic', 'notebook_launcher.json')))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + SET "FAILURE_MESSAGE=Application shortcut creation failed." + GOTO :fail +) + +SET "ARP_KEY=HKCU\Software\Microsoft\Windows\CurrentVersion\Uninstall\VLab4Mic" +SET "UNINSTALL_EXE=%PREFIX%\Uninstall-VLab4Mic.exe" +SET "DISPLAY_ICON=%PROJECT_ROOT%\logo.ico" +SET "DISPLAY_VERSION=0.1.3" +SET "PUBLISHER=CellMigrationLab" +echo Registering VLab4Mic in Windows Apps list >> "%LOG_FILE%" +reg add "%ARP_KEY%" /v DisplayName /d "VLab4Mic" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v DisplayVersion /d "%DISPLAY_VERSION%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v Publisher /d "%PUBLISHER%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v InstallLocation /d "%PREFIX%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v DisplayIcon /d "%DISPLAY_ICON%" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v UninstallString /d "\"%UNINSTALL_EXE%\"" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v QuietUninstallString /d "\"%UNINSTALL_EXE%\" /S" /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v NoModify /t REG_DWORD /d 1 /f >> "%LOG_FILE%" 2>&1 +reg add "%ARP_KEY%" /v NoRepair /t REG_DWORD /d 1 /f >> "%LOG_FILE%" 2>&1 + +echo Post-install completed successfully. >> "%LOG_FILE%" +echo Post-install completed successfully. +ENDLOCAL +EXIT /B 0 + +:fail +IF EXIST "%CERTIFI_PATH_FILE%" DEL /Q "%CERTIFI_PATH_FILE%" >NUL 2>&1 +IF NOT DEFINED FAILURE_MESSAGE SET "FAILURE_MESSAGE=Post-install failed for an unknown reason." +echo ERROR: %FAILURE_MESSAGE% >> "%LOG_FILE%" +echo ERROR: %FAILURE_MESSAGE% +echo See "%LOG_FILE%" for details. +ENDLOCAL +EXIT /B 1 + +:detect_nvidia_smi +FOR %%P IN ( + "%ProgramFiles%\NVIDIA Corporation\NVSMI\nvidia-smi.exe" + "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvidia-smi.exe" + "%SystemRoot%\System32\nvidia-smi.exe" + "%SystemRoot%\Sysnative\nvidia-smi.exe" +) DO ( + IF NOT DEFINED NVIDIA_SMI IF EXIST "%%~P" SET "NVIDIA_SMI=%%~P" +) +IF DEFINED NVIDIA_SMI GOTO :EOF + +FOR /F "delims=" %%I IN ('where.exe nvidia-smi.exe 2^>NUL') DO ( + IF NOT DEFINED NVIDIA_SMI SET "NVIDIA_SMI=%%~fI" +) +GOTO :EOF + +:install_lc_tools +REM Optional (LabConstrictor tools bridge): install labconstrictor-tools when the app's package ships a module named _lc_tools. +REM Called before the import check of the package, which may import labconstrictor_tools at module level. +REM This step must never fail the installation: a failure is logged as a WARNING and LC_TOOLS_READY stays 0 (no registration later). +REM LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +REM Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), +REM because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. +SET "LC_TOOLS_PIN=v0.2.0-rc.1" +IF NOT DEFINED LC_TOOLS_SPEC SET "LC_TOOLS_SPEC=https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/%LC_TOOLS_PIN%.zip" +SET "LC_TOOLS_READY=0" +"%PYTHON_EXE%" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('vlab4mic_lc_tools') else 1)" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 GOTO :EOF +echo Found vlab4mic_lc_tools: installing labconstrictor-tools before the import check of VLab4Mic. >> "%LOG_FILE%" +REM LC_TOOLS_SPEC goes to pip as one argument through Python: cmd.exe never re-parses it (a quote or & in it cannot run anything). +"%PYTHON_EXE%" -c "import os, subprocess, sys; sys.exit(subprocess.call([sys.executable, '-m', 'pip', 'install', os.environ['LC_TOOLS_SPEC']]))" >> "%LOG_FILE%" 2>&1 +IF ERRORLEVEL 1 ( + echo WARNING: labconstrictor-tools could not be installed - see the pip output above in this file; the tools of VLab4Mic will not be registered ^(the import check fails only if the package needs labconstrictor-tools at import time^). >> "%LOG_FILE%" + GOTO :EOF +) +SET "LC_TOOLS_READY=1" +GOTO :EOF diff --git a/tests/fixtures/template_0_1_17_apps/vlab4mic/post_install.sh b/tests/fixtures/template_0_1_17_apps/vlab4mic/post_install.sh new file mode 100644 index 0000000..b13c562 --- /dev/null +++ b/tests/fixtures/template_0_1_17_apps/vlab4mic/post_install.sh @@ -0,0 +1,134 @@ +#!/bin/bash +set -euo pipefail + +LOG_FILE="$PREFIX/menuinst_debug.log" +PYTHON_EXE="$PREFIX/bin/python" +PROJECT_ROOT="$PREFIX/VLab4Mic" +BASE_REQUIREMENTS="$PROJECT_ROOT/requirements.txt" +GPU_REQUIREMENTS="$PROJECT_ROOT/requirements_gpu.txt" +SELECTED_REQUIREMENTS="$BASE_REQUIREMENTS" + +echo "Running post_install" > "$LOG_FILE" + +fail() { + echo "ERROR: $1" | tee -a "$LOG_FILE" >&2 + exit 1 +} + +[ -x "$PYTHON_EXE" ] || fail "Bundled Python executable was not found at $PYTHON_EXE." +[ -f "$PROJECT_ROOT/launch_jupyter.py" ] || fail "TLS-resilient launcher was not found at $PROJECT_ROOT/launch_jupyter.py." +[ -f "$BASE_REQUIREMENTS" ] || fail "Base requirements file was not found at $BASE_REQUIREMENTS." + +# pip 24.2+ uses system certificates by default. Opt out for installer pip +# operations and use an explicit verified CA bundle instead. +export PIP_USE_DEPRECATED=legacy-certs +CA_BUNDLE="$("$PYTHON_EXE" "$PROJECT_ROOT/launch_jupyter.py" --print-ca-bundle 2>> "$LOG_FILE")" +[ -s "$CA_BUNDLE" ] || fail "No valid verified CA bundle could be selected: $CA_BUNDLE" + +export PIP_CERT="$CA_BUNDLE" +export REQUESTS_CA_BUNDLE="$CA_BUNDLE" +export CURL_CA_BUNDLE="$CA_BUNDLE" +echo "Using verified CA bundle for installer network operations: $CA_BUNDLE" >> "$LOG_FILE" + +if [ -f "$GPU_REQUIREMENTS" ]; then + if [[ "${OSTYPE:-}" == "darwin"* ]]; then + echo "macOS detected, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" + elif command -v nvidia-smi >/dev/null 2>&1; then + echo "NVIDIA GPU detected, installing GPU requirements from $GPU_REQUIREMENTS" >> "$LOG_FILE" + SELECTED_REQUIREMENTS="$GPU_REQUIREMENTS" + else + echo "NVIDIA GPU not detected, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" + fi +else + echo "GPU requirements file not found, installing CPU requirements from $BASE_REQUIREMENTS" >> "$LOG_FILE" +fi + +echo "Installing requirements from $SELECTED_REQUIREMENTS" >> "$LOG_FILE" +"$PYTHON_EXE" -m pip install -r "$SELECTED_REQUIREMENTS" >> "$LOG_FILE" 2>&1 + +if [[ "${OSTYPE:-}" == "darwin"* ]]; then + echo "Detected macOS platform" >> "$LOG_FILE" + if [ -f "$PROJECT_ROOT/requirements-macos.txt" ]; then + "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-macos.txt" >> "$LOG_FILE" 2>&1 + fi +elif [[ "${OSTYPE:-}" == "linux-gnu"* ]]; then + echo "Detected Linux platform" >> "$LOG_FILE" + if [ -f "$PROJECT_ROOT/requirements-linux.txt" ]; then + "$PYTHON_EXE" -m pip install -r "$PROJECT_ROOT/requirements-linux.txt" >> "$LOG_FILE" 2>&1 + fi +else + echo "Unknown platform: ${OSTYPE:-unset}" >> "$LOG_FILE" +fi + +# External Python code is optional. Verify the generated package only when the +# constructor bundled setup.py and src/. +if [ -f "$PROJECT_ROOT/setup.py" ]; then + echo "Found setup.py, installing VLab4Mic package locally without build isolation" >> "$LOG_FILE" + "$PYTHON_EXE" -m pip install --no-deps --no-build-isolation "$PROJECT_ROOT" >> "$LOG_FILE" 2>&1 + # Tools first, only for an app whose package ships a _lc_tools module: the app package may import labconstrictor_tools + # at module level, so labconstrictor-tools must be installed BEFORE the import check below. A failed install is only a WARNING + # (it never fails the installation by itself). LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). + # Default source: the GitHub archive of the labconstrictor-tools release TAG below (a plain zip: no git needed on the user's computer), + # because the package is not on PyPI yet. Once it is, use "labconstrictor-tools" here. To move the pin, change only this tag. + LC_TOOLS_PIN="v0.2.0-rc.1" + LC_TOOLS_MODULE="vlab4mic_lc_tools" + LC_TOOLS_READY=0 + if "$PYTHON_EXE" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('$LC_TOOLS_MODULE') else 1)" >> "$LOG_FILE" 2>&1; then + echo "Found $LC_TOOLS_MODULE: installing labconstrictor-tools before the import check of VLab4Mic" >> "$LOG_FILE" || : + if "$PYTHON_EXE" -m pip install "${LC_TOOLS_SPEC:-https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/${LC_TOOLS_PIN}.zip}" >> "$LOG_FILE" 2>&1; then + LC_TOOLS_READY=1 + else + echo "WARNING: labconstrictor-tools could not be installed - see the pip output above in this file; the tools of VLab4Mic will not be registered (the import check below fails only if the package needs labconstrictor-tools at import time)." >> "$LOG_FILE" || : + fi + fi + "$PYTHON_EXE" -c "import vlab4mic; print('VLab4Mic import successful:', vlab4mic.__file__)" >> "$LOG_FILE" 2>&1 +else + echo "No setup.py detected; this project does not bundle an optional Python package." >> "$LOG_FILE" +fi + + +# --- Optional: expose the app's tools to Napari, Fiji and the command line (LabConstrictor tools bridge) ------------------- +# If the app's package ships a module named _lc_tools, install labconstrictor-tools and register that module. +# This step must never fail the installation. LC_TOOLS_SPEC can point to another source (wheel, git URL, mirror). +# labconstrictor-tools is installed above, before the import check of the package (LC_TOOLS_PIN, LC_TOOLS_MODULE and LC_TOOLS_READY are set there). +if [ -f "$PROJECT_ROOT/setup.py" ] && "$PYTHON_EXE" -c "import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('$LC_TOOLS_MODULE') else 1)" >> "$LOG_FILE" 2>&1; then + lc_note() { echo "$*" >> "$LOG_FILE" || :; } # a full or unwritable log must not fail the installation + lc_note "Found $LC_TOOLS_MODULE: registering the tools of VLab4Mic for Napari and Fiji" + LC_APP_VERSION="$(sed -n 's/^version: *//p' "$PROJECT_ROOT/construct.yaml" 2>/dev/null | head -1 | tr -d "\"'\r" || true)" + if [ -z "$LC_APP_VERSION" ]; then + lc_note "WARNING: no version: line at the start of a line in construct.yaml - registering the tools with version 0." + LC_APP_VERSION=0 + fi + if [ "${LC_TOOLS_READY:-0}" = 1 ] \ + && "$PYTHON_EXE" -m labconstrictor_tools register --name "VLab4Mic" --prefix "$PREFIX" \ + --module "$LC_TOOLS_MODULE" --version "$LC_APP_VERSION" --display-name "VLab4Mic" >> "$LOG_FILE" 2>&1; then + lc_note "Tools registered (labconstrictor-tools list shows them)." + else + lc_note "WARNING: tool registration failed - see the pip and register output above in this file; VLab4Mic itself is installed." + fi +fi + +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PROJECT_ROOT/notebook_launcher.json" --keyword "BASE_PATH_KEYWORD" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PREFIX/pre_uninstall.sh" --keyword "BASE_PATH" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/include_path.py" --path "$PREFIX" --files "$PREFIX/uninstall.sh" --keyword "BASE_PATH" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" "$PROJECT_ROOT/hide_code_cells.py" "$PROJECT_ROOT" >> "$LOG_FILE" 2>&1 + +# Keep conda verification enabled while avoiding the system certificate store +# that failed. Conda accepts a CA bundle path for ssl_verify. +if ! "$PYTHON_EXE" -m conda config --system --set ssl_verify "$CA_BUNDLE" >> "$LOG_FILE" 2>&1; then + echo "WARNING: Could not configure conda with the selected CA bundle. The launcher still has its own verified fallback." >> "$LOG_FILE" +fi + +# Do not create the application shortcut unless launcher preflight succeeds. +"$PYTHON_EXE" "$PROJECT_ROOT/launch_jupyter.py" --self-test >> "$LOG_FILE" 2>&1 +echo "Launcher preflight completed successfully." >> "$LOG_FILE" + +"$PYTHON_EXE" -c "import os, sys; print('Python:', sys.executable); print('Prefix:', os.environ.get('PREFIX'))" >> "$LOG_FILE" 2>&1 +"$PYTHON_EXE" -c "from menuinst.api import install; import os; print(install(os.path.join('$PREFIX', 'VLab4Mic', 'notebook_launcher.json')))" >> "$LOG_FILE" 2>&1 + +echo "Post-install completed successfully." >> "$LOG_FILE" + +if [ -t 0 ]; then + echo + read -rp "Press Enter to close the installer..." _ +fi diff --git a/tests/fixtures/template_0_1_17_docs/README.md b/tests/fixtures/template_0_1_17_docs/README.md new file mode 100644 index 0000000..0153a70 --- /dev/null +++ b/tests/fixtures/template_0_1_17_docs/README.md @@ -0,0 +1 @@ +The install.md and tools_bridge.md of template 0.1.17 (commit 2dbc5ef), used by tests/test_template_sync_0_1_18.py as the "previous version" a migration may update. diff --git a/tests/fixtures/template_0_1_17_docs/install.md b/tests/fixtures/template_0_1_17_docs/install.md new file mode 100644 index 0000000..247e071 --- /dev/null +++ b/tests/fixtures/template_0_1_17_docs/install.md @@ -0,0 +1,32 @@ +# Install an app + +Installing takes about 6-8 minutes and you only do it once. You need an internet connection while installing: the installer downloads the app's Python packages. Download the installer for your system from the app's page in the [App Centre](https://labconstrictor.cellmig.org/apps/) (or from the app's GitHub release) and follow the steps for your system. + +> **Check where it comes from.** The installers are not code-signed yet, so your system will warn you. Download only from the app's own GitHub release (the address starts with `https://github.com/` followed by the app's owner and repository, as shown on the app's page), and run it only if you trust that owner. A listing in the App Centre checks that the files come from the app's repository; it is not a security audit. + +> A command prompt window opens during installation on Windows; this is normal, do not close it. + +## Install on Windows + +1. Download the `.exe` file and double-click it. +2. If Windows shows "Windows protected your PC", click "More info" and then "Run anyway". The installers are not code-signed yet. + + ![Windows protected your PC: click More info, then Run anyway](https://github.com/CellMigrationLab/LabConstrictor/blob/doc_source/Windows_Protected_your_PC.png) + +3. Follow the on-screen prompts. Launch the app from the Start Menu or your desktop. + +## Install on a Mac + +1. Choose your file: open the Apple menu, then About This Mac. If the Chip or Processor says Intel, download the Intel installer; if it says M1, M2 or similar, download the Apple Silicon one. +2. Double-click the `.pkg` file. If macOS refuses to open it, go to System Settings, Privacy & Security, scroll down to the message about the package and click "Open Anyway" (it appears only after you tried to open the file once). If you do not see it, try opening the file again, then check this page again. +3. Follow the prompts (we recommend "Install only for me"). Launch the app from your Applications folder. + +## Install on Linux + +1. Download the `.sh` file. +2. Open a terminal in the download folder and run `bash` followed by the name of the file you downloaded, for example: `bash MyApp-0.1.0-Linux-x86_64.sh` +3. Follow the prompts, then launch the app from your applications menu. + +## Next + +Open the app and start working: [Use notebooks after installation](notebook_usage.md). If something goes wrong, see [Troubleshooting an installed app](troubleshooting_installed_app.md). diff --git a/tests/fixtures/template_0_1_17_docs/tools_bridge.md b/tests/fixtures/template_0_1_17_docs/tools_bridge.md new file mode 100644 index 0000000..236e827 --- /dev/null +++ b/tests/fixtures/template_0_1_17_docs/tools_bridge.md @@ -0,0 +1,58 @@ +# Expose your app's tools to Napari and Fiji (optional, experimental) + +If your repository ships Python code (see [Upload external code](external_code_upload.md)), you can declare some of its functions +as **tools**. After installation they appear in a generic **Napari** widget and a generic **Fiji** command, and work from the +command line, with forms generated automatically from the function signatures. Nothing changes for apps that do not declare tools. + +1. Add a module named `_lc_tools` next to your package under `src/` (so `src/mytool_lc_tools/__init__.py` for + the package `mytool`). Declare tools in it with typed functions, importing your science code **inside** the functions: + + ```python + from typing import Annotated + from labconstrictor_tools import Image, ImageOut, Min, tool + + @tool("Gaussian blur") + def blur(image: Image, sigma: Annotated[float, Min(0)] = 2.0) -> ImageOut: + from mytool.filters import gaussian + return gaussian(image, sigma) + ``` + + Keep the module light: it is imported to read the tool list, so heavy imports at the top slow every start. + **Import `labconstrictor_tools` only in the tools module if you can.** Since template 0.1.17 the installer installs `labconstrictor-tools` *before* it checks that + your package can be imported, so a package that imports `labconstrictor_tools` at module level no longer stops the installation. It is still better practice to import it lazily (inside functions) + or only in `_lc_tools`: the rest of your package then also works where Tools is not installed (a notebook run on Colab, a copy of the package used elsewhere, a failed Tools install). +2. The installer (`post_install`) detects the module, installs `labconstrictor-tools` and registers the app + (`labconstrictor-tools register ...`). `pre_uninstall` unregisters it. A failure in this step is logged to + `menuinst_debug.log` and never fails the installation. + **Order in `post_install`.** (1) your package is installed, (2) if it ships `_lc_tools`, `labconstrictor-tools` is installed, (3) the package import check runs, (4) the app is registered. + Installing Tools before the import check is what lets your package import `labconstrictor_tools` at module level. A failed Tools install is a `WARNING` in `menuinst_debug.log` + and the registration is skipped; the installation continues, unless your package needs Tools to be imported (then the import check fails, as it should). + **Where `labconstrictor-tools` comes from.** It is not on PyPI yet, so by default the installer downloads the GitHub archive of a **pinned release tag**, + `https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/v0.2.0-rc.1.zip` (a plain zip: git is not needed on the user's computer). + Set the environment variable `LC_TOOLS_SPEC` when the installer runs to install it from somewhere else (a wheel, a git URL, a mirror, or the + name `labconstrictor-tools` once it is published on PyPI); it overrides the pin completely. + **Pinned (important).** Two installers built from the same app release get the same version of the tools package, and a change to Tools `main` + does not reach new installations until the pin is moved on purpose. + **How to move the pin.** The tag name is written in one place per script: the variable `LC_TOOLS_PIN` (`LC_TOOLS_PIN="v0.2.0-rc.1"` in + `app/bash_bat_scripts/post_install.sh`, `SET "LC_TOOLS_PIN=v0.2.0-rc.1"` in `post_install.bat`). To use a new release: create and test the tag in + LabConstrictor-Tools (procedure in its `docs/OPERATIONS.md`), change that string in both scripts and in the copies under + `.template_sync/resources/` of the latest migration (the test `test_resources_are_identical_to_the_template_files` fails when they differ), + update the tag in this paragraph, and bump `template_version` so apps receive the change. Apps already migrated need a new migration to change the + string in their own scripts. +3. Test your tools before releasing: `labconstrictor-tools check --module mytool_lc_tools` and + `labconstrictor-tools test --module mytool_lc_tools --cases lc_tests/cases.json`. +4. Install the Napari plugin (`napari-labconstrictor`) or the Fiji plugin (`labconstrictor-fiji`) once per machine. Neither is on PyPI or a Fiji update site yet: see the READMEs of + [napari-labconstrictor](https://github.com/CellMigrationLab/napari-labconstrictor) and [LabConstrictor-Fiji](https://github.com/CellMigrationLab/LabConstrictor-Fiji) for the install commands. +5. Optional: make a long form readable and rich with markers (the authoring guide in the Tools repository lists them all): + - layout: `Group("...")`, `Advanced()`, `Collapsed()`, `EnabledWhen(...)`; + - input widgets: `Widget("slider")` for a bounded number and `Widget("radio")` for a short list of choices; + - `PickChannel()` to let the person choose one channel of a multi-channel image (your function receives that channel as a 2D array); + - `ChoicesFrom("other_tool")` to fill a dropdown from another tool of yours (for example the available devices); + - `RegionOf("image")` (optional labels input) to run on the selection in the host (Napari Shapes layer, Fiji ROI, QuPath annotations); + - outputs: `ImageOut`, `LabelsOut`, `TableOut`, `Scalars`, `MessageOut`, `PointsOut` and `ShapesOut` (polygons: Napari shapes layer, Fiji ROIs, QuPath annotations); + - say "nothing found" with `ToolError("no_match", ...)`. + Napari, Fiji and QuPath do not support every feature in the same way: the table "What each host does with each feature" in `docs/HOST_FEATURES.md` of the Tools repository is the reference. +Documentation and source: https://github.com/CellMigrationLab/LabConstrictor-Tools. +The registered version is read from the `version:` line of `construct.yaml`. The Windows hook (`post_install.bat`) has not yet been run on a real Windows computer; if you can help, follow the +[human test protocol](https://github.com/CellMigrationLab/LabConstrictor-Tools/blob/main/docs/HUMAN_TEST_PROTOCOL.md). +Troubleshooting: `labconstrictor-tools doctor`, `labconstrictor-tools logs`, and `~/.labconstrictor/logs/labconstrictor.log`. diff --git a/tests/fixtures/template_0_1_17_workflows/check_app_manifest.yml b/tests/fixtures/template_0_1_17_workflows/check_app_manifest.yml new file mode 100644 index 0000000..6f5e4b6 --- /dev/null +++ b/tests/fixtures/template_0_1_17_workflows/check_app_manifest.yml @@ -0,0 +1,46 @@ +name: Check App Centre manifest + +# Checks labconstrictor-app.yaml (the description of the app for the LabConstrictor App Centre). +# A missing file is a warning, an invalid file fails this check. It never blocks building installers: +# it runs on its own, also when a release is published. +on: + pull_request: + paths: + - labconstrictor-app.yaml + - .tools/schema/** + - .tools/python/check_app_manifest.py + - .github/workflows/check_app_manifest.yml + push: + branches: + - main + paths: + - labconstrictor-app.yaml + - .tools/schema/** + - .tools/python/check_app_manifest.py + - .github/workflows/check_app_manifest.yml + release: + types: + - published + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + name: Check labconstrictor-app.yaml + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install pyyaml==6.0.2 jsonschema==4.23.0 + + - name: Check the manifest + run: python .tools/python/check_app_manifest.py diff --git a/tests/fixtures/template_0_1_17_workflows/release_constructor.yml b/tests/fixtures/template_0_1_17_workflows/release_constructor.yml new file mode 100644 index 0000000..6410111 --- /dev/null +++ b/tests/fixtures/template_0_1_17_workflows/release_constructor.yml @@ -0,0 +1,188 @@ +name: Build Release Installers + +on: + release: + types: + - published + workflow_dispatch: + inputs: + version: + description: "Version to package (e.g. 0.0.4). Defaults to release tag when empty." + required: false + +# This enables to run a single workflow at a time +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: write + +jobs: + bump-version: + name: Bump version on main branch + runs-on: ubuntu-latest + if: github.event_name == 'release' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Determine release version + id: version + run: | + set -euxo pipefail + RAW="${{ github.event.release.tag_name }}" + RAW="${RAW#v}" + echo "release_version=$RAW" >> "$GITHUB_OUTPUT" + echo "RELEASE_VERSION=$RAW" >> "$GITHUB_ENV" + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Sync versioned files + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_version.py" \ + .tools/python/bump_version.py \ + python .tools/python/bump_version.py "${RELEASE_VERSION}" + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_constructor.py" \ + .tools/python/bump_constructor.py \ + python .tools/python/bump_constructor.py + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_Welcome_notebook.py" \ + .tools/python/bump_Welcome_notebook.py \ + python .tools/python/bump_Welcome_notebook.py "${{ github.repository_owner }}" "${{ github.event.repository.name }}" + + + - name: Commit updated versioned files + uses: stefanzweifel/git-auto-commit-action@v5 + with: + branch: main + commit_message: GitHub Action - Update version to ${{ github.event.release.tag_name }} + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@v7 + with: + name: release-bump-version-logs + path: artifacts/error_log_document.log + if-no-files-found: ignore + archive: false + + build-installers: + name: Build installers (${{ matrix.os_name }}) + runs-on: ${{ matrix.runner }} + needs: bump-version + strategy: + fail-fast: false + matrix: + include: + - os_name: linux + runner: ubuntu-latest + - os_name: macos-arm64 + runner: macos-15 + - os_name: macos-intel + runner: macos-15-intel + - os_name: windows + runner: windows-latest + defaults: + run: + shell: bash -l {0} + env: + CONSTRUCTOR_ENV: constructor-env + OUTPUT_DIR: dist/${{ matrix.os_name }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Determine release version + id: version + run: | + set -euxo pipefail + if [ "${{ github.event_name }}" = "release" ]; then + RAW="${{ github.event.release.tag_name }}" + else + RAW="${{ inputs.version }}" + fi + + if [ -z "$RAW" ]; then + echo "Version is required when running manually." >&2 + exit 1 + fi + + RAW="${RAW#v}" + echo "release_version=$RAW" >> "$GITHUB_OUTPUT" + echo "RELEASE_VERSION=$RAW" >> "$GITHUB_ENV" + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Sync versioned files and populate Welcome notebook + run: | + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document.log" \ + "bump_version.py" \ + .tools/python/bump_version.py \ + python .tools/python/bump_version.py "${RELEASE_VERSION}" + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document.log" \ + "bump_constructor.py" \ + .tools/python/bump_constructor.py \ + python .tools/python/bump_constructor.py + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document.log" \ + "bump_Welcome_notebook.py" \ + .tools/python/bump_Welcome_notebook.py \ + python .tools/python/bump_Welcome_notebook.py "${{ github.repository_owner }}" "${{ github.event.repository.name }}" + + - name: Set up Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-activate: true + miniforge-version: latest + use-mamba: true + + - name: Create constructor environment + run: | + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document_${{ matrix.os_name }}.log" \ + "create_constructor_environment.sh" \ + .tools/bash/create_constructor_environment.sh \ + bash .tools/bash/create_constructor_environment.sh + + - name: Build installer with constructor + run: | + bash .tools/bash/run_with_context.sh \ + "artifacts/error_log_document_${{ matrix.os_name }}.log" \ + "build_installer_with_constructor.sh" \ + .tools/bash/build_installer_with_constructor.sh \ + bash .tools/bash/build_installer_with_constructor.sh + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@v7 + with: + name: build-installers-${{ matrix.os_name }}-logs + path: artifacts/error_log_document_${{ matrix.os_name }}.log + if-no-files-found: ignore + archive: false + + - name: Upload installers to release + uses: softprops/action-gh-release@v2 + with: + files: dist/${{ matrix.os_name }}/* + fail_on_unmatched_files: true diff --git a/tests/fixtures/template_0_1_17_workflows/sync_template.yml b/tests/fixtures/template_0_1_17_workflows/sync_template.yml new file mode 100644 index 0000000..22dc7e9 --- /dev/null +++ b/tests/fixtures/template_0_1_17_workflows/sync_template.yml @@ -0,0 +1,170 @@ +name: Sync with Template Repository + +on: + schedule: + # Run weekly on Monday at 9 AM UTC + - cron: '0 9 * * 1' + workflow_dispatch: + +# This enables only one template synchronization workflow at a time. +concurrency: + group: template-sync-main + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + +jobs: + sync-template: + name: Sync with template repository + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + # Do not leave the default GITHUB_TOKEN in the local Git configuration. + # The pull-request step authenticates separately with the repository-scoped + # LABCONSTRICTOR_SYNC_TOKEN secret. + persist-credentials: false + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Add upstream remote + run: | + git remote add upstream https://github.com/CellMigrationLab/LabConstrictor.git + + - name: Fetch upstream changes + run: | + git fetch upstream main + + - name: Read local template version + id: local-version + run: | + local_version=$(python3 - <<'PY' + from pathlib import Path + + manifest = Path(".template_sync/manifest.yaml") + if not manifest.exists(): + print("0.0.0") + raise SystemExit + + for line in manifest.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if stripped.startswith("template_version:"): + print(stripped.split(":", 1)[1].strip()) + break + else: + raise SystemExit("template_version not found in .template_sync/manifest.yaml") + PY + ) + echo "version=$local_version" >> "$GITHUB_OUTPUT" + + - name: Read upstream template version + id: upstream-version + run: | + upstream_version=$(python3 - <<'PY' + import subprocess + + manifest = subprocess.run( + ["git", "show", "upstream/main:.template_sync/manifest.yaml"], + check=True, + capture_output=True, + text=True, + ).stdout + for line in manifest.splitlines(): + stripped = line.strip() + if stripped.startswith("template_version:"): + print(stripped.split(":", 1)[1].strip()) + break + else: + raise SystemExit( + "template_version not found in upstream .template_sync/manifest.yaml" + ) + PY + ) + echo "version=$upstream_version" >> "$GITHUB_OUTPUT" + + - name: Check whether template sync is needed + id: version-check + env: + LOCAL_VERSION: ${{ steps.local-version.outputs.version }} + UPSTREAM_VERSION: ${{ steps.upstream-version.outputs.version }} + run: | + python3 - <<'PY' + import os + from pathlib import Path + + def parse_version(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in version.split(".")) + + local_version = os.environ["LOCAL_VERSION"] + upstream_version = os.environ["UPSTREAM_VERSION"] + sync_required = parse_version(upstream_version) > parse_version(local_version) + + output_path = Path(os.environ["GITHUB_OUTPUT"]) + with output_path.open("a", encoding="utf-8") as fh: + fh.write(f"sync_required={'true' if sync_required else 'false'}\n") + PY + + - name: Check synchronization token setup + if: steps.version-check.outputs.sync_required == 'true' + env: + LABCONSTRICTOR_SYNC_TOKEN: ${{ secrets.LABCONSTRICTOR_SYNC_TOKEN }} + run: | + if [[ -z "$LABCONSTRICTOR_SYNC_TOKEN" ]]; then + { + echo "## One-time LabConstrictor setup required" + echo + echo "A template update is available, but this repository does not have the" + echo '`LABCONSTRICTOR_SYNC_TOKEN` repository secret configured.' + echo + echo "Follow the beginner-friendly guide in:" + echo + echo '`.tools/docs/template_synchronization.md`' + } >> "$GITHUB_STEP_SUMMARY" + + echo "::error title=Synchronization setup required::Configure the LABCONSTRICTOR_SYNC_TOKEN repository secret by following .tools/docs/template_synchronization.md." + exit 1 + fi + + - name: Fetch template sync bootstrap + if: steps.version-check.outputs.sync_required == 'true' + run: | + git checkout upstream/main -- .template_sync + + - name: Run template migrations + if: steps.version-check.outputs.sync_required == 'true' + run: | + python3 .template_sync/sync.py \ + --from-version "${{ steps.local-version.outputs.version }}" \ + --to-version "${{ steps.upstream-version.outputs.version }}" \ + --repository-root "$GITHUB_WORKSPACE" + + - name: Check for changes + id: check-changes + run: | + if [[ -n "$(git status --short)" ]]; then + echo "has_changes=true" >> "$GITHUB_OUTPUT" + else + echo "has_changes=false" >> "$GITHUB_OUTPUT" + fi + + - name: Create pull request + if: steps.check-changes.outputs.has_changes == 'true' + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.LABCONSTRICTOR_SYNC_TOKEN }} + branch: template-sync-${{ steps.local-version.outputs.version }}-to-${{ steps.upstream-version.outputs.version }} + delete-branch: true + commit-message: Sync template to version ${{ steps.upstream-version.outputs.version }} + title: Sync template to version ${{ steps.upstream-version.outputs.version }} + body: | + This PR updates `.template_sync` from the template repository and applies the migrations needed to move this repository from template version `${{ steps.local-version.outputs.version }}` to `${{ steps.upstream-version.outputs.version }}`. + + The branch is created with the repository-scoped `LABCONSTRICTOR_SYNC_TOKEN` secret so that updates to GitHub workflow files can be included safely. diff --git a/tests/fixtures/template_0_1_17_workflows/update_on_notebook_change.yml b/tests/fixtures/template_0_1_17_workflows/update_on_notebook_change.yml new file mode 100644 index 0000000..1ecd580 --- /dev/null +++ b/tests/fixtures/template_0_1_17_workflows/update_on_notebook_change.yml @@ -0,0 +1,154 @@ +name: Updates on notebook change + +on: + push: + branches: [ main, master ] + paths: + - 'notebooks/**/*.ipynb' + - 'notebooks/notebook_latest_versions.yaml' + - 'notebooks/**/requirements.yaml' + workflow_dispatch: + +# This enables to run a single workflow at a time +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon notebook change, update manifests and changelogs + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install requirements + run: pip install -r .tools/requirements.txt + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v44 + with: + separator: "," + files: | + notebooks/**/*.ipynb + notebooks/notebook_latest_versions.yaml + notebooks/**/requirements.yaml + + - name: Syncronize notebook versions + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "sync_notebook_versions.py" \ + .tools/python/sync_notebook_versions.py \ + python .tools/python/sync_notebook_versions.py + + - name: Update CHANGELOG + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_changelog.py" \ + .tools/python/update_changelog.py \ + python .tools/python/update_changelog.py notebooks/notebook_latest_versions.yaml + + - name: Update last commit in notebook_latest_versions.yaml + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "update_version_last_commit.py" \ + .tools/python/update_version_last_commit.py \ + python .tools/python/update_version_last_commit.py notebooks/notebook_latest_versions.yaml + + - name: Hide code cells in notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "hide_code_cells.py" \ + app/python_scripts/hide_code_cells.py \ + python app/python_scripts/hide_code_cells.py . + + - name: Index notebooks + if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "bump_notebook_README.py" \ + .tools/python/bump_notebook_README.py \ + python .tools/python/bump_notebook_README.py + + - name: Run merge script + if: github.event_name == 'workflow_dispatch' || contains(steps.changed-files.outputs.all_changed_files, 'requirements.yaml') + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "merge_requirements.py" \ + .tools/python/merge_requirements.py \ + python .tools/python/merge_requirements.py --source-dir notebooks --output requirements.txt + + - name: Check for changes + id: check-changes + run: | + if [ -z "$(git status --porcelain)" ]; then + echo "has_changes=false" >> $GITHUB_OUTPUT + else + echo "has_changes=true" >> $GITHUB_OUTPUT + fi + + - name: Set up Miniconda for requirements test + if: steps.check-changes.outputs.has_changes == 'true' + uses: conda-incubator/setup-miniconda@v3 + with: + auto-activate: true + miniforge-version: latest + use-mamba: true + + - name: Verify merged requirements inside conda + id: test-conda-env + if: steps.check-changes.outputs.has_changes == 'true' + env: + SKIP_CONDA_ENV_TEST: ${{ vars.SKIP_CONDA_ENV_TEST }} + run: | + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "test_conda_environment.sh" \ + .tools/bash/test_conda_environment.sh \ + bash .tools/bash/test_conda_environment.sh + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@v7 + with: + name: notebook-debug-bundle + path: artifacts/error_log_document.log + archive: false + + - name: Commit updated files + id: auto-commit + if: steps.check-changes.outputs.has_changes == 'true' && steps.test-conda-env.outcome == 'success' + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: GitHub Action - Updated files because of notebook and/or requirements changes + file_pattern: requirements*.txt environment.yaml notebooks/* setup.py + + - name: Revert user commit that broke requirements + if: failure() && github.event_name == 'push' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git reset --hard + git clean -fd + git revert "${{ github.sha }}" -m 1 --no-edit + git push origin HEAD:${{ github.ref_name }} diff --git a/tests/fixtures/template_0_1_17_workflows/update_on_src_change.yaml b/tests/fixtures/template_0_1_17_workflows/update_on_src_change.yaml new file mode 100644 index 0000000..acc7c34 --- /dev/null +++ b/tests/fixtures/template_0_1_17_workflows/update_on_src_change.yaml @@ -0,0 +1,41 @@ +name: Updates on external code change + +on: + push: + branches: [ main, master ] + paths: + - 'src/**' + workflow_dispatch: + +# This enables to run a single workflow at a time +concurrency: + group: notebooks-global-lock-${{ github.ref }} + cancel-in-progress: false + +jobs: + updates-version_change: + name: Upon external code update the code version + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Update src change file + run: | + cat > .tools/meta/src_change.yaml </apps/DemoApp.json`. + +Cases: the real manifests of the apps (tests/fixtures/manifests), a manifest with a quoted hosts key and a BOM, a manifest without a hosts: line and no manifest at all. For each, the `hosts` +of the entry must equal the `hosts:` line of the manifest (in the declared order), and for the last two the entry must have no `hosts` and +the installer log must carry the sentence "declares no hosts". Anything else is a failure, never a skip. + +What is stood in for (the rest is the real script): the TLS launcher (prints a CA bundle, answers --self-test), include_path.py, +hide_code_cells.py and menuinst (no shortcuts), and on Windows the venv layout (python is in Scripts\\, so the script's PYTHON_EXE line is pointed there). + +Usage: python tests/smoke/post_install_hosts.py (env: LC_TOOLS_SPEC optional; SMOKE_CA_BUNDLE optional, a CA file for pip behind a proxy) +""" +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "app/bash_bat_scripts" +MANIFESTS = ROOT / "tests/fixtures/manifests" +WINDOWS = os.name == "nt" +APP = "DemoApp" +NO_HOSTS = "declares no hosts, so it is listed in every host" +TOKENS = ( + ("UNDERSCORED_PROJECT_NAME", "Demo_App"), + ("PYTHON_PROJ_NAME", "demoapp"), + ("PROJECT_NAME", APP), + ("VERSION_NUMBER", "1.2.3"), + ("GITHUB_OWNER", "CellMigrationLab"), +) +SETUP_PY = 'from setuptools import find_packages, setup\nsetup(name="demoapp", version="1.2.3", package_dir={"": "src"}, packages=find_packages("src"))\n' +TOOLS_MODULE = ( + "from labconstrictor_tools import Scalars, tool\n\n\n" + '@tool("Double a number")\n' + "def double(x: float = 2.0) -> Scalars:\n" + ' return {"y": x * 2}\n' +) +LAUNCHER = ( + "import sys\n" + "if '--print-ca-bundle' in sys.argv:\n" + " import os\n" + " from pip._vendor import certifi\n" + " print(os.environ.get('SMOKE_CA_BUNDLE') or certifi.where())\n" +) +MENUINST_API = "def install(path):\n return path\n" +HOSTS_LINE = re.compile(r"(?m)^hosts[ \t]*:[ \t]*\[(.*)\]") + + +def venv_python(prefix: Path) -> Path: + return prefix / ("Scripts/python.exe" if WINDOWS else "bin/python") + + +def declared_hosts(manifest: Path) -> list[str]: + """The names of the `hosts: [a, b]` line of a manifest, in order (the fixtures write it as a flow list).""" + match = HOSTS_LINE.search(manifest.read_text(encoding="utf-8")) + if match is None: + raise ValueError(f"{manifest} has no hosts: [..] line") + return [name.strip() for name in match.group(1).split(",") if name.strip()] + + +def run(command: list[str], env: dict[str, str] | None = None, stdin: int | None = None) -> None: + print("+", " ".join(command), flush=True) + subprocess.run(command, check=True, env=env, stdin=stdin) + + +def make_prefix(base: Path) -> Path: + """A fake install prefix: a venv with setuptools, a stub menuinst, and the project folder of the app (DemoApp) without a manifest.""" + prefix = base / "prefix" + run([sys.executable, "-m", "venv", str(prefix)]) + python = str(venv_python(prefix)) + ca = os.environ.get("SMOKE_CA_BUNDLE") + extra = ["--cert", ca] if ca else [] + run([python, "-m", "pip", "install", "--quiet", "pip>=24.2", "setuptools>=70.1", "wheel", *extra]) + site = Path(subprocess.check_output([python, "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], text=True).strip()) + (site / "menuinst").mkdir() + (site / "menuinst/__init__.py").write_text("", encoding="utf-8") + (site / "menuinst/api.py").write_text(MENUINST_API, encoding="utf-8") + write_project(prefix / APP) + return prefix + + +def write_project(root: Path) -> None: + (root / "src/demoapp").mkdir(parents=True) + (root / "src/demoapp_lc_tools").mkdir(parents=True) + (root / "src/demoapp/__init__.py").write_text("", encoding="utf-8") + (root / "src/demoapp_lc_tools/__init__.py").write_text(TOOLS_MODULE, encoding="utf-8") + (root / "setup.py").write_text(SETUP_PY, encoding="utf-8") + (root / "launch_jupyter.py").write_text(LAUNCHER, encoding="utf-8") + for helper in ("include_path.py", "hide_code_cells.py"): + (root / helper).write_text("", encoding="utf-8") + (root / "requirements.txt").write_text("setuptools\n", encoding="utf-8") + (root / "construct.yaml").write_text("name: Demo_App\nversion: 1.2.3\n", encoding="utf-8") + (root / "notebook_launcher.json").write_text("{}", encoding="utf-8") + + +def render_script(base: Path) -> Path: + """The live script of this OS with the app's names filled in (on Windows also pointed at the venv's python).""" + name = "post_install.bat" if WINDOWS else "post_install.sh" + text = (SCRIPTS / name).read_bytes().decode().replace("\r\n", "\n") + for token, value in TOKENS: + text = text.replace(token, value) + if WINDOWS: + text = text.replace('SET "PYTHON_EXE=%PREFIX%\\python.exe"', 'SET "PYTHON_EXE=%PREFIX%\\Scripts\\python.exe"') + text = text.replace("\n", "\r\n") + path = base / name + path.write_bytes(text.encode()) + return path + + +def install(script: Path, prefix: Path, lc_home: Path) -> str: + """Run the script as the installer does (PREFIX in the environment); return the installer log.""" + env = {**os.environ, "PREFIX": str(prefix), "LC_HOME": str(lc_home)} + command = ["cmd", "/c", str(script)] if WINDOWS else ["bash", str(script)] + log_file = prefix / "menuinst_debug.log" + try: + run(command, env=env, stdin=subprocess.DEVNULL) + except subprocess.CalledProcessError: + print("The installer failed; its log:\n" + log_file.read_text(encoding="utf-8", errors="replace"), flush=True) + raise + return log_file.read_text(encoding="utf-8", errors="replace") + + +def registry_entry(lc_home: Path) -> dict[str, object]: + path = lc_home / "apps" / f"{APP}.json" + if not path.is_file(): + raise AssertionError(f"no registry entry at {path}") + entry: dict[str, object] = json.loads(path.read_text(encoding="utf-8")) + return entry + + +def check_declared(name: str, manifest: Path, entry: dict[str, object], log: str) -> None: + expected = declared_hosts(manifest) + if entry.get("hosts") != expected: + raise AssertionError(f"{name}: registry hosts {entry.get('hosts')!r} != hosts line of the manifest {expected!r}") + warnings = [line for line in log.splitlines() if "WARNING" in line and "conda" not in line] # the stand-in prefix has no conda + if warnings: + raise AssertionError(f"{name}: the installer log has a WARNING: {warnings}") + print(f"OK {name}: hosts {expected}", flush=True) + + +def check_undeclared(name: str, entry: dict[str, object], log: str) -> None: + if "hosts" in entry: + raise AssertionError(f"{name}: the entry must have no hosts, it has {entry['hosts']!r}") + if f"'{APP}' {NO_HOSTS}" not in log.replace("^", ""): + raise AssertionError(f"{name}: the installer log does not say that the app declares no hosts:\n{log}") + print(f"OK {name}: no hosts, and the log says so", flush=True) + + +def install_case(script: Path, prefix: Path, lc_home: Path) -> tuple[dict[str, object], str]: + """Install once into a fresh registry home: (the registry entry of the app, the installer log).""" + log = install(script, prefix, lc_home) + return registry_entry(lc_home), log + + +def main() -> int: + manifests = sorted(MANIFESTS.glob("*_labconstrictor-app.yaml")) + if len(manifests) < 5: + raise AssertionError(f"expected the real manifests of the apps in {MANIFESTS}, found {len(manifests)}") + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + prefix = make_prefix(base) + script = render_script(base) + target = prefix / APP / "labconstrictor-app.yaml" + for manifest in manifests: + target.write_bytes(manifest.read_bytes()) + entry, log = install_case(script, prefix, base / f"home_{manifest.stem}") + check_declared(manifest.name, manifest, entry, log) + target.write_bytes(b"\xef\xbb\xbfschema: 1\nname: Demo\n\"hosts\": [notebook, napari]\n") + entry, log = install_case(script, prefix, base / "home_bom_quoted_key") + if entry.get("hosts") != ["notebook", "napari"]: + raise AssertionError(f"BOM and quoted hosts key: registry hosts {entry.get('hosts')!r}") + print("OK manifest with a BOM and a quoted hosts key: hosts ['notebook', 'napari']", flush=True) + target.write_text("schema: 1\nname: Demo\n", encoding="utf-8") + entry, log = install_case(script, prefix, base / "home_no_hosts_line") + check_undeclared("manifest without a hosts: line", entry, log) + target.unlink() + entry, log = install_case(script, prefix, base / "home_no_manifest") + check_undeclared("no manifest in the install", entry, log) + print("post_install hosts smoke: all cases passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_post_install_hosts.py b/tests/test_post_install_hosts.py new file mode 100644 index 0000000..7fa41bb --- /dev/null +++ b/tests/test_post_install_hosts.py @@ -0,0 +1,163 @@ +"""The live post_install.sh of the template passes the app's hosts to `labconstrictor-tools register` (and says so when it cannot). + +The script runs for real (bash) in a fake install prefix whose "python" is a small shell stub that records the arguments of `register`: the +registry entry itself is checked by the smoke install of the template CI (tests/smoke/post_install_hosts.py, which needs Tools). +`bump_constructor.py` is tested here too: it bundles labconstrictor-app.yaml in extra_files only when the file exists. +""" +from __future__ import annotations + +import importlib.util +import os +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path +from types import ModuleType + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "app/bash_bat_scripts/post_install.sh" +BAT = ROOT / "app/bash_bat_scripts/post_install.bat" +NO_HOSTS = "'DemoApp' declares no hosts, so it is listed in every host; register it with --app-manifest (or --hosts) to choose" +STUB_PYTHON = """#!/bin/bash +# stands for the bundled python: records `register` calls, answers the module check with "found", does nothing else +case "$*" in + *--print-ca-bundle*) echo "$STUB_CA" ;; + *"-m labconstrictor_tools register"*) echo "$@" >> "$STUB_REGISTER_ARGS" ;; +esac +exit 0 +""" + + +def render(text: str) -> str: + for token, value in (("UNDERSCORED_PROJECT_NAME", "Demo_App"), ("PYTHON_PROJ_NAME", "demoapp"), ("PROJECT_NAME", "DemoApp")): + text = text.replace(token, value) + return text + + +class LiveScriptTests(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + base = Path(self.tmp.name) + self.prefix = base / "prefix" + self.root = self.prefix / "DemoApp" + (self.prefix / "bin").mkdir(parents=True) + self.root.mkdir() + python = self.prefix / "bin/python" + python.write_text(STUB_PYTHON, encoding="utf-8") + python.chmod(python.stat().st_mode | stat.S_IEXEC) + for name in ("launch_jupyter.py", "requirements.txt", "setup.py"): + (self.root / name).write_text("", encoding="utf-8") + (self.root / "construct.yaml").write_text("name: Demo_App\nversion: 1.2.3\n", encoding="utf-8") + self.ca = base / "ca.pem" + self.ca.write_text("ca", encoding="utf-8") + self.recorded = base / "register_args.txt" + self.script = base / "post_install.sh" + self.script.write_text(render(SCRIPT.read_text(encoding="utf-8")), encoding="utf-8") + + def tearDown(self) -> None: + self.tmp.cleanup() + + def install(self) -> str: + env = {**os.environ, "PREFIX": str(self.prefix), "STUB_CA": str(self.ca), "STUB_REGISTER_ARGS": str(self.recorded)} + subprocess.run(["bash", str(self.script)], check=True, env=env, stdin=subprocess.DEVNULL, capture_output=True) + return (self.prefix / "menuinst_debug.log").read_text(encoding="utf-8") + + def registered(self) -> str: + return self.recorded.read_text(encoding="utf-8").strip() + + def test_the_bundled_manifest_goes_to_register(self) -> None: + manifest = self.root / "labconstrictor-app.yaml" + manifest.write_text("schema: 1\nname: Demo\nhosts: [notebook, napari]\n", encoding="utf-8") + log = self.install() + self.assertTrue(self.registered().endswith(f"--app-manifest {manifest}"), self.registered()) + self.assertIn("--name DemoApp", self.registered()) + self.assertIn(f"Passing the hosts of {manifest} to Tools (--app-manifest); Tools logs the same sentence if the file declares no hosts.", log) + self.assertNotIn("WARNING", log) + + def test_a_missing_manifest_is_logged_with_the_sentence_and_registers_without_hosts(self) -> None: + log = self.install() + self.assertNotIn("--app-manifest", self.registered()) + self.assertNotIn("--hosts", self.registered()) + self.assertIn(f"WARNING: {self.root}/labconstrictor-app.yaml was not found", log) + self.assertIn("add labconstrictor-app.yaml to extra_files in construct.yaml", log) + self.assertIn(NO_HOSTS, log) + + def test_a_manifest_without_a_hosts_line_is_still_passed_and_tools_decides(self) -> None: + manifest = self.root / "labconstrictor-app.yaml" + manifest.write_text("schema: 1\nname: Demo\n", encoding="utf-8") + log = self.install() + self.assertIn(f"--app-manifest {manifest}", self.registered()) + self.assertNotIn("WARNING", log, "the hook has no opinion of the file's content: Tools reads it and logs its own sentence") + + def test_a_manifest_with_crlf_and_a_spaced_hosts_key_is_passed(self) -> None: + manifest = self.root / "labconstrictor-app.yaml" + manifest.write_bytes(b"schema: 1\r\nhosts : [notebook]\r\n") + log = self.install() + self.assertIn("Passing the hosts of", log) + self.assertNotIn("WARNING", log) + + +class TemplateTextTests(unittest.TestCase): + def test_the_bat_script_has_the_same_sentence_and_both_register_lines(self) -> None: + text = BAT.read_text(encoding="utf-8").replace("^", "") + self.assertEqual(text.count("'PROJECT_NAME' declares no hosts, so it is listed in every host; register it with --app-manifest (or --hosts) to choose"), 1) + self.assertEqual(text.count('--app-manifest "%LC_APP_MANIFEST%"'), 1) + self.assertEqual(text.count("labconstrictor_tools register"), 2) + + def test_the_sh_script_passes_the_argument_with_an_idiom_that_works_in_bash_3(self) -> None: + text = SCRIPT.read_text(encoding="utf-8") + self.assertIn('${LC_HOSTS_ARGS[@]+"${LC_HOSTS_ARGS[@]}"}', text, "macOS bash 3.2 fails on an empty array under set -u without it") + self.assertEqual(text.count("labconstrictor_tools register"), 1) + + +def load(path: Path, name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class BumpConstructorTests(unittest.TestCase): + def setUp(self) -> None: + self.bump = load(ROOT / ".tools/python/bump_constructor.py", "bump_constructor_hosts") + self.tmp = tempfile.TemporaryDirectory() + self.manifest = Path(self.tmp.name) / "labconstrictor-app.yaml" + + def tearDown(self) -> None: + self.tmp.cleanup() + + @staticmethod + def data() -> dict: + return {"extra_files": [{"app/menuinst/Welcome.ipynb": "DemoApp/notebooks/Welcome.ipynb"}]} + + def test_the_mapping_is_added_when_the_file_exists_and_only_once(self) -> None: + self.manifest.write_text("schema: 1\n", encoding="utf-8") + data = self.data() + self.assertTrue(self.bump.ensure_app_manifest_in_extra_files(data, self.manifest)) + self.assertIn({"labconstrictor-app.yaml": "DemoApp/labconstrictor-app.yaml"}, data["extra_files"]) + self.assertFalse(self.bump.ensure_app_manifest_in_extra_files(data, self.manifest)) + self.assertEqual(sum("labconstrictor-app.yaml" in item for item in data["extra_files"]), 1) + + def test_no_mapping_is_added_for_a_missing_file(self) -> None: + data = self.data() + self.assertFalse(self.bump.ensure_app_manifest_in_extra_files(data, self.manifest)) + self.assertEqual(data, self.data()) + + def test_a_mapping_whose_file_is_gone_is_kept_and_reported(self) -> None: + data = self.data() + data["extra_files"].append({"labconstrictor-app.yaml": "Mine/labconstrictor-app.yaml"}) + from contextlib import redirect_stdout + from io import StringIO + + out = StringIO() + with redirect_stdout(out): + self.assertFalse(self.bump.ensure_app_manifest_in_extra_files(data, self.manifest)) + self.assertIn("WARNING: extra_files maps labconstrictor-app.yaml but the file does not exist", out.getvalue()) + self.assertIn({"labconstrictor-app.yaml": "Mine/labconstrictor-app.yaml"}, data["extra_files"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_template_sync_0_1_16.py b/tests/test_template_sync_0_1_16.py index 2a6956e..1cc4f50 100644 --- a/tests/test_template_sync_0_1_16.py +++ b/tests/test_template_sync_0_1_16.py @@ -272,7 +272,7 @@ def test_the_sync_entry_point_runs_the_chain_step(self) -> None: self.install_old_scripts() shutil.copytree(ROOT / ".template_sync", self.repo / ".template_sync") manifest = self.repo / ".template_sync/manifest.yaml" # sync.py checks the target against the manifest: stop the chain at this step - manifest.write_text(manifest.read_text(encoding="utf-8").replace("template_version: 0.1.17", "template_version: 0.1.16"), encoding="utf-8") + manifest.write_text(manifest.read_text(encoding="utf-8").replace("template_version: 0.1.18", "template_version: 0.1.16"), encoding="utf-8") result = subprocess.run( [sys.executable, str(self.repo / ".template_sync/sync.py"), "--from-version", "0.1.15", "--to-version", "0.1.16", "--repository-root", str(self.repo)], capture_output=True, text=True, @@ -311,8 +311,8 @@ def test_the_fixtures_are_the_unpinned_bridge_test_text(self) -> None: self.assertNotIn("LC_TOOLS_PIN", text, f"{app}/{name}") def test_the_unpinned_source_of_the_real_apps_is_rewritten_to_the_pinned_tag(self) -> None: - template_sh = (ROOT / "app/bash_bat_scripts/post_install.sh").read_text(encoding="utf-8") - self.assertIn(f'LC_TOOLS_PIN="{PIN}"', template_sh, "the tests expect the pin the template scripts carry") + template_sh = (self.migration.RESOURCE_ROOT / "app/bash_bat_scripts/post_install.sh").read_text(encoding="utf-8") + self.assertIn(f'LC_TOOLS_PIN="{PIN}"', template_sh, "the tests expect the pin the scripts of this migration carry") for app in self.APPS: for crlf in (False, True): with self.subTest(app=app, crlf=crlf): @@ -585,7 +585,7 @@ def test_the_default_tools_source_is_the_pinned_tag_archive(self) -> None: def test_the_pin_is_written_once_per_script_and_never_a_branch(self) -> None: for name in ("post_install.sh", "post_install.bat"): - text = (ROOT / "app/bash_bat_scripts" / name).read_text(encoding="utf-8") + text = (self.migration.RESOURCE_ROOT / "app/bash_bat_scripts" / name).read_text(encoding="utf-8") self.assertEqual(text.count("v0.2.0-rc.1"), 1, name) self.assertNotIn("refs/heads/", text, name) self.assertIn("refs/tags/", text, name) diff --git a/tests/test_template_sync_0_1_17.py b/tests/test_template_sync_0_1_17.py index fd8566a..8491b17 100644 --- a/tests/test_template_sync_0_1_17.py +++ b/tests/test_template_sync_0_1_17.py @@ -17,6 +17,7 @@ from io import StringIO from pathlib import Path +from _hosts import strip_hosts ROOT = Path(__file__).resolve().parents[1] MIGRATION_PATH = ROOT / ".template_sync/migrations/v0_1_16_to_v0_1_17.py" @@ -27,6 +28,8 @@ APPS_0_1_16 = FIXTURES / "template_0_1_16_apps" BRIDGE_TEST = FIXTURES / "bridge_test_hooks" SCRIPT_NAMES = ("post_install.sh", "post_install.bat") +OLD_PIN = "v0.2.0-rc.1" # the pin of template 0.1.17 (the resources of this migration) +TEMPLATE_PIN = "v0.3.0-alpha.1" # the pin of the live template since 0.1.18 CONSTRUCT = """name: Demo_App version: 1.2.3 @@ -95,7 +98,8 @@ def install_0_1_16_template(self) -> None: self.script(name).write_bytes(self.render(self.old_migration.RESOURCE_ROOT / "app/bash_bat_scripts" / name)) def expected(self, name: str) -> bytes: - return self.render(ROOT / "app/bash_bat_scripts" / name) + # what the 0.1.17 migration converges to: the scripts of template 0.1.17 (its resources). The live template moved on (0.1.18 pin). + return self.render(self.migration.RESOURCE_ROOT / "app/bash_bat_scripts" / name) def run_migration(self) -> str: buffer = StringIO() @@ -106,18 +110,16 @@ def run_migration(self) -> str: class TemplateConsistencyTests(unittest.TestCase): def test_resources_are_identical_to_the_template_files(self) -> None: + """Since template 0.1.18 the live files carry the new Tools pin, so the resources of 0.1.17 (frozen: they are what 0.1.17 delivers) + must equal the live files except for that pin and for the app-hosts lines (a 0.1.18 step, removed by tests/_hosts.py); the byte-for-byte check against the live files is made for the 0.1.18 resources.""" migration = load(MIGRATION_PATH, "migration_0_1_17_resources") for relative in migration.MANAGED_FILES: with self.subTest(file=str(relative)): - self.assertEqual( - (migration.RESOURCE_ROOT / relative).read_bytes(), - (ROOT / relative).read_bytes(), - "update .template_sync/resources/v0_1_16_to_v0_1_17 when the template file changes", - ) + live = strip_hosts(relative.name, (ROOT / relative).read_bytes()).replace(TEMPLATE_PIN.encode(), OLD_PIN.encode()) + self.assertEqual((migration.RESOURCE_ROOT / relative).read_bytes(), live) def test_the_manifest_lists_the_migration_and_the_new_version(self) -> None: text = (ROOT / ".template_sync/manifest.yaml").read_text(encoding="utf-8") - self.assertIn("template_version: 0.1.17", text) self.assertIn("script: migrations/v0_1_16_to_v0_1_17.py", text) def test_chains_from_every_older_version_reach_0_1_17(self) -> None: @@ -144,10 +146,12 @@ def test_the_template_scripts_install_tools_once_before_the_import_check(self) - self.assertLess(install, check) self.assertLess(check, register) self.assertEqual(sh.count("-m pip install \"${LC_TOOLS_SPEC"), 1) - self.assertEqual(sh.count("v0.2.0-rc.1"), 1) + self.assertEqual(sh.count(TEMPLATE_PIN), 1) + self.assertNotIn(OLD_PIN, sh) bat = (ROOT / "app/bash_bat_scripts/post_install.bat").read_bytes().decode() self.assertEqual(bat.count("\n"), bat.count("\r\n"), "post_install.bat must stay CRLF") - self.assertEqual(bat.count("v0.2.0-rc.1"), 1) + self.assertEqual(bat.count(TEMPLATE_PIN), 1) + self.assertNotIn(OLD_PIN, bat) self.assertLess(bat.index("CALL :install_lc_tools"), bat.index('-c "import PYTHON_PROJ_NAME;')) self.assertLess(bat.index('-c "import PYTHON_PROJ_NAME;'), bat.index("-m labconstrictor_tools register")) self.assertEqual(bat.count("\n:install_lc_tools\r\n"), 1) @@ -197,6 +201,8 @@ def test_the_whole_chain_from_0_1_15_reaches_the_0_1_17_template(self) -> None: newline = "\r\n" if "\r\n" in source else "\n" self.script(name).write_bytes(self.helpers.render_template(source, self.replacements, newline).encode()) shutil.copytree(ROOT / ".template_sync", self.repo / ".template_sync") + manifest = self.repo / ".template_sync/manifest.yaml" # sync.py checks the target against the manifest: stop the chain at this step + manifest.write_text(manifest.read_text(encoding="utf-8").replace("template_version: 0.1.18", "template_version: 0.1.17"), encoding="utf-8") result = subprocess.run( [sys.executable, str(self.repo / ".template_sync/sync.py"), "--from-version", "0.1.15", "--to-version", "0.1.17", "--repository-root", str(self.repo)], capture_output=True, text=True, @@ -268,7 +274,7 @@ def test_the_tools_guide_is_refreshed_only_if_it_exists(self) -> None: self.assertFalse((self.repo / ".tools/docs/tools_bridge.md").exists()) (self.repo / ".tools/docs/tools_bridge.md").write_text("old guide\n", encoding="utf-8") self.run_migration() - self.assertEqual((self.repo / ".tools/docs/tools_bridge.md").read_bytes(), self.render(ROOT / ".tools/docs/tools_bridge.md", "\n")) + self.assertEqual((self.repo / ".tools/docs/tools_bridge.md").read_bytes(), self.render(self.migration.RESOURCE_ROOT / ".tools/docs/tools_bridge.md", "\n")) class RealAppScriptsTests(MigrationCase): diff --git a/tests/test_template_sync_0_1_18.py b/tests/test_template_sync_0_1_18.py new file mode 100644 index 0000000..c0e9555 --- /dev/null +++ b/tests/test_template_sync_0_1_18.py @@ -0,0 +1,504 @@ +"""Tests of the 0.1.18 migration: new Tools pin, GitHub Actions pinned by SHA, install rules in the install guide.""" +from __future__ import annotations + +import difflib +import hashlib +import importlib.util +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path +from types import ModuleType + +from _hosts import strip_hosts + + +ROOT = Path(__file__).resolve().parents[1] +MIGRATION_PATH = ROOT / ".template_sync/migrations/v0_1_17_to_v0_1_18.py" +SYNC_PATH = ROOT / ".template_sync/sync.py" +FIXTURES = Path(__file__).resolve().parent / "fixtures" +APPS_0_1_17 = FIXTURES / "template_0_1_17_apps" +WORKFLOWS_0_1_17 = FIXTURES / "template_0_1_17_workflows" +DOCS_0_1_17 = FIXTURES / "template_0_1_17_docs" +OLD_SCRIPTS = FIXTURES / "template_0_1_15_scripts" +BRIDGE_TEST = FIXTURES / "bridge_test_hooks" +SCRIPT_NAMES = ("post_install.sh", "post_install.bat") +WORKFLOW_NAMES = ("check_app_manifest.yml", "release_constructor.yml", "sync_template.yml", "update_on_notebook_change.yml", "update_on_src_change.yaml") +OLD_PIN = "v0.2.0-rc.1" +NEW_PIN = "v0.3.0-alpha.1" + +# The commit SHA of each third-party action tag the template workflows use (the same pins the apps carry after their own pinning pull requests). +PINS = { + ("actions/checkout", "v4"): "11d5960a326750d5838078e36cf38b85af677262", + ("actions/setup-python", "v4"): "7f4fc3e22c37d6ff65e88745f38bd3157c663f7c", + ("actions/upload-artifact", "v7"): "cf430e030ddbb5b0abf93d22962f4752f3646cd9", + ("tj-actions/changed-files", "v44"): "2d756ea4c53f7f6b397767d8723b3a10a9f35bf2", + ("conda-incubator/setup-miniconda", "v3"): "fc2d68f6413eb2d87b895e92f8584b5b94a10167", + ("stefanzweifel/git-auto-commit-action", "v5"): "b863ae1933cb653a53c021fe36dbb774e1fb9403", + ("softprops/action-gh-release", "v2"): "3bb12739c298aeb8a4eeaf626c5b8d85266b0e65", + ("peter-evans/create-pull-request", "v6"): "c5a7806660adbe173f04e3e038b0ccdcd758773c", +} +USES = re.compile(r"^\s*(?:-\s+)?uses:\s*(\S+)@(\S+)(?:[ \t]+#[ \t]*(\S+))?\s*$") + +# The three install rules, word for word. Every doc that states a rule must use exactly this sentence. +RULE_PATH = "The install path must contain no space: the installer refuses a path with a space. Use a path such as `C:\\LabConstrictor\\` on Windows or `~/labconstrictor/` on macOS and Linux." +RULE_UPDATE = "On macOS and Windows, update an app by uninstalling the old version first and then installing the new one: the installer does not install over an existing install." +RULE_LINUX = "On Linux, running the installer with `-u` updates in place only if the previous install finished; otherwise delete `/pkgs` or uninstall the app and install it again." + +CONSTRUCT = """name: Demo_App +version: 1.2.3 +icon_image: app/logo/demo.png +extra_files: +- app/menuinst/Welcome.ipynb: DemoApp/notebooks/Welcome.ipynb +- app/menuinst/notebook_launcher.json: DemoApp/notebook_launcher.json +- requirements.txt: DemoApp/requirements.txt +""" + + +def load(path: Path, name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def snapshot(root: Path) -> dict[str, str]: + return {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(root.rglob("*")) if p.is_file()} + + +def pin_view(root: Path) -> dict[str, str]: + """`snapshot` of the pin tests: the post_install scripts are seen without the lines of the hosts step, which has its own tests.""" + view = {} + for path in sorted(root.rglob("*")): + if path.is_file(): + data = path.read_bytes() + view[str(path.relative_to(root))] = hashlib.sha256(strip_hosts(path.name, data) if path.name.startswith("post_install.") else data).hexdigest() + return view + + +def to_crlf(data: bytes) -> bytes: + return data.replace(b"\r\n", b"\n").replace(b"\n", b"\r\n") + + +def to_lf(data: bytes) -> bytes: + return data.replace(b"\r\n", b"\n") + + +def uses_lines(path: Path) -> list[tuple[str, str, str | None]]: + found = [] + for line in path.read_text(encoding="utf-8").splitlines(): + match = USES.match(line) + if match: + found.append((match.group(1), match.group(2), match.group(3))) + return found + + +class TemplateConsistencyTests(unittest.TestCase): + def test_resources_are_identical_to_the_template_files(self) -> None: + migration = load(MIGRATION_PATH, "migration_0_1_18_resources") + for relative in migration.MANAGED_FILES: + with self.subTest(file=str(relative)): + self.assertEqual( + (migration.RESOURCE_ROOT / relative).read_bytes(), + (ROOT / relative).read_bytes(), + "update .template_sync/resources/v0_1_17_to_v0_1_18 when the template file changes", + ) + + def test_the_manifest_lists_the_migration_and_the_new_version(self) -> None: + text = (ROOT / ".template_sync/manifest.yaml").read_text(encoding="utf-8") + self.assertIn("template_version: 0.1.18", text) + self.assertIn("script: migrations/v0_1_17_to_v0_1_18.py", text) + + def test_chains_from_every_older_version_reach_0_1_18(self) -> None: + sync = load(SYNC_PATH, "sync_for_0_1_18") + manifest = sync.parse_manifest(ROOT / ".template_sync/manifest.yaml") + for start in ("0.0.0", "0.1.9", "0.1.14", "0.1.15", "0.1.16", "0.1.17"): + chain = sync.resolve_migration_chain(manifest["migrations"], start, "0.1.18") + self.assertEqual(chain[-1]["to"], "0.1.18", start) + for entry in manifest["migrations"]: + self.assertTrue((ROOT / ".template_sync" / entry["script"]).is_file(), entry["script"]) + + def test_the_docs_are_linked_and_name_the_pinned_tag(self) -> None: + self.assertIn("(migration_0_1_18.md)", (ROOT / ".tools/docs/README.md").read_text(encoding="utf-8")) + self.assertTrue((ROOT / ".tools/docs/migration_0_1_18.md").is_file()) + bridge = (ROOT / ".tools/docs/tools_bridge.md").read_text(encoding="utf-8") + self.assertIn(f"archive/refs/tags/{NEW_PIN}.zip", bridge) + self.assertIn(f'LC_TOOLS_PIN="{NEW_PIN}"', bridge) + self.assertNotIn(OLD_PIN, bridge) + + def test_the_template_scripts_pin_the_new_tag_once_per_script(self) -> None: + for name in SCRIPT_NAMES: + text = (ROOT / "app/bash_bat_scripts" / name).read_bytes().decode() + with self.subTest(script=name): + self.assertEqual(text.count(NEW_PIN), 1) + self.assertNotIn(OLD_PIN, text) + self.assertNotIn("refs/heads/", text) + bat = (ROOT / "app/bash_bat_scripts/post_install.bat").read_bytes().decode() + self.assertEqual(bat.count("\n"), bat.count("\r\n"), "post_install.bat must stay CRLF") + + def test_every_third_party_action_of_the_workflows_is_pinned_by_sha(self) -> None: + files = sorted((ROOT / ".github/workflows").glob("*.y*ml")) + sorted((ROOT / ".template_sync/resources").glob("*/.github/workflows/*.y*ml")) + self.assertGreaterEqual(len(files), 10) + for path in files: + lines = uses_lines(path) + self.assertTrue(lines, path) + for action, ref, tag in lines: + with self.subTest(file=str(path.relative_to(ROOT)), action=action): + self.assertRegex(ref, r"^[0-9a-f]{40}$", "pin by the 40-character commit SHA") + self.assertIsNotNone(tag, "the tag stays as a comment") + self.assertEqual(PINS.get((action, tag or "")), ref, f"unknown pin for {action} {tag}") + + def test_the_tests_workflow_of_the_template(self) -> None: + text = (ROOT / ".github/workflows/tests.yml").read_text(encoding="utf-8") + self.assertRegex(text, r"(?m)^on:\n pull_request:\n") + self.assertRegex(text, r"(?m)^ push:\n branches: \[main\]") + self.assertNotIn("paths:", text, "the checks run for every change: no path filter") + self.assertRegex(text, r"(?m)^permissions:\n contents: read\n") + self.assertIn("github.repository == 'CellMigrationLab/LabConstrictor'", text, "it does not run in the apps created from the template") + for command in ("pytest tests", "ruff check", "mypy .template_sync .tools/python tests"): + self.assertIn(command, text) + self.assertEqual(sum(1 for _ in uses_lines(ROOT / ".github/workflows/tests.yml")), 4, "two jobs, two actions each") + self.assertIn("python tests/smoke/post_install_hosts.py", text, "the smoke install of post_install runs in CI") + self.assertNotIn("LC_TOOLS_SPEC", text, "the smoke job installs the tag pinned by post_install: no Tools source override in CI") + + def test_the_install_rules_are_in_the_install_guide_with_one_wording(self) -> None: + guide = (ROOT / ".tools/docs/install.md").read_text(encoding="utf-8") + for rule in (RULE_PATH, RULE_UPDATE, RULE_LINUX): + self.assertIn(rule, guide) + markers = (("must contain no space", RULE_PATH), ("uninstalling the old version first", RULE_UPDATE), ("-u` updates in place", RULE_LINUX)) + docs = [*sorted((ROOT / ".tools/docs").glob("*.md")), ROOT / "README.md", *sorted((ROOT / ".tools/templates").glob("*.md"))] + for path in docs: + text = path.read_text(encoding="utf-8") + for marker, rule in markers: + if marker in text: + with self.subTest(doc=str(path.relative_to(ROOT)), rule=marker): + self.assertIn(rule, text, "a doc that states a rule uses the exact sentence of install.md") + + +class MigrationCase(unittest.TestCase): + def setUp(self) -> None: + self.migration = load(MIGRATION_PATH, "migration_0_1_18") + self.helpers = self.migration._load_helpers() + self.tmp = tempfile.TemporaryDirectory() + self.repo = Path(self.tmp.name) / "app_repo" + (self.repo / "app/bash_bat_scripts").mkdir(parents=True) + (self.repo / "src/demoapp").mkdir(parents=True) + (self.repo / "src/demoapp/__init__.py").write_text("", encoding="utf-8") + (self.repo / "construct.yaml").write_text(CONSTRUCT, encoding="utf-8") + (self.repo / ".tools/docs").mkdir(parents=True) + (self.repo / ".github/workflows").mkdir(parents=True) + + def tearDown(self) -> None: + self.tmp.cleanup() + + def script(self, name: str) -> Path: + return self.repo / "app/bash_bat_scripts" / name + + def workflow(self, name: str) -> Path: + return self.repo / ".github/workflows" / name + + def resource(self, relative: str) -> bytes: + return (self.migration.RESOURCE_ROOT / relative).read_bytes() + + def run_migration(self) -> str: + buffer = StringIO() + with redirect_stdout(buffer): + self.migration.migrate(self.repo, {}) + return buffer.getvalue() + + def put_app(self, folder: Path, crlf_sh: bool = False) -> dict[str, bytes]: + originals = {} + for name in SCRIPT_NAMES: + data = (folder / name).read_bytes() + if crlf_sh and name.endswith(".sh"): + data = to_crlf(data) + self.script(name).write_bytes(data) + originals[name] = data + return originals + + +class PinTests(MigrationCase): + APPS = ("playground", "nucleisky", "vlab4mic", "guessthecondition") + + def test_real_apps_move_to_the_new_pin_and_nothing_else_changes(self) -> None: + for app in self.APPS: + for crlf in (False, True): + with self.subTest(app=app, crlf=crlf): + before = self.put_app(APPS_0_1_17 / app, crlf) + output = self.run_migration() + self.assertEqual(output.count(f"updated: Tools pin {OLD_PIN} -> {NEW_PIN}"), 2, output) + for name, data in before.items(): + after = strip_hosts(name, self.script(name).read_bytes()) + self.assertEqual(after, data.replace(OLD_PIN.encode(), NEW_PIN.encode()), name) + self.assertEqual(after.count(NEW_PIN.encode()), 1, name) + self.assertNotIn(OLD_PIN.encode(), after, name) + changed = [l for l in difflib.ndiff(data.decode().splitlines(), after.decode().splitlines()) if l[:2] in ("- ", "+ ")] + self.assertEqual(len(changed), 2, f"{name}: exactly the pin line changes, nothing else") + snap = pin_view(self.repo) + self.assertIn("no repository changes were required", self.run_migration()) + self.assertEqual(pin_view(self.repo), snap, "second run: nothing changes") + + def test_the_new_template_scripts_of_the_apps_equal_the_live_template_after_the_move(self) -> None: + """The scripts of an app are the template scripts with the app's names filled in: after the move they are the live template scripts.""" + for name in SCRIPT_NAMES: + self.script(name).write_bytes((APPS_0_1_17 / "nucleisky" / name).read_bytes()) + self.run_migration() + replacements = self.helpers.load_replacements(self.repo) + replacements.update({"PROJECT_NAME": "NucleiSky", "PYTHON_PROJ_NAME": "nucleisky"}) + sh = self.script("post_install.sh").read_text(encoding="utf-8") + live = self.helpers.render_template((ROOT / "app/bash_bat_scripts/post_install.sh").read_text(encoding="utf-8"), replacements, "\n") + self.assertEqual(sh.count(f'LC_TOOLS_PIN="{NEW_PIN}"'), 1) + self.assertIn(f'LC_TOOLS_PIN="{NEW_PIN}"', live) + + def test_an_app_with_its_own_pin_is_left_alone_with_a_note(self) -> None: + for name in SCRIPT_NAMES: + data = (APPS_0_1_17 / "nucleisky" / name).read_bytes().replace(OLD_PIN.encode(), b"v0.2.0-rc.2") + self.script(name).write_bytes(data) + before = pin_view(self.repo) + output = self.run_migration() + self.assertEqual(pin_view(self.repo), before) + self.assertEqual(output.count("has its own pin v0.2.0-rc.2: left as it is"), 2, output) + + def test_unpinned_hooks_stay_as_they_are_with_a_note(self) -> None: + for app in ("vlab4mic", "ns_pr32"): + with self.subTest(app=app): + self.put_app(BRIDGE_TEST / app) + before = pin_view(self.repo) + output = self.run_migration() + self.assertEqual(pin_view(self.repo), before) + self.assertEqual(output.count("has no LC_TOOLS_PIN line"), 2, output) + + def test_an_app_that_already_has_the_new_pin_is_unchanged(self) -> None: + for name in SCRIPT_NAMES: + self.script(name).write_bytes((APPS_0_1_17 / "nucleisky" / name).read_bytes().replace(OLD_PIN.encode(), NEW_PIN.encode())) + before = pin_view(self.repo) + output = self.run_migration() + self.assertEqual(pin_view(self.repo), before) + self.assertEqual(output.count(f"already pins Tools {NEW_PIN}: unchanged"), 2, output) + + def test_two_pin_lines_are_ambiguous_and_left_alone(self) -> None: + sh = self.script("post_install.sh") + text = (APPS_0_1_17 / "nucleisky/post_install.sh").read_text(encoding="utf-8") + sh.write_text(text + f'\nLC_TOOLS_PIN="{OLD_PIN}"\n', encoding="utf-8") + before = sh.read_bytes() + output = self.run_migration() + self.assertEqual(strip_hosts(sh.name, sh.read_bytes()), before) + self.assertIn("has 2 LC_TOOLS_PIN lines", output) + + def test_only_the_exact_old_template_line_is_changed(self) -> None: + sh = self.script("post_install.sh") + text = (APPS_0_1_17 / "nucleisky/post_install.sh").read_text(encoding="utf-8") + # the old value elsewhere (a comment, another variable) is not a pin line and stays; the pin line keeps its indent and trailing comment + text = text.replace(f'LC_TOOLS_PIN="{OLD_PIN}"', f'LC_TOOLS_PIN="{OLD_PIN}" # my comment') + f'# was {OLD_PIN}\nMY_VAR="{OLD_PIN}"\n' + sh.write_text(text, encoding="utf-8") + self.run_migration() + after = strip_hosts(sh.name, sh.read_bytes()).decode() + self.assertIn(f'LC_TOOLS_PIN="{NEW_PIN}" # my comment', after) + self.assertIn(f"# was {OLD_PIN}\n", after) + self.assertIn(f'MY_VAR="{OLD_PIN}"\n', after) + + def test_scripts_without_the_hook_and_missing_files_are_not_touched_or_created(self) -> None: + output = self.run_migration() + self.assertEqual([p.name for p in (self.repo / "app/bash_bat_scripts").iterdir()], []) + self.assertEqual(output.count("SKIPPED, file not found"), 2) + self.script("post_install.sh").write_text('#!/bin/bash\necho mine\nLC_TOOLS_PIN="v0.2.0-rc.1"\n', encoding="utf-8") + self.script("post_install.bat").write_bytes(b'@ECHO OFF\r\nSET "LC_TOOLS_PIN=v0.2.0-rc.1"\r\nEXIT /B 0\r\n') + before = pin_view(self.repo) + output = self.run_migration() + self.assertEqual(pin_view(self.repo), before) + self.assertEqual(output.count("nothing to move"), 2) + + +class WorkflowTests(MigrationCase): + def test_a_workflow_identical_to_the_previous_template_version_gets_the_pinned_file(self) -> None: + for crlf in (False, True): + for name in WORKFLOW_NAMES: + with self.subTest(workflow=name, crlf=crlf): + old = (WORKFLOWS_0_1_17 / name).read_bytes() + old = to_crlf(old) if crlf else to_lf(old) if name != "update_on_src_change.yaml" else old + self.workflow(name).write_bytes(old) + output = self.run_migration() + self.assertIn(f".github/workflows/{name}: updated: every third-party action is pinned by commit SHA", output) + expected = self.resource(f".github/workflows/{name}") + expected = to_crlf(expected) if b"\r\n" in old else to_lf(expected) + self.assertEqual(self.workflow(name).read_bytes(), expected, "pinned, line endings kept") + # only `uses:` lines differ from the previous version + changed = [l for l in difflib.ndiff(to_lf(old).decode().splitlines(), to_lf(self.workflow(name).read_bytes()).decode().splitlines()) if l[:2] in ("- ", "+ ")] + self.assertTrue(changed and all("uses:" in l for l in changed), changed) + self.workflow(name).unlink() + + def test_all_five_in_one_run_and_the_second_run_changes_nothing(self) -> None: + for name in WORKFLOW_NAMES: + self.workflow(name).write_bytes((WORKFLOWS_0_1_17 / name).read_bytes()) + output = self.run_migration() + self.assertEqual(output.count("every third-party action is pinned by commit SHA"), 5, output) + snap = snapshot(self.repo) + output = self.run_migration() + self.assertEqual(snapshot(self.repo), snap) + self.assertEqual(output.count("already the pinned template file: unchanged"), 5, output) + self.assertIn("no repository changes were required", output) + + def test_workflows_the_apps_pinned_themselves_are_unchanged(self) -> None: + """The apps pinned the actions by their own pull requests (same SHAs): their files equal the resources and are left alone.""" + for name in WORKFLOW_NAMES: + self.workflow(name).write_bytes(self.resource(f".github/workflows/{name}")) + before = snapshot(self.repo) + output = self.run_migration() + self.assertEqual(snapshot(self.repo), before) + self.assertEqual(output.count("already the pinned template file: unchanged"), 5, output) + + def test_a_customised_and_already_pinned_workflow_is_left_alone(self) -> None: + """Playground: release_constructor.yml without the Intel Mac build, pinned by the app.""" + target = self.workflow("release_constructor.yml") + target.write_bytes((APPS_0_1_17 / "playground/release_constructor.yml").read_bytes()) + before = target.read_bytes() + self.assertNotEqual(before, self.resource(".github/workflows/release_constructor.yml")) + output = self.run_migration() + self.assertEqual(target.read_bytes(), before) + self.assertIn("release_constructor.yml: customised and already pinned: left as it is", output) + + def test_a_customised_unpinned_workflow_is_skipped_and_the_unpinned_actions_are_listed(self) -> None: + target = self.workflow("update_on_notebook_change.yml") + target.write_text((WORKFLOWS_0_1_17 / "update_on_notebook_change.yml").read_text(encoding="utf-8") + "# my own addition\n", encoding="utf-8") + before = target.read_bytes() + output = self.run_migration() + self.assertEqual(target.read_bytes(), before) + self.assertIn("update_on_notebook_change.yml: SKIPPED: customised", output) + self.assertIn("tj-actions/changed-files@v44", output) + self.assertIn("migration_0_1_18.md", output) + + def test_a_workflow_pinned_in_part_by_the_app_is_customised_and_skipped(self) -> None: + target = self.workflow("sync_template.yml") + text = (WORKFLOWS_0_1_17 / "sync_template.yml").read_text(encoding="utf-8") + text = text.replace("actions/checkout@v4", "actions/checkout@" + PINS[("actions/checkout", "v4")] + " # v4") + target.write_text(text, encoding="utf-8") + before = target.read_bytes() + output = self.run_migration() + self.assertEqual(target.read_bytes(), before) + self.assertIn("sync_template.yml: SKIPPED: customised", output) + self.assertIn("peter-evans/create-pull-request@v6", output) + + def test_missing_workflows_are_not_created(self) -> None: + output = self.run_migration() + self.assertEqual(list((self.repo / ".github/workflows").iterdir()), []) + self.assertEqual(output.count("not present in this repository (nothing was created)"), 5 + 2, output) # five workflows, install.md, tools_bridge.md + + +class DocsTests(MigrationCase): + def test_the_install_guide_of_0_1_17_gets_both_sections_and_nothing_is_removed(self) -> None: + for crlf in (False, True): + with self.subTest(crlf=crlf): + old = (DOCS_0_1_17 / "install.md").read_bytes() + old = to_crlf(old) if crlf else old + target = self.repo / ".tools/docs/install.md" + target.write_bytes(old) + output = self.run_migration() + self.assertIn("install.md: updated: added 'Before you install' and 'Update an installed app'", output) + expected = self.resource(".tools/docs/install.md") + self.assertEqual(target.read_bytes(), to_crlf(expected) if crlf else expected) + removed = [l for l in difflib.ndiff(to_lf(old).decode().splitlines(), to_lf(target.read_bytes()).decode().splitlines()) if l.startswith("- ")] + self.assertEqual(removed, [], "nothing is deleted") + snap = snapshot(self.repo) + self.assertIn("install.md: already has the install rules: unchanged", self.run_migration()) + self.assertEqual(snapshot(self.repo), snap) + + def test_a_customised_install_guide_gets_only_the_missing_section(self) -> None: + target = self.repo / ".tools/docs/install.md" + text = (DOCS_0_1_17 / "install.md").read_text(encoding="utf-8").replace("## Install on Windows", "## Install on Windows (my wording)") + target.write_text(text, encoding="utf-8") + output = self.run_migration() + self.assertIn("SKIPPED: no line '## Install on Windows' to insert '## Before you install' before", output) + self.assertEqual(target.read_text(encoding="utf-8"), text, "a step that cannot be done changes nothing in that file") + + def test_the_rules_are_in_the_inserted_text_word_for_word(self) -> None: + target = self.repo / ".tools/docs/install.md" + target.write_bytes((DOCS_0_1_17 / "install.md").read_bytes()) + self.run_migration() + text = target.read_text(encoding="utf-8") + for rule in (RULE_PATH, RULE_UPDATE, RULE_LINUX): + self.assertIn(rule, text) + + def test_the_tools_guide_is_refreshed_only_if_it_is_the_0_1_17_text(self) -> None: + target = self.repo / ".tools/docs/tools_bridge.md" + target.write_bytes((DOCS_0_1_17 / "tools_bridge.md").read_bytes()) + self.assertIn(OLD_PIN, target.read_text(encoding="utf-8")) + self.assertIn("tools_bridge.md: updated: written", self.run_migration()) + self.assertEqual(target.read_bytes(), self.resource(".tools/docs/tools_bridge.md")) + self.assertIn("tools_bridge.md: unchanged", self.run_migration()) + custom = (DOCS_0_1_17 / "tools_bridge.md").read_text(encoding="utf-8") + "\nOur own note.\n" + target.write_text(custom, encoding="utf-8") + output = self.run_migration() + self.assertIn("tools_bridge.md: SKIPPED: not the 0.1.17 text", output) + self.assertEqual(target.read_text(encoding="utf-8"), custom) + + +class ChainTests(MigrationCase): + def sync(self, start: str, end: str = "0.1.18") -> subprocess.CompletedProcess[str]: + shutil.copytree(ROOT / ".template_sync", self.repo / ".template_sync", dirs_exist_ok=True) + return subprocess.run( + [sys.executable, str(self.repo / ".template_sync/sync.py"), "--from-version", start, "--to-version", end, "--repository-root", str(self.repo)], + capture_output=True, + text=True, + ) + + def test_the_sync_script_runs_the_migration_from_0_1_17(self) -> None: + self.put_app(APPS_0_1_17 / "playground") + for name in WORKFLOW_NAMES: + self.workflow(name).write_bytes((WORKFLOWS_0_1_17 / name).read_bytes()) + result = self.sync("0.1.17") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("[migration 0.1.18]", result.stdout) + for name in WORKFLOW_NAMES: + self.assertEqual(to_lf(self.workflow(name).read_bytes()), to_lf(self.resource(f".github/workflows/{name}")), name) + self.assertIn(NEW_PIN.encode(), self.script("post_install.sh").read_bytes()) + again = self.sync("0.1.17") + self.assertIn("no repository changes were required", again.stdout) + + def test_the_whole_chain_from_0_1_15_reaches_the_live_template_scripts(self) -> None: + for name in (*SCRIPT_NAMES, "pre_uninstall.sh", "pre_uninstall.bat", "uninstall.sh"): + source = self.helpers.read_text(OLD_SCRIPTS / name) + replacements = self.helpers.load_replacements(self.repo) + newline = "\r\n" if "\r\n" in source else "\n" + self.script(name).write_bytes(self.helpers.render_template(source, replacements, newline).encode()) + result = self.sync("0.1.15") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + replacements = self.helpers.load_replacements(self.repo) + for name in SCRIPT_NAMES: + source = self.helpers.read_text(ROOT / "app/bash_bat_scripts" / name) + newline = "\r\n" if "\r\n" in source else "\n" + self.assertEqual(self.script(name).read_bytes(), self.helpers.render_template(source, replacements, newline).encode(), name) + + def test_a_hand_integrated_unpinned_app_ends_at_the_new_pin_through_the_chain(self) -> None: + """VLab4Mic and NucleiSky pull request 32: unpinned source; 0.1.16 pins it to the then-template tag, 0.1.18 moves that tag.""" + for app in ("vlab4mic", "ns_pr32"): + with self.subTest(app=app): + self.put_app(BRIDGE_TEST / app) + result = self.sync("0.1.15") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(f"Tools pin {OLD_PIN} -> {NEW_PIN}", result.stdout) + for name in SCRIPT_NAMES: + data = self.script(name).read_bytes() + self.assertEqual(data.count(NEW_PIN.encode()), 1, name) + self.assertNotIn(OLD_PIN.encode(), data, name) + self.assertNotIn(b"refs/heads/", data, name) + + @unittest.skipUnless(shutil.which("bash"), "bash is not available") + def test_migrated_bash_scripts_of_real_apps_have_valid_syntax_and_pin_the_new_tag_in_the_download_url(self) -> None: + for app in PinTests.APPS: + with self.subTest(app=app): + self.put_app(APPS_0_1_17 / app) + self.run_migration() + path = self.script("post_install.sh") + result = subprocess.run(["bash", "-n", str(path)], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('${LC_TOOLS_SPEC:-https://github.com/CellMigrationLab/LabConstrictor-Tools/archive/refs/tags/${LC_TOOLS_PIN}.zip}', path.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_template_sync_0_1_18_hosts.py b/tests/test_template_sync_0_1_18_hosts.py new file mode 100644 index 0000000..9a8bb8a --- /dev/null +++ b/tests/test_template_sync_0_1_18_hosts.py @@ -0,0 +1,225 @@ +"""Tests of the app-hosts step of the 0.1.18 migration: labconstrictor-app.yaml bundled by the installer, passed to `register --app-manifest`. + +Real files of apps at template 0.1.17 (tests/fixtures/template_0_1_17_apps: scripts and construct.yaml of each app's main, 13-14 Oct 2026) and the +real manifests of the apps (tests/fixtures/manifests), plus a CellTracksColab-like app (hosts: [notebook]). +""" +from __future__ import annotations + +import hashlib +import importlib.util +import re +import subprocess +import tempfile +import unittest +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path +from types import ModuleType + +from _hosts import strip_hosts + + +ROOT = Path(__file__).resolve().parents[1] +MIGRATION_PATH = ROOT / ".template_sync/migrations/v0_1_17_to_v0_1_18.py" +FIXTURES = Path(__file__).resolve().parent / "fixtures" +APPS = FIXTURES / "template_0_1_17_apps" +MANIFESTS = FIXTURES / "manifests" +SCRIPT_NAMES = ("post_install.sh", "post_install.bat") +MANIFEST_NAME = "labconstrictor-app.yaml" +ARG = "--app-manifest" +# app folder -> (manifest fixture, project folder of its extra_files) +REAL_APPS = { + "playground": ("playground_labconstrictor-app.yaml", "LabConstrictorPlayground"), + "nucleisky": ("nucleisky_labconstrictor-app.yaml", "NucleiSky"), + "vlab4mic": ("vlab4mic_labconstrictor-app.yaml", "VLab4Mic"), + "guessthecondition": ("guessthecondition_labconstrictor-app.yaml", "GuessTheCondition"), +} +NO_HOSTS_SENTENCE = "declares no hosts, so it is listed in every host; register it with --app-manifest (or --hosts) to choose" + + +def load(path: Path, name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def tree(root: Path) -> dict[str, str]: + return {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(root.rglob("*")) if p.is_file()} + + +def to_crlf(data: bytes) -> bytes: + return data.replace(b"\r\n", b"\n").replace(b"\n", b"\r\n") + + +class HostsCase(unittest.TestCase): + def setUp(self) -> None: + self.migration = load(MIGRATION_PATH, "migration_0_1_18_hosts") + self.tmp = tempfile.TemporaryDirectory() + self.repo = Path(self.tmp.name) / "app_repo" + (self.repo / "app/bash_bat_scripts").mkdir(parents=True) + + def tearDown(self) -> None: + self.tmp.cleanup() + + def script(self, name: str) -> Path: + return self.repo / "app/bash_bat_scripts" / name + + def put_app(self, app: str, manifest: str | None, crlf_sh: bool = False) -> None: + """An app repository as it is at 0.1.17: its scripts, its construct.yaml and (optionally) its labconstrictor-app.yaml.""" + for name in SCRIPT_NAMES: + data = (APPS / app / name).read_bytes() + self.script(name).write_bytes(to_crlf(data) if crlf_sh and name.endswith(".sh") else data) + (self.repo / "construct.yaml").write_bytes((APPS / app / "construct.yaml").read_bytes()) + if manifest is not None: + (self.repo / MANIFEST_NAME).write_bytes((MANIFESTS / manifest).read_bytes()) + + def run_migration(self) -> str: + buffer = StringIO() + with redirect_stdout(buffer): + self.migration.migrate(self.repo, {}) + return buffer.getvalue() + + def hosts_of(self, manifest: str) -> str: + match = re.search(r"(?m)^hosts:.*$", (MANIFESTS / manifest).read_text(encoding="utf-8")) + assert match is not None + return match.group(0) + + +class RealAppsTests(HostsCase): + def test_the_installer_bundles_the_manifest_and_the_hook_passes_it(self) -> None: + for app, (manifest, folder) in REAL_APPS.items(): + for crlf in (False, True): + with self.subTest(app=app, crlf=crlf): + self.put_app(app, manifest, crlf) + before = {n: self.script(n).read_bytes() for n in SCRIPT_NAMES} + construct_before = (self.repo / "construct.yaml").read_bytes() + output = self.run_migration() + self.assertIn("construct.yaml: updated: extra_files bundles labconstrictor-app.yaml", output) + self.assertEqual(output.count("updated: the Tools hook passes --app-manifest"), 2, output) + construct = (self.repo / "construct.yaml").read_bytes() + added = [l for l in construct.decode().splitlines() if "labconstrictor-app.yaml" in l] + self.assertEqual(added, [f"- {MANIFEST_NAME}: {folder}/{MANIFEST_NAME}"]) + self.assertEqual(len(construct.splitlines()), len(construct_before.splitlines()) + 1, "one line added, nothing else") + for name in SCRIPT_NAMES: + after = self.script(name).read_bytes() + self.assertIn(ARG.encode(), after) + # nothing but the hosts lines changed: without them (and with the pin moved) the script is what it was + self.assertEqual(strip_hosts(name, after), before[name].replace(b"v0.2.0-rc.1", b"v0.3.0-alpha.1")) + for name in SCRIPT_NAMES: + after = self.script(name).read_bytes() + self.assertEqual(b"\r\n" in after, b"\r\n" in before[name], f"{name}: line endings kept") + if b"\r\n" in after: + self.assertEqual(after.count(b"\n"), after.count(b"\r\n"), f"{name}: every line ends with CRLF") + + def test_the_migrated_scripts_name_the_app_in_the_sentence_and_are_valid_shell(self) -> None: + for app, (manifest, _) in REAL_APPS.items(): + with self.subTest(app=app): + self.put_app(app, manifest) + self.run_migration() + sh = self.script("post_install.sh") + name = re.search(r'--name "([^"]+)"', sh.read_text(encoding="utf-8")) + assert name is not None + self.assertIn(f"'{name.group(1)}' {NO_HOSTS_SENTENCE}", sh.read_text(encoding="utf-8")) + self.assertIn(f"'{name.group(1)}' {NO_HOSTS_SENTENCE}", self.script("post_install.bat").read_text(encoding="utf-8").replace("^", "")) + subprocess.run(["bash", "-n", str(sh)], check=True) + + def test_a_second_run_changes_nothing(self) -> None: + for app, (manifest, _) in REAL_APPS.items(): + with self.subTest(app=app): + self.put_app(app, manifest) + self.run_migration() + snap = tree(self.repo) + output = self.run_migration() + self.assertEqual(tree(self.repo), snap) + self.assertEqual(output.count("already passes the hosts of the app to Tools: unchanged"), 2, output) + self.assertIn("construct.yaml: already bundles labconstrictor-app.yaml: unchanged", output) + + def test_the_manifest_hosts_line_is_the_one_the_fixtures_declare(self) -> None: + """The matrix of the apps (a guard on the fixtures the other tests rely on): the hosts line of each real manifest.""" + expected = { + "playground": "hosts: [notebook, napari, fiji, qupath]", + "vlab4mic": "hosts: [notebook, napari, fiji]", + "celltrackscolab": "hosts: [notebook]", + } + for app, line in expected.items(): + self.assertEqual(self.hosts_of(f"{app}_labconstrictor-app.yaml"), line, app) + + def test_a_celltrackscolab_like_app_with_hosts_notebook_only(self) -> None: + self.put_app("nucleisky", "celltrackscolab_labconstrictor-app.yaml") + output = self.run_migration() + self.assertIn("construct.yaml: updated: extra_files bundles labconstrictor-app.yaml", output) + self.assertIn(f"- {MANIFEST_NAME}: NucleiSky/{MANIFEST_NAME}", (self.repo / "construct.yaml").read_text(encoding="utf-8")) + self.assertIn(ARG, self.script("post_install.sh").read_text(encoding="utf-8")) + + +class SafetyTests(HostsCase): + def test_without_a_manifest_construct_is_skipped_with_a_note_and_the_hook_still_logs(self) -> None: + self.put_app("vlab4mic", None) + construct = (self.repo / "construct.yaml").read_bytes() + output = self.run_migration() + self.assertEqual((self.repo / "construct.yaml").read_bytes(), construct) + self.assertRegex(output, r"construct\.yaml: SKIPPED: the repository has no labconstrictor-app\.yaml, so it is not bundled") + self.assertIn(ARG, self.script("post_install.sh").read_text(encoding="utf-8"), "the hook logs the missing file at install time") + + def test_nothing_is_ever_created_or_deleted(self) -> None: + self.put_app("guessthecondition", "guessthecondition_labconstrictor-app.yaml") + before = set(tree(self.repo)) + self.run_migration() + self.assertEqual(set(tree(self.repo)), before) + empty = Path(self.tmp.name) / "empty" + empty.mkdir() + with redirect_stdout(StringIO()): + self.migration.migrate(empty, {}) + self.assertEqual(list(empty.iterdir()), []) + + def test_unknown_shapes_are_skipped_with_a_note_and_left_unchanged(self) -> None: + self.put_app("nucleisky", "nucleisky_labconstrictor-app.yaml") + flow = "name: X\nversion: 1\nextra_files: [a, b]\n" + (self.repo / "construct.yaml").write_text(flow, encoding="utf-8") + sh = self.script("post_install.sh") + sh.write_text(sh.read_text(encoding="utf-8").replace("lc_note()", "my_note()"), encoding="utf-8") + bat = self.script("post_install.bat") + bat.write_bytes(bat.read_bytes().replace(b"IF NOT ERRORLEVEL 1 \"%PYTHON_EXE%\" -m labconstrictor_tools register", b"CALL \"%PYTHON_EXE%\" -m labconstrictor_tools register")) + before = tree(self.repo) + output = self.run_migration() + self.assertEqual(tree(self.repo)["construct.yaml"], before["construct.yaml"]) + self.assertRegex(output, r"construct\.yaml: SKIPPED: expected one `extra_files:` block") + self.assertRegex(output, r"post_install\.sh: SKIPPED: lc_note is not defined before the register line") + self.assertRegex(output, r"post_install\.bat: SKIPPED: the register line does not have the known shape") + self.assertNotIn(ARG, sh.read_text(encoding="utf-8")) + self.assertNotIn(ARG, bat.read_text(encoding="utf-8")) + + def test_a_hook_that_registers_with_its_own_hosts_is_left_alone(self) -> None: + self.put_app("nucleisky", "nucleisky_labconstrictor-app.yaml") + sh = self.script("post_install.sh") + sh.write_text(sh.read_text(encoding="utf-8").replace('--display-name "NucleiSky"', '--display-name "NucleiSky" --hosts notebook'), encoding="utf-8") + before = sh.read_bytes() + output = self.run_migration() + self.assertEqual(sh.read_bytes().replace(b"v0.2.0-rc.1", b"v0.3.0-alpha.1"), before.replace(b"v0.2.0-rc.1", b"v0.3.0-alpha.1")) + self.assertIn("post_install.sh: already passes the hosts of the app to Tools: unchanged", output) + + def test_a_mapping_that_exists_is_not_added_twice(self) -> None: + self.put_app("nucleisky", "nucleisky_labconstrictor-app.yaml") + construct = self.repo / "construct.yaml" + construct.write_text(construct.read_text(encoding="utf-8").replace("extra_files:\n", f"extra_files:\n- {MANIFEST_NAME}: Mine/{MANIFEST_NAME}\n", 1), encoding="utf-8") + before = construct.read_bytes() + output = self.run_migration() + self.assertEqual(construct.read_bytes(), before) + self.assertIn("construct.yaml: already bundles labconstrictor-app.yaml: unchanged", output) + + def test_the_hand_integrated_hooks_of_vlab4mic_and_nucleisky_pr32_get_the_argument(self) -> None: + for app in ("vlab4mic", "ns_pr32"): + with self.subTest(app=app): + for name in SCRIPT_NAMES: + self.script(name).write_bytes((FIXTURES / "bridge_test_hooks" / app / name).read_bytes()) + (self.repo / "construct.yaml").write_text("name: X\nextra_files:\n- a/Welcome.ipynb: X/Welcome.ipynb\n", encoding="utf-8") + self.run_migration() + for name in SCRIPT_NAMES: + self.assertIn(ARG, self.script(name).read_text(encoding="utf-8"), name) + subprocess.run(["bash", "-n", str(self.script("post_install.sh"))], check=True) + + +if __name__ == "__main__": + unittest.main()