diff --git a/.template_sync/manifest.yaml b/.template_sync/manifest.yaml index 6aef785..b509e28 100644 --- a/.template_sync/manifest.yaml +++ b/.template_sync/manifest.yaml @@ -1,4 +1,4 @@ -template_version: 0.1.21 +template_version: 0.1.22 migrations: - from: 0.0.0 to: 0.0.1 @@ -69,3 +69,6 @@ migrations: - from: 0.1.20 to: 0.1.21 script: migrations/v0_1_20_to_v0_1_21.py + - from: 0.1.21 + to: 0.1.22 + script: migrations/v0_1_21_to_v0_1_22.py diff --git a/.template_sync/migrations/v0_1_21_to_v0_1_22.py b/.template_sync/migrations/v0_1_21_to_v0_1_22.py new file mode 100644 index 0000000..14969fb --- /dev/null +++ b/.template_sync/migrations/v0_1_21_to_v0_1_22.py @@ -0,0 +1,167 @@ +"""Template 0.1.21 -> 0.1.22: Windows computers with an NVIDIA driver can install TensorFlow apps; the version cell reads the version file. + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_22.md + +What this changes in an app repository (every step is independent, idempotent and never deletes anything): + +1. .tools/python/merge_requirements.py. The GPU requirements file rewrote `tensorflow` to `tensorflow[and-cuda]` for every system. The + extra pulls nvidia-* wheels that exist only for Linux and carries no platform marker, so the universal lock pinned them for Windows + too, and `pip install` failed on every Windows computer with an NVIDIA driver (post_install.bat picks the GPU file there). The GPU + file now has `tensorflow[and-cuda]==X ; sys_platform == "linux"` and `tensorflow==X ; sys_platform != "linux"`. Three known blocks + are changed (docstring, a new function before build_gpu_lines, the tensorflow branch of build_gpu_lines). +2. .tools/docs/notebook_upload.md. The version cell snippet looked the notebook up at the top level of notebook_latest_versions.yaml, + which sync_notebook_versions.py writes grouped by folder, so Colab users always read "not listed". The one known line is replaced. + Notebooks that already contain the old snippet are not edited (see the design note). + +Rules this migration keeps: surgical (only known blocks), idempotent (a second run reports no changes), never raises for a customised +file (the sync pull request must still be created), the line endings of each file are kept, nothing is deleted. +""" +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any + + +DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_22.md" +TAG = "[migration 0.1.22]" +MERGE = Path(".tools/python/merge_requirements.py") +UPLOAD_DOC = Path(".tools/docs/notebook_upload.md") + +# merge_requirements.py: (0.1.21 text, 0.1.22 text), LF. +MERGE_DOC_OLD = ( + "GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` and adds the PyTorch\n" + "CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present.\n" +) +MERGE_DOC_NEW = ( + "GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` on Linux only (TensorFlow's CUDA\n" + "wheels exist only for Linux; other systems keep plain `tensorflow`) and adds the\n" + "PyTorch CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present.\n" +) +MERGE_ANCHOR = "def build_gpu_lines(final_lines: list[str]) -> list[str]:\n" +MERGE_FUNCTION = ( + "# TensorFlow's `and-cuda` extra pulls nvidia-* wheels that exist only for Linux (nvidia-nccl-cu12 has never had a Windows or macOS\n" + '# wheel), and the extra itself carries no platform marker. Without a marker the universal lock of requirements_gpu.txt pins them for\n' + '# every system, so `pip install` fails on a Windows computer with an NVIDIA driver (the installer picks the GPU file there). TensorFlow\n' + '# has had no native Windows GPU build since 2.10, so Windows and macOS keep plain `tensorflow` in the GPU file.\n' + 'GPU_TENSORFLOW_PLATFORM = \'sys_platform == "linux"\'\n' + 'NON_GPU_TENSORFLOW_PLATFORM = \'sys_platform != "linux"\'\n' + '\n' + '\n' + 'def _with_platform(marker: str, platform: str) -> str:\n' + ' """Return ` ; `, combined with the line\'s own marker (`; `) when it has one."""\n' + ' existing = marker.split(";", 1)[1].strip() if ";" in marker else ""\n' + ' return f" ; ({existing}) and {platform}" if existing else f" ; {platform}"\n' + '\n' + '\n' + 'def tensorflow_gpu_lines(line: str) -> list[str]:\n' + ' """Return the GPU-file lines for a tensorflow requirement: `[and-cuda]` on Linux, the plain requirement elsewhere.\n' + '\n' + ' A line that is not a tensorflow requirement is returned unchanged (as a one-item list).\n' + ' """\n' + ' match = pkg_re.match(line)\n' + ' if match:\n' + ' if match.group("name").lower() != "tensorflow":\n' + ' return [line]\n' + ' name, extras = match.group("name"), match.group("extras") or ""\n' + ' spec = f"{match.group(\'op\')}{match.group(\'ver\')}"\n' + ' else:\n' + ' match = PLAIN_PKG_RE.match(line)\n' + ' if not match or match.group("name").lower() != "tensorflow":\n' + ' return [line]\n' + ' name, extras, spec = match.group("name"), match.group("extras") or "", ""\n' + ' marker = match.group("marker") or ""\n' + ' cuda_extras = merge_requirement_extras(extras, "and-cuda")\n' + ' return [\n' + ' f"{name}{cuda_extras}{spec}{_with_platform(marker, GPU_TENSORFLOW_PLATFORM)}",\n' + ' f"{name}{extras}{spec}{_with_platform(marker, NON_GPU_TENSORFLOW_PLATFORM)}",\n' + ' ]\n' + '\n' + '\n' +) +MERGE_BODY_OLD = ( + ' if package_name == "tensorflow":\n' + " line = rewrite_tensorflow_for_gpu(line)\n" + " if package_name in TORCH_GPU_PACKAGES:\n" + " has_torch_package = True\n" +) +MERGE_BODY_NEW = ( + " if package_name in TORCH_GPU_PACKAGES:\n" + " has_torch_package = True\n" + ' if package_name == "tensorflow":\n' + " gpu_lines.extend(tensorflow_gpu_lines(line))\n" + " continue\n" +) +MERGE_DONE = "def tensorflow_gpu_lines(" + +UPLOAD_OLD = ' latest_version = config.get(notebook_name, "")\n' +UPLOAD_NEW = ( + ' # notebook_latest_versions.yaml groups the notebooks by folder: {"": {"": ""}}\n' + " latest_version = next((str(group[notebook_name]) for group in (config or {}).values()\n" + ' if isinstance(group, dict) and notebook_name in group), "")\n' +) +UPLOAD_DONE = "groups the notebooks by folder" + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def _load_helpers() -> Any: + """Reuse the helpers of the 0.1.10 migration (newline-preserving file IO).""" + path = Path(__file__).resolve().parent / "v0_1_9_to_v0_1_10.py" + spec = importlib.util.spec_from_file_location("template_sync_helpers_0_1_22", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _patch_merge(text: str, newline: str) -> tuple[str, str]: + """Return (new text, status) for merge_requirements.py; the code blocks change together or not at all.""" + if MERGE_DONE in text: + return text, "unchanged: the GPU file already limits tensorflow[and-cuda] to Linux" + anchor, body_old, body_new = (s.replace("\n", newline) for s in (MERGE_ANCHOR, MERGE_BODY_OLD, MERGE_BODY_NEW)) + if text.count(anchor) != 1 or text.count(body_old) != 1: + return text, f"SKIPPED: the 0.1.21 build_gpu_lines was not found once (customised file); see {DOCS_URL} for the change to make by hand" + function = MERGE_FUNCTION.replace("\n", newline) + text = text.replace(anchor, function + anchor, 1).replace(body_old, body_new, 1) + doc_old, doc_new = MERGE_DOC_OLD.replace("\n", newline), MERGE_DOC_NEW.replace("\n", newline) + if text.count(doc_old) == 1: + text = text.replace(doc_old, doc_new, 1) + return text, "updated: tensorflow[and-cuda] only on Linux in the GPU file" + + +def _patch_upload(text: str, newline: str) -> tuple[str, str]: + if UPLOAD_DONE in text: + return text, "unchanged: the version cell already reads the grouped version file" + old = UPLOAD_OLD.replace("\n", newline) + if text.count(old) != 1: + return text, f"SKIPPED: the 0.1.21 version lookup line was not found once (customised file). See {DOCS_URL}" + return text.replace(old, UPLOAD_NEW.replace("\n", newline), 1), "updated: the version cell reads the grouped version file" + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """Apply both changes (idempotent, surgical, never raises for a customised or missing file).""" + _ = context + helpers = _load_helpers() + changed = 0 + for relative, patch in ((MERGE, _patch_merge), (UPLOAD_DOC, _patch_upload)): + path = repo_root / relative + if not path.is_file(): + log(f"{relative}: SKIPPED, file not found in this repository (nothing was created).") + continue + try: + text = helpers.read_text(path) + new_text, status = patch(text, "\r\n" if "\r\n" in text else "\n") + if new_text != text: + helpers.write_text(path, new_text) + changed += 1 + except (OSError, ValueError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + log(f"{relative}: {status}") + if changed: + log(f"done: {changed} file(s) changed. Push a notebook change (or run 'Updates on notebook change') to regenerate the GPU lock, then release a new installer.") + else: + log("done: no repository changes were required.") diff --git a/.tools/docs/README.md b/.tools/docs/README.md index 5598b0f..e95da60 100644 --- a/.tools/docs/README.md +++ b/.tools/docs/README.md @@ -33,6 +33,7 @@ This file temporarily lists all internal LabConstrictor documentation. After you - [Template 0.1.19: src_change.yaml in every installer, Python series in setup.py](migration_0_1_19.md) - [Template 0.1.20: lock files, no lost workflow pushes, a launcher that starts fresh](migration_0_1_20.md) - [Template 0.1.21: no administrator prompt when uninstalling on Windows](migration_0_1_21.md) +- [Template 0.1.22: TensorFlow apps install on Windows computers with an NVIDIA GPU; the version cell finds the notebook](migration_0_1_22.md) ## Notebook authoring tips diff --git a/.tools/docs/migration_0_1_22.md b/.tools/docs/migration_0_1_22.md new file mode 100644 index 0000000..94643fb --- /dev/null +++ b/.tools/docs/migration_0_1_22.md @@ -0,0 +1,68 @@ +# Template 0.1.22: TensorFlow apps install on Windows computers with an NVIDIA GPU + +Design note for the 0.1.22 migration (`.template_sync/migrations/v0_1_21_to_v0_1_22.py`). How a change reaches an installed copy is described in [migration_0_1_16.md](migration_0_1_16.md): a new installer is needed, nothing edits an existing install. + +## 1. The problems + +### 1.1 Windows + NVIDIA GPU: the installation of a TensorFlow app fails + +`post_install.bat` installs `requirements_gpu.txt` (its lock file) when it finds `nvidia-smi`. `merge_requirements.py` wrote that file by rewriting `tensorflow` to `tensorflow[and-cuda]` for every system. + +* TensorFlow's `and-cuda` extra lists `nvidia-*` packages with **no platform marker**. +* Some of them exist only as Linux wheels: `nvidia-nccl-cu12` has never had a Windows or macOS wheel, and the newest `nvidia-cublas-cu12` (12.9.3.13) is Linux-only. +* `lock_requirements.py` locks with `uv pip compile --universal`, which pins them for every system. +* On Windows, `pip install -r requirements_gpu.lock.txt` then stops with `No matching distribution found for nvidia-...`, and the installer ends with "Application requirements installation failed". + +TensorFlow has had no native Windows GPU build since 2.10, so `[and-cuda]` never helped on Windows anyway. macOS was not affected: `post_install.sh` always installs the CPU file there. + +Found while planning a StarDist 2D (TensorFlow 2.20) app. Reproduced with uv 0.13 for `x86_64-pc-windows-msvc`: + +| Lock file | Result | +|---|---| +| 0.1.21 GPU lock | `No solution found` | +| 0.1.22 GPU lock | installs (106 packages, no `nvidia-*`) | + +On Linux the 0.1.22 lock still installs `tensorflow` with the CUDA packages (119 packages). + +### 1.2 The version cell always says "not listed" on Colab + +The snippet in [notebook_upload.md](notebook_upload.md) did `config.get(notebook_name, "")` on `notebooks/notebook_latest_versions.yaml`. `sync_notebook_versions.py` writes that file grouped by folder (`"": {"": ""}`), so the lookup always returned `""`. + +## 2. What changes + +| Part | Change | +|---|---| +| `.tools/python/merge_requirements.py` | In the GPU file, each `tensorflow` line becomes two lines: `tensorflow[and-cuda]==X ; sys_platform == "linux"` and `tensorflow==X ; sys_platform != "linux"`. A line's own marker is kept and combined with `and`. uv then marks every `nvidia-*` package `sys_platform == 'linux'`. | +| `.tools/docs/notebook_upload.md` | The version cell looks the notebook up inside its folder group. | +| `.tools/docs/notebook_requirements.md` | One paragraph on what the GPU file does for TensorFlow. | + +## 3. What the migration does + +* **merge_requirements.py:** + * The new function is inserted before the known 0.1.21 `build_gpu_lines`. + * The tensorflow branch of `build_gpu_lines` is replaced. + * The module docstring sentence is updated when it is still the 0.1.21 text. + * The two code changes are made together or not at all. + * A file that already has `tensorflow_gpu_lines` is `unchanged`. + * A customised file is `SKIPPED`; apply section 2 by hand. +* **notebook_upload.md:** the one known lookup line is replaced; a customised file is `SKIPPED`. +* **Missing files** are not created. Line endings are kept. +* **Notebooks are not edited.** A notebook that already contains the old snippet keeps it. Replace the line `latest_version = config.get(notebook_name, "")` in its version cell with the two lines from [notebook_upload.md](notebook_upload.md). + +## 4. Adopting it + +1. Take this template version (the synchronization pull request). +2. Regenerate the GPU lock: push a notebook change, or run **Actions > Updates on notebook change > Run workflow**. +3. Build and release a new installer. + +An installer made with an older template still fails on Windows computers with an NVIDIA GPU. A workaround for those users: rename or hide `nvidia-smi.exe` during the installation, so that the CPU file is used. + +## 5. What is not verified here + +* The template tests check: + * the new GPU lines; + * the migration on the 0.1.21 files (LF and CRLF); + * idempotence; + * customised files; + * the chain from older versions. +* The lock results above come from uv's resolver for each platform (`uv pip install --dry-run --python-platform ...`), not from an installer run on a Windows computer with an NVIDIA GPU. diff --git a/.tools/docs/notebook_requirements.md b/.tools/docs/notebook_requirements.md index 9fea2b4..a79d331 100644 --- a/.tools/docs/notebook_requirements.md +++ b/.tools/docs/notebook_requirements.md @@ -70,6 +70,8 @@ Follow the instructions provided inside the notebook to validate your file. You When you push a notebook with its `requirements.yaml`, the workflow **Updates on notebook change** merges the requirements of every notebook into `requirements.txt` (plus `requirements_gpu.txt` and, if a notebook needs them, `requirements-windows.txt`, `requirements-linux.txt` or `requirements-macos.txt`). +The installer uses `requirements_gpu.txt` when it finds an NVIDIA GPU on Windows or Linux. In that file `tensorflow` becomes `tensorflow[and-cuda]` on Linux only: TensorFlow has had no Windows GPU build since version 2.10, and its CUDA packages exist only for Linux, so Windows keeps plain `tensorflow` (CPU). + It then writes a **lock file** next to each of them (`requirements.lock.txt`, ...). A lock file lists every package the installer installs, including the packages your packages depend on, each with its exact version, for Windows, macOS and Linux. The installer installs the lock file, so every installation of a release gets the same versions. * A lock file changes only when its requirements file, the Python version or the JupyterLab version changes. Pushing a notebook without new requirements does not move any version. diff --git a/.tools/docs/notebook_upload.md b/.tools/docs/notebook_upload.md index cb6b818..41a83d0 100644 --- a/.tools/docs/notebook_upload.md +++ b/.tools/docs/notebook_upload.md @@ -109,7 +109,9 @@ try: content = version_response.json()['content'] decoded_content = base64.b64decode(content).decode('utf-8') config = yaml.safe_load(decoded_content) - latest_version = config.get(notebook_name, "") + # notebook_latest_versions.yaml groups the notebooks by folder: {"": {"": ""}} + latest_version = next((str(group[notebook_name]) for group in (config or {}).values() + if isinstance(group, dict) and notebook_name in group), "") output_area.value = (f"Notebook version: `{current_version}`
" f"Latest version available: `{latest_version}`
") diff --git a/.tools/python/merge_requirements.py b/.tools/python/merge_requirements.py index 8b41c68..1fccecb 100644 --- a/.tools/python/merge_requirements.py +++ b/.tools/python/merge_requirements.py @@ -13,8 +13,9 @@ lists, in addition to the common file. Alongside the common requirements file, it emits a GPU-targeted variant. The -GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` and adds the PyTorch -CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present. +GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` on Linux only (TensorFlow's CUDA +wheels exist only for Linux; other systems keep plain `tensorflow`) and adds the +PyTorch CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present. It resolves simple version conflicts by choosing the newest version seen for a given package (and emits a pinned `pkg==`). It still preserves @@ -336,6 +337,44 @@ def rewrite_tensorflow_for_gpu(line: str) -> str: return line +# TensorFlow's `and-cuda` extra pulls nvidia-* wheels that exist only for Linux (nvidia-nccl-cu12 has never had a Windows or macOS +# wheel), and the extra itself carries no platform marker. Without a marker the universal lock of requirements_gpu.txt pins them for +# every system, so `pip install` fails on a Windows computer with an NVIDIA driver (the installer picks the GPU file there). TensorFlow +# has had no native Windows GPU build since 2.10, so Windows and macOS keep plain `tensorflow` in the GPU file. +GPU_TENSORFLOW_PLATFORM = 'sys_platform == "linux"' +NON_GPU_TENSORFLOW_PLATFORM = 'sys_platform != "linux"' + + +def _with_platform(marker: str, platform: str) -> str: + """Return ` ; `, combined with the line's own marker (`; `) when it has one.""" + existing = marker.split(";", 1)[1].strip() if ";" in marker else "" + return f" ; ({existing}) and {platform}" if existing else f" ; {platform}" + + +def tensorflow_gpu_lines(line: str) -> list[str]: + """Return the GPU-file lines for a tensorflow requirement: `[and-cuda]` on Linux, the plain requirement elsewhere. + + A line that is not a tensorflow requirement is returned unchanged (as a one-item list). + """ + match = pkg_re.match(line) + if match: + if match.group("name").lower() != "tensorflow": + return [line] + name, extras = match.group("name"), match.group("extras") or "" + spec = f"{match.group('op')}{match.group('ver')}" + else: + match = PLAIN_PKG_RE.match(line) + if not match or match.group("name").lower() != "tensorflow": + return [line] + name, extras, spec = match.group("name"), match.group("extras") or "", "" + marker = match.group("marker") or "" + cuda_extras = merge_requirement_extras(extras, "and-cuda") + return [ + f"{name}{cuda_extras}{spec}{_with_platform(marker, GPU_TENSORFLOW_PLATFORM)}", + f"{name}{extras}{spec}{_with_platform(marker, NON_GPU_TENSORFLOW_PLATFORM)}", + ] + + def build_gpu_lines(final_lines: list[str]) -> list[str]: gpu_lines: list[str] = [] has_torch_package = False @@ -348,10 +387,11 @@ def build_gpu_lines(final_lines: list[str]) -> list[str]: match = pkg_re.match(line) or PLAIN_PKG_RE.match(line) if match: package_name = match.group("name").lower() - if package_name == "tensorflow": - line = rewrite_tensorflow_for_gpu(line) if package_name in TORCH_GPU_PACKAGES: has_torch_package = True + if package_name == "tensorflow": + gpu_lines.extend(tensorflow_gpu_lines(line)) + continue gpu_lines.append(line) diff --git a/tests/test_template_sync_0_1_21.py b/tests/test_template_sync_0_1_21.py index edf6632..cbaf230 100644 --- a/tests/test_template_sync_0_1_21.py +++ b/tests/test_template_sync_0_1_21.py @@ -2,6 +2,7 @@ from __future__ import annotations import importlib.util +import re import shutil import subprocess import sys @@ -43,7 +44,8 @@ def old_text() -> str: class TemplateConsistencyTests(unittest.TestCase): def test_the_manifest_lists_the_migration_and_the_new_version(self) -> None: text = (ROOT / ".template_sync/manifest.yaml").read_text(encoding="utf-8") - self.assertIn("template_version: 0.1.21", text) + # 0.1.21 or any later version (the chain test below checks that 0.1.21 stays reachable). + self.assertRegex(text, r"template_version: 0\.1\.(2[1-9]|[3-9][0-9])\b") self.assertIn("script: migrations/v0_1_20_to_v0_1_21.py", text) def test_chains_from_every_older_version_reach_0_1_21(self) -> None: @@ -146,6 +148,9 @@ class ChainTests(MigrationCase): def test_the_sync_script_runs_the_migration_from_0_1_20(self) -> None: path = self.put(old_text()) shutil.copytree(ROOT / ".template_sync", self.repo / ".template_sync") + # sync.py refuses a target other than the manifest version: pin the copy to 0.1.21 (later templates add migrations). + manifest = self.repo / ".template_sync/manifest.yaml" + manifest.write_text(re.sub(r"template_version: [0-9.]+", "template_version: 0.1.21", manifest.read_text(encoding="utf-8")), encoding="utf-8") sync = str(self.repo / ".template_sync/sync.py") command = [sys.executable, sync, "--from-version", "0.1.20", "--to-version", "0.1.21", "--repository-root", str(self.repo)] result = subprocess.run(command, capture_output=True, text=True) diff --git a/tests/test_template_sync_0_1_22.py b/tests/test_template_sync_0_1_22.py new file mode 100644 index 0000000..4be8b7c --- /dev/null +++ b/tests/test_template_sync_0_1_22.py @@ -0,0 +1,214 @@ +"""Tests of the 0.1.22 migration: tensorflow[and-cuda] only on Linux in the GPU file; the version cell reads the grouped version file.""" +from __future__ import annotations + +import importlib.util +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path +from types import ModuleType + + +ROOT = Path(__file__).resolve().parents[1] +MIGRATION_PATH = ROOT / ".template_sync/migrations/v0_1_21_to_v0_1_22.py" +SYNC_PATH = ROOT / ".template_sync/sync.py" + + +def load(path: Path, name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +MIGRATION = load(MIGRATION_PATH, "migration_0_1_22") +MERGE_MODULE = load(ROOT / ".tools/python/merge_requirements.py", "merge_requirements_0_1_22") +MERGE = MIGRATION.MERGE +UPLOAD_DOC = MIGRATION.UPLOAD_DOC + + +def live(relative: Path) -> str: + return (ROOT / relative).read_text(encoding="utf-8") + + +def old_merge() -> str: + """The 0.1.21 merge_requirements.py: the live file with the three 0.1.22 blocks put back.""" + text = live(MERGE).replace(MIGRATION.MERGE_FUNCTION, "", 1) + text = text.replace(MIGRATION.MERGE_BODY_NEW, MIGRATION.MERGE_BODY_OLD, 1) + return text.replace(MIGRATION.MERGE_DOC_NEW, MIGRATION.MERGE_DOC_OLD, 1) + + +def old_upload() -> str: + return live(UPLOAD_DOC).replace(MIGRATION.UPLOAD_NEW, MIGRATION.UPLOAD_OLD, 1) + + +class TemplateConsistencyTests(unittest.TestCase): + def test_the_manifest_lists_the_migration_and_the_new_version(self) -> None: + text = live(Path(".template_sync/manifest.yaml")) + self.assertRegex(text, r"template_version: 0\.1\.(2[2-9]|[3-9][0-9])\b") + self.assertIn("script: migrations/v0_1_21_to_v0_1_22.py", text) + + def test_chains_from_every_older_version_reach_0_1_22(self) -> None: + sync = load(SYNC_PATH, "sync_for_0_1_22") + manifest = sync.parse_manifest(ROOT / ".template_sync/manifest.yaml") + for start in ("0.0.0", "0.1.9", "0.1.15", "0.1.20", "0.1.21"): + chain = sync.resolve_migration_chain(manifest["migrations"], start, "0.1.22") + self.assertEqual(chain[-1]["to"], "0.1.22", start) + + def test_the_design_note_is_linked(self) -> None: + self.assertIn("(migration_0_1_22.md)", live(Path(".tools/docs/README.md"))) + self.assertTrue((ROOT / ".tools/docs/migration_0_1_22.md").is_file()) + + def test_the_live_files_hold_each_new_block_once(self) -> None: + merge = live(MERGE) + self.assertEqual(merge.count(MIGRATION.MERGE_FUNCTION), 1) + self.assertEqual(merge.count(MIGRATION.MERGE_BODY_NEW), 1) + self.assertEqual(merge.count(MIGRATION.MERGE_DOC_NEW), 1) + self.assertNotIn(MIGRATION.MERGE_BODY_OLD, merge) + self.assertEqual(live(UPLOAD_DOC).count(MIGRATION.UPLOAD_NEW), 1) + self.assertNotIn(MIGRATION.UPLOAD_OLD, live(UPLOAD_DOC)) + + +class GpuLinesTests(unittest.TestCase): + def test_a_pinned_tensorflow_gets_cuda_on_linux_only(self) -> None: + self.assertEqual( + MERGE_MODULE.build_gpu_lines(["numpy==2.1.3", "tensorflow==2.20.0"]), + ["numpy==2.1.3", 'tensorflow[and-cuda]==2.20.0 ; sys_platform == "linux"', 'tensorflow==2.20.0 ; sys_platform != "linux"'], + ) + + def test_extras_and_markers_are_kept(self) -> None: + lines = MERGE_MODULE.tensorflow_gpu_lines('tensorflow[foo]>=2.16 ; python_version >= "3.10"') + self.assertEqual( + lines, + [ + 'tensorflow[foo,and-cuda]>=2.16 ; (python_version >= "3.10") and sys_platform == "linux"', + 'tensorflow[foo]>=2.16 ; (python_version >= "3.10") and sys_platform != "linux"', + ], + ) + + def test_an_unpinned_tensorflow_and_other_packages(self) -> None: + self.assertEqual( + MERGE_MODULE.tensorflow_gpu_lines("tensorflow"), + ['tensorflow[and-cuda] ; sys_platform == "linux"', 'tensorflow ; sys_platform != "linux"'], + ) + self.assertEqual(MERGE_MODULE.tensorflow_gpu_lines("tensorflow-io==0.37.1"), ["tensorflow-io==0.37.1"]) + + def test_the_torch_index_is_still_added(self) -> None: + lines = MERGE_MODULE.build_gpu_lines(["tensorflow==2.20.0", "torch==2.5.0"]) + self.assertTrue(lines[0].startswith("--extra-index-url https://download.pytorch.org/whl/")) + self.assertIn("torch==2.5.0", lines) + + def test_every_gpu_line_is_a_valid_requirement(self) -> None: + pattern = re.compile(r'^tensorflow(\[[a-z,-]+\])?(==2\.20\.0)? ; sys_platform [!=]= "linux"$') + for line in MERGE_MODULE.tensorflow_gpu_lines("tensorflow==2.20.0"): + self.assertRegex(line, pattern) + + +class VersionCellSnippetTests(unittest.TestCase): + def _lookup(self, config: object, notebook_name: str) -> str: + """Run the lookup lines of the live snippet.""" + code = "\n".join(line.strip() for line in MIGRATION.UPLOAD_NEW.splitlines() if not line.strip().startswith("#")) + code = code.replace("\n", " ", 1) # the generator expression spans two lines + scope: dict[str, object] = {"config": config, "notebook_name": notebook_name} + exec(code, scope) # noqa: S102 - the snippet's own two lines, from this repository + return str(scope["latest_version"]) + + def test_the_grouped_file_written_by_sync_notebook_versions_is_read(self) -> None: + self.assertEqual(self._lookup({"StarDist_2D": {"StarDist_2D_ZeroCostDL4Mic": "1.0.0"}}, "StarDist_2D_ZeroCostDL4Mic"), "1.0.0") + + def test_a_missing_notebook_or_an_empty_file_gives_an_empty_version(self) -> None: + self.assertEqual(self._lookup({"A": {"x": "1"}}, "y"), "") + self.assertEqual(self._lookup(None, "y"), "") + + +class MigrationCase(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.repo = Path(self._tmp.name) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def put(self, relative: Path, text: str, newline: str = "\n") -> Path: + path = self.repo / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(text.replace("\n", newline).encode("utf-8")) + return path + + def run_migration(self) -> str: + buffer = StringIO() + with redirect_stdout(buffer): + MIGRATION.migrate(self.repo, {}) + return buffer.getvalue() + + +class MigrationBehaviourTests(MigrationCase): + def test_a_0_1_21_app_gets_both_changes_and_a_second_run_changes_nothing(self) -> None: + merge = self.put(MERGE, old_merge()) + upload = self.put(UPLOAD_DOC, old_upload()) + output = self.run_migration() + self.assertIn(f"{MERGE}: updated", output) + self.assertIn(f"{UPLOAD_DOC}: updated", output) + self.assertEqual(merge.read_text(encoding="utf-8"), live(MERGE)) + self.assertEqual(upload.read_text(encoding="utf-8"), live(UPLOAD_DOC)) + before = (merge.read_bytes(), upload.read_bytes()) + again = self.run_migration() + self.assertIn("unchanged", again) + self.assertIn("no repository changes were required", again) + self.assertEqual((merge.read_bytes(), upload.read_bytes()), before) + + def test_crlf_files_keep_their_line_endings(self) -> None: + merge = self.put(MERGE, old_merge(), newline="\r\n") + self.assertIn(f"{MERGE}: updated", self.run_migration()) + data = merge.read_bytes() + self.assertNotIn(b"\n", data.replace(b"\r\n", b"")) + self.assertEqual(data.decode("utf-8").replace("\r\n", "\n"), live(MERGE)) + + def test_a_customised_merge_script_is_skipped_and_left_as_it_is(self) -> None: + custom = old_merge().replace(" line = rewrite_tensorflow_for_gpu(line)\n", " line = my_rewrite(line)\n") + merge = self.put(MERGE, custom) + output = self.run_migration() + self.assertIn(f"{MERGE}: SKIPPED", output) + self.assertEqual(merge.read_text(encoding="utf-8"), custom) + + def test_a_customised_docstring_does_not_block_the_code_change(self) -> None: + custom = old_merge().replace(MIGRATION.MERGE_DOC_OLD, "My own description.\n") + merge = self.put(MERGE, custom) + self.assertIn(f"{MERGE}: updated", self.run_migration()) + self.assertIn(MIGRATION.MERGE_DONE, merge.read_text(encoding="utf-8")) + + def test_missing_files_are_not_created(self) -> None: + output = self.run_migration() + self.assertIn(f"{MERGE}: SKIPPED, file not found", output) + self.assertFalse((self.repo / MERGE).exists()) + self.assertFalse((self.repo / UPLOAD_DOC).exists()) + + def test_the_migrated_merge_script_compiles_and_writes_the_new_gpu_lines(self) -> None: + self.put(MERGE, old_merge()) + self.run_migration() + module = load(self.repo / MERGE, "migrated_merge_0_1_22") + self.assertIn('tensorflow==2.20.0 ; sys_platform != "linux"', module.build_gpu_lines(["tensorflow==2.20.0"])) + + +class ChainTests(MigrationCase): + def test_the_sync_script_runs_the_migration_from_0_1_21(self) -> None: + merge = self.put(MERGE, old_merge()) + shutil.copytree(ROOT / ".template_sync", self.repo / ".template_sync") + sync = str(self.repo / ".template_sync/sync.py") + command = [sys.executable, sync, "--from-version", "0.1.21", "--to-version", "0.1.22", "--repository-root", str(self.repo)] + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("[migration 0.1.22]", result.stdout) + self.assertEqual(merge.read_text(encoding="utf-8"), live(MERGE)) + again = subprocess.run(command, capture_output=True, text=True) + self.assertIn("no repository changes were required", again.stdout) + + +if __name__ == "__main__": + unittest.main()