Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .template_sync/manifest.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
template_version: 0.1.21
template_version: 0.1.22
migrations:
- from: 0.0.0
to: 0.0.1
Expand Down Expand Up @@ -69,3 +69,6 @@ migrations:
- from: 0.1.20
to: 0.1.21
script: migrations/v0_1_20_to_v0_1_21.py
- from: 0.1.21
to: 0.1.22
script: migrations/v0_1_21_to_v0_1_22.py
167 changes: 167 additions & 0 deletions .template_sync/migrations/v0_1_21_to_v0_1_22.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
"""Template 0.1.21 -> 0.1.22: Windows computers with an NVIDIA driver can install TensorFlow apps; the version cell reads the version file.

Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_22.md

What this changes in an app repository (every step is independent, idempotent and never deletes anything):

1. .tools/python/merge_requirements.py. The GPU requirements file rewrote `tensorflow` to `tensorflow[and-cuda]` for every system. The
extra pulls nvidia-* wheels that exist only for Linux and carries no platform marker, so the universal lock pinned them for Windows
too, and `pip install` failed on every Windows computer with an NVIDIA driver (post_install.bat picks the GPU file there). The GPU
file now has `tensorflow[and-cuda]==X ; sys_platform == "linux"` and `tensorflow==X ; sys_platform != "linux"`. Three known blocks
are changed (docstring, a new function before build_gpu_lines, the tensorflow branch of build_gpu_lines).
2. .tools/docs/notebook_upload.md. The version cell snippet looked the notebook up at the top level of notebook_latest_versions.yaml,
which sync_notebook_versions.py writes grouped by folder, so Colab users always read "not listed". The one known line is replaced.
Notebooks that already contain the old snippet are not edited (see the design note).

Rules this migration keeps: surgical (only known blocks), idempotent (a second run reports no changes), never raises for a customised
file (the sync pull request must still be created), the line endings of each file are kept, nothing is deleted.
"""
from __future__ import annotations

import importlib.util
from pathlib import Path
from typing import Any


DOCS_URL = "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/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_22.md"
TAG = "[migration 0.1.22]"
MERGE = Path(".tools/python/merge_requirements.py")
UPLOAD_DOC = Path(".tools/docs/notebook_upload.md")

# merge_requirements.py: (0.1.21 text, 0.1.22 text), LF.
MERGE_DOC_OLD = (
"GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` and adds the PyTorch\n"
"CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present.\n"
)
MERGE_DOC_NEW = (
"GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` on Linux only (TensorFlow's CUDA\n"
"wheels exist only for Linux; other systems keep plain `tensorflow`) and adds the\n"
"PyTorch CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present.\n"
)
MERGE_ANCHOR = "def build_gpu_lines(final_lines: list[str]) -> list[str]:\n"
MERGE_FUNCTION = (
"# TensorFlow's `and-cuda` extra pulls nvidia-* wheels that exist only for Linux (nvidia-nccl-cu12 has never had a Windows or macOS\n"
'# wheel), and the extra itself carries no platform marker. Without a marker the universal lock of requirements_gpu.txt pins them for\n'
'# every system, so `pip install` fails on a Windows computer with an NVIDIA driver (the installer picks the GPU file there). TensorFlow\n'
'# has had no native Windows GPU build since 2.10, so Windows and macOS keep plain `tensorflow` in the GPU file.\n'
'GPU_TENSORFLOW_PLATFORM = \'sys_platform == "linux"\'\n'
'NON_GPU_TENSORFLOW_PLATFORM = \'sys_platform != "linux"\'\n'
'\n'
'\n'
'def _with_platform(marker: str, platform: str) -> str:\n'
' """Return ` ; <platform>`, combined with the line\'s own marker (`; <marker>`) when it has one."""\n'
' existing = marker.split(";", 1)[1].strip() if ";" in marker else ""\n'
' return f" ; ({existing}) and {platform}" if existing else f" ; {platform}"\n'
'\n'
'\n'
'def tensorflow_gpu_lines(line: str) -> list[str]:\n'
' """Return the GPU-file lines for a tensorflow requirement: `[and-cuda]` on Linux, the plain requirement elsewhere.\n'
'\n'
' A line that is not a tensorflow requirement is returned unchanged (as a one-item list).\n'
' """\n'
' match = pkg_re.match(line)\n'
' if match:\n'
' if match.group("name").lower() != "tensorflow":\n'
' return [line]\n'
' name, extras = match.group("name"), match.group("extras") or ""\n'
' spec = f"{match.group(\'op\')}{match.group(\'ver\')}"\n'
' else:\n'
' match = PLAIN_PKG_RE.match(line)\n'
' if not match or match.group("name").lower() != "tensorflow":\n'
' return [line]\n'
' name, extras, spec = match.group("name"), match.group("extras") or "", ""\n'
' marker = match.group("marker") or ""\n'
' cuda_extras = merge_requirement_extras(extras, "and-cuda")\n'
' return [\n'
' f"{name}{cuda_extras}{spec}{_with_platform(marker, GPU_TENSORFLOW_PLATFORM)}",\n'
' f"{name}{extras}{spec}{_with_platform(marker, NON_GPU_TENSORFLOW_PLATFORM)}",\n'
' ]\n'
'\n'
'\n'
)
MERGE_BODY_OLD = (
' if package_name == "tensorflow":\n'
" line = rewrite_tensorflow_for_gpu(line)\n"
" if package_name in TORCH_GPU_PACKAGES:\n"
" has_torch_package = True\n"
)
MERGE_BODY_NEW = (
" if package_name in TORCH_GPU_PACKAGES:\n"
" has_torch_package = True\n"
' if package_name == "tensorflow":\n'
" gpu_lines.extend(tensorflow_gpu_lines(line))\n"
" continue\n"
)
MERGE_DONE = "def tensorflow_gpu_lines("

UPLOAD_OLD = ' latest_version = config.get(notebook_name, "")\n'
UPLOAD_NEW = (
' # notebook_latest_versions.yaml groups the notebooks by folder: {"<folder>": {"<notebook>": "<version>"}}\n'
" latest_version = next((str(group[notebook_name]) for group in (config or {}).values()\n"
' if isinstance(group, dict) and notebook_name in group), "")\n'
)
UPLOAD_DONE = "groups the notebooks by folder"


def log(message: str) -> None:
print(f"{TAG} {message}")


def _load_helpers() -> Any:
"""Reuse the helpers of the 0.1.10 migration (newline-preserving file IO)."""
path = Path(__file__).resolve().parent / "v0_1_9_to_v0_1_10.py"
spec = importlib.util.spec_from_file_location("template_sync_helpers_0_1_22", path)
if spec is None or spec.loader is None:
raise RuntimeError(f"Unable to load {path}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module


def _patch_merge(text: str, newline: str) -> tuple[str, str]:
"""Return (new text, status) for merge_requirements.py; the code blocks change together or not at all."""
if MERGE_DONE in text:
return text, "unchanged: the GPU file already limits tensorflow[and-cuda] to Linux"
anchor, body_old, body_new = (s.replace("\n", newline) for s in (MERGE_ANCHOR, MERGE_BODY_OLD, MERGE_BODY_NEW))
if text.count(anchor) != 1 or text.count(body_old) != 1:
return text, f"SKIPPED: the 0.1.21 build_gpu_lines was not found once (customised file); see {DOCS_URL} for the change to make by hand"
function = MERGE_FUNCTION.replace("\n", newline)
text = text.replace(anchor, function + anchor, 1).replace(body_old, body_new, 1)
doc_old, doc_new = MERGE_DOC_OLD.replace("\n", newline), MERGE_DOC_NEW.replace("\n", newline)
if text.count(doc_old) == 1:
text = text.replace(doc_old, doc_new, 1)
return text, "updated: tensorflow[and-cuda] only on Linux in the GPU file"


def _patch_upload(text: str, newline: str) -> tuple[str, str]:
if UPLOAD_DONE in text:
return text, "unchanged: the version cell already reads the grouped version file"
old = UPLOAD_OLD.replace("\n", newline)
if text.count(old) != 1:
return text, f"SKIPPED: the 0.1.21 version lookup line was not found once (customised file). See {DOCS_URL}"
return text.replace(old, UPLOAD_NEW.replace("\n", newline), 1), "updated: the version cell reads the grouped version file"


def migrate(repo_root: Path, context: dict[str, Any]) -> None:
"""Apply both changes (idempotent, surgical, never raises for a customised or missing file)."""
_ = context
helpers = _load_helpers()
changed = 0
for relative, patch in ((MERGE, _patch_merge), (UPLOAD_DOC, _patch_upload)):
path = repo_root / relative
if not path.is_file():
log(f"{relative}: SKIPPED, file not found in this repository (nothing was created).")
continue
try:
text = helpers.read_text(path)
new_text, status = patch(text, "\r\n" if "\r\n" in text else "\n")
if new_text != text:
helpers.write_text(path, new_text)
changed += 1
except (OSError, ValueError, UnicodeDecodeError) as exc:
status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}"
log(f"{relative}: {status}")
if changed:
log(f"done: {changed} file(s) changed. Push a notebook change (or run 'Updates on notebook change') to regenerate the GPU lock, then release a new installer.")
else:
log("done: no repository changes were required.")
1 change: 1 addition & 0 deletions .tools/docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ This file temporarily lists all internal LabConstrictor documentation. After you
- [Template 0.1.19: src_change.yaml in every installer, Python series in setup.py](migration_0_1_19.md)
- [Template 0.1.20: lock files, no lost workflow pushes, a launcher that starts fresh](migration_0_1_20.md)
- [Template 0.1.21: no administrator prompt when uninstalling on Windows](migration_0_1_21.md)
- [Template 0.1.22: TensorFlow apps install on Windows computers with an NVIDIA GPU; the version cell finds the notebook](migration_0_1_22.md)

## Notebook authoring tips

Expand Down
68 changes: 68 additions & 0 deletions .tools/docs/migration_0_1_22.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
# Template 0.1.22: TensorFlow apps install on Windows computers with an NVIDIA GPU

Design note for the 0.1.22 migration (`.template_sync/migrations/v0_1_21_to_v0_1_22.py`). How a change reaches an installed copy is described in [migration_0_1_16.md](migration_0_1_16.md): a new installer is needed, nothing edits an existing install.

## 1. The problems

### 1.1 Windows + NVIDIA GPU: the installation of a TensorFlow app fails

`post_install.bat` installs `requirements_gpu.txt` (its lock file) when it finds `nvidia-smi`. `merge_requirements.py` wrote that file by rewriting `tensorflow` to `tensorflow[and-cuda]` for every system.

* TensorFlow's `and-cuda` extra lists `nvidia-*` packages with **no platform marker**.
* Some of them exist only as Linux wheels: `nvidia-nccl-cu12` has never had a Windows or macOS wheel, and the newest `nvidia-cublas-cu12` (12.9.3.13) is Linux-only.
* `lock_requirements.py` locks with `uv pip compile --universal`, which pins them for every system.
* On Windows, `pip install -r requirements_gpu.lock.txt` then stops with `No matching distribution found for nvidia-...`, and the installer ends with "Application requirements installation failed".

TensorFlow has had no native Windows GPU build since 2.10, so `[and-cuda]` never helped on Windows anyway. macOS was not affected: `post_install.sh` always installs the CPU file there.

Found while planning a StarDist 2D (TensorFlow 2.20) app. Reproduced with uv 0.13 for `x86_64-pc-windows-msvc`:

| Lock file | Result |
|---|---|
| 0.1.21 GPU lock | `No solution found` |
| 0.1.22 GPU lock | installs (106 packages, no `nvidia-*`) |

On Linux the 0.1.22 lock still installs `tensorflow` with the CUDA packages (119 packages).

### 1.2 The version cell always says "not listed" on Colab

The snippet in [notebook_upload.md](notebook_upload.md) did `config.get(notebook_name, "")` on `notebooks/notebook_latest_versions.yaml`. `sync_notebook_versions.py` writes that file grouped by folder (`"<folder>": {"<notebook>": "<version>"}`), so the lookup always returned `""`.

## 2. What changes

| Part | Change |
|---|---|
| `.tools/python/merge_requirements.py` | In the GPU file, each `tensorflow` line becomes two lines: `tensorflow[and-cuda]==X ; sys_platform == "linux"` and `tensorflow==X ; sys_platform != "linux"`. A line's own marker is kept and combined with `and`. uv then marks every `nvidia-*` package `sys_platform == 'linux'`. |
| `.tools/docs/notebook_upload.md` | The version cell looks the notebook up inside its folder group. |
| `.tools/docs/notebook_requirements.md` | One paragraph on what the GPU file does for TensorFlow. |

## 3. What the migration does

* **merge_requirements.py:**
* The new function is inserted before the known 0.1.21 `build_gpu_lines`.
* The tensorflow branch of `build_gpu_lines` is replaced.
* The module docstring sentence is updated when it is still the 0.1.21 text.
* The two code changes are made together or not at all.
* A file that already has `tensorflow_gpu_lines` is `unchanged`.
* A customised file is `SKIPPED`; apply section 2 by hand.
* **notebook_upload.md:** the one known lookup line is replaced; a customised file is `SKIPPED`.
* **Missing files** are not created. Line endings are kept.
* **Notebooks are not edited.** A notebook that already contains the old snippet keeps it. Replace the line `latest_version = config.get(notebook_name, "")` in its version cell with the two lines from [notebook_upload.md](notebook_upload.md).

## 4. Adopting it

1. Take this template version (the synchronization pull request).
2. Regenerate the GPU lock: push a notebook change, or run **Actions > Updates on notebook change > Run workflow**.
3. Build and release a new installer.

An installer made with an older template still fails on Windows computers with an NVIDIA GPU. A workaround for those users: rename or hide `nvidia-smi.exe` during the installation, so that the CPU file is used.

## 5. What is not verified here

* The template tests check:
* the new GPU lines;
* the migration on the 0.1.21 files (LF and CRLF);
* idempotence;
* customised files;
* the chain from older versions.
* The lock results above come from uv's resolver for each platform (`uv pip install --dry-run --python-platform ...`), not from an installer run on a Windows computer with an NVIDIA GPU.
2 changes: 2 additions & 0 deletions .tools/docs/notebook_requirements.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ Follow the instructions provided inside the notebook to validate your file. You

When you push a notebook with its `requirements.yaml`, the workflow **Updates on notebook change** merges the requirements of every notebook into `requirements.txt` (plus `requirements_gpu.txt` and, if a notebook needs them, `requirements-windows.txt`, `requirements-linux.txt` or `requirements-macos.txt`).

The installer uses `requirements_gpu.txt` when it finds an NVIDIA GPU on Windows or Linux. In that file `tensorflow` becomes `tensorflow[and-cuda]` on Linux only: TensorFlow has had no Windows GPU build since version 2.10, and its CUDA packages exist only for Linux, so Windows keeps plain `tensorflow` (CPU).

It then writes a **lock file** next to each of them (`requirements.lock.txt`, ...). A lock file lists every package the installer installs, including the packages your packages depend on, each with its exact version, for Windows, macOS and Linux. The installer installs the lock file, so every installation of a release gets the same versions.

* A lock file changes only when its requirements file, the Python version or the JupyterLab version changes. Pushing a notebook without new requirements does not move any version.
Expand Down
4 changes: 3 additions & 1 deletion .tools/docs/notebook_upload.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,9 @@ try:
content = version_response.json()['content']
decoded_content = base64.b64decode(content).decode('utf-8')
config = yaml.safe_load(decoded_content)
latest_version = config.get(notebook_name, "")
# notebook_latest_versions.yaml groups the notebooks by folder: {"<folder>": {"<notebook>": "<version>"}}
latest_version = next((str(group[notebook_name]) for group in (config or {}).values()
if isinstance(group, dict) and notebook_name in group), "")

output_area.value = (f"<b>Notebook version:</b> `{current_version}`<br>"
f"<b>Latest version available:</b> `{latest_version}`<br>")
Expand Down
48 changes: 44 additions & 4 deletions .tools/python/merge_requirements.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,9 @@
lists, in addition to the common file.

Alongside the common requirements file, it emits a GPU-targeted variant. The
GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` and adds the PyTorch
CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present.
GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` on Linux only (TensorFlow's CUDA
wheels exist only for Linux; other systems keep plain `tensorflow`) and adds the
PyTorch CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present.

It resolves simple version conflicts by choosing the newest version seen for a
given package (and emits a pinned `pkg==<newest>`). It still preserves
Expand Down Expand Up @@ -336,6 +337,44 @@ def rewrite_tensorflow_for_gpu(line: str) -> str:
return line


# TensorFlow's `and-cuda` extra pulls nvidia-* wheels that exist only for Linux (nvidia-nccl-cu12 has never had a Windows or macOS
# wheel), and the extra itself carries no platform marker. Without a marker the universal lock of requirements_gpu.txt pins them for
# every system, so `pip install` fails on a Windows computer with an NVIDIA driver (the installer picks the GPU file there). TensorFlow
# has had no native Windows GPU build since 2.10, so Windows and macOS keep plain `tensorflow` in the GPU file.
GPU_TENSORFLOW_PLATFORM = 'sys_platform == "linux"'
NON_GPU_TENSORFLOW_PLATFORM = 'sys_platform != "linux"'


def _with_platform(marker: str, platform: str) -> str:
"""Return ` ; <platform>`, combined with the line's own marker (`; <marker>`) when it has one."""
existing = marker.split(";", 1)[1].strip() if ";" in marker else ""
return f" ; ({existing}) and {platform}" if existing else f" ; {platform}"


def tensorflow_gpu_lines(line: str) -> list[str]:
"""Return the GPU-file lines for a tensorflow requirement: `[and-cuda]` on Linux, the plain requirement elsewhere.

A line that is not a tensorflow requirement is returned unchanged (as a one-item list).
"""
match = pkg_re.match(line)
if match:
if match.group("name").lower() != "tensorflow":
return [line]
name, extras = match.group("name"), match.group("extras") or ""
spec = f"{match.group('op')}{match.group('ver')}"
else:
match = PLAIN_PKG_RE.match(line)
if not match or match.group("name").lower() != "tensorflow":
return [line]
name, extras, spec = match.group("name"), match.group("extras") or "", ""
marker = match.group("marker") or ""
cuda_extras = merge_requirement_extras(extras, "and-cuda")
return [
f"{name}{cuda_extras}{spec}{_with_platform(marker, GPU_TENSORFLOW_PLATFORM)}",
f"{name}{extras}{spec}{_with_platform(marker, NON_GPU_TENSORFLOW_PLATFORM)}",
]


def build_gpu_lines(final_lines: list[str]) -> list[str]:
gpu_lines: list[str] = []
has_torch_package = False
Expand All @@ -348,10 +387,11 @@ def build_gpu_lines(final_lines: list[str]) -> list[str]:
match = pkg_re.match(line) or PLAIN_PKG_RE.match(line)
if match:
package_name = match.group("name").lower()
if package_name == "tensorflow":
line = rewrite_tensorflow_for_gpu(line)
if package_name in TORCH_GPU_PACKAGES:
has_torch_package = True
if package_name == "tensorflow":
gpu_lines.extend(tensorflow_gpu_lines(line))
continue

gpu_lines.append(line)

Expand Down
Loading
Loading