From d4543b26a911ef8e9bc8379a731e5497e1b48d83 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 07:54:53 +0000 Subject: [PATCH 1/4] PP-8d: the notebook runs on Google Colab and follows LabConstrictor's portability guide - Code cells start with "# @title" (all hidden: an app-like widget notebook). - Step 0: Google Drive mounted and Colab's custom widget manager enabled inside `if "google.colab" in sys.modules:`; locally an environment report. The GitHub install is pinned to RELEASE_TAG v0.4.0rc2; current_version/notebook_name match notebooks/notebook_latest_versions.yaml. ipywidgets 7 is upgraded, or a restart asked for when it is already loaded. - Colab / JupyterLab callouts at Steps 0 and 1; a stale-lock entry in Common problems; the removed TIFF conversion card is no longer described. - Browse... without a dialog (Colab) says how to copy a path (notebook_style no_dialog_status). - tests/test_notebook_file.py: the portability rules, the pin, and the notebook run in a real kernel (nbclient, ipykernel in the dev extra). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- notebooks/MorphoBricks_Preprocessing.ipynb | 150 ++++++++++--- notebooks/notebook_latest_versions.yaml | 3 + pyproject.toml | 3 + .../notebook_style.py | 31 +++ src/morphobricks_preprocessing/notebook_ui.py | 12 +- tests/test_notebook_file.py | 199 ++++++++++++++++++ tests/test_notebook_setup.py | 73 ++++++- tests/test_notebook_ui.py | 82 +------- tools/code_rules_baseline.json | 5 +- 9 files changed, 431 insertions(+), 127 deletions(-) create mode 100644 notebooks/notebook_latest_versions.yaml create mode 100644 tests/test_notebook_file.py diff --git a/notebooks/MorphoBricks_Preprocessing.ipynb b/notebooks/MorphoBricks_Preprocessing.ipynb index f24ac29d..05894695 100644 --- a/notebooks/MorphoBricks_Preprocessing.ipynb +++ b/notebooks/MorphoBricks_Preprocessing.ipynb @@ -79,6 +79,16 @@ "Click the cell below and press **Shift + Enter** (or the ▶ button in the toolbar). A green box\n", "appears when everything is ready.\n", "\n", + "> **Colab users:** Step 0 connects your Google Drive (allow access in the window that opens)\n", + "> and installs MorphoBricks preprocessing **v0.4.0rc2** from GitHub, which takes a minute or\n", + "> two. Keep your experiment folder and your results on Google Drive: in this notebook your Drive\n", + "> is the folder **/content/drive/MyDrive**. Everything else on the runtime is wiped when it\n", + "> disconnects; then run Step 0 again.\n", + ">\n", + "> **JupyterLab users:** Step 0 uses MorphoBricks preprocessing as installed for this notebook's\n", + "> kernel (or the copy of the repository around the notebook), installs what is missing, and\n", + "> prints the Python version and working folder. Nothing is mounted.\n", + "\n", "> 💡 **Tip:** the code of each cell is hidden; you only need to run the cells, from top to bottom,\n", "> one after the other." ] @@ -99,16 +109,69 @@ }, "outputs": [], "source": [ - "# @markdown ### Run to start the preprocessing notebook\n", + "# @title Run to start the preprocessing notebook\n", + "import platform\n", "import subprocess\n", "import sys\n", "from importlib import import_module, invalidate_caches\n", + "from importlib.metadata import PackageNotFoundError, version\n", "from importlib.util import find_spec\n", "from pathlib import Path\n", "\n", - "# Current version of the notebook the user is running\n", - "current_version = \"0.0.1\"\n", + "# The notebook's version and name, as LabConstrictor's version check reads them\n", + "# (notebooks/notebook_latest_versions.yaml lists the latest version).\n", + "current_version = \"0.4.0rc2\"\n", "notebook_name = \"MorphoBricks_Preprocessing\"\n", + "# The release installed from GitHub when the package is missing (Google Colab, or a\n", + "# downloaded notebook). Set to the new tag, with current_version, before each release is\n", + "# tagged, so a tagged notebook installs its own release (docs/plan.md, release checklist).\n", + "RELEASE_TAG = \"v0.4.0rc2\"\n", + "REPOSITORY = \"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/CellMigrationLab/morphobricks-preprocessing.git\"\n", + "\n", + "if \"google.colab\" in sys.modules:\n", + " # Colab only: Google Drive holds the data and the results, which outlive the runtime.\n", + " from google.colab import drive, output\n", + "\n", + " print(\n", + " \"Google Colab: connecting Google Drive. Allow access in the window that opens; \"\n", + " \"your Drive is then the folder /content/drive/MyDrive.\",\n", + " flush=True,\n", + " )\n", + " try:\n", + " drive.mount(\"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/content/drive\")\n", + " except Exception as exc: # a refused or interrupted authorisation\n", + " raise RuntimeError(\n", + " \"Google Drive could not be connected. Run Step 0 again and allow access to \"\n", + " \"your Drive; your experiment and output folders must be on Drive.\"\n", + " ) from exc\n", + " # Colab ships ipywidgets 7; this notebook's widgets need ipywidgets 8's front end.\n", + " output.enable_custom_widget_manager()\n", + "else:\n", + " print(\n", + " f\"Local Jupyter: Python {platform.python_version()} on {platform.platform()}\\n\"\n", + " f\"Working folder: {Path.cwd()}\",\n", + " flush=True,\n", + " )\n", + "\n", + "\n", + "def _missing_requirements() -> list[str]:\n", + " \"\"\"Return the notebook's requirements this kernel cannot import (or too old).\"\"\"\n", + " required = (\n", + " \"morphobricks_preprocessing\",\n", + " \"morphobricks_normalization\",\n", + " \"numpy\",\n", + " \"tifffile\",\n", + " \"ipywidgets\",\n", + " \"matplotlib\",\n", + " )\n", + " missing = [name for name in required if find_spec(name) is None]\n", + " try:\n", + " widgets_major = int(version(\"ipywidgets\").split(\".\")[0])\n", + " except (PackageNotFoundError, ValueError):\n", + " widgets_major = 8\n", + " if widgets_major < 8 and \"ipywidgets\" not in missing:\n", + " missing.append(\"ipywidgets 8\")\n", + " return missing\n", "\n", "\n", "def _start_preprocessing_notebook() -> object:\n", @@ -144,23 +207,15 @@ " origin = f\"the local checkout at {checkout}\"\n", " elif find_spec(\"morphobricks_preprocessing\") is not None:\n", " install_target = [\"morphobricks-preprocessing[notebook]\"]\n", - " origin = \"the installed package\"\n", + " try:\n", + " origin = f\"the installed package, version {version('morphobricks-preprocessing')}\"\n", + " except PackageNotFoundError:\n", + " origin = \"the installed package\"\n", " else:\n", - " install_target = [\n", - " \"morphobricks-preprocessing[notebook] @ \"\n", - " \"git+https://github.com/CellMigrationLab/morphobricks-preprocessing.git\"\n", - " ]\n", - " origin = \"the GitHub installation\"\n", + " install_target = [f\"morphobricks-preprocessing[notebook] @ git+{REPOSITORY}@{RELEASE_TAG}\"]\n", + " origin = f\"release {RELEASE_TAG} from GitHub\"\n", "\n", - " required = (\n", - " \"morphobricks_preprocessing\",\n", - " \"morphobricks_normalization\",\n", - " \"numpy\",\n", - " \"tifffile\",\n", - " \"ipywidgets\",\n", - " \"matplotlib\",\n", - " )\n", - " missing = [name for name in required if find_spec(name) is None]\n", + " missing = _missing_requirements()\n", " if missing:\n", " print(f\"Installing notebook requirements ({', '.join(missing)})...\", flush=True)\n", " try:\n", @@ -182,14 +237,23 @@ " \"Installing from GitHub also needs Git.\"\n", " ) from exc\n", " invalidate_caches()\n", - " missing = [name for name in required if find_spec(name) is None]\n", + " missing = _missing_requirements()\n", " if missing:\n", " raise RuntimeError(\n", " f\"Installation finished, but this kernel still cannot find {', '.join(missing)}. \"\n", " \"Restart the kernel and run Step 0 again.\"\n", " )\n", + " loaded_widgets = str(getattr(sys.modules.get(\"ipywidgets\"), \"__version__\", \"8\")).split(\".\")[0]\n", + " if loaded_widgets.isdigit() and int(loaded_widgets) < 8:\n", + " raise RuntimeError(\n", + " \"This kernel loaded ipywidgets 7 before ipywidgets 8 was installed. Restart the \"\n", + " \"kernel (on Colab: Runtime > Restart session), then run Step 0 again.\"\n", + " )\n", "\n", - " print(f\"Using {origin}.\\nNotebook Python: {sys.executable}\", flush=True)\n", + " print(\n", + " f\"Using {origin}.\\nNotebook {notebook_name} {current_version}; Python: {sys.executable}\",\n", + " flush=True,\n", + " )\n", " try:\n", " interface = import_module(\"morphobricks_preprocessing.notebook_ui\")\n", " except ImportError as exc:\n", @@ -244,6 +308,23 @@ " what to check. Different numbers of channels, or different channel names at the same channel\n", " number, are errors: every image needs the same channels in the same order.\n", "\n", + "> **Colab users:**\n", + "> - Paste paths that start with **/content/drive/MyDrive/**. **Browse…** cannot open a dialog\n", + "> on Colab: open the **Files** panel in the left bar, right-click a folder, choose\n", + "> **Copy path** and paste it.\n", + "> - Keep the **Output folder** on Google Drive and outside the experiment folder, so the results\n", + "> survive when the runtime disconnects. The default, next to the experiment folder, is right\n", + "> when the experiment is on Drive.\n", + "> - Reading from Google Drive is slower than from the runtime's own disk. For a large dataset you\n", + "> may copy the experiment folder to the runtime's disk first (see the user guide, \"Google\n", + "> Colab\"); then type an **Output folder** on Drive, because the default would be next to the\n", + "> copy and lost with the runtime.\n", + "> - If the runtime disconnected during a run, the next check may say another run is using the\n", + "> output folder: see \"Common problems\" at the end.\n", + ">\n", + "> **JupyterLab users:** paste the path or use **Browse…**, which opens your computer's folder\n", + "> dialog. The default output folder is next to the experiment folder.\n", + "\n", "> 💡 **Tip:** to change a label, rename the folder and click **Check folder** again.\n", ">\n", "> 🩹 **A damaged image?** Click **Exclude these images and check again** to leave it out, or list\n", @@ -266,7 +347,7 @@ }, "outputs": [], "source": [ - "# @markdown ### Run to show the folder check\n", + "# @title Run to show the folder check\n", "notebook.folder_step()" ] }, @@ -330,7 +411,7 @@ }, "outputs": [], "source": [ - "# @markdown ### Run to review the axis roles\n", + "# @title Run to review the axis roles\n", "notebook.axis_step()" ] }, @@ -352,12 +433,6 @@ "> **Use the recommended settings** to change the number of bins, the memory limit, or to recompute\n", "> everything.\n", ">\n", - "> 🗂️ **(Optional) Image conversion:** by default the images are read directly and nothing extra is\n", - "> saved. You can instead **convert the TIFF images to OME-Zarr** first; the histograms are then computed\n", - "> from the converted copies, which are stored in the **ome_zarr** folder of the output. ⚠️ This stores\n", - "> the images a second time, so the space they occupy is duplicated; the card shows how much space is\n", - "> needed and lets you choose the chunk size, compression and pyramid levels. It needs the zarr package.\n", - ">\n", "> 🛡️ Nothing is written unless the whole run succeeds.\n", "\n", "After the run, **Intensity checks** compares the images using the table just written (no image is\n", @@ -393,7 +468,7 @@ }, "outputs": [], "source": [ - "# @markdown ### Run to show the settings and the run button\n", + "# @title Run to show the settings and the run button\n", "notebook.run_step()" ] }, @@ -436,7 +511,7 @@ }, "outputs": [], "source": [ - "# @markdown ### Run to show the statistics\n", + "# @title Run to show the statistics\n", "notebook.results_step()" ] }, @@ -452,8 +527,6 @@ " **run_manifest.json**.\n", "- [ ] Keep these files together; the next MorphoBricks brick reads them.\n", "- [ ] Rerun this notebook whenever you add, change or remove images.\n", - "- [ ] If you chose to convert the TIFF images, the output folder also has an **ome_zarr** folder\n", - " with one OME-Zarr copy per TIFF image.\n", "\n", "### Common problems\n", "\n", @@ -499,6 +572,17 @@ "intensity check, never an error. Look at the image: it may be out of focus, contain debris or\n", "have been acquired with other settings. If it is faulty, exclude it and run again.\n", "\n", + "**\"Another run is using this output folder\"**: wait for that run to finish. If no run is going on\n", + "(on Colab, the runtime disconnected during a run), remove its lock by running this in a new code\n", + "cell, with your output folder, then check the folder again. Never do this while another computer or\n", + "Colab session may be using the same output folder.\n", + "\n", + "```python\n", + "from morphobricks_preprocessing.run import remove_stale_lock\n", + "\n", + "remove_stale_lock(\"path/to/your/output_folder\")\n", + "```\n", + "\n", "**The preview looks like stripes**: X or Y is on the wrong dimension; change the roles in step 2.\n", "\n", "More help: the [user guide](../docs/user_guide.md)." @@ -507,7 +591,7 @@ ], "metadata": { "kernelspec": { - "display_name": "morpho_pre", + "display_name": "Python 3", "language": "python", "name": "python3" }, diff --git a/notebooks/notebook_latest_versions.yaml b/notebooks/notebook_latest_versions.yaml new file mode 100644 index 00000000..8420b39d --- /dev/null +++ b/notebooks/notebook_latest_versions.yaml @@ -0,0 +1,3 @@ +# Latest version of each notebook, read by LabConstrictor's version check by notebook_name. +# Equal to the notebook's current_version and its RELEASE_TAG without the "v" (README, "Notebook"). +"MorphoBricks_Preprocessing": "0.4.0rc2" diff --git a/pyproject.toml b/pyproject.toml index da511bd1..ca9717ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,9 @@ zarr = [ dev = [ "pytest>=7.4", "ruff>=0.6", + # Running the notebook file in a real kernel (tests/test_notebook_file.py). + "nbclient>=0.7", + "ipykernel>=6", ] [project.scripts] diff --git a/src/morphobricks_preprocessing/notebook_style.py b/src/morphobricks_preprocessing/notebook_style.py index a495392f..21d69f03 100644 --- a/src/morphobricks_preprocessing/notebook_style.py +++ b/src/morphobricks_preprocessing/notebook_style.py @@ -384,3 +384,34 @@ def ask_open_file(initial: str = "", filetypes=(("All files", "*.*"),)) -> str | finally: root.destroy() return os.path.normpath(chosen) if chosen else "" + + +def no_dialog_status(dialog: str, target: str) -> str: + """HTML of the status card shown when no system dialog can open (Google Colab, a server). + + The Browse… buttons open a dialog on the computer running the kernel, which a hosted + runtime such as Google Colab does not have; the paths are pasted instead. + + Parameters + ---------- + dialog + What the dialog chooses, ``"folder"`` or ``"file"``. + target + What to paste the path of, e.g. ``"the folder"``. + + Returns + ------- + str + An amber status card naming how to copy a path on Google Colab. + """ + return status( + "warn", + f"The {dialog} dialog could not be opened here.", + [ + f"Paste the path of {target} into the box instead.", + f"On Google Colab: open the Files panel in the left bar, right-click the " + f"{dialog}, choose Copy path and paste it. Google Drive is the folder " + "/content/drive/MyDrive.", + ], + raw=True, + ) diff --git a/src/morphobricks_preprocessing/notebook_ui.py b/src/morphobricks_preprocessing/notebook_ui.py index 8224f42a..8d106918 100644 --- a/src/morphobricks_preprocessing/notebook_ui.py +++ b/src/morphobricks_preprocessing/notebook_ui.py @@ -1088,11 +1088,7 @@ def _update_output_hint(self) -> None: def _browse(self, target: widgets.Text) -> None: chosen = ui.ask_directory(ui.clean_path(target.value)) if chosen is None: - self.folder_status.value = ui.status( - "warn", - "The folder dialog could not be opened here.", - ["Paste the path of the folder into the box instead."], - ) + self.folder_status.value = ui.no_dialog_status("folder", "the folder") elif chosen: target.value = chosen @@ -2196,11 +2192,7 @@ def _browse_csv(self) -> None: start = str(Path(current).parent) if current else ui.clean_path(self.folder.value) chosen = ui.ask_open_file(start, (("Histogram tables", "*.csv"), ("All files", "*.*"))) if chosen is None: - self.stats_status.value = ui.status( - "warn", - "The file dialog could not be opened here.", - ["Paste the path of the plane_histograms.csv file into the box instead."], - ) + self.stats_status.value = ui.no_dialog_status("file", "the plane_histograms.csv file") elif chosen: self.csv_file.value = chosen diff --git a/tests/test_notebook_file.py b/tests/test_notebook_file.py new file mode 100644 index 00000000..5fa93bf6 --- /dev/null +++ b/tests/test_notebook_file.py @@ -0,0 +1,199 @@ +"""The notebook file: LabConstrictor's portability rules, its release pin, and that it runs. + +The rules follow LabConstrictor's notebook portability guide (plan §12 question 52): every +code cell starts with ``# @title``; Colab-only code runs inside +``if "google.colab" in sys.modules:``; no shell or line magics; ``current_version`` and +``notebook_name`` as its version check reads them. This notebook is an app-like widget +notebook, so every code cell's source is hidden (LabConstrictor's code-hiding guide). +""" + +import ast +import json +import re +from pathlib import Path + +import pytest + +ROOT = Path(__file__).parents[1] +NOTEBOOK = ROOT / "notebooks" / "MorphoBricks_Preprocessing.ipynb" +VERSIONS = ROOT / "notebooks" / "notebook_latest_versions.yaml" +HIDDEN = { + "cellView": "form", + "collapsed": False, + "jupyter": {"source_hidden": True}, + "jupyterlabHideCode": {"locked": True}, +} +STEP_CALLS = [ + "notebook.folder_step()", + "notebook.axis_step()", + "notebook.run_step()", + "notebook.results_step()", +] + + +def _notebook(): + return json.loads(NOTEBOOK.read_text(encoding="utf-8")) + + +def _code_cells(): + return ["".join(c["source"]) for c in _notebook()["cells"] if c["cell_type"] == "code"] + + +def _markdown(): + return "\n".join( + "".join(c["source"]) for c in _notebook()["cells"] if c["cell_type"] == "markdown" + ) + + +def _setting(name): + match = re.search(rf'^{name} = "([^"]*)"$', _code_cells()[0], re.M) + assert match is not None, f"the setup cell sets no {name}" + return match.group(1) + + +def _colab_guard_lines(source): + """Return the line numbers inside the setup cell's top-level Colab guard.""" + guards = [ + node + for node in ast.parse(source).body + if isinstance(node, ast.If) and ast.unparse(node.test) == "'google.colab' in sys.modules" + ] + assert len(guards) == 1, "the setup cell needs one `if 'google.colab' in sys.modules:` block" + (guard,) = guards + return set(range(guard.body[0].lineno, guard.body[-1].end_lineno + 1)) + + +def test_notebook_is_valid_and_has_no_outputs(): + notebook = _notebook() + assert notebook["nbformat"] == 4 + code = [cell for cell in notebook["cells"] if cell["cell_type"] == "code"] + assert all(cell["outputs"] == [] and cell["execution_count"] is None for cell in code) + + +def test_every_code_cell_is_titled_and_hidden(): + for cell in _notebook()["cells"]: + if cell["cell_type"] == "code": + first_line = "".join(cell["source"]).splitlines()[0] + assert first_line.startswith("# @title Run to "), first_line + assert cell["metadata"] == HIDDEN, first_line + + +def test_colab_only_code_is_guarded_and_no_magics_are_used(): + for number, source in enumerate(_code_cells()): + assert not re.search(r"^\s*[!%]", source, re.M), "magics only run in IPython" + assert "get_ipython" not in source + guarded = _colab_guard_lines(source) if number == 0 else set() + for line_number, line in enumerate(source.splitlines(), start=1): + colab = "google.colab" in line and "sys.modules" not in line + if colab or "/content" in line: + assert line_number in guarded, f"Colab-only code outside the guard: {line!r}" + + +def test_colab_guard_mounts_drive_and_enables_the_widget_manager(): + source = _code_cells()[0] + guarded = "\n".join( + line + for number, line in enumerate(source.splitlines(), start=1) + if number in _colab_guard_lines(source) + ) + assert 'drive.mount("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/content/drive")' in guarded + assert "output.enable_custom_widget_manager()" in guarded + + +def test_github_install_is_pinned_to_the_release_tag(): + tag = _setting("RELEASE_TAG") + assert re.fullmatch(r"v\d+\.\d+\.\d+(rc\d+)?", tag), tag + setup = _code_cells()[0] + assert setup.count("git+") == 1 + assert '@ git+{REPOSITORY}@{RELEASE_TAG}"' in setup + assert _setting("REPOSITORY") == ( + "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/CellMigrationLab/morphobricks-preprocessing.git" + ) + # The notebook's version is its release, and the Colab callout names the tag installed. + assert _setting("current_version") == tag.removeprefix("v") + assert f"**{tag}**" in _markdown() + + +def test_version_metadata_matches_labconstrictor_index(): + assert _setting("notebook_name") == NOTEBOOK.stem + index = dict( + re.fullmatch(r'"([^"]+)": "([^"]+)"', line).groups() + for line in VERSIONS.read_text(encoding="utf-8").splitlines() + if line and not line.startswith("#") + ) + assert index == {NOTEBOOK.stem: _setting("current_version")} + + +def test_notebook_keeps_processing_in_the_package(): + sources = [source.split("\n", 1)[-1].strip() for source in _code_cells()] + # Step 0 must work before the package or third-party dependencies are installed. + setup_imports = set() + for node in ast.walk(ast.parse(sources[0])): + if isinstance(node, ast.Import): + setup_imports.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + setup_imports.add(node.module.split(".")[0]) + assert setup_imports <= {"subprocess", "sys", "importlib", "pathlib", "platform", "google"} + assert sources[1:] == STEP_CALLS + + +def test_notebook_names_buttons_that_exist(): + pytest.importorskip("ipywidgets") + pytest.importorskip("matplotlib") + from morphobricks_preprocessing.notebook_ui import PreprocessingNotebook + + text = "".join("".join(c["source"]) for c in _notebook()["cells"]) + nb = PreprocessingNotebook() + for button in ( + nb.check_button, + nb.labels_confirmed.button, + nb.preview_button, + nb.preview_checked.button, + nb.missing_as_singleton.button, + nb.confirm_button, + nb.image_type_checked.button, + nb.run_button, + nb.statistics_button, + ): + assert f"**{button.description.strip()}**" in text + + +def test_notebook_cells_run(capsys): + pytest.importorskip("ipywidgets") + pytest.importorskip("matplotlib") + from morphobricks_preprocessing.notebook_ui import PreprocessingNotebook + + namespace: dict = {} + for source in _code_cells(): + exec(source, namespace) # noqa: S102 - our own notebook + assert isinstance(namespace["notebook"], PreprocessingNotebook) + assert "Local Jupyter: Python" in capsys.readouterr().out + + +def test_notebook_runs_in_a_kernel(): + # The whole file, as Jupyter runs it: a fresh kernel started in the notebooks folder. + pytest.importorskip("ipywidgets") + pytest.importorskip("matplotlib") + pytest.importorskip("ipykernel", reason="the dev extra installs ipykernel") + nbclient = pytest.importorskip("nbclient", reason="the dev extra installs nbclient") + nbformat = pytest.importorskip("nbformat") + + notebook = nbformat.read(NOTEBOOK, as_version=4) + client = nbclient.NotebookClient( + notebook, + timeout=300, + kernel_name="python3", + resources={"metadata": {"path": str(NOTEBOOK.parent)}}, + ) + client.execute() + code = [cell for cell in notebook.cells if cell.cell_type == "code"] + outputs = [output for cell in code for output in cell.outputs] + assert not [output for output in outputs if output.output_type == "error"] + setup_text = "".join(o.get("text", "") for o in code[0].outputs) + assert "Local Jupyter: Python" in setup_text + assert "Using the local checkout" in setup_text + for cell in code: # Step 0's ready card and every step's page are widgets + assert any( + "application/vnd.jupyter.widget-view+json" in output.get("data", {}) + for output in cell.outputs + ), cell.source.splitlines()[0] diff --git a/tests/test_notebook_setup.py b/tests/test_notebook_setup.py index 6476eedd..2efe83db 100644 --- a/tests/test_notebook_setup.py +++ b/tests/test_notebook_setup.py @@ -1,6 +1,7 @@ """Exercise the standalone setup cell without installing into the test environment.""" import importlib +import importlib.metadata import importlib.util import json import subprocess @@ -51,9 +52,9 @@ def import_interface(name): def start(): notebook = json.loads(NOTEBOOK.read_text(encoding="utf-8")) source = next(c["source"] for c in notebook["cells"] if c["cell_type"] == "code") - namespace = {} - exec("".join(source), namespace) - return namespace + state.namespace = {} + exec("".join(source), state.namespace) + return state.namespace state.start = start return state @@ -82,12 +83,17 @@ def test_setup_installs_missing_package_from_github_in_this_kernel(setup_environ state.start() (command,) = state.installs assert command[:5] == [sys.executable, "-m", "pip", "install", "--disable-pip-version-check"] + namespace = state.namespace assert command[5:] == [ "morphobricks-preprocessing[notebook] @ " "git+https://github.com/CellMigrationLab/morphobricks-preprocessing.git" + f"@{namespace['RELEASE_TAG']}" ] assert state.welcomes == ["ready"] - assert "Installing notebook requirements" in capsys.readouterr().out + out = capsys.readouterr().out + assert "Installing notebook requirements" in out + assert f"Using release {namespace['RELEASE_TAG']} from GitHub." in out + assert f"MorphoBricks_Preprocessing {namespace['current_version']}" in out def test_setup_repairs_missing_notebook_dependencies(setup_environment): @@ -162,13 +168,64 @@ def test_setup_moves_an_existing_source_path_to_the_front(setup_environment, tmp assert sys.path.count(source) == 1 -def test_setup_reports_colab_without_importing_or_mounting_drive( +def test_setup_mounts_drive_and_enables_the_widget_manager_on_colab( setup_environment, monkeypatch, capsys ): - monkeypatch.setitem(sys.modules, "google.colab", SimpleNamespace()) + calls = [] + colab = SimpleNamespace( + drive=SimpleNamespace(mount=lambda path: calls.append(("mount", path))), + output=SimpleNamespace(enable_custom_widget_manager=lambda: calls.append("widgets")), + ) + monkeypatch.setitem(sys.modules, "google.colab", colab) + setup_environment.start() + assert calls == [("mount", "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/content/drive"), "widgets"] + out = capsys.readouterr().out + assert "connecting Google Drive" in out and "Checking the Google Colab environment" in out + assert "Local Jupyter" not in out + assert setup_environment.welcomes == ["ready"] + + +def test_setup_stops_when_drive_cannot_be_mounted(setup_environment, monkeypatch): + def refuse(path): + raise ValueError("mount failed") + + colab = SimpleNamespace(drive=SimpleNamespace(mount=refuse), output=SimpleNamespace()) + monkeypatch.setitem(sys.modules, "google.colab", colab) + with pytest.raises(RuntimeError, match="Google Drive could not be connected"): + setup_environment.start() + assert setup_environment.installs == setup_environment.welcomes == [] + + +def test_setup_reports_the_local_environment(setup_environment, capsys): setup_environment.start() - assert "Checking the Google Colab environment" in capsys.readouterr().out - assert setup_environment.installs == [] + out = capsys.readouterr().out + assert "Local Jupyter: Python" in out and "Working folder:" in out + assert "Google Drive" not in out + + +def test_setup_upgrades_ipywidgets_7(setup_environment, monkeypatch, capsys): + real = importlib.metadata.version + widgets = {"version": "7.7.1"} + + def version(name): + return widgets["version"] if name == "ipywidgets" else real(name) + + def install(command, *, check): + setup_environment.installs.append(command) + widgets["version"] = "8.1.5" + + monkeypatch.setattr(importlib.metadata, "version", version) + monkeypatch.setattr(subprocess, "run", install) + setup_environment.start() + assert setup_environment.installs[0][-1] == "morphobricks-preprocessing[notebook]" + assert "Installing notebook requirements (ipywidgets 8)" in capsys.readouterr().out + + +def test_setup_asks_for_a_restart_when_ipywidgets_7_is_loaded(setup_environment, monkeypatch): + monkeypatch.setitem(sys.modules, "ipywidgets", SimpleNamespace(__version__="7.7.1")) + with pytest.raises(RuntimeError, match="Restart session"): + setup_environment.start() + assert setup_environment.imports == setup_environment.welcomes == [] def test_setup_rejects_old_python_before_installing(setup_environment, monkeypatch): diff --git a/tests/test_notebook_ui.py b/tests/test_notebook_ui.py index ddcda3a8..2048c3fa 100644 --- a/tests/test_notebook_ui.py +++ b/tests/test_notebook_ui.py @@ -1,5 +1,3 @@ -import ast -import json import os from pathlib import Path @@ -30,8 +28,6 @@ from morphobricks_preprocessing.validation import error, warning # noqa: E402 from tiff_tree import write_data, write_file # noqa: E402 -NOTEBOOK = Path(__file__).parents[1] / "notebooks" / "MorphoBricks_Preprocessing.ipynb" - @pytest.fixture def experiment(tmp_path): @@ -487,6 +483,7 @@ def test_browse_fills_the_path_or_explains(experiment, monkeypatch): monkeypatch.setattr(ui, "ask_directory", lambda initial: None) # no dialog possible nb.folder_browse.click() assert "could not be opened" in nb.folder_status.value + assert "Copy path" in nb.folder_status.value # how to paste on Google Colab assert nb.folder.value == str(experiment) @@ -1417,7 +1414,8 @@ def test_statistics_of_another_table(experiment, monkeypatch): assert later.stats_next.value == "" monkeypatch.setattr(ui, "ask_open_file", lambda initial, filetypes: None) later.csv_browse.click() - assert "could not be opened" in later.stats_status.value + assert "file dialog could not be opened" in later.stats_status.value + assert "right-click the file" in later.stats_status.value def test_step_4_is_optional_and_keeps_its_controls_in_the_group_card(): @@ -1502,7 +1500,12 @@ def test_tiles_table_and_checklist(): @pytest.mark.parametrize( ("text", "expected"), - [(' "C:/data/exp" ', "C:/data/exp"), ("‘/data’", "/data"), ("", "")], + [ + (' "C:/data/exp" ', "C:/data/exp"), + ("‘/data’", "/data"), + ("", ""), + ("'/content/drive/MyDrive/exp 1' ", "/content/drive/MyDrive/exp 1"), # Colab Copy path + ], ) def test_clean_path(text, expected): assert ui.clean_path(text) == expected @@ -1512,48 +1515,6 @@ def test_clean_path_expands_home(): assert ui.clean_path("~/data") == os.path.expanduser("~/data") -# The notebook file ------------------------------------------------------------------------- - - -def _notebook(): - return json.loads(NOTEBOOK.read_text(encoding="utf-8")) - - -def test_notebook_is_valid_and_has_no_outputs(): - notebook = _notebook() - assert notebook["nbformat"] == 4 - code = [cell for cell in notebook["cells"] if cell["cell_type"] == "code"] - assert all(cell["outputs"] == [] and cell["execution_count"] is None for cell in code) - - -def test_notebook_code_cells_are_hidden_and_titled(): - for cell in _notebook()["cells"]: - if cell["cell_type"] == "code": - assert cell["metadata"]["jupyter"]["source_hidden"] is True - assert cell["metadata"]["jupyterlabHideCode"]["locked"] is True - assert cell["source"][0].startswith("# @markdown ### Run to ") - - -def test_notebook_keeps_processing_in_the_package(): - sources = [ - "".join(c["source"][1:]).strip() for c in _notebook()["cells"] if c["cell_type"] == "code" - ] - # Step 0 must work before the package or third-party dependencies are installed. - setup_imports = set() - for node in ast.walk(ast.parse(sources[0])): - if isinstance(node, ast.Import): - setup_imports.update(alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, ast.ImportFrom): - setup_imports.add(node.module.split(".")[0]) - assert setup_imports <= {"subprocess", "sys", "importlib", "pathlib"} - assert sources[1:] == [ - "notebook.folder_step()", - "notebook.axis_step()", - "notebook.run_step()", - "notebook.results_step()", - ] - - def test_widgets_use_99_percent_width_and_bold_paths(experiment): import morphobricks_preprocessing.notebook_style as style_module import morphobricks_preprocessing.notebook_ui as ui_module @@ -1566,31 +1527,6 @@ def test_widgets_use_99_percent_width_and_bold_paths(experiment): assert "" in nb.output_hint.value and "" not in nb.labels_table.value -def test_notebook_names_buttons_that_exist(): - text = "".join("".join(c["source"]) for c in _notebook()["cells"]) - nb = PreprocessingNotebook() - for button in ( - nb.check_button, - nb.labels_confirmed.button, - nb.preview_button, - nb.preview_checked.button, - nb.missing_as_singleton.button, - nb.confirm_button, - nb.image_type_checked.button, - nb.run_button, - nb.statistics_button, - ): - assert f"**{button.description.strip()}**" in text - - -def test_notebook_cells_run(): - namespace: dict = {} - for cell in _notebook()["cells"]: - if cell["cell_type"] == "code": - exec("".join(cell["source"]), namespace) # noqa: S102 - our own notebook - assert isinstance(namespace["notebook"], PreprocessingNotebook) - - def test_preview_caption_of_a_reduced_plane(): from morphobricks_preprocessing import BlockReader diff --git a/tools/code_rules_baseline.json b/tools/code_rules_baseline.json index 45ff541f..684d65b7 100644 --- a/tools/code_rules_baseline.json +++ b/tools/code_rules_baseline.json @@ -32,7 +32,6 @@ "src/morphobricks_preprocessing/metadata.py: # lines (max #)": 779, "src/morphobricks_preprocessing/notebook_ui.py __init__: # executable lines (max #)": 110, "src/morphobricks_preprocessing/notebook_ui.py _bins_help: private function of # lines without a docstring": 12, - "src/morphobricks_preprocessing/notebook_ui.py _browse_csv: private function of # lines without a docstring": 11, "src/morphobricks_preprocessing/notebook_ui.py _build_axis_step: # executable lines (max #)": 82, "src/morphobricks_preprocessing/notebook_ui.py _build_axis_step: private function of # lines without a docstring": 82, "src/morphobricks_preprocessing/notebook_ui.py _build_design_card: private function of # lines without a docstring": 53, @@ -69,7 +68,7 @@ "src/morphobricks_preprocessing/notebook_ui.py _toggle_confirmation: private function of # lines without a docstring": 48, "src/morphobricks_preprocessing/notebook_ui.py _update_output_hint: private function of # lines without a docstring": 20, "src/morphobricks_preprocessing/notebook_ui.py run: # executable lines (max #)": 63, - "src/morphobricks_preprocessing/notebook_ui.py: # lines (max #)": 3051, + "src/morphobricks_preprocessing/notebook_ui.py: # lines (max #)": 3043, "src/morphobricks_preprocessing/physical_sizes.py _imagej_sizes: private function of # lines without a docstring": 27, "src/morphobricks_preprocessing/physical_sizes.py _ome_sizes: private function of # lines without a docstring": 20, "src/morphobricks_preprocessing/physical_sizes.py: # lines (max #)": 706, @@ -85,7 +84,7 @@ "tests/test_discovery.py: # lines (max #)": 826, "tests/test_masks.py: # lines (max #)": 612, "tests/test_metadata.py: # lines (max #)": 629, - "tests/test_notebook_ui.py: # lines (max #)": 1604, + "tests/test_notebook_ui.py: # lines (max #)": 1540, "tests/test_planes.py: # lines (max #)": 818, "tests/test_run.py: # lines (max #)": 982, "tests/test_zarr_input.py: # lines (max #)": 869, From f766c1b58420f63e4f3f6e77f8e97dba259e862d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 08:00:41 +0000 Subject: [PATCH 2/4] PP-8d docs: Colab section of the user guide, README notebook section, plan question 52 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - README: "Notebook: Google Colab and JupyterLab" (the portability rules, the release pin and how it moves at a release). - User guide §1 "Google Colab": Drive, output folder, copying a large dataset to the runtime's disk (same path every session), and the limits of Google Drive (lock only within one runtime, a stale lock after a disconnect, files read again). - Plan: question 52 (supersedes "Local Jupyter (not Colab-first)"), the v0.4.0 (PP-13) release checklist with the notebook pin, milestone 5 note. - AGENTS.md and development.md: the notebook rules and tests. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- AGENTS.md | 24 +++++++++++--- README.md | 28 ++++++++++++++++ docs/development.md | 3 ++ docs/plan.md | 33 +++++++++++++++++++ docs/user_guide.md | 79 ++++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 157 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4b5870a9..f469b0ee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -152,6 +152,7 @@ src/morphobricks_preprocessing/ show_output and show_design for step 1) so other bricks can embed them (plan §12 Q26, Q31) notebooks/MorphoBricks_Preprocessing.ipynb thin notebook: every code cell calls notebook_ui only +notebooks/notebook_latest_versions.yaml the notebook's latest version, for LabConstrictor's version check tests/ pytest with simulated TIFFs (tests/tiff_tree.py, tests/simulation.py); the schema-version-2 fixture (tests/v2_fixture.py, tests/data) experiments/binning_study/ the study behind the binning decisions (not part of the package) @@ -514,7 +515,7 @@ Technical details go to `logging` (shown in the notebook's "Technical details" p | RGB `S` axis (Q3) | Prefilled as C, confirmed like any role | | Significant bits (Q4) | Manifest only, information only; `bit_depth` and binning unchanged | | Corrupt files (Q6) | Explicit exclusion list, recorded in manifest, rows removed | -| Notebook target | Local Jupyter (not Colab-first) | +| Notebook target | Google Colab and JupyterLab from one file, following LabConstrictor's notebook portability guide (plan Q52; was local Jupyter only): `# @title` cells, Colab-only code (Drive mount, custom widget manager) inside `if "google.colab" in sys.modules:`, no magics, GitHub install pinned to `RELEASE_TAG` | | CLI | `check ` (summary, label tree, issues, suggested `--axes`) and `run --axes ROLES` with `--output`, `--exclude`, `--bins`, `--histogram-range`, `--robust-quantiles`, `--memory-limit 2GiB`, `--missing-as-singleton`, `--mode restart|full`, `--no-masks`, `--no-exclude`, `--remove-stale-lock`, `--log`, and the conversion options above. Axis roles are always stated explicitly, never taken silently from metadata. Exit 1 on errors | | Large planes (Q15) | Planes over the memory limit are read in blocks automatically, announced by the warning `planes_read_in_blocks` and `read_in_blocks` in the manifest. Block size derived from the limit (whole strips/tiles/chunks, uncompressed pieces and chunks also by rows). Compressed single-strip/tile planes keep the `plane_over_memory_limit` error. Two-pass histograms identical to whole planes. (Up to v0.3, OME-Zarr copies were written block by block; conversion was removed in v0.4.0.) Axis preview reduced (every n-th pixel) with a caption saying so | | Normalization brick (P1) | The modules `dtypes`, `histogram`, `records`, `csv_io`, `pooling` and `normalization` moved unchanged to `morphobricks_normalization` (normalization plan N1, N2, N6) and are imported from there. Up to v0.3 their public names stayed importable from `morphobricks_preprocessing` and the old submodules, with a `DeprecationWarning`; those stand-ins (`_moved.py`) were removed in v0.4.0 (plan Q46). The package's own code never uses the old paths. Outputs unchanged (CSV schema 2); the manifest's `versions` adds `morphobricks-normalization`; Step 0 checks for it; contract §3–§7 text moved to the normalization contract (§1.2, now §6.2 for the CSV) | @@ -599,10 +600,14 @@ Technical details go to `logging` (shown in the notebook's "Technical details" p tests skip without zarr; CI's Python 3.11 jobs pin zarr 2.18 and 3.13 jobs get zarr 3. - **The notebook file is tested too:** - valid nbformat 4, with no outputs; - - code cells hidden and titled `# @markdown ### Run to …`; + - code cells hidden and titled `# @title Run to …`; + - Colab-only code and `/content` paths only inside the Colab guard; no magics; + - the GitHub install pinned to `RELEASE_TAG`, equal to `current_version` and the + version index; - Step 0 bootstraps the environment; later code cells only call `notebook_ui`; - the buttons it names exist; - - its cells execute. + - its cells execute, by `exec` and in a real kernel (`nbclient`, `tests/test_notebook_file.py`; + Step 0's alternatives in `tests/test_notebook_setup.py`). --- @@ -627,12 +632,21 @@ plainer version "not interactive enough". Descriptive, guided and forgiving is t - **Code cells:** - Hidden, with metadata `{"cellView": "form", "collapsed": false, "jupyter": {"source_hidden": true}, "jupyterlabHideCode": {"locked": true}}`. - - The first line is a Colab-style title: `# @markdown ### Run to …`. + - The first line is a Colab title: `# @title Run to …` (LabConstrictor portability + guide; every code cell, all hidden, plan Q52). Longer explanations go in Markdown cells. - Outputs are stripped. - Step 0 is a standalone standard-library bootstrap: nearby checkout, installed package, - or GitHub installation, with missing requirements installed using the kernel's Python. + or GitHub installation of the release `RELEASE_TAG`, with missing requirements + installed using the kernel's Python (`subprocess`, never `!pip`). Colab-only code + (Drive mount, `output.enable_custom_widget_manager()`) stays inside + `if "google.colab" in sys.modules:`; no `/content` path outside it. `current_version` + and `notebook_name` are LabConstrictor's names; at each release the pin moves + (plan §10, release checklist). - Later cells are one call each, e.g. `notebook.folder_step()`. - **Step 0** starts the notebook and shows a green "ready" card with the package version. +- **Runtime callouts:** Steps 0 and 1 carry "**Colab users:**" / "**JupyterLab users:**" + blockquotes (Drive, output folder outside the experiment, slower Drive reads, runtime + wiped on disconnect, Browse… only locally). - **Closing cell:** "✅ After running" checklist and "Common problems" that quote the real message texts (check they still match the code). - **Text:** use bold, not inline code, in the markdown (fenced text blocks such as the diff --git a/README.md b/README.md index 1feaa23e..97d17f78 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,34 @@ The notebook's Step 0 can set this up automatically: it uses a nearby checkout o installed package, and installs missing requirements into the active kernel. A standalone download installs from GitHub if needed (internet access and Git required). +### Notebook: Google Colab and JupyterLab + +[`notebooks/MorphoBricks_Preprocessing.ipynb`](notebooks/MorphoBricks_Preprocessing.ipynb) +runs unchanged in Google Colab and in JupyterLab (also as a LabConstrictor app). It follows +LabConstrictor's +[notebook portability guide](https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/notebook_portability.md): + +1. Every code cell starts with `# @title …` (Colab's cell header) and is hidden: the notebook is + driven by widgets, so no code is edited. Explanations are in Markdown cells. +2. Colab-only setup runs in Step 0 inside `if "google.colab" in sys.modules:`: it mounts + Google Drive at `/content/drive` and enables Colab's custom widget manager (the notebook + needs ipywidgets 8). Locally Step 0 prints the Python version, platform and working folder. + No shell or line magics are used; pip runs with the kernel's Python. +3. When the package is missing (always on a fresh Colab runtime), Step 0 installs the release + named by `RELEASE_TAG` (currently `v0.4.0rc2`), never the moving `main` branch, and prints + which source and version it uses. A checkout around the notebook, or an installed package, + is used first. +4. `current_version` and `notebook_name` in Step 0 are the names LabConstrictor's version check + reads; [`notebooks/notebook_latest_versions.yaml`](notebooks/notebook_latest_versions.yaml) + lists the latest version. At each release, `RELEASE_TAG`, `current_version`, the index and + the tag named in the Step 0 Colab callout move to the new tag before it is tagged + ([plan](docs/plan.md) §10, release checklist); `tests/test_notebook_file.py` checks that + they agree, that the rules above hold, and runs the notebook in a local kernel. The Colab + run itself is checked by hand. +5. Steps 0 and 1 have *Colab users* / *JupyterLab users* callouts. On Colab, keep the data and + the output folder on Google Drive; the [user guide](docs/user_guide.md) §1 explains why, and + the limits of Google Drive. + From a terminal: ```bash diff --git a/docs/development.md b/docs/development.md index fa314cd9..51a796c4 100644 --- a/docs/development.md +++ b/docs/development.md @@ -25,6 +25,9 @@ functions the tests use. The notebook's Step 0 contains a standalone bootstrap using only the standard library, so it can locate a checkout or install missing requirements before importing the package. Its environment alternatives are tested without installing packages during the test suite. +Colab-only setup sits inside `if "google.colab" in sys.modules:` and the GitHub installation +is pinned to the release in `RELEASE_TAG` (plan §12 question 52); +`tests/test_notebook_file.py` checks these rules and runs the notebook in a real kernel. ## Style diff --git a/docs/plan.md b/docs/plan.md index 127d7de8..18e050e8 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -299,6 +299,8 @@ API, CLI). The notebook and CLI only call the API. *Setup feedback: standalone Step 0 detects the environment, prefers a nearby checkout, reuses an installed package, or installs missing requirements into the active kernel.* *Step-1 feedback: centre the folder-label table's headers and contents consistently.* + *v0.4.0 (PP-8d, question 52): the notebook runs on Google Colab and in JupyterLab following + LabConstrictor's portability guide; its GitHub installation is pinned to a release tag.* *Notebook guidance feedback: separate "What next?" boxes show the next action in larger text, following the current confirmations. Step-1 message details are collapsed with persistent orange warning and red error counts. Step-3 conversion settings use an @@ -381,6 +383,15 @@ API, CLI). The notebook and CLI only call the API. 35.* - **Release v0.3.1:** `position_id` names an image with the other image extensions bricks read (MorphoBricks D85, PR #33). *To be tagged, see question 36.* +- **Release v0.4.0 (PP-13) checklist**, for every tag of v0.4.0 including release + candidates, in the release PR before the tag is pushed: + - set `version` in `pyproject.toml`; + - set the notebook's pin (question 52): `RELEASE_TAG` in Step 0 to the new tag, + `current_version` to the tag without "v", the same version in + `notebooks/notebook_latest_versions.yaml`, and the tag named in the Step 0 Colab callout + (`tests/test_notebook_file.py` checks that the four agree); + - run the notebook by hand on Google Colab with the PR #118 checklist (the tests run it + only in a local kernel). ## 11. Tests and acceptance criteria @@ -1158,3 +1169,25 @@ Raised during review; to be decided before the milestones that need them: OME-Zarr image whose `labels` group cannot be read is its own refusal naming the image, not a silent pass and not an "unrecorded" file; (4) the version stays 0.3.4 until PP-13, and v0.4.0rc2 is tagged by the owner after this PR.* +52. How does the notebook run on Google Colab, and which release does it install (PP-8d)? + *Decided (2 November 2026, owner's PP-8d brief; supersedes the "Local Jupyter (not + Colab-first)" target): one notebook file for Google Colab and JupyterLab, following + LabConstrictor's notebook portability guide. Every code cell starts with `# @title`, and + explanations are in Markdown cells. Every code cell keeps its source hidden: the notebook + is an app-like widget notebook in which nothing is edited (LabConstrictor's code-hiding + guide), so this differs from fiberfeatures, whose settings cells stay visible. Colab-only + code runs inside `if "google.colab" in sys.modules:` in Step 0: Google Drive is mounted + at `/content/drive` and Colab's custom widget manager is enabled (the package needs + ipywidgets 8; Colab ships 7). Locally Step 0 prints the Python version, platform and + working folder. No shell or line magics; pip runs through `subprocess` with the kernel's + Python. When the package is missing, Step 0 installs the release named by `RELEASE_TAG` + (v0.4.0rc2 first), never an unpinned `main`; a nearby checkout and an installed package + are still used first, and Step 0 prints which one and its version. `current_version` + equals the tag without "v" and `notebook_name` the file name, as LabConstrictor's + version check reads them, with `notebooks/notebook_latest_versions.yaml` as the index; + the pin is moved at every release (§10, release checklist). Steps 0 and 1 have "Colab + users" / "JupyterLab users" callouts (data and results on Drive, output folder outside + the experiment, slower Drive reads, a runtime wiped on disconnect). The run's semantics + are unchanged on Google Drive; its known limits there (the lock is only exclusive within + one runtime, fingerprints of copies) are in the user guide §1 and were put to the owner + in PR #118.* diff --git a/docs/user_guide.md b/docs/user_guide.md index ccc22c80..85e3d079 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -20,12 +20,14 @@ run **Step 0**. It detects local Jupyter or Google Colab and checks these option subfolders, use that checkout's source code, including local edits. 2. **Installed package:** otherwise, use the package already available to this kernel. 3. **GitHub installation:** if the package is missing, install it with the notebook - dependencies from the project's GitHub repository. + dependencies from the project's GitHub repository, at the release the notebook names + (`RELEASE_TAG` in Step 0, for example `v0.4.0rc2`), never the changing `main` branch. Missing notebook dependencies are installed into the same Python environment as the kernel. A local checkout is installed in editable mode when it needs dependencies; an installed package's notebook extra is used when its UI dependencies are missing. -Setup prints the selected source and Python executable. It does not upgrade an already +Setup prints the selected source with its version or release, the notebook's version and the +Python executable. It does not upgrade an already usable installation. Installation needs internet access, and the GitHub option needs Git. If another copy of the package is already loaded, restart the kernel to switch checkouts. @@ -40,9 +42,76 @@ pip install jupyterlab To read OME-Zarr images, install the `zarr` extra too: `pip install "morphobricks-preprocessing[notebook,zarr] @ git+https://github.com/CellMigrationLab/morphobricks-preprocessing.git"`. -Start JupyterLab with `jupyter lab` and open the notebook. Google Colab is detected by the -setup cell, but native **Browse…** dialogs are only available with a local desktop: paste -paths on the runtime filesystem when using a hosted notebook. +Start JupyterLab with `jupyter lab` and open the notebook. **Browse…** opens your computer's +folder dialog; where no dialog can open (Google Colab, a remote server), it says so and you +paste the path instead. + +### Google Colab + +Open the notebook in Google Colab (**File › Open notebook › GitHub**, or upload the file) and +run **Step 0**. On Colab it: + +- connects your Google Drive: allow access in the window that opens. Your Drive is then the + folder `/content/drive/MyDrive`; +- installs MorphoBricks preprocessing at the release the notebook names, with its notebook + dependencies, which takes a minute or two (ipywidgets 8 replaces Colab's ipywidgets 7; + if a message asks you to, choose **Runtime › Restart session** and run Step 0 again); +- enables Colab's widget support for these widgets. + +To read OME-Zarr images on Colab, also install the `zarr` extra: in a new code cell run +`import subprocess, sys` and then +`subprocess.run([sys.executable, "-m", "pip", "install", "zarr>=2.18"], check=True)`. + +**Where to keep the data.** A Colab runtime is a temporary computer: everything on it except +Google Drive is wiped when it disconnects (after some idle time, or after several hours). + +- Put the experiment folder on Google Drive and paste its path in step 1, for example + `/content/drive/MyDrive/experiment_2026_10`. **Browse…** cannot open a dialog on Colab: open + the **Files** panel in the left bar, right-click the folder, choose **Copy path** and paste. +- Keep the output folder on Google Drive too, outside the experiment folder. The default, + next to the experiment folder, is right when the experiment is on Drive. +- Google Drive is slower to read than the runtime's own disk. For a large dataset you may + copy the experiment folder to the runtime's disk first, keeping the modification times, + for example in a new code cell: + + ```python + import shutil + + shutil.copytree("/content/drive/MyDrive/experiment_2026_10", "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/content/experiment_2026_10") + ``` + + Then type an output folder on Drive in step 1 (for example + `/content/drive/MyDrive/experiment_2026_10_MorphoBricks_Preprocessing`): the default would + be next to the copy and lost with the runtime. The output records where the images were + read and each image's size and modification time. A later run reuses the rows only when + the images are read from the same place: copy to the same path in every session (as + above; the copy keeps the times) and only new or changed images are read again. Reading + from another place (Drive one time, the copy another) recomputes every row, and plain + stacks must be confirmed again. Later bricks must be given the experiment folder they + should read, because the recorded location is the copy's, which a new runtime no longer + has until you copy again. + +**Limits of Google Drive** (the run behaves the same; these are things to know): + +- **One session per output folder.** The output folder's lock stops a second run on the + same Colab runtime. Google Drive copies files between computers with a delay, so the lock + does not stop a run started at the same time from another Colab session or from a + computer that syncs the same Drive folder. Never run two sessions on one output folder. +- **A lock left by a disconnected runtime.** If the runtime disconnects during a check or a + run, its lock stays in the output folder and the next check says "Another run is using + this output folder". When you are sure no other session uses the folder, remove it in a + new code cell, then check the folder again: + + ```python + from morphobricks_preprocessing.run import remove_stale_lock + + remove_stale_lock("/content/drive/MyDrive/experiment_2026_10_MorphoBricks_Preprocessing") + ``` + +- **Unchanged images read again.** An image counts as unchanged when its size and + modification time are the same as at the last run. If Google Drive reports a file's + time differently later (for example after the file was synced again from another + computer), the image is read again. That costs time but never reuses wrong rows. ## 2. Organising the experiment folder From 823a7ca9f82c094c832d769cc6cdcb658411d14d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 09:22:33 +0000 Subject: [PATCH 3/4] PP-8d review: Colab installs the zarr extra; setup tests independent of order - Owner decision (2 November 2026): on Colab, Step 0 installs the notebook and zarr extras, so OME-Zarr images can be read there; zarr is then a requirement Step 0 checks, so an installed package without it gets the extra. Locally unchanged (notebook extra). Tests: release from GitHub with [notebook,zarr] on Colab, zarr added to an installed package on Colab, zarr neither required nor installed locally. - Colab callouts: data and output stay on Drive at the same place (no advice to copy to the runtime's disk), and one session per output folder, because the lock cannot guard it across sessions on Drive. - The setup tests failed (15 of 17) when run on their own: Step 0 calls platform.platform(), which runs uname through subprocess the first time, and the fixture fakes subprocess.run for pip. The fixture now fills platform's cache before faking pip. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- notebooks/MorphoBricks_Preprocessing.ipynb | 18 ++++---- tests/test_notebook_setup.py | 49 +++++++++++++++++++++- 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/notebooks/MorphoBricks_Preprocessing.ipynb b/notebooks/MorphoBricks_Preprocessing.ipynb index 05894695..4d2a44c5 100644 --- a/notebooks/MorphoBricks_Preprocessing.ipynb +++ b/notebooks/MorphoBricks_Preprocessing.ipynb @@ -127,6 +127,8 @@ "# tagged, so a tagged notebook installs its own release (docs/plan.md, release checklist).\n", "RELEASE_TAG = \"v0.4.0rc2\"\n", "REPOSITORY = \"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/CellMigrationLab/morphobricks-preprocessing.git\"\n", + "# The package's extras to install; on Colab also zarr, to read OME-Zarr images.\n", + "EXTRAS = \"notebook\"\n", "\n", "if \"google.colab\" in sys.modules:\n", " # Colab only: Google Drive holds the data and the results, which outlive the runtime.\n", @@ -146,6 +148,7 @@ " ) from exc\n", " # Colab ships ipywidgets 7; this notebook's widgets need ipywidgets 8's front end.\n", " output.enable_custom_widget_manager()\n", + " EXTRAS = \"notebook,zarr\"\n", "else:\n", " print(\n", " f\"Local Jupyter: Python {platform.python_version()} on {platform.platform()}\\n\"\n", @@ -163,6 +166,7 @@ " \"tifffile\",\n", " \"ipywidgets\",\n", " \"matplotlib\",\n", + " *((\"zarr\",) if \"zarr\" in EXTRAS.split(\",\") else ()),\n", " )\n", " missing = [name for name in required if find_spec(name) is None]\n", " try:\n", @@ -203,16 +207,16 @@ " # A local source tree works immediately, including in a git worktree.\n", " sys.path[:] = [str(source), *(entry for entry in sys.path if entry != str(source))]\n", " invalidate_caches()\n", - " install_target = [\"-e\", f\"{checkout}[notebook]\"]\n", + " install_target = [\"-e\", f\"{checkout}[{EXTRAS}]\"]\n", " origin = f\"the local checkout at {checkout}\"\n", " elif find_spec(\"morphobricks_preprocessing\") is not None:\n", - " install_target = [\"morphobricks-preprocessing[notebook]\"]\n", + " install_target = [f\"morphobricks-preprocessing[{EXTRAS}]\"]\n", " try:\n", " origin = f\"the installed package, version {version('morphobricks-preprocessing')}\"\n", " except PackageNotFoundError:\n", " origin = \"the installed package\"\n", " else:\n", - " install_target = [f\"morphobricks-preprocessing[notebook] @ git+{REPOSITORY}@{RELEASE_TAG}\"]\n", + " install_target = [f\"morphobricks-preprocessing[{EXTRAS}] @ git+{REPOSITORY}@{RELEASE_TAG}\"]\n", " origin = f\"release {RELEASE_TAG} from GitHub\"\n", "\n", " missing = _missing_requirements()\n", @@ -315,10 +319,10 @@ "> - Keep the **Output folder** on Google Drive and outside the experiment folder, so the results\n", "> survive when the runtime disconnects. The default, next to the experiment folder, is right\n", "> when the experiment is on Drive.\n", - "> - Reading from Google Drive is slower than from the runtime's own disk. For a large dataset you\n", - "> may copy the experiment folder to the runtime's disk first (see the user guide, \"Google\n", - "> Colab\"); then type an **Output folder** on Drive, because the default would be next to the\n", - "> copy and lost with the runtime.\n", + "> - Keep the experiment folder on Drive too, at the same place every session: a later run\n", + "> reuses its results only when the images are read from the same place.\n", + "> - Use one Colab session at a time per output folder. On Google Drive the output folder's lock\n", + "> cannot guard it across machines or sessions: it only stops a second run in the same runtime.\n", "> - If the runtime disconnected during a run, the next check may say another run is using the\n", "> output folder: see \"Common problems\" at the end.\n", ">\n", diff --git a/tests/test_notebook_setup.py b/tests/test_notebook_setup.py index 2efe83db..6e416718 100644 --- a/tests/test_notebook_setup.py +++ b/tests/test_notebook_setup.py @@ -4,6 +4,7 @@ import importlib.metadata import importlib.util import json +import platform import subprocess import sys from pathlib import Path @@ -20,14 +21,19 @@ "ipywidgets", "matplotlib", } +ZARR = "zarr" # required on Colab only, where Step 0 installs the zarr extra +GITHUB = "@ git+https://github.com/CellMigrationLab/morphobricks-preprocessing.git@" @pytest.fixture def setup_environment(tmp_path, monkeypatch): + # Step 0 prints platform.platform(), which may run `uname -p` through subprocess the + # first time; computing (and caching) it now keeps that away from the faked pip below. + platform.platform() monkeypatch.chdir(tmp_path) monkeypatch.setattr(sys, "path", sys.path.copy()) monkeypatch.delitem(sys.modules, "morphobricks_preprocessing", raising=False) - state = SimpleNamespace(available=REQUIRED.copy(), installs=[], welcomes=[], imports=[]) + state = SimpleNamespace(available={*REQUIRED, ZARR}, installs=[], welcomes=[], imports=[]) def find_spec(name): return object() if name in state.available else None @@ -35,7 +41,7 @@ def find_spec(name): def install(command, *, check): assert check is True state.installs.append(command) - state.available.update(REQUIRED) + state.available.update({*REQUIRED, ZARR}) def import_interface(name): state.imports.append(name) @@ -185,6 +191,45 @@ def test_setup_mounts_drive_and_enables_the_widget_manager_on_colab( assert setup_environment.welcomes == ["ready"] +def colab(monkeypatch): + fake = SimpleNamespace( + drive=SimpleNamespace(mount=lambda path: None), + output=SimpleNamespace(enable_custom_widget_manager=lambda: None), + ) + monkeypatch.setitem(sys.modules, "google.colab", fake) + + +def test_colab_installs_the_release_with_the_zarr_extra(setup_environment, monkeypatch, capsys): + colab(monkeypatch) + state = setup_environment + state.available.clear() + namespace = state.start() + (command,) = state.installs + assert command[5:] == [ + f"morphobricks-preprocessing[notebook,zarr] {GITHUB}{namespace['RELEASE_TAG']}" + ] + assert state.welcomes == ["ready"] + + +def test_colab_adds_zarr_to_an_installed_package(setup_environment, monkeypatch, capsys): + colab(monkeypatch) + state = setup_environment + state.available.remove(ZARR) + state.start() + assert state.installs[0][5:] == ["morphobricks-preprocessing[notebook,zarr]"] + assert "Installing notebook requirements (zarr)" in capsys.readouterr().out + + +def test_locally_zarr_is_neither_required_nor_installed(setup_environment): + state = setup_environment + state.available.discard(ZARR) + state.start() + assert state.installs == [] + state.available.clear() + state.start() + assert state.installs[0][5:][0].startswith(f"morphobricks-preprocessing[notebook] {GITHUB}") + + def test_setup_stops_when_drive_cannot_be_mounted(setup_environment, monkeypatch): def refuse(path): raise ValueError("mount failed") From 40376f5fb273ebfa1f5d209e3c53c82b72d19e58 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 09:22:33 +0000 Subject: [PATCH 4/4] PP-8d review docs: owner decisions on the Colab questions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User guide, README, AGENTS: the lock on Google Drive is documented only (one session per output folder; it cannot guard across machines or sessions); no advice to copy datasets to the runtime's disk (reuse depends on the absolute path); Colab installs the zarr extra (the manual pip line is gone). Plan: Q58 records the decisions; §10 gains the PP-10 note for a remove-the-lock button. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- AGENTS.md | 7 ++++--- README.md | 6 ++++-- docs/plan.md | 20 ++++++++++++++++---- docs/user_guide.md | 43 +++++++++++++------------------------------ 4 files changed, 37 insertions(+), 39 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c50a1809..0f5272e9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -637,7 +637,8 @@ plainer version "not interactive enough". Descriptive, guided and forgiving is t - Outputs are stripped. - Step 0 is a standalone standard-library bootstrap: nearby checkout, installed package, or GitHub installation of the release `RELEASE_TAG`, with missing requirements - installed using the kernel's Python (`subprocess`, never `!pip`). Colab-only code + installed using the kernel's Python (`subprocess`, never `!pip`); on Colab with the + `zarr` extra too. Colab-only code (Drive mount, `output.enable_custom_widget_manager()`) stays inside `if "google.colab" in sys.modules:`; no `/content` path outside it. `current_version` and `notebook_name` are LabConstrictor's names; at each release the pin moves @@ -645,8 +646,8 @@ plainer version "not interactive enough". Descriptive, guided and forgiving is t - Later cells are one call each, e.g. `notebook.folder_step()`. - **Step 0** starts the notebook and shows a green "ready" card with the package version. - **Runtime callouts:** Steps 0 and 1 carry "**Colab users:**" / "**JupyterLab users:**" - blockquotes (Drive, output folder outside the experiment, slower Drive reads, runtime - wiped on disconnect, Browse… only locally). + blockquotes (Drive, output folder outside the experiment, one session per output folder + on Drive, runtime wiped on disconnect, Browse… only locally). - **Closing cell:** "✅ After running" checklist and "Common problems" that quote the real message texts (check they still match the code). - **Text:** use bold, not inline code, in the markdown (fenced text blocks such as the diff --git a/README.md b/README.md index 05e839cf..9c9f68ed 100644 --- a/README.md +++ b/README.md @@ -106,9 +106,11 @@ LabConstrictor's ([plan](docs/plan.md) §10, release checklist); `tests/test_notebook_file.py` checks that they agree, that the rules above hold, and runs the notebook in a local kernel. The Colab run itself is checked by hand. -5. Steps 0 and 1 have *Colab users* / *JupyterLab users* callouts. On Colab, keep the data and +5. On Colab, Step 0 installs the `notebook` and `zarr` extras; locally it installs `notebook`. +6. Steps 0 and 1 have *Colab users* / *JupyterLab users* callouts. On Colab, keep the data and the output folder on Google Drive; the [user guide](docs/user_guide.md) §1 explains why, and - the limits of Google Drive. + the limits of Google Drive: on Drive the lock cannot guard an output folder across machines + or sessions, so use one session at a time per output folder. From a terminal: diff --git a/docs/plan.md b/docs/plan.md index f316d231..63cb1784 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -383,6 +383,11 @@ API, CLI). The notebook and CLI only call the API. 35.* - **Release v0.3.1:** `position_id` names an image with the other image extensions bricks read (MorphoBricks D85, PR #33). *To be tagged, see question 36.* +- **PP-10 (notebook cards) notes:** a button on the `output_locked` card that removes the + lock of a stopped run (owner, 2 November 2026, PP-8d review: Colab disconnects leave + locks behind). Same rule as `--remove-stale-lock`: only on that request, never the lock + of a process still running on the same computer. Until then the notebook's "Common + problems" gives a `remove_stale_lock(...)` code cell. *To do in PP-10.* - **Release v0.4.0 (PP-13) checklist**, for every tag of v0.4.0 including release candidates, in the release PR before the tag is pushed: - set `version` in `pyproject.toml`; @@ -1264,7 +1269,14 @@ Raised during review; to be decided before the milestones that need them: version check reads them, with `notebooks/notebook_latest_versions.yaml` as the index; the pin is moved at every release (§10, release checklist). Steps 0 and 1 have "Colab users" / "JupyterLab users" callouts (data and results on Drive, output folder outside - the experiment, slower Drive reads, a runtime wiped on disconnect). The run's semantics - are unchanged on Google Drive; its known limits there (the lock is only exclusive within - one runtime, fingerprints of copies) are in the user guide §1 and were put to the owner - in PR #118.* + the experiment, one session per output folder, a runtime wiped on disconnect). The run's + semantics are unchanged on Google Drive; its limits there are in the user guide §1.* + *Decided in review (2 November 2026, owner, PR #118 questions): (1) the lock on Google + Drive is documented only: one session per output folder, because the lock cannot guard + an output folder across machines or sessions on Drive; (2) a button that removes the + lock of a stopped run comes in PP-10 (§10, PP-10 notes); until then a documented code + cell; (3) no advice to copy the dataset to the runtime's own disk, because reuse + depends on the absolute path of the images: data and output stay on Drive; (4) on + Colab, Step 0 installs the `notebook` and `zarr` extras (locally `notebook`, unchanged), + so OME-Zarr images can be read there; (5) `RELEASE_TAG` stays `v0.4.0rc2`; the owner + tags it after PP-8c and PP-8d are merged.* diff --git a/docs/user_guide.md b/docs/user_guide.md index a7ffea0d..03097f00 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -54,14 +54,11 @@ run **Step 0**. On Colab it: - connects your Google Drive: allow access in the window that opens. Your Drive is then the folder `/content/drive/MyDrive`; - installs MorphoBricks preprocessing at the release the notebook names, with its notebook - dependencies, which takes a minute or two (ipywidgets 8 replaces Colab's ipywidgets 7; - if a message asks you to, choose **Runtime › Restart session** and run Step 0 again); + dependencies and the `zarr` extra (to read OME-Zarr images), which takes a minute or two + (ipywidgets 8 replaces Colab's ipywidgets 7; if a message asks you to, choose + **Runtime › Restart session** and run Step 0 again); - enables Colab's widget support for these widgets. -To read OME-Zarr images on Colab, also install the `zarr` extra: in a new code cell run -`import subprocess, sys` and then -`subprocess.run([sys.executable, "-m", "pip", "install", "zarr>=2.18"], check=True)`. - **Where to keep the data.** A Colab runtime is a temporary computer: everything on it except Google Drive is wiped when it disconnects (after some idle time, or after several hours). @@ -70,33 +67,19 @@ Google Drive is wiped when it disconnects (after some idle time, or after severa the **Files** panel in the left bar, right-click the folder, choose **Copy path** and paste. - Keep the output folder on Google Drive too, outside the experiment folder. The default, next to the experiment folder, is right when the experiment is on Drive. -- Google Drive is slower to read than the runtime's own disk. For a large dataset you may - copy the experiment folder to the runtime's disk first, keeping the modification times, - for example in a new code cell: - - ```python - import shutil - - shutil.copytree("/content/drive/MyDrive/experiment_2026_10", "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/content/experiment_2026_10") - ``` - - Then type an output folder on Drive in step 1 (for example - `/content/drive/MyDrive/experiment_2026_10_MorphoBricks_Preprocessing`): the default would - be next to the copy and lost with the runtime. The output records where the images were - read and each image's size and modification time. A later run reuses the rows only when - the images are read from the same place: copy to the same path in every session (as - above; the copy keeps the times) and only new or changed images are read again. Reading - from another place (Drive one time, the copy another) recomputes every row, and plain - stacks must be confirmed again. Later bricks must be given the experiment folder they - should read, because the recorded location is the copy's, which a new runtime no longer - has until you copy again. +- Keep both there for every session. The output records where the images were read (their + full path) and each image's size and modification time; a later run reuses the rows only + when the images are read from the same place. Do not copy the experiment to the runtime's + own disk to read it faster: the copy is lost when the runtime disconnects, and reading it + from another place recomputes every row. **Limits of Google Drive** (the run behaves the same; these are things to know): -- **One session per output folder.** The output folder's lock stops a second run on the - same Colab runtime. Google Drive copies files between computers with a delay, so the lock - does not stop a run started at the same time from another Colab session or from a - computer that syncs the same Drive folder. Never run two sessions on one output folder. +- **One session per output folder.** On Google Drive the lock cannot guard an output folder + across machines or sessions. It only stops a second run in the same Colab runtime. Google + Drive copies files between computers with a delay, so a run started at the same time from + another Colab session, or from a computer that syncs the same Drive folder, is not + stopped. Use one session at a time per output folder on Drive. - **A lock left by a disconnected runtime.** If the runtime disconnects during a check or a run, its lock stays in the output folder and the next check says "Another run is using this output folder". When you are sure no other session uses the folder, remove it in a