From 04b0b9d5eb46e81bd04eaf002cf0a2859a4da207 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 16:20:16 +0000 Subject: [PATCH] Notebook: progress for every wait, progress callbacks from the readers, quick scan first Owner test of v0.5.0 (10 November 2026), items (c) and (d). Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- CHANGELOG.md | 29 ++ pyproject.toml | 2 + src/morphobricks_preprocessing/discovery.py | 38 +-- src/morphobricks_preprocessing/image_names.py | 25 +- .../masks/checks.py | 8 +- src/morphobricks_preprocessing/masks/files.py | 10 +- .../notebook_convert.py | 10 +- .../notebook_folder.py | 50 +++- .../notebook_folder_previews.py | 13 +- .../notebook_mask_step.py | 16 +- .../notebook_masks.py | 11 +- .../notebook_progress.py | 189 +++++++++++++ .../notebook_quick_scan.py | 255 ++++++++++++++++++ .../notebook_results.py | 14 +- .../notebook_run.py | 17 +- .../notebook_steps.py | 15 +- .../notebook_style.py | 6 + .../run/preflight.py | 24 +- tests/test_notebook_progress.py | 245 +++++++++++++++++ tests/test_notebook_quick_scan.py | 192 +++++++++++++ tests/test_notebook_ui.py | 2 +- tools/code_rules_baseline.json | 1 - 22 files changed, 1106 insertions(+), 66 deletions(-) create mode 100644 src/morphobricks_preprocessing/notebook_progress.py create mode 100644 src/morphobricks_preprocessing/notebook_quick_scan.py create mode 100644 tests/test_notebook_progress.py create mode 100644 tests/test_notebook_quick_scan.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d4333b95..5982f910 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,35 @@ v0.5.1: no run before v0.5.1 is read.) ## Unreleased (v0.5.1) +### Notebook redesign 2: progress for every wait, and a quick scan before the headers + +Owner test of v0.5.0 (10 November 2026), items (c) and (d). + +- **Every wait shows what is done of the total, the item being read, the time elapsed and, + once it can be measured, the time left** (`notebook_progress.Wait`, `progress_html`, + `time_left`): the folder walk ("Listing the folders: 14 so far"), the image headers + ("Reading image headers: 34 of 120, CTRL/rep1/a.tif"), the mask headers, the check report, + the pictures per folder, **Check the masks**, the run, the mask step, the named-axis copies + and the statistics (their stage of four). "Checking the folders and image headers…" is gone. + The time left is said only after one item and one second; uncounted work shows the elapsed + time alone. +- **Progress callbacks from the readers.** `discover_experiment(..., progress=)` is called with + each folder before it is listed; `masks.find_masks(..., progress=)` and + `run.preflight.remasked(..., progress)` before each mask header; `preflight`'s `progress` + now also reports `("folders", 0, 1, folder)` and `("masks", done, total, mask_path)`. The + first and last call of each phase, and every result, are unchanged. +- **Quick scan first (the shipped notebook's first step).** **Scan the folders** walks the + folders only (the folder check's own walk; no image header is read), shows every folder with + its labels and image count, and opens one image chosen at random, with its path and seed, for + looking only (drawn from that file's header by the rules of the folder pictures). Nothing of + the scan is kept. The folder check is then the explicit click **Looks right: read all image + headers**; it is disabled, with a line saying why, until the folders are scanned as they are + typed now, so changing a folder, the output or masks folder or the exclusions asks for a new + scan. `folder_step()` for other bricks is unchanged: **Check folder**, one click. +- `is_tiff_name`, `is_system_name`, `TIFF_SUFFIXES` and `SYSTEM_NAMES` moved from `discovery` + to `image_names` (the functions are still importable from `discovery`), keeping + `discovery.py` under the 600-line rule (its baseline entry is gone). + ### Notebook redesign 1: fixes from the owner's test of v0.5.0 (10 November 2026) - **Lists of files in a message draw no empty bullets.** `notebook_style.path_list` returns diff --git a/pyproject.toml b/pyproject.toml index 81b56876..4d6a5f2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,6 +139,8 @@ baseline = "tools/code_rules_baseline.json" "morphobricks_preprocessing.mask_record" = "io" "morphobricks_preprocessing.run_labels" = "io" "morphobricks_preprocessing.notebook_axes" = "ui" +"morphobricks_preprocessing.notebook_progress" = "ui" +"morphobricks_preprocessing.notebook_quick_scan" = "ui" "morphobricks_preprocessing.notebook_axes_quick" = "ui" "morphobricks_preprocessing.notebook_channel_thumbs" = "ui" "morphobricks_preprocessing.notebook_channels" = "ui" diff --git a/src/morphobricks_preprocessing/discovery.py b/src/morphobricks_preprocessing/discovery.py index 8b16201a..052e5700 100644 --- a/src/morphobricks_preprocessing/discovery.py +++ b/src/morphobricks_preprocessing/discovery.py @@ -25,14 +25,14 @@ import logging import os -from collections.abc import Iterable, Sequence +from collections.abc import Callable, Iterable, Sequence from dataclasses import dataclass from pathlib import Path import numpy as np from morphobricks_normalization.records import LEVELS, SINGLE -from .image_names import position_id +from .image_names import is_system_name, is_tiff_name, position_id from .links import LinkGuard, relative_inside from .mask_names import mask_name_issues from .validation import ValidationIssue, agree, error, has_errors, plural, warning @@ -40,13 +40,6 @@ logger = logging.getLogger(__name__) -TIFF_SUFFIXES = (".tif", ".tiff") - -# Files and folders created by operating systems and tools, not by users. They -# are skipped without a warning so that copying an experiment from a Mac or -# opening it in Jupyter does not flood the preflight with messages. -SYSTEM_NAMES = frozenset({"thumbs.db", "desktop.ini", "__macosx"}) - OUTPUT_FOLDER_SUFFIX = "_MorphoBricks_Preprocessing" ROOT = "." @@ -189,20 +182,6 @@ def _same_spelling(seen: dict[str, str], value: str, field: str, scope: str) -> ) -def is_system_name(name: str) -> bool: - """Return True for system files and folders that are skipped silently. - - These are dot-files and dot-folders (``.DS_Store``, ``._image.tif``, - ``.ipynb_checkpoints``), ``Thumbs.db``, ``desktop.ini`` and ``__MACOSX``. - """ - return name.startswith(".") or name.lower() in SYSTEM_NAMES - - -def is_tiff_name(name: str) -> bool: - """Return True if ``name`` has a TIFF extension (``.tif`` or ``.tiff``, any case).""" - return name.lower().endswith(TIFF_SUFFIXES) - - def is_ome_zarr_folder(entry: os.DirEntry) -> bool: """Return True for a folder that is an OME-Zarr image (its name ends in ``.zarr``). @@ -315,7 +294,10 @@ def check_output_folder( def discover_experiment( - experiment_root: str | os.PathLike, output_folder: str | os.PathLike | None = None + experiment_root: str | os.PathLike, + output_folder: str | os.PathLike | None = None, + *, + progress: Callable[[str], None] | None = None, ) -> DiscoveryResult: """Walk an experiment folder, find its leaf folders and check its structure. @@ -330,6 +312,10 @@ def discover_experiment( The planned output folder, if known. It is checked with :func:`check_output_folder` and, if it lies inside the experiment, it is not scanned. + progress + Called with each folder's relative path (``"."`` for the experiment folder) before it + is listed, so a caller can show how far the walk is. It changes nothing about the + result. Returns ------- @@ -346,6 +332,7 @@ def discover_experiment( return DiscoveryResult(root=root, issues=(issue,)) walker = _Walker(root) + walker.progress = progress if output_folder is not None: walker.issues.extend(found := check_output_folder(root, output_folder)) walker.excluded = root.joinpath(found[0].paths[0]) if found else None # walk's name @@ -361,6 +348,7 @@ def __init__(self, root: Path) -> None: self.root = root self.links = LinkGuard(root, "experiment folder") self.excluded: Path | None = None + self.progress: Callable[[str], None] | None = None self.leaves: list[LeafFolder] = [] self.issues: list[ValidationIssue] = [] self.reserved: list[str] = [] @@ -380,6 +368,8 @@ def relative(self, path: Path) -> str: def visit(self, folder: Path, names: tuple[str, ...]) -> None: """Sort the entries of ``folder`` (labels ``names``), following links by the guard.""" + if self.progress is not None: + self.progress(self.relative(folder)) try: entries = sorted(os.scandir(folder), key=lambda entry: entry.name) except OSError: diff --git a/src/morphobricks_preprocessing/image_names.py b/src/morphobricks_preprocessing/image_names.py index fe6c56f2..8be814b1 100644 --- a/src/morphobricks_preprocessing/image_names.py +++ b/src/morphobricks_preprocessing/image_names.py @@ -1,6 +1,7 @@ """An image's position identifier, from its name (contract §1, MorphoBricks D02, D85). -Moved from ``discovery.py`` (PP-5), so that the mask-name screen can use it. +Moved from ``discovery.py`` (PP-5), so that the mask-name screen can use it; the names of +TIFF images and of system files (``is_tiff_name``, ``is_system_name``) are here too. """ from __future__ import annotations @@ -57,3 +58,25 @@ def position_id(image: str) -> str: ) return name[: -len(suffix)] return name + + +TIFF_SUFFIXES = (".tif", ".tiff") + +# Files and folders created by operating systems and tools, not by users. They +# are skipped without a warning so that copying an experiment from a Mac or +# opening it in Jupyter does not flood the preflight with messages. +SYSTEM_NAMES = frozenset({"thumbs.db", "desktop.ini", "__macosx"}) + + +def is_system_name(name: str) -> bool: + """Return True for system files and folders that are skipped silently. + + These are dot-files and dot-folders (``.DS_Store``, ``._image.tif``, + ``.ipynb_checkpoints``), ``Thumbs.db``, ``desktop.ini`` and ``__MACOSX``. + """ + return name.startswith(".") or name.lower() in SYSTEM_NAMES + + +def is_tiff_name(name: str) -> bool: + """Return True if ``name`` has a TIFF extension (``.tif`` or ``.tiff``, any case).""" + return name.lower().endswith(TIFF_SUFFIXES) diff --git a/src/morphobricks_preprocessing/masks/checks.py b/src/morphobricks_preprocessing/masks/checks.py index cbe231e8..1601df79 100644 --- a/src/morphobricks_preprocessing/masks/checks.py +++ b/src/morphobricks_preprocessing/masks/checks.py @@ -6,7 +6,7 @@ from __future__ import annotations import logging -from collections.abc import Iterable, Mapping +from collections.abc import Callable, Iterable, Mapping from dataclasses import replace from pathlib import Path @@ -36,6 +36,7 @@ def _header_issues( unreadable: Iterable[str] = (), stated: Mapping[str, str] | None = None, bind: bool = False, + progress: Callable[[int, int, str], None] | None = None, ) -> list[ValidationIssue]: """Read each mask's header (axes, shape, pixel type) and check it against its image. @@ -46,10 +47,13 @@ def _header_issues( decision of 5 November 2026). ``unreadable`` are paths already known not to open (OME-Zarr ``labels`` groups). With ``bind``, each mask is fingerprinted before and after its header is read (:attr:`.model.MaskFile.bound`, issue #160); a mask whose two - fingerprints differ is ``mask_changed_during_check``. + fingerprints differ is ``mask_changed_during_check``. ``progress(done, total, path)`` is + called before each mask's header is read. """ found: list[tuple[str, str, str]] = [("unreadable_mask", path, "") for path in unreadable] for index, mask in enumerate(files): + if progress is not None: + progress(index, len(files), mask.path) where = mask.location(folder, root) if bind else None before = _fingerprint(where) checked, problems = _check_header(mask, images[mask.image], folder, root, stated or {}) diff --git a/src/morphobricks_preprocessing/masks/files.py b/src/morphobricks_preprocessing/masks/files.py index 3223ab95..10b41fd1 100644 --- a/src/morphobricks_preprocessing/masks/files.py +++ b/src/morphobricks_preprocessing/masks/files.py @@ -7,7 +7,7 @@ import logging import os -from collections.abc import Collection, Iterable, Mapping +from collections.abc import Callable, Collection, Iterable, Mapping from pathlib import Path from ..discovery import ( @@ -46,6 +46,7 @@ def find_masks( mask_axes: Mapping[str, str] | None = None, output_folder: str | os.PathLike | None = None, bind: bool = False, + progress: Callable[[int, int, str], None] | None = None, ) -> MaskInventory: """Pair every image with its masks and check them from their headers, without pixels. @@ -76,6 +77,9 @@ def find_masks( fingerprint with it (:attr:`.model.MaskFile.bound`, issue #160), so a run or the mask step refuses a mask that is no longer the one its header described; a mask that changed while its header was read is the error ``mask_changed_during_check``. + progress + Called as ``progress(done, total, mask_path)`` before each mask's header is read. It + changes nothing about the result. Returns ------- @@ -100,7 +104,9 @@ def find_masks( files.sort(key=lambda f: (f.image, f.mask_type)) issues.extend(_duplicate_issues(files)) stated = dict(mask_axes or {}) - issues.extend(_header_issues(files, images, folder, discovery.root, unreadable, stated, bind)) + issues.extend( + _header_issues(files, images, folder, discovery.root, unreadable, stated, bind, progress) + ) types = tuple(sorted({f.mask_type for f in files})) issues.extend(_unknown_type_issues(stated, types)) issues.extend(_missing_issues(types, files, images)) diff --git a/src/morphobricks_preprocessing/notebook_convert.py b/src/morphobricks_preprocessing/notebook_convert.py index 7b73c823..cd8aeb46 100644 --- a/src/morphobricks_preprocessing/notebook_convert.py +++ b/src/morphobricks_preprocessing/notebook_convert.py @@ -39,6 +39,7 @@ from .conversion_record import RECORD_FILENAME from .notebook_options import _WIDE from .notebook_output import _LockCard +from .notebook_progress import Wait from .notebook_render import blocked_line, issues_html from .planes import format_bytes from .run.output_guard import recorded_masks_of @@ -46,6 +47,7 @@ _LABEL = " Make named-axis copies" _ICON = "copy" +COPYING = "Copying images" _RECHECK_LABEL = " Check the copies in step 1" _COPY_HOLDERS = ( "Another conversion is writing into this folder, or one was stopped (for example Colab " @@ -283,7 +285,8 @@ def _on_convert(self) -> ConversionResult | None: self._busy(self.convert_button, "Making the copies…") ui.set_visible(self.convert_progress, True) self.convert_progress.value, self.convert_progress.bar_style = 0, "info" - self.convert_status.value = ui.working("Checking the images and the folder…") + self._convert_wait = Wait(self.convert_status, bar=False) # the bar widget is above + self._convert_wait.start("Checking the images and the folder") try: with self.log.attached(): result = convert_experiment( @@ -314,8 +317,11 @@ def _on_convert(self) -> ConversionResult | None: def _convert_progressed(self, done: int, total: int, image: str) -> None: self.convert_progress.max = max(total, 1) self.convert_progress.value = done + wait = self._convert_wait + if image and wait.label != COPYING: + wait.phase(COPYING, total) if image: - self.convert_status.value = ui.working(f"Copying image {done + 1} of {total}: {image}") + wait.step(done, total, image) def _show_conversion(self, result: ConversionResult) -> None: """Show a refusal with its card, or the copies, the record and the next step.""" diff --git a/src/morphobricks_preprocessing/notebook_folder.py b/src/morphobricks_preprocessing/notebook_folder.py index b9de0aa6..e9212d67 100644 --- a/src/morphobricks_preprocessing/notebook_folder.py +++ b/src/morphobricks_preprocessing/notebook_folder.py @@ -3,6 +3,7 @@ from __future__ import annotations import html +from collections.abc import Callable from pathlib import Path import ipywidgets as widgets @@ -12,8 +13,9 @@ from .dataset import CONSISTENCY_CODES from .discovery import check_output_folder, default_output_folder from .manifest import MANIFEST_FILENAME -from .notebook_folder_previews import LabelPreviews +from .notebook_folder_previews import DRAWN_AT_ONCE, LabelPreviews from .notebook_options import _WIDE, EXCLUDABLE_CODES +from .notebook_progress import Wait from .notebook_render import ( _folder_tiles, _labels_explanation, @@ -26,6 +28,38 @@ from .unit_consistency import unit_checked from .validation import ValidationIssue, plural +CHECK_LABEL = (" Check folder", "search") +CHECK_PHASES = { + "folders": "Listing the folders", + "headers": "Reading image headers", + "masks": "Reading mask headers", + "acquisitions": "Comparing the images", +} +"""What the folder check's wait says in each phase of :data:`.run.phases.PHASES`.""" + + +def check_progress(wait: Wait) -> Callable[[str, int, int, str], None]: + """Return the folder check's progress callback: it words each phase in ``wait``. + + The headers and the mask headers are counted (done of total, the file being read); the + folder walk says how many folders were listed so far, since its total is not known. + """ + state = {"phase": "", "listed": 0} + + def report(phase: str, done: int, total: int, path: str) -> None: + if phase != state["phase"]: + state["phase"] = phase + wait.phase(CHECK_PHASES[phase], total if phase == "headers" else 0) + if not path: + return + if phase == "folders": + state["listed"] += 1 + wait.step(state["listed"], 0, path) + else: + wait.step(done, total, path) + + return report + class _FolderStep: """Step 1's folder and labels: check the folder, show its issues, confirm the labels.""" @@ -203,7 +237,8 @@ def check_folder(self) -> Preflight | None: return None self._busy(self.check_button, "Checking…") self.folder_next.value = "" - self.folder_status.value = ui.working("Checking the folders and image headers…") + wait = self._folder_wait = Wait(self.folder_status) + wait.start(CHECK_PHASES["folders"]) try: with self.log.attached(): masks, mask_axes = self._check_masks_arguments(Path(folder).resolve()) @@ -216,15 +251,17 @@ def check_folder(self) -> Preflight | None: # Each header is bound to the file's fingerprint: every picture and the # save are of the images as this check read them (audit batch 33). bind_headers=True, + progress=check_progress(wait), ) pre = unit_checked(pre) # a described run checks inside each unit self._note_edits() # a check fills only what the user left as filled self.missing_as_singleton.set(self._recorded_missing_as_singleton(), notify=False) # The report reads images lacking some axes by that answer, as 'check # --missing-as-singleton' does (F10); step 2 asks again before the run. + wait.phase("Writing the check report") self._write_check(pre, self.missing_as_singleton.value) finally: - self._idle(self.check_button, " Check folder", "search") + self._idle(self.check_button, *getattr(self, "_check_label", CHECK_LABEL)) self.preflight = pre self._checked_folder = folder self.mapping = pre.suggested_mapping @@ -244,7 +281,12 @@ def _show_folder_results(self, pre: Preflight) -> None: ui.set_visible(self.design_card, False) self._hide_description() self.labels_table.value = labels_html(pre) - self.label_previews.show(pre) + wait = getattr(self, "_folder_wait", None) + if wait is not None: + wait.phase( + "Drawing a picture per folder", min(len(pre.discovery.leaves), DRAWN_AT_ONCE) + ) + self.label_previews.show(pre, None if wait is None else wait.step) ui.set_visible(self.labels_table, False) ui.set_visible(self.label_previews.widget, True) self.labels_confirmed.set(False, notify=False) diff --git a/src/morphobricks_preprocessing/notebook_folder_previews.py b/src/morphobricks_preprocessing/notebook_folder_previews.py index a9f9e404..4e17d254 100644 --- a/src/morphobricks_preprocessing/notebook_folder_previews.py +++ b/src/morphobricks_preprocessing/notebook_folder_previews.py @@ -12,7 +12,7 @@ from __future__ import annotations import html -from collections.abc import Mapping +from collections.abc import Callable, Mapping from pathlib import Path import ipywidgets as widgets @@ -97,8 +97,11 @@ def clear(self) -> None: self._waiting = [] ui.set_visible(self.more, False) - def show(self, pre: Preflight) -> None: - """Draw the table of a new check: labels, image count, folder and picture per folder.""" + def show(self, pre: Preflight, progress: Callable[[int, int, str], None] | None = None) -> None: + """Draw the table of a new check: labels, image count, folder and picture per folder. + + ``progress(done, total, folder)`` is called before each picture is drawn. + """ self.clear() self._root, self._headers = Path(pre.discovery.root), first_headers(pre) self._bound = pre.header_fingerprints # drawn only as the check read them @@ -106,7 +109,9 @@ def show(self, pre: Preflight) -> None: top = widgets.GridBox([header, widgets.HTML("Preview")], layout=self._grid()) leaves = pre.discovery.leaves rows = [self._row(leaf) for leaf in leaves[:DRAWN_AT_ONCE]] - for leaf in leaves[:DRAWN_AT_ONCE]: + for done, leaf in enumerate(leaves[:DRAWN_AT_ONCE]): + if progress is not None: + progress(done, min(len(leaves), DRAWN_AT_ONCE), leaf.relative_path) self.draw(leaf.relative_path) # The other folders are one HTML block until drawn: a widget per row costs time. self._waiting = list(leaves[DRAWN_AT_ONCE:]) diff --git a/src/morphobricks_preprocessing/notebook_mask_step.py b/src/morphobricks_preprocessing/notebook_mask_step.py index 3ed5a8c2..853f91e7 100644 --- a/src/morphobricks_preprocessing/notebook_mask_step.py +++ b/src/morphobricks_preprocessing/notebook_mask_step.py @@ -37,6 +37,7 @@ mask_type_rows, ) from .notebook_output import _LockCard +from .notebook_progress import Wait from .notebook_render import issues_html from .run import MaskStep, MaskStepResult, RecordedMasks, mask_step from .validation import has_errors, plural @@ -45,6 +46,7 @@ """Marks the step as describing no run (a run is going on): the next refresh reads it again.""" _LABEL = " Describe the masks" _ICON = "sitemap" +READING_MASKS = "Reading the masks, one image at a time" _SUGGESTIONS = ( "These are suggestions from the masks' pixels: check the relationships before using them. " "Each mask keeps the ID it was given on Masks (step 10)." @@ -151,7 +153,8 @@ def _on_describe_masks(self) -> MaskStepResult | None: return None self._busy(self.mask_step_button, "Reading the masks…") self._clear_mask_step() - self.mask_step_status.value = ui.working("Pairing the masks the run recorded…") + self._mask_wait = Wait(self.mask_step_status, bar=False) # the bar is the widget above + self._mask_wait.start("Pairing the masks the run recorded") try: with self.log.attached(): return self._describe_recorded_masks() @@ -205,10 +208,13 @@ def _describe_recorded_masks(self) -> MaskStepResult | None: def _mask_step_progressed(self, done: int, total: int, image: str) -> None: self.mask_step_progress.max = max(total, 1) self.mask_step_progress.value = done - if image: - self.mask_step_status.value = ui.working( - f"Reading the masks of image {done + 1} of {total}: {image}" - ) + wait = self._mask_wait + if not image: + wait.phase("Writing the description") + return + if wait.label != READING_MASKS: + wait.phase(READING_MASKS, total) + wait.step(done, total, image) def _mask_step_failed(self, title: str, lines: list[str]) -> None: """Show a failure as a red card; nothing was written.""" diff --git a/src/morphobricks_preprocessing/notebook_masks.py b/src/morphobricks_preprocessing/notebook_masks.py index a4fce000..7779e60f 100644 --- a/src/morphobricks_preprocessing/notebook_masks.py +++ b/src/morphobricks_preprocessing/notebook_masks.py @@ -36,6 +36,7 @@ from .masks import MaskInventory from .metadata import TiffMetadata from .notebook_options import _WIDE +from .notebook_progress import Wait from .notebook_render import issue_count_chips, issues_html from .run.planned import PlannedMasks, planned_masks from .run.preflight import remasked @@ -102,6 +103,7 @@ def _build_masks_card(self) -> None: icon="refresh", tooltip="Pair the masks again with the masks folder and the mask axes above", ) + self.masks_wait = widgets.HTML() self.masks_counts = widgets.HTML() self.masks_issues = widgets.HTML() self.masks_card = ui.card( @@ -111,6 +113,7 @@ def _build_masks_card(self) -> None: self.masks_table, self.mask_axes_box, self.masks_button, + self.masks_wait, self.masks_counts, self.masks_issues, ], @@ -160,13 +163,13 @@ def _check_masks(self) -> None: root = self.preflight.discovery.root plan = self._planned_masks(root) output = self._output_folder() or default_output_folder(root) + wait = Wait(self.masks_wait) + wait.start("Reading mask headers") with self.log.attached(): self.preflight = remasked( - self.preflight, - plan.folder, - plan.mask_axes or None, - output, + self.preflight, plan.folder, plan.mask_axes or None, output, wait.step ) + self.masks_wait.value = "" self._paired_masks = _pairing(plan) self._show_issue_summary() self._refresh_run_step() diff --git a/src/morphobricks_preprocessing/notebook_progress.py b/src/morphobricks_preprocessing/notebook_progress.py new file mode 100644 index 00000000..f711de57 --- /dev/null +++ b/src/morphobricks_preprocessing/notebook_progress.py @@ -0,0 +1,189 @@ +"""Progress of a wait in the notebook: what is done of the total, elapsed time, time left. + +Every wait that can take more than about a second shows one (owner test of v0.5.0, 10 November +2026): the folder walk, the image headers, the mask headers, the pictures, the save, the run, +the mask step and the group statistics. A :class:`Wait` is started with a label, told +``step(done, total, item)`` by the reader's progress callback, and renders one status line +with a bar. The time left is said only once it can be measured: after at least one item and +one second (:func:`time_left`); before that, and for work that cannot be counted, only the +elapsed time is shown. Nothing here reads a file or decides anything. +""" + +from __future__ import annotations + +import html +import time +from collections.abc import Callable + +import ipywidgets as widgets + +MEASURABLE_SECONDS = 1.0 +"""No estimate of the time left before this much time was spent on at least one item.""" +MIN_INTERVAL = 0.1 +"""Seconds between two redraws of a wait (the first and the last are always drawn).""" + + +def clock_text(seconds: float) -> str: + """Return a duration as ``m:ss`` (``h:mm:ss`` from an hour on).""" + whole = max(int(round(seconds)), 0) + hours, rest = divmod(whole, 3600) + minutes, secs = divmod(rest, 60) + return f"{hours}:{minutes:02d}:{secs:02d}" if hours else f"{minutes}:{secs:02d}" + + +def time_left(done: int, total: int, elapsed: float) -> float | None: + """Return the estimated seconds left, or None while it cannot be measured. + + The estimate is the mean time per item done so far times the items left. It needs at + least one item done, :data:`MEASURABLE_SECONDS` elapsed and a known total. + """ + if total <= 0 or done <= 0 or done > total or elapsed < MEASURABLE_SECONDS: + return None + return elapsed / done * (total - done) + + +def progress_html( + label: str, + done: int, + total: int, + item: str, + elapsed: float, + *, + bar: bool = True, + measured: float | None = None, +) -> str: + """Return the status line of a wait (HTML): label, count, item, elapsed and time left. + + Parameters + ---------- + label + What is being done, e.g. ``"Reading image headers"``. + done, total + Items done and to do; a ``total`` of 0 means the work cannot be counted (``done`` + is then shown alone when it is not 0, e.g. folders listed so far). + item + The item being read now (a relative path), or an empty string. + elapsed + Seconds since the wait started. + bar + False leaves the bar out, for a wait that already has a bar widget. + measured + Seconds spent on these ``done`` items, when the wait had other parts before them + (the estimate of the time left uses it); None means ``elapsed``. + """ + parts = [f"{html.escape(label)}"] + if total > 0: + parts.append(f"{min(done, total):,} of {total:,}") + elif done > 0: + parts.append(f"{done:,} so far") + if item: + parts.append(html.escape(item)) + times = [f"{clock_text(elapsed)} elapsed"] + left = time_left(done, total, elapsed if measured is None else measured) + if left is not None: + times.append(f"about {clock_text(left)} left") + head = ( + f"
{' · '.join(parts)}" + f"{' · '.join(times)}
" + ) + if not bar: + return f"
{head}
" + if total > 0: + fill = ( + f"
" + ) + else: + fill = "
" + return f"
{head}
{fill}
" + + +class Wait: + """One wait shown in an HTML widget: started, stepped by a progress callback, finished. + + Parameters + ---------- + target + The HTML widget the line is drawn in (a step's status line, usually). + bar + False for a wait that has its own bar widget beside the line. + clock + Returns the time in seconds (monotonic); replaced in tests. + + Attributes + ---------- + drawn + How many times the line was drawn since :meth:`start` (tests, and the throttle). + """ + + def __init__( + self, + target: widgets.HTML, + *, + bar: bool = True, + clock: Callable[[], float] = time.monotonic, + ) -> None: + self.target = target + self.bar = bar + self.clock = clock + self.label = "" + self.drawn = 0 + self._started = self._phase_started = self._last = 0.0 + self._state: tuple[int, int, str] = (0, 0, "") + + @property + def elapsed(self) -> float: + """Seconds since :meth:`start`.""" + return self.clock() - self._started + + def start(self, label: str, total: int = 0) -> None: + """Start the wait: the clock is set and the line drawn with nothing done.""" + self.label = label + self._started = self._phase_started = self.clock() + self.drawn = 0 + self._draw(0, total, "") + + def phase(self, label: str, total: int = 0, done: int = 0) -> None: + """Name the next part of the same wait; the elapsed time goes on. + + ``done`` of ``total`` says which part this is when the parts themselves are counted. + """ + self.label = label + self._phase_started = self.clock() + self._draw(done, total, "") + + def step(self, done: int, total: int = 0, item: str = "") -> None: + """Report progress; redrawn at most every :data:`MIN_INTERVAL` seconds, and at the end.""" + self._state = (done, total, item) + last = total > 0 and done >= total + if last or self.clock() - self._last >= MIN_INTERVAL: + self._draw(done, total, item) + + def _draw(self, done: int, total: int, item: str) -> None: + self._state = (done, total, item) + self._last = self.clock() + self.drawn += 1 + self.target.value = self.html(done, total, item) + + def html(self, done: int, total: int, item: str) -> str: + """Return the line as it would be drawn now. + + The time left is estimated from the current part's own items (a part starts with + :meth:`start` or :meth:`phase`); the elapsed time shown is that of the whole wait. + """ + now = self.clock() + return progress_html( + self.label, + done, + total, + item, + now - self._started, + bar=self.bar, + measured=now - self._phase_started, + ) + + def took(self) -> str: + """Return how long the wait took, for the message that replaces it (``"0:12"``).""" + return clock_text(self.elapsed) + + +__all__ = ["MEASURABLE_SECONDS", "MIN_INTERVAL", "Wait", "clock_text", "progress_html", "time_left"] diff --git a/src/morphobricks_preprocessing/notebook_quick_scan.py b/src/morphobricks_preprocessing/notebook_quick_scan.py new file mode 100644 index 00000000..82fd2364 --- /dev/null +++ b/src/morphobricks_preprocessing/notebook_quick_scan.py @@ -0,0 +1,255 @@ +"""The quick scan of the shipped notebook's first step: look before every header is read. + +Owner decision (10 November 2026): the first step is *choose the folders*, then a **quick +scan** that only walks the folders (the same walk as the folder check: labels and image counts, +no image header is read) and opens **one** image chosen at random, so the user sees that this is +the right experiment before waiting for every header. Only then the explicit click **Looks +right: read all image headers** starts the folder check (the "deep scan"), with its progress. + +The random image is for looking only. Its path and the seed that chose it are shown; it is drawn +from its own header by the rules of the folder pictures (:mod:`.header_preview`: only when the +header names Y and X, bound to the file as read). Nothing of the quick scan is kept: no label, +axis, header or picture of it feeds the check, the draft or the run. Changing a folder makes the +scan stale, and the check is not offered until the folders are scanned again. +""" + +from __future__ import annotations + +import html +import random +import secrets +from pathlib import Path + +import ipywidgets as widgets + +from . import notebook_style as ui +from .discovery import DiscoveryResult, discover_experiment +from .fingerprint import current_fingerprint +from .header_preview import HEADER_NOTE, HeaderPreview, header_preview +from .metadata import inspect_image +from .notebook_progress import Wait +from .notebook_render import blocked_line, issues_html +from .notebook_thumb_tile import ThumbTile +from .run.recorded import normalise_relative_path +from .validation import plural + +SCAN_LABEL, DEEP_LABEL = " Scan the folders", " Looks right: read all image headers" +SEED_RANGE = 1_000_000 +EXAMPLE_SIDE = 320 +"""The largest side of the quick scan's picture, in pixels.""" + + +def tree_html(discovery: DiscoveryResult, excluded: frozenset[str]) -> str: + """Return the folders found as a table: labels and image count per folder (HTML).""" + rows = [("Condition", "Biological repeat", "Technical repeat", "Images")] + for leaf in discovery.leaves: + count = sum(image not in excluded for image in leaf.images) + rows.append((*leaf.labels, f"{count:,}")) + return ui.table(rows, numeric_from=3) + + +def scan_summary(discovery: DiscoveryResult, excluded: frozenset[str]) -> str: + """Return one line of counts: images, folders, conditions and biological repeats.""" + leaves = discovery.leaves + images = sum(image not in excluded for image in discovery.images) + conditions = {leaf.condition for leaf in leaves} + repeats = {(leaf.condition, leaf.biological_repeat) for leaf in leaves} + parts = [ + plural(images, "image"), + plural(len(leaves), "folder"), + plural(len(conditions), "condition"), + plural(len(repeats), "biological repeat"), + ] + left_out = len(discovery.images) - images + if left_out: + parts.append(f"{plural(left_out, 'image')} excluded") + return " · ".join(parts) + + +def random_image(images: list[str], seed: int) -> str | None: + """Return the image the seed chooses among ``images`` (sorted first), None without one.""" + return random.Random(seed).choice(sorted(images)) if images else None + + +def example_preview(root: Path, image: str) -> HeaderPreview: + """Return the picture of one image from its own header, or why there is none. + + The header is read here, for this picture only, and the file is fingerprinted before and + after, so the picture is of the file as that header describes it. + """ + path = root / image + try: + before = current_fingerprint(path) + meta = inspect_image(path, image) + after = current_fingerprint(path) + except (OSError, ValueError) as exc: + return HeaderPreview(image, problem=f"cannot be read for a preview: {exc}") + bound = {image: before if before is not None and before == after else None} + return header_preview(root, meta, EXAMPLE_SIDE, bound=bound) + + +class _QuickScan: + """The quick scan card of the stepper's first step, and the gate of the folder check.""" + + def _build_quick_scan(self) -> None: + """Build the scan button, its results and the line saying what blocks the check.""" + self._scanned: tuple | None = None + self._scan: DiscoveryResult | None = None + self.scan_seed: int | None = None + self.scan_image: str | None = None + self.scan_button = ui.button( + SCAN_LABEL, + style="primary", + icon="search", + tooltip="List the folders and show one image; no image header is read", + main=True, + ) + self.scan_status = widgets.HTML() + self.scan_tree = widgets.HTML() + self.scan_example = widgets.VBox() + self.scan_another = ui.button( + " Show another image", icon="random", tooltip="Choose another image at random" + ) + self.scan_box = widgets.HBox( + [ + widgets.VBox([self.scan_tree], layout=widgets.Layout(flex="1 1 0", min_width="0")), + widgets.VBox( + [self.scan_example, self.scan_another], + layout=widgets.Layout(flex="0 0 auto", margin="0 0 0 16px"), + ), + ], + layout=widgets.Layout(width="99%"), + ) + self.scan_box.add_class("mb-scan") + ui.set_visible(self.scan_box, False) + self.check_blocker = widgets.HTML() + self.scan_button.on_click(lambda _: self._quick_scan()) + self.scan_another.on_click(lambda _: self._show_scan_example(None)) + self._check_label = (DEEP_LABEL, "check") + self.check_button.description, self.check_button.icon = self._check_label + self.check_button.button_style = "success" + self.check_button.tooltip = "Read every image header and check the experiment" + for box in (self.folder, self.output, self.masks_folder, self.exclude): + box.observe(lambda _: self._refresh_scan_gate(), names="value") + self._refresh_scan_gate() + + def _scan_inputs(self) -> tuple: + """Return the folders and exclusions as typed now (what a scan is a scan of).""" + return ( + ui.clean_path(self.folder.value), + self._output_folder(), + ui.clean_path(self.masks_folder.value), + tuple(self._excluded()), + ) + + def _scan_current(self) -> bool: + """Return whether the folders shown were scanned as they are typed now.""" + return self._scanned is not None and self._scanned == self._scan_inputs() + + def _refresh_scan_gate(self) -> None: + """Offer the folder check only for folders scanned as typed; say what blocks it.""" + current = self._scan_current() + if self._scanned is not None and not current: + ui.set_visible(self.scan_box, False) # the tree and picture are of other folders + self.scan_status.value = ui.status( + "info", "The folders changed.", ["Scan the folders again."] + ) + usable = current and self._scan is not None and bool(self._scan.images) + self.check_button.disabled = not usable + if usable: + reasons: list[str] = [] + elif not ui.clean_path(self.folder.value): + reasons = ["no experiment folder is given"] + elif not current: + reasons = ["the folders are not scanned as they are now: click Scan the folders"] + else: + reasons = ["the scan found no image to read"] + self.check_blocker.value = blocked_line(reasons) + + def _quick_scan(self) -> DiscoveryResult | None: + """Walk the folders (no header is read), show the tree and one random image. + + Returns + ------- + DiscoveryResult or None + What the walk found; None when no experiment folder is given. + """ + folder = ui.clean_path(self.folder.value) + if not folder: + self.scan_status.value = ui.status( + "warn", "Enter the experiment folder first.", ["Paste its path or use Browse…."] + ) + return None + self._busy(self.scan_button, "Scanning…") + wait = Wait(self.scan_status) + wait.start("Listing the folders") + seen = [0] + + def listed(path: str) -> None: + seen[0] += 1 + wait.step(seen[0], 0, path) + + try: + with self.log.attached(): + found = discover_experiment(folder, self._output_folder(), progress=listed) + finally: + self._idle(self.scan_button, SCAN_LABEL, "search") + self._scan, self._scanned = found, self._scan_inputs() + wait.phase("Opening one image") + self._show_scan_example(None, status=False) + excluded = frozenset(normalise_relative_path(path) for path in self._excluded()) + self.scan_tree.value = tree_html(found, excluded) + errors = [issue for issue in found.issues if issue.is_error] + self.scan_status.value = ui.status( + "error" if errors else "ok" if found.images else "warn", + f"Found {scan_summary(found, excluded)} (listed in {wait.took()}).", + ["No image header was read. If this is your experiment, read the headers below."], + ) + (issues_html(errors) if errors else "") + ui.set_visible(self.scan_box, bool(found.leaves)) + self._refresh_scan_gate() + self._show_log() + return found + + def _show_scan_example(self, seed: int | None, *, status: bool = True) -> None: + """Show one image chosen at random by ``seed`` (a new seed when None), display only.""" + found = self._scan + if found is None: + return + excluded = {normalise_relative_path(path) for path in self._excluded()} + images = [image for image in found.images if image not in excluded] + self.scan_seed = secrets.randbelow(SEED_RANGE) if seed is None else seed + self.scan_image = random_image(images, self.scan_seed) + ui.set_visible(self.scan_another, len(images) > 1) + if self.scan_image is None: + self.scan_example.children = [] + return + if status: + self._busy(self.scan_another, "Opening…") + try: + preview = example_preview(Path(found.root), self.scan_image) + finally: + if status: + self._idle(self.scan_another, " Show another image", "random") + tile = ThumbTile( + preview.pictures, + caption=f"{preview.text}; {HEADER_NOTE}" if preview.pictures is not None else "", + problem=preview.problem, + colour=preview.colour, + width=f"{EXAMPLE_SIDE}px", + ) + chosen = ( + "
One image chosen at random (seed " + f"{self.scan_seed}), for looking only:
{html.escape(self.scan_image)}
" + ) + self.scan_example.children = [widgets.HTML(chosen), tile.widget] + + +__all__ = [ + "DEEP_LABEL", + "SCAN_LABEL", + "_QuickScan", + "example_preview", + "random_image", + "scan_summary", + "tree_html", +] diff --git a/src/morphobricks_preprocessing/notebook_results.py b/src/morphobricks_preprocessing/notebook_results.py index c73825ac..9af4c27c 100644 --- a/src/morphobricks_preprocessing/notebook_results.py +++ b/src/morphobricks_preprocessing/notebook_results.py @@ -13,9 +13,13 @@ from . import notebook_style as ui from .notebook_options import _WIDE, LEVEL_OPTIONS from .notebook_plots import IntensityPanel, distribution_png +from .notebook_progress import Wait from .report import LEVEL_NAMES, group_summaries, group_table from .validation import plural +STAGES = 4 +"""The parts of the statistics: read the table, pool, plot, check.""" + class _ResultsStep: """Step 4: pooled statistics and distributions per group of any histogram table.""" @@ -186,7 +190,8 @@ def show_statistics(self) -> None: ) return self._busy(self.statistics_button, "Calculating…") - self.stats_status.value = ui.working("Reading the histogram table…") + self._stats_wait = Wait(self.stats_status) + self._stats_wait.start("Reading the histogram table") self.stats_tiles.value = self.statistics.value = "" ui.set_visible(self.statistics_card, False) ui.set_visible(self.distribution_card, False) @@ -212,7 +217,8 @@ def _calculate_statistics(self, path: Path, percentiles: Sequence[float]) -> Non except (CsvFormatError, OSError) as exc: self.stats_status.value = ui.status("error", "The table cannot be read.", [str(exc)]) return - self.stats_status.value = ui.working("Calculating statistics for each group…") + wait = self._stats_wait + wait.phase("Pooling the planes of each group", STAGES, 1) level = self.level.value summaries = group_summaries(records, level, percentiles) groups = group_records(records, level) @@ -233,11 +239,11 @@ def _calculate_statistics(self, path: Path, percentiles: Sequence[float]) -> Non # The labels and the channel column are text; the counts and statistics numbers. self.statistics.value = ui.table(rows, numeric_from=depth + 1) ui.set_visible(self.statistics_card, True) - self.stats_status.value = ui.working("Drawing the intensity distributions…") + wait.phase("Drawing the intensity distributions", STAGES, 2) png = distribution_png(groups) self.distribution.value = png or b"" ui.set_visible(self.distribution_card, bool(png)) - self.stats_status.value = ui.working("Checking the intensities of every image…") + wait.phase("Checking the intensities of every image", STAGES, 3) self.table_checks.show(records) self.stats_status.value = ui.status( "ok", diff --git a/src/morphobricks_preprocessing/notebook_run.py b/src/morphobricks_preprocessing/notebook_run.py index 2d1cca25..dd5593df 100644 --- a/src/morphobricks_preprocessing/notebook_run.py +++ b/src/morphobricks_preprocessing/notebook_run.py @@ -26,6 +26,7 @@ VERIFY_CONTENT_HELP, ) from .notebook_plots import IntensityPanel +from .notebook_progress import Wait from .notebook_render import _bins_help, _label, blocked_line, change_lines, issues_html from .planes import DEFAULT_MEMORY_LIMIT from .report import issue_counts, rebuild_line @@ -33,6 +34,8 @@ from .run_labels import recorded_labels from .validation import has_errors, plural +READING = "Reading images" + class _RunStep: """Step 3: settings, what will happen, and the run.""" @@ -315,7 +318,8 @@ def _start_run(self) -> None: self._mask_step_stale() ui.set_visible(self.progress, True) self.progress.value, self.progress.max, self.progress.bar_style = 0, 1, "info" - self.run_status.value = ui.working("Preparing: checking everything again…") + self._run_wait = Wait(self.run_status, bar=False) # the bar is the widget above + self._run_wait.start("Preparing: checking the folders and the image headers again") self.run_tiles.value = self.run_issues.value = "" def _call_run(self, histogram_settings: HistogramSettings) -> RunResult: @@ -457,7 +461,10 @@ def _show_run_checks(self, path: Path) -> None: def _progress(self, done: int, total: int, path: str) -> None: self.progress.max = max(total, 1) self.progress.value = done - if path: - self.run_status.value = ui.working(f"Reading image {done + 1} of {total}: {path}") - else: - self.run_status.value = ui.working("Writing the table…") + wait = self._run_wait + if not path: + wait.phase("Writing the table") + return + if wait.label != READING: + wait.phase(READING, total) + wait.step(done, total, path) diff --git a/src/morphobricks_preprocessing/notebook_steps.py b/src/morphobricks_preprocessing/notebook_steps.py index 0ae630bf..66d68b72 100644 --- a/src/morphobricks_preprocessing/notebook_steps.py +++ b/src/morphobricks_preprocessing/notebook_steps.py @@ -24,14 +24,16 @@ from .notebook_channels_type import ImageTypeRows from .notebook_check_save import INTRO as CHECK_INTRO from .notebook_masks_screen import INTRO as MASKS_INTRO +from .notebook_quick_scan import _QuickScan from .notebook_step_states import _StepStates from .notebook_stepper import Step, Stepper from .run import Preflight FOLDER_INTRO = ( - "What happens here: the folders are scanned, the labels are taken from the folder " - "names (condition / biological repeat / technical repeat) and every image header is " - "checked. Problems are listed with what to do. Nothing in the folder is changed." + "What happens here: first the folders are listed (labels from the folder names: " + "condition / biological repeat / technical repeat) and one image is shown, so you can see " + "that this is the right experiment. Then every image header is read and checked; problems " + "are listed with what to do. Nothing in the folder is changed." ) AXES_INTRO = ( "Why this matters: every image is cut into flat 2D planes, one per depth slice, " @@ -55,7 +57,7 @@ """The steps (0-based) that show a card the saved reopening folds.""" -class _StepsFlow(_QuickAxes, _StepStates): +class _StepsFlow(_QuickScan, _QuickAxes, _StepStates): """``steps()``: the stepper of the shipped notebook, and the hooks that keep it current.""" def steps(self) -> widgets.Widget: @@ -81,6 +83,7 @@ def steps(self) -> widgets.Widget: self.type_rows.reopen = self.channel_grid.reopen = self._reopen_units self.overview_box = widgets.VBox() self._build_quick_axes() + self._build_quick_scan() self.consistency_slot = widgets.VBox([self.consistency_card]) self._stepper = Stepper(self._step_list(), self._step_shown) self.overview_box.children = [self._stepper.overview(OVERVIEW_STEPS)] @@ -132,7 +135,11 @@ def _images_page(self) -> list[widgets.Widget]: self.output_card, self.masks_input_card, self.exclude_box, + self.scan_button, + self.scan_status, + self.scan_box, self.check_button, + self.check_blocker, self.folder_status, self.folder_lock.card, self.check_report.widget, diff --git a/src/morphobricks_preprocessing/notebook_style.py b/src/morphobricks_preprocessing/notebook_style.py index f2ef6cc6..24a97e67 100644 --- a/src/morphobricks_preprocessing/notebook_style.py +++ b/src/morphobricks_preprocessing/notebook_style.py @@ -88,6 +88,12 @@ border: 2px solid #93C5FD; border-top-color: #2563EB; border-radius: 50%; animation: mb-spin 0.8s linear infinite; vertical-align: -1px; } @keyframes mb-spin { to { transform: rotate(360deg); } } +.mb-wait-head { display: flex; justify-content: space-between; gap: 12px; flex-wrap: wrap; } +.mb-wait-time { color: #475569; font-variant-numeric: tabular-nums; white-space: nowrap; } +.mb-bar { height: 8px; background: #DBEAFE; border-radius: 4px; margin-top: 6px; overflow: hidden; } +.mb-bar-fill { height: 8px; background: #2563EB; border-radius: 4px; } +.mb-bar-unknown { width: 30%; animation: mb-slide 1.2s ease-in-out infinite alternate; } +@keyframes mb-slide { from { margin-left: 0; } to { margin-left: 70%; } } .mb-root .widget-button { border-radius: 6px; } .mb-root .widget-button:disabled { opacity: 0.45; cursor: not-allowed; } diff --git a/src/morphobricks_preprocessing/run/preflight.py b/src/morphobricks_preprocessing/run/preflight.py index 8d4f27dd..0b576765 100644 --- a/src/morphobricks_preprocessing/run/preflight.py +++ b/src/morphobricks_preprocessing/run/preflight.py @@ -203,7 +203,9 @@ def preflight( Called as ``progress(phase, done, total, path)`` while the check runs, with a phase of :data:`.phases.PHASES` (``folders``, ``headers``, ``masks``, ``acquisitions``): a phase is announced as ``(phase, 0, total, "")`` and ends with ``(phase, total, total, "")``; - the headers phase also reports ``(phase, done, total, path)`` before each image. + the headers phase also reports ``(phase, done, total, path)`` before each image, the + masks phase ``("masks", done, total, mask_path)`` before each mask's header, and the + folders phase ``("folders", 0, 1, folder)`` before each folder is listed. :func:`.phases.progress_text` words it. It changes nothing about the result. Returns @@ -214,7 +216,9 @@ def preflight( root = Path(experiment_root).resolve() output = default_output_folder(root) if output_folder is None else Path(output_folder) report("folders", 0, 1, "") - discovery = discover_experiment(root, output) + discovery = discover_experiment( + root, output, progress=lambda folder: report("folders", 0, 1, folder) + ) # Every run is described: a folder label that is not NFC cannot be a source folder. discovery = replace( discovery, issues=(*discovery.issues, *non_nfc_folders_issue(discovery.leaves)) @@ -226,7 +230,15 @@ def preflight( inspected, bound = _inspected(root, images, bind_headers, progress) metadata, calibration = _calibrated(inspected, discovery.images, output) report("masks", 0, 1, "") - masks = _masks(discovery, metadata, masks_folder, mask_axes, output, bind_headers) + masks = _masks( + discovery, + metadata, + masks_folder, + mask_axes, + output, + bind_headers, + lambda done, total, path: report("masks", done, total, path), + ) report("masks", 1, 1, "") report("acquisitions", 0, 1, "") result = Preflight( @@ -288,6 +300,7 @@ def remasked( masks_folder: str | os.PathLike | None, mask_axes: Mapping[str, str] | None, output_folder: str | os.PathLike, + progress: Callable[[int, int, str], None] | None = None, ) -> Preflight: """Return the preflight with its masks paired again, as :func:`preflight` pairs them. @@ -306,6 +319,8 @@ def remasked( As :func:`preflight`. output_folder The output folder (it may not be inside the masks folder). + progress + Called as ``progress(done, total, mask_path)`` before each mask's header is read. Returns ------- @@ -319,6 +334,7 @@ def remasked( mask_axes, Path(output_folder), bind=bool(pre.header_fingerprints), + progress=progress, ) return replace(pre, masks=masks) @@ -330,6 +346,7 @@ def _masks( mask_axes: Mapping[str, str] | None, output: Path, bind: bool = False, + progress: Callable[[int, int, str], None] | None = None, ) -> MaskInventory | None: """Pair the analysed images with their masks; None without masks folder or label image.""" masks = find_masks( @@ -339,6 +356,7 @@ def _masks( mask_axes=mask_axes, output_folder=output, bind=bind, + progress=progress, ) if masks_folder is None and not masks.files and not masks.issues: return None # no masks folder and no label images: the run has no masks diff --git a/tests/test_notebook_progress.py b/tests/test_notebook_progress.py new file mode 100644 index 00000000..6733ef81 --- /dev/null +++ b/tests/test_notebook_progress.py @@ -0,0 +1,245 @@ +"""Progress for every wait of the notebook, and the readers' progress callbacks (owner, (c)). + +A wait shows what is done of the total, the item being read, the elapsed time and, once it +can be measured, the time left. The folder walk and the mask headers report their progress. +""" + +import re + +import pytest + +from morphobricks_preprocessing.discovery import discover_experiment +from morphobricks_preprocessing.run import preflight +from morphobricks_preprocessing.run.preflight import remasked +from test_notebook_check_save import answer_all, exp # noqa: F401 + +pytest.importorskip("ipywidgets") +pytest.importorskip("matplotlib") + +import ipywidgets as widgets # noqa: E402 + +from morphobricks_preprocessing.notebook_folder import CHECK_PHASES, check_progress # noqa: E402 +from morphobricks_preprocessing.notebook_progress import ( # noqa: E402 + MIN_INTERVAL, + Wait, + clock_text, + progress_html, + time_left, +) +from morphobricks_preprocessing.run.phases import PHASES # noqa: E402 +from test_notebook_axes_quick import stepper # noqa: E402 + + +def _text(html): + return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html)).strip() + + +def _history(widget): + seen = [] + widget.observe(lambda change: seen.append(_text(change["new"])), names="value") + return seen + + +class Clock: + def __init__(self): + self.now = 100.0 + + def __call__(self): + return self.now + + +# --- the pure parts ------------------------------------------------------------------------------ + + +@pytest.mark.parametrize( + ("seconds", "text"), [(0, "0:00"), (9.4, "0:09"), (75, "1:15"), (3725, "1:02:05"), (-3, "0:00")] +) +def test_clock_text(seconds, text): + assert clock_text(seconds) == text + + +def test_time_left_only_once_it_can_be_measured(): + assert time_left(0, 10, 5.0) is None # nothing done yet + assert time_left(3, 10, 0.5) is None # too early to measure + assert time_left(3, 0, 5.0) is None # the total is not known + assert time_left(11, 10, 5.0) is None + assert time_left(2, 10, 4.0) == pytest.approx(16.0) # 2 s per item, 8 left + assert time_left(10, 10, 4.0) == 0 + + +def test_progress_html_says_done_of_total_item_elapsed_and_time_left(): + said = _text(progress_html("Reading image headers", 34, 120, "CTRL/rep1/.tif", 17.0)) + assert said == ( + "Reading image headers · 34 of 120 · CTRL/rep1/<a>.tif 0:17 elapsed · about 0:43 left" + ) + html = progress_html("Reading image headers", 30, 120, "", 2.0) + assert "width:25.0%" in html and "mb-bar-unknown" not in html + early = _text(progress_html("Reading image headers", 0, 120, "a.tif", 0.2)) + assert "0 of 120" in early and "elapsed" in early and "left" not in early + + +def test_progress_html_of_uncounted_work_has_no_total_and_no_estimate(): + html = progress_html("Listing the folders", 7, 0, "CTRL/rep1", 3.0) + assert "mb-bar-unknown" in html + assert _text(html) == "Listing the folders · 7 so far · CTRL/rep1 0:03 elapsed" + assert "mb-bar" not in progress_html("Reading images", 1, 3, "a.tif", 3.0, bar=False) + + +def test_a_wait_is_redrawn_at_most_every_interval_and_at_the_end(): + clock, target = Clock(), widgets.HTML() + wait = Wait(target, clock=clock) + wait.start("Reading image headers", 1000) + assert wait.drawn == 1 and "0 of 1,000" in _text(target.value) + for done in range(1, 500): + wait.step(done, 1000, f"{done}.tif") # no time passes: nothing is redrawn + assert wait.drawn == 1 + clock.now += 2 * MIN_INTERVAL + wait.step(500, 1000, "500.tif") + assert wait.drawn == 2 and "500 of 1,000 · 500.tif" in _text(target.value) + wait.step(1000, 1000, "") # the end is always drawn + assert wait.drawn == 3 and "1,000 of 1,000" in _text(target.value) + + +def test_a_wait_estimates_from_its_current_part_and_keeps_the_whole_elapsed_time(): + clock, target = Clock(), widgets.HTML() + wait = Wait(target, clock=clock) + wait.start("Listing the folders") + clock.now += 10 + wait.phase("Reading image headers", 100) + clock.now += 5 + wait.step(10, 100, "a.tif") + said = _text(target.value) + assert "0:15 elapsed" in said # the whole wait + assert "about 0:45 left" in said # 0.5 s per header, 90 headers left + assert wait.took() == "0:15" + + +# --- the readers' callbacks ---------------------------------------------------------------------- + + +def test_the_folder_walk_reports_each_folder_and_finds_the_same(exp): # noqa: F811 + root = exp[0] + seen = [] + found = discover_experiment(root, exp[2], progress=seen.append) + assert found == discover_experiment(root, exp[2]) + assert seen[0] == "." and seen == sorted(seen, key=lambda p: (p != ".", p)) + assert {leaf.relative_path for leaf in found.leaves} <= set(seen) + assert len(seen) == len(set(seen)) # every folder once + + +def test_the_check_reports_each_mask_header_and_each_folder(exp): # noqa: F811 + root, masks, out = exp + calls = [] + pre = preflight(root, out, masks_folder=masks, progress=lambda *call: calls.append(call)) + same = preflight(root, out, masks_folder=masks) + assert pre.masks == same.masks and pre.metadata == same.metadata + order = [phase for phase, *_ in calls] + assert order == sorted(order, key=PHASES.index) + reading = [ + (done, total, path) for phase, done, total, path in calls if phase == "masks" and path + ] + assert [done for done, _, _ in reading] == list(range(len(pre.masks.files))) + assert {total for _, total, _ in reading} == {len(pre.masks.files)} + assert [path for _, _, path in reading] == [mask.path for mask in pre.masks.files] + folders = [path for phase, _, _, path in calls if phase == "folders" and path] + assert folders[0] == "." and len(folders) > len(pre.discovery.leaves) + for phase in PHASES: # the documented first and last call of every phase are unchanged + mine = [call for call in calls if call[0] == phase] + assert mine[0][1] == 0 and mine[-1][1] == mine[-1][2] + + +def test_pairing_the_masks_again_reports_each_mask(exp): # noqa: F811 + root, masks, out = exp + pre = preflight(root, out, masks_folder=masks) + calls = [] + again = remasked(pre, masks, None, out, lambda *call: calls.append(call)) + assert again.masks == pre.masks + assert [call[0] for call in calls] == list(range(len(pre.masks.files))) + + +def test_check_progress_words_each_phase(): + clock, target = Clock(), widgets.HTML() + wait = Wait(target, clock=clock) + wait.start(CHECK_PHASES["folders"]) + report = check_progress(wait) + seen = _history(target) + report("folders", 0, 1, "") + for folder in (".", "ctrl", "ctrl/rep1"): + clock.now += 1 + report("folders", 0, 1, folder) + report("folders", 1, 1, "") + report("headers", 0, 2, "") + clock.now += 1 + report("headers", 1, 2, "ctrl/rep1/b.tif") + report("headers", 2, 2, "") + report("masks", 0, 1, "") + clock.now += 1 + report("masks", 3, 8, "Nuclei/ctrl/rep1/b.tif") + report("acquisitions", 0, 1, "") + assert any(line.startswith("Listing the folders · 3 so far · ctrl/rep1") for line in seen) + assert any(line.startswith("Reading image headers · 0 of 2") for line in seen) + assert any("Reading image headers · 1 of 2 · ctrl/rep1/b.tif" in line for line in seen) + assert any("Reading mask headers · 3 of 8 · Nuclei/ctrl/rep1/b.tif" in line for line in seen) + assert seen[-1].startswith("Comparing the images") + assert set(CHECK_PHASES) == set(PHASES) + + +# --- the notebook's waits ------------------------------------------------------------------------ + + +def test_the_folder_check_shows_its_phases_with_counts(exp): # noqa: F811 + nb = stepper(*exp) + seen = _history(nb.folder_status) + nb.check_button.click() + waits = [line for line in seen if "elapsed" in line] + for label in ("Listing the folders", "Reading image headers", "Reading mask headers"): + assert any(line.startswith(label) for line in waits), label + assert any(re.match(r"Reading image headers · 0 of \d+", line) for line in waits) + assert any(line.startswith("Writing the check report") for line in waits) + assert any(re.match(r"Drawing a picture per folder · 0 of \d+", line) for line in waits) + assert "elapsed" not in _text(nb.folder_status.value) # the result replaced the wait + assert not any("Checking the folders and image headers" in line for line in seen) + + +def test_the_run_and_the_mask_step_show_what_is_read_of_how_many(exp): # noqa: F811 + nb = stepper(*exp) + nb.quick_confirm.click() + answer_all(nb) + assert nb.save_experiment() + seen = _history(nb.run_status) + assert nb.run().written + assert seen[0].startswith("Preparing: checking the folders and the image headers again") + assert any(re.match(r"Reading images · 0 of \d+ .*elapsed", line) for line in seen) + assert any(line.startswith("Writing the table") for line in seen) + assert nb.progress.value == nb.progress.max and nb.progress.bar_style == "success" + seen = _history(nb.mask_step_status) + nb.mask_step_button.click() + assert seen[0].startswith("Pairing the masks the run recorded") + assert any( + re.match(r"Reading the masks, one image at a time · 0 of \d+", line) for line in seen + ) + assert nb.mask_step_progress.bar_style == "success" + + +def test_check_the_masks_shows_the_mask_headers_read(exp): # noqa: F811 + nb = stepper(*exp) + seen = _history(nb.masks_wait) + nb.masks_button.click() + assert seen[0].startswith("Reading mask headers") and seen[-1] == "" + assert nb.preflight.masks is not None and nb.preflight.masks.files + + +def test_the_statistics_show_their_stage_of_four(exp): # noqa: F811 + nb = stepper(*exp) + nb.quick_confirm.click() + answer_all(nb) + assert nb.save_experiment() + assert nb.run().written + seen = _history(nb.stats_status) + nb.statistics_button.click() + assert seen[0].startswith("Reading the histogram table") + for stage, label in enumerate( + ("Pooling the planes of each group", "Drawing the intensity distributions"), start=1 + ): + assert any(line.startswith(f"{label} · {stage} of 4") for line in seen) + assert "calculated." in seen[-1] diff --git a/tests/test_notebook_quick_scan.py b/tests/test_notebook_quick_scan.py new file mode 100644 index 00000000..d35885fa --- /dev/null +++ b/tests/test_notebook_quick_scan.py @@ -0,0 +1,192 @@ +"""The quick scan of the first step: look at the folders and one image, then read the headers. + +Owner decision (10 November 2026, item (d)): choosing the folders, a quick scan (the folder +tree with labels and counts, no header read, and one randomly chosen image shown for looking +only), then the explicit click that reads every header. Changing a folder redoes the scan. +""" + +import re + +import pytest + +from morphobricks_preprocessing.discovery import discover_experiment +from test_notebook_check_save import add_folder, exp # noqa: F401 + +pytest.importorskip("ipywidgets") +pytest.importorskip("matplotlib") + +from morphobricks_preprocessing import metadata, notebook_folder, notebook_quick_scan # noqa: E402 +from morphobricks_preprocessing import notebook_style as ui # noqa: E402 +from morphobricks_preprocessing.notebook_quick_scan import ( # noqa: E402 + DEEP_LABEL, + random_image, + scan_summary, + tree_html, +) +from morphobricks_preprocessing.notebook_ui import PreprocessingNotebook # noqa: E402 + + +def _text(html): + return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html)).strip() + + +def fresh(root, masks, out): + nb = PreprocessingNotebook(str(root), output_folder=str(out)) + nb.steps() + nb.masks_folder.value = str(masks) + return nb + + +def test_the_check_waits_for_a_scan_and_says_so(exp): # noqa: F811 + nb = fresh(*exp) + assert nb.check_button.description == DEEP_LABEL and nb.check_button.disabled + assert "click Scan the folders" in _text(nb.check_blocker.value) + assert not ui.is_visible(nb.scan_box) and nb.preflight is None + nb.folder.value = "" + assert "no experiment folder is given" in _text(nb.check_blocker.value) + nb.scan_button.click() + assert "Enter the experiment folder first" in nb.scan_status.value and nb.check_button.disabled + + +def test_the_scan_reads_no_header_but_the_one_image_it_shows(exp, monkeypatch): # noqa: F811 + nb = fresh(*exp) + opened = [] + real = metadata.inspect_image + + def one_header(path, relative_path=None): + opened.append(relative_path) + return real(path, relative_path) + + def no_check(*args, **kwargs): + raise AssertionError("the folder check ran") + + monkeypatch.setattr(notebook_quick_scan, "inspect_image", one_header) + monkeypatch.setattr(metadata, "inspect_image", no_check) + monkeypatch.setattr(notebook_folder, "preflight", no_check) + nb.scan_button.click() + found = discover_experiment(exp[0], exp[2]) + assert opened == [nb.scan_image] and nb.scan_image in found.images + assert nb.preflight is None and nb.mapping is None and nb.draft is None # nothing is kept + assert not list(exp[2].glob("*")) if exp[2].exists() else True # and nothing written + assert ui.is_visible(nb.scan_box) and not nb.check_button.disabled + assert nb.check_blocker.value == "" + said = _text(nb.scan_status.value) + assert ( + f"Found {scan_summary(found, frozenset())}" in said and "No image header was read" in said + ) + assert nb.scan_tree.value == tree_html(found, frozenset()) + + +def test_the_tree_lists_every_folder_with_its_labels_and_image_count(exp): # noqa: F811 + found = discover_experiment(exp[0], exp[2]) + html = tree_html(found, frozenset()) + for leaf in found.leaves: + for label in leaf.labels: + assert f"{label}" in html + assert html.count("") == len(found.leaves) + 1 + first = found.leaves[0] + without = tree_html(found, frozenset(first.images)) + assert "0" in without # excluded images are not counted + summary = scan_summary(found, frozenset(first.images[:1])) + assert summary.endswith("1 image excluded") and f"{len(found.images) - 1} images" in summary + + +def test_the_random_image_is_named_with_its_seed_and_is_reproducible(exp): # noqa: F811 + nb = fresh(*exp) + nb.scan_button.click() + images = list(discover_experiment(exp[0], exp[2]).images) + assert 0 <= nb.scan_seed < notebook_quick_scan.SEED_RANGE + assert nb.scan_image == random_image(images, nb.scan_seed) + shown = _text(nb.scan_example.children[0].value) + assert ( + f"seed {nb.scan_seed}" in shown and nb.scan_image in shown and "for looking only" in shown + ) + assert "from the file header" in _text(nb.scan_example.children[1].children[1].value) + nb._show_scan_example(7) + assert nb.scan_seed == 7 and nb.scan_image == random_image(images, 7) + assert random_image(images, 7) == random_image(list(reversed(images)), 7) # order-free + assert random_image([], 7) is None + seeds = set() + for _ in range(4): + nb.scan_another.click() + seeds.add(nb.scan_seed) + assert len(seeds) > 1 # another image is another draw + + +def test_an_excluded_image_is_never_the_one_shown(exp): # noqa: F811 + nb = fresh(*exp) + images = sorted(discover_experiment(exp[0], exp[2]).images) + nb.exclude.value = "\n".join(images[1:]) + nb.scan_button.click() + assert nb.scan_image == images[0] + assert not ui.is_visible(nb.scan_another) # one image left: nothing else to draw + + +def test_changing_a_folder_makes_the_scan_stale_and_blocks_the_check(exp, tmp_path): # noqa: F811 + nb = fresh(*exp) + nb.scan_button.click() + assert not nb.check_button.disabled + for box, value in ( + (nb.output, str(tmp_path / "elsewhere")), + (nb.masks_folder, ""), + (nb.exclude, "ctrl/rep1/a.tif"), + (nb.folder, str(tmp_path / "exp") + "/"), + ): + nb.scan_button.click() + assert not nb.check_button.disabled + box.value = value + assert nb.check_button.disabled, value + assert "click Scan the folders" in _text(nb.check_blocker.value) + assert "The folders changed" in nb.scan_status.value and not ui.is_visible(nb.scan_box) + nb.scan_button.click() + assert not nb.check_button.disabled and ui.is_visible(nb.scan_box) + + +def test_after_the_scan_the_explicit_click_reads_every_header_as_before(exp): # noqa: F811 + nb = fresh(*exp) + nb.scan_button.click() + nb.check_button.click() + plain = PreprocessingNotebook(str(exp[0]), output_folder=str(exp[2].parent / "plain")) + plain.masks_folder.value = str(exp[1]) + plain.check_button.click() # a notebook without the stepper: one click, no scan + assert nb.preflight is not None and nb.preflight.ok + assert nb.preflight.metadata == plain.preflight.metadata + assert nb.preflight.discovery.leaves == plain.preflight.discovery.leaves + assert nb.mapping == plain.mapping and nb.confirmed_mapping is None + assert plain.check_button.description == " Check folder" and not plain.check_button.disabled + assert nb.check_button.description == DEEP_LABEL and not nb.check_button.disabled + + +def test_a_folder_without_images_or_a_missing_folder_is_said_and_not_checked(tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + nb = PreprocessingNotebook(str(empty), output_folder=str(tmp_path / "out")) + nb.steps() + nb.scan_button.click() + assert nb.check_button.disabled + assert "the scan found no image to read" in _text(nb.check_blocker.value) + nb.folder.value = str(tmp_path / "missing") + nb.scan_button.click() + assert "experiment not found" in _text(nb.scan_status.value) + assert nb.check_button.disabled + + +def test_the_scan_shows_its_progress_folder_by_folder(exp): # noqa: F811 + nb = fresh(*exp) + seen = [] + nb.scan_status.observe(lambda change: seen.append(_text(change["new"])), names="value") + nb.scan_button.click() + assert seen[0].startswith("Listing the folders") and "elapsed" in seen[0] + assert any(line.startswith("Opening one image") for line in seen) + assert seen[-1].startswith("Found ") + + +def test_a_new_folder_appears_only_after_scanning_again(exp): # noqa: F811 + root, masks, _ = exp + nb = fresh(*exp) + nb.scan_button.click() + before = nb.scan_tree.value + add_folder(root, masks, "ctrl/rep9", 9) + assert nb.scan_tree.value == before + nb.scan_button.click() + assert "rep9" in nb.scan_tree.value and "rep9" not in before diff --git a/tests/test_notebook_ui.py b/tests/test_notebook_ui.py index 5aef2e52..ef252e22 100644 --- a/tests/test_notebook_ui.py +++ b/tests/test_notebook_ui.py @@ -919,7 +919,7 @@ def while_busy(*args, stage=stage, original=original, **kwargs): assert nb.statistics_button.disabled assert nb.statistics_button.description.strip() == "Calculating…" assert nb.statistics_button.icon == "spinner" - assert "mb-spinner" in nb.stats_status.value + assert "mb-wait" in nb.stats_status.value # the wait line with its bar assert nb.stats_next.value == "" assert "calculated." not in nb.stats_status.value stages.append(stage) diff --git a/tools/code_rules_baseline.json b/tools/code_rules_baseline.json index 7596a5f5..22a9196f 100644 --- a/tools/code_rules_baseline.json +++ b/tools/code_rules_baseline.json @@ -16,7 +16,6 @@ "src/morphobricks_preprocessing/design.py _parse_row: private function of # lines without a docstring": 36, "src/morphobricks_preprocessing/discovery.py _check_names: private function of # lines without a docstring": 14, "src/morphobricks_preprocessing/discovery.py finish: # executable lines (max #)": 145, - "src/morphobricks_preprocessing/discovery.py: # lines (max #)": 609, "src/morphobricks_preprocessing/image_type.py _rgb_problem: private function of # lines without a docstring": 12, "src/morphobricks_preprocessing/manifest.py file_entry_for: # parameters (max #)": 9, "src/morphobricks_preprocessing/masks/files.py _duplicate_issues: private function of # lines without a docstring": 19,