Notebook redesign 2: progress for every wait, progress callbacks from the readers, quick scan first

CHANGELOG.md

@@ -17,6 +17,35 @@ OME-XML description is parsed (which images are marked is unchanged; a malformed
 debug line without a traceback), and while the capture is attached the package's DEBUG records
 go to the "Technical details" panel only, with WARNING and above still reaching the root handlers.
 
+### Notebook redesign 2: progress for every wait, and a quick scan before the headers
+
+Owner test of v0.5.0 (10 November 2026), items (c) and (d).
+
+- **Every wait shows what is done of the total, the item being read, the time elapsed and,
+  once it can be measured, the time left** (`notebook_progress.Wait`, `progress_html`,
+  `time_left`): the folder walk ("Listing the folders: 14 so far"), the image headers
+  ("Reading image headers: 34 of 120, CTRL/rep1/a.tif"), the mask headers, the check report,
+  the pictures per folder, **Check the masks**, the run, the mask step, the named-axis copies
+  and the statistics (their stage of four). "Checking the folders and image headers…" is gone.
+  The time left is said only after one item and one second; uncounted work shows the elapsed
+  time alone.
+- **Progress callbacks from the readers.** `discover_experiment(..., progress=)` is called with
+  each folder before it is listed; `masks.find_masks(..., progress=)` and
+  `run.preflight.remasked(..., progress)` before each mask header; `preflight`'s `progress`
+  now also reports `("folders", 0, 1, folder)` and `("masks", done, total, mask_path)`. The
+  first and last call of each phase, and every result, are unchanged.
+- **Quick scan first (the shipped notebook's first step).** **Scan the folders** walks the
+  folders only (the folder check's own walk; no image header is read), shows every folder with
+  its labels and image count, and opens one image chosen at random, with its path and seed, for
+  looking only (drawn from that file's header by the rules of the folder pictures). Nothing of
+  the scan is kept. The folder check is then the explicit click **Looks right: read all image
+  headers**; it is disabled, with a line saying why, until the folders are scanned as they are
+  typed now, so changing a folder, the output or masks folder or the exclusions asks for a new
+  scan. `folder_step()` for other bricks is unchanged: **Check folder**, one click.
+- `is_tiff_name`, `is_system_name`, `TIFF_SUFFIXES` and `SYSTEM_NAMES` moved from `discovery`
+  to `image_names` (the functions are still importable from `discovery`), keeping
+  `discovery.py` under the 600-line rule (its baseline entry is gone).
+
 ### Notebook redesign 1: fixes from the owner's test of v0.5.0 (10 November 2026)
 
 - **Lists of files in a message draw no empty bullets.** `notebook_style.path_list` returns

pyproject.toml

@@ -139,6 +139,8 @@ baseline = "tools/code_rules_baseline.json"
 "morphobricks_preprocessing.mask_record" = "io"
 "morphobricks_preprocessing.run_labels" = "io"
 "morphobricks_preprocessing.notebook_axes" = "ui"
+"morphobricks_preprocessing.notebook_progress" = "ui"
+"morphobricks_preprocessing.notebook_quick_scan" = "ui"
 "morphobricks_preprocessing.notebook_axes_quick" = "ui"
 "morphobricks_preprocessing.notebook_channel_thumbs" = "ui"
 "morphobricks_preprocessing.notebook_channels" = "ui"

src/morphobricks_preprocessing/discovery.py

@@ -25,28 +25,21 @@
 
 import logging
 import os
-from collections.abc import Iterable, Sequence
+from collections.abc import Callable, Iterable, Sequence
 from dataclasses import dataclass
 from pathlib import Path
 
 import numpy as np
 from morphobricks_normalization.records import LEVELS, SINGLE
 
-from .image_names import position_id
+from .image_names import is_system_name, is_tiff_name, position_id
 from .links import LinkGuard, relative_inside
 from .mask_names import mask_name_issues
 from .validation import ValidationIssue, agree, error, has_errors, plural, warning
 from .zarr_input import is_zarr_name
 
 logger = logging.getLogger(__name__)
 
-TIFF_SUFFIXES = (".tif", ".tiff")
-
-# Files and folders created by operating systems and tools, not by users. They
-# are skipped without a warning so that copying an experiment from a Mac or
-# opening it in Jupyter does not flood the preflight with messages.
-SYSTEM_NAMES = frozenset({"thumbs.db", "desktop.ini", "__macosx"})
-
 OUTPUT_FOLDER_SUFFIX = "_MorphoBricks_Preprocessing"
 
 ROOT = "."
@@ -189,20 +182,6 @@ def _same_spelling(seen: dict[str, str], value: str, field: str, scope: str) ->
         )
 
 
-def is_system_name(name: str) -> bool:
-    """Return True for system files and folders that are skipped silently.
-
-    These are dot-files and dot-folders (``.DS_Store``, ``._image.tif``,
-    ``.ipynb_checkpoints``), ``Thumbs.db``, ``desktop.ini`` and ``__MACOSX``.
-    """
-    return name.startswith(".") or name.lower() in SYSTEM_NAMES
-
-
-def is_tiff_name(name: str) -> bool:
-    """Return True if ``name`` has a TIFF extension (``.tif`` or ``.tiff``, any case)."""
-    return name.lower().endswith(TIFF_SUFFIXES)
-
-
 def is_ome_zarr_folder(entry: os.DirEntry) -> bool:
     """Return True for a folder that is an OME-Zarr image (its name ends in ``.zarr``).
 
@@ -315,7 +294,10 @@ def check_output_folder(
 
 
 def discover_experiment(
-    experiment_root: str | os.PathLike, output_folder: str | os.PathLike | None = None
+    experiment_root: str | os.PathLike,
+    output_folder: str | os.PathLike | None = None,
+    *,
+    progress: Callable[[str], None] | None = None,
 ) -> DiscoveryResult:
     """Walk an experiment folder, find its leaf folders and check its structure.
 
@@ -330,6 +312,10 @@ def discover_experiment(
         The planned output folder, if known. It is checked with
         :func:`check_output_folder` and, if it lies inside the experiment, it is
         not scanned.
+    progress
+        Called with each folder's relative path (``"."`` for the experiment folder) before it
+        is listed, so a caller can show how far the walk is. It changes nothing about the
+        result.
 
     Returns
     -------
@@ -346,6 +332,7 @@ def discover_experiment(
         return DiscoveryResult(root=root, issues=(issue,))
 
     walker = _Walker(root)
+    walker.progress = progress
     if output_folder is not None:
         walker.issues.extend(found := check_output_folder(root, output_folder))
         walker.excluded = root.joinpath(found[0].paths[0]) if found else None  # walk's name
@@ -361,6 +348,7 @@ def __init__(self, root: Path) -> None:
         self.root = root
         self.links = LinkGuard(root, "experiment folder")
         self.excluded: Path | None = None
+        self.progress: Callable[[str], None] | None = None
         self.leaves: list[LeafFolder] = []
         self.issues: list[ValidationIssue] = []
         self.reserved: list[str] = []
@@ -380,6 +368,8 @@ def relative(self, path: Path) -> str:
 
     def visit(self, folder: Path, names: tuple[str, ...]) -> None:
         """Sort the entries of ``folder`` (labels ``names``), following links by the guard."""
+        if self.progress is not None:
+            self.progress(self.relative(folder))
         try:
             entries = sorted(os.scandir(folder), key=lambda entry: entry.name)
         except OSError:

src/morphobricks_preprocessing/image_names.py

@@ -1,6 +1,7 @@
 """An image's position identifier, from its name (contract §1, MorphoBricks D02, D85).
 
-Moved from ``discovery.py`` (PP-5), so that the mask-name screen can use it.
+Moved from ``discovery.py`` (PP-5), so that the mask-name screen can use it; the names of
+TIFF images and of system files (``is_tiff_name``, ``is_system_name``) are here too.
 """
 
 from __future__ import annotations
@@ -57,3 +58,25 @@ def position_id(image: str) -> str:
                 )
             return name[: -len(suffix)]
     return name
+
+
+TIFF_SUFFIXES = (".tif", ".tiff")
+
+# Files and folders created by operating systems and tools, not by users. They
+# are skipped without a warning so that copying an experiment from a Mac or
+# opening it in Jupyter does not flood the preflight with messages.
+SYSTEM_NAMES = frozenset({"thumbs.db", "desktop.ini", "__macosx"})
+
+
+def is_system_name(name: str) -> bool:
+    """Return True for system files and folders that are skipped silently.
+
+    These are dot-files and dot-folders (``.DS_Store``, ``._image.tif``,
+    ``.ipynb_checkpoints``), ``Thumbs.db``, ``desktop.ini`` and ``__MACOSX``.
+    """
+    return name.startswith(".") or name.lower() in SYSTEM_NAMES
+
+
+def is_tiff_name(name: str) -> bool:
+    """Return True if ``name`` has a TIFF extension (``.tif`` or ``.tiff``, any case)."""
+    return name.lower().endswith(TIFF_SUFFIXES)

src/morphobricks_preprocessing/masks/checks.py

@@ -6,7 +6,7 @@
 from __future__ import annotations
 
 import logging
-from collections.abc import Iterable, Mapping
+from collections.abc import Callable, Iterable, Mapping
 from dataclasses import replace
 from pathlib import Path
 
@@ -36,6 +36,7 @@ def _header_issues(
     unreadable: Iterable[str] = (),
     stated: Mapping[str, str] | None = None,
     bind: bool = False,
+    progress: Callable[[int, int, str], None] | None = None,
 ) -> list[ValidationIssue]:
     """Read each mask's header (axes, shape, pixel type) and check it against its image.
 
@@ -46,10 +47,13 @@ def _header_issues(
     decision of 5 November 2026). ``unreadable`` are paths already known not to open
     (OME-Zarr ``labels`` groups). With ``bind``, each mask is fingerprinted before and
     after its header is read (:attr:`.model.MaskFile.bound`, issue #160); a mask whose two
-    fingerprints differ is ``mask_changed_during_check``.
+    fingerprints differ is ``mask_changed_during_check``. ``progress(done, total, path)`` is
+    called before each mask's header is read.
     """
     found: list[tuple[str, str, str]] = [("unreadable_mask", path, "") for path in unreadable]
     for index, mask in enumerate(files):
+        if progress is not None:
+            progress(index, len(files), mask.path)
         where = mask.location(folder, root) if bind else None
         before = _fingerprint(where)
         checked, problems = _check_header(mask, images[mask.image], folder, root, stated or {})

src/morphobricks_preprocessing/masks/files.py

@@ -7,7 +7,7 @@
 
 import logging
 import os
-from collections.abc import Collection, Iterable, Mapping
+from collections.abc import Callable, Collection, Iterable, Mapping
 from pathlib import Path
 
 from ..discovery import (
@@ -46,6 +46,7 @@ def find_masks(
     mask_axes: Mapping[str, str] | None = None,
     output_folder: str | os.PathLike | None = None,
     bind: bool = False,
+    progress: Callable[[int, int, str], None] | None = None,
 ) -> MaskInventory:
     """Pair every image with its masks and check them from their headers, without pixels.
 
@@ -76,6 +77,9 @@ def find_masks(
         fingerprint with it (:attr:`.model.MaskFile.bound`, issue #160), so a run or the
         mask step refuses a mask that is no longer the one its header described; a mask
         that changed while its header was read is the error ``mask_changed_during_check``.
+    progress
+        Called as ``progress(done, total, mask_path)`` before each mask's header is read. It
+        changes nothing about the result.
 
     Returns
     -------
@@ -100,7 +104,9 @@ def find_masks(
     files.sort(key=lambda f: (f.image, f.mask_type))
     issues.extend(_duplicate_issues(files))
     stated = dict(mask_axes or {})
-    issues.extend(_header_issues(files, images, folder, discovery.root, unreadable, stated, bind))
+    issues.extend(
+        _header_issues(files, images, folder, discovery.root, unreadable, stated, bind, progress)
+    )
     types = tuple(sorted({f.mask_type for f in files}))
     issues.extend(_unknown_type_issues(stated, types))
     issues.extend(_missing_issues(types, files, images))

src/morphobricks_preprocessing/notebook_convert.py

@@ -39,13 +39,15 @@
 from .conversion_record import RECORD_FILENAME
 from .notebook_options import _WIDE
 from .notebook_output import _LockCard
+from .notebook_progress import Wait
 from .notebook_render import blocked_line, issues_html
 from .planes import format_bytes
 from .run.output_guard import recorded_masks_of
 from .validation import plural
 
 _LABEL = " Make named-axis copies"
 _ICON = "copy"
+COPYING = "Copying images"
 _RECHECK_LABEL = " Check the copies in step 1"
 _COPY_HOLDERS = (
     "Another conversion is writing into this folder, or one was stopped (for example Colab "
@@ -283,7 +285,8 @@ def _on_convert(self) -> ConversionResult | None:
         self._busy(self.convert_button, "Making the copies…")
         ui.set_visible(self.convert_progress, True)
         self.convert_progress.value, self.convert_progress.bar_style = 0, "info"
-        self.convert_status.value = ui.working("Checking the images and the folder…")
+        self._convert_wait = Wait(self.convert_status, bar=False)  # the bar widget is above
+        self._convert_wait.start("Checking the images and the folder")
         try:
             with self.log.attached():
                 result = convert_experiment(
@@ -314,8 +317,11 @@ def _on_convert(self) -> ConversionResult | None:
     def _convert_progressed(self, done: int, total: int, image: str) -> None:
         self.convert_progress.max = max(total, 1)
         self.convert_progress.value = done
+        wait = self._convert_wait
+        if image and wait.label != COPYING:
+            wait.phase(COPYING, total)
         if image:
-            self.convert_status.value = ui.working(f"Copying image {done + 1} of {total}: {image}")
+            wait.step(done, total, image)
 
     def _show_conversion(self, result: ConversionResult) -> None:
         """Show a refusal with its card, or the copies, the record and the next step."""

src/morphobricks_preprocessing/notebook_folder.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import html
+from collections.abc import Callable
 from pathlib import Path
 
 import ipywidgets as widgets
@@ -12,8 +13,9 @@
 from .dataset import CONSISTENCY_CODES
 from .discovery import check_output_folder, default_output_folder
 from .manifest import MANIFEST_FILENAME
-from .notebook_folder_previews import LabelPreviews
+from .notebook_folder_previews import DRAWN_AT_ONCE, LabelPreviews
 from .notebook_options import _WIDE, EXCLUDABLE_CODES
+from .notebook_progress import Wait
 from .notebook_render import (
     _folder_tiles,
     _labels_explanation,
@@ -26,6 +28,38 @@
 from .unit_consistency import unit_checked
 from .validation import ValidationIssue, plural
 
+CHECK_LABEL = (" Check folder", "search")
+CHECK_PHASES = {
+    "folders": "Listing the folders",
+    "headers": "Reading image headers",
+    "masks": "Reading mask headers",
+    "acquisitions": "Comparing the images",
+}
+"""What the folder check's wait says in each phase of :data:`.run.phases.PHASES`."""
+
+
+def check_progress(wait: Wait) -> Callable[[str, int, int, str], None]:
+    """Return the folder check's progress callback: it words each phase in ``wait``.
+
+    The headers and the mask headers are counted (done of total, the file being read); the
+    folder walk says how many folders were listed so far, since its total is not known.
+    """
+    state = {"phase": "", "listed": 0}
+
+    def report(phase: str, done: int, total: int, path: str) -> None:
+        if phase != state["phase"]:
+            state["phase"] = phase
+            wait.phase(CHECK_PHASES[phase], total if phase == "headers" else 0)
+        if not path:
+            return
+        if phase == "folders":
+            state["listed"] += 1
+            wait.step(state["listed"], 0, path)
+        else:
+            wait.step(done, total, path)
+
+    return report
+
 
 class _FolderStep:
     """Step 1's folder and labels: check the folder, show its issues, confirm the labels."""
@@ -203,7 +237,8 @@ def check_folder(self) -> Preflight | None:
             return None
         self._busy(self.check_button, "Checking…")
         self.folder_next.value = ""
-        self.folder_status.value = ui.working("Checking the folders and image headers…")
+        wait = self._folder_wait = Wait(self.folder_status)
+        wait.start(CHECK_PHASES["folders"])
         try:
             with self.log.attached():
                 masks, mask_axes = self._check_masks_arguments(Path(folder).resolve())
@@ -216,15 +251,17 @@ def check_folder(self) -> Preflight | None:
                     # Each header is bound to the file's fingerprint: every picture and the
                     # save are of the images as this check read them (audit batch 33).
                     bind_headers=True,
+                    progress=check_progress(wait),
                 )
                 pre = unit_checked(pre)  # a described run checks inside each unit
                 self._note_edits()  # a check fills only what the user left as filled
                 self.missing_as_singleton.set(self._recorded_missing_as_singleton(), notify=False)
                 # The report reads images lacking some axes by that answer, as 'check
                 # --missing-as-singleton' does (F10); step 2 asks again before the run.
+                wait.phase("Writing the check report")
                 self._write_check(pre, self.missing_as_singleton.value)
         finally:
-            self._idle(self.check_button, " Check folder", "search")
+            self._idle(self.check_button, *getattr(self, "_check_label", CHECK_LABEL))
         self.preflight = pre
         self._checked_folder = folder
         self.mapping = pre.suggested_mapping
@@ -244,7 +281,12 @@ def _show_folder_results(self, pre: Preflight) -> None:
         ui.set_visible(self.design_card, False)
         self._hide_description()
         self.labels_table.value = labels_html(pre)
-        self.label_previews.show(pre)
+        wait = getattr(self, "_folder_wait", None)
+        if wait is not None:
+            wait.phase(
+                "Drawing a picture per folder", min(len(pre.discovery.leaves), DRAWN_AT_ONCE)
+            )
+        self.label_previews.show(pre, None if wait is None else wait.step)
         ui.set_visible(self.labels_table, False)
         ui.set_visible(self.label_previews.widget, True)
         self.labels_confirmed.set(False, notify=False)