Discovery and header additions for the experiment description (Stage 1, PR 6)

CHANGELOG.md

@@ -8,6 +8,23 @@ are in [docs/plan.md](docs/plan.md) §12. Upgrading from v0.3: see the
 
 ## Unreleased
 
+- **Discovery and header additions for the experiment description (Stage 1, PR 6).**
+  Additive; a run without a description behaves as before. (1) `preflight` and `remasked`
+  take `with_description=False`; with it, a condition, repeat or technical-repeat folder that
+  is not Unicode NFC is the error `R_FOLDER_NOT_NFC`, a discovered mask type that breaks the
+  Label rules (`NA`, ` cells`) is `R_MASK_TYPE_NAME_INVALID` and one that is not NFC is
+  `R_FOLDER_NOT_NFC` (new module `name_issues`, also used by `find_masks(with_description=)`);
+  nothing is renamed or normalized (owner decision Q1: without a description these names are
+  accepted as in v0.4). (2) `TiffMetadata.raw_channel_names` and `FileEntry.raw_channel_names`
+  hold the header channel names exactly as read, untrimmed; `channel_names` stays trimmed,
+  `same_source` ignores them, and the manifest file entry gains the optional key
+  `raw_channel_names`, written only when the header names a channel. The OME header reading
+  moved from `metadata` to `ome_header`. (3) `preflight(progress=...)` calls
+  `progress(phase, done, total, path)` for the phases `folders`, `headers` (one call per
+  image), `masks` and `acquisitions`; `run.phases.progress_text` words them ("2/4 headers
+  120/480"). Not used by the notebook or the command line yet. Tests:
+  `tests/test_spec_discovery.py` (conformance P21, P26 recorded names, P29) and
+  `tests/test_preflight_progress.py`.
 - **Content hashing of the sources is on by default (#159; owner decision of 7 November 2026).**
   A run records each image's content SHA-256 and reuses a row only for the same content, so an
   image edited in place with its size and modification time kept is read again (manifest

pyproject.toml

@@ -129,6 +129,9 @@ baseline = "tools/code_rules_baseline.json"
 "morphobricks_preprocessing.axis_names" = "pure"
 "morphobricks_preprocessing.image_names" = "pure"
 "morphobricks_preprocessing.mask_names" = "pure"
+"morphobricks_preprocessing.name_issues" = "pure"
+"morphobricks_preprocessing.run.phases" = "pure"
+"morphobricks_preprocessing.ome_header" = "io"
 "morphobricks_preprocessing.axis_review" = "pure"
 "morphobricks_preprocessing.mask_checks" = "pure"
 "morphobricks_preprocessing.mask_review" = "pure"

src/morphobricks_preprocessing/manifest.py

@@ -38,10 +38,12 @@
 from .inventory import INVENTORY_KEY, source_signature
 from .inventory import source_fingerprint as source_fingerprint  # re-exported (moved, PP-12)
 from .manifest_fields import (
+    RAW_NAMES_KEY,
     optional_text,
     recorded_flag,
     recorded_masks,
     recorded_position_id,
+    recorded_raw_names,
     recorded_roles,
 )
 from .manifest_versions import (
@@ -130,21 +132,19 @@ class FileEntry:
     significant_bits
         Significant bits from metadata; None when not given.
     channel_names_trimmed
-        True when leading or trailing whitespace was removed from the image's
-        channel names as they were read (names are recorded trimmed).
+        True when whitespace was removed around the channel names (names are recorded trimmed).
     read_in_blocks
-        True when the image's planes are too large for the memory limit and are
-        read in blocks. For information only: the rows are the same either way.
+        True when the planes are too large for the memory limit and are read in blocks.
+        For information only: the rows are the same either way.
     position_id
         The image's position identifier, its name without the extension
         (:func:`~morphobricks_preprocessing.image_names.position_id`). Required in manifest
         version 2, and equal to that rule (round-2 audit A-m5); None in older manifests.
     roles
-        The role of each of the file's axes (``"X"``, ``"Y"``, ``"Z"``, ``"C"``,
-        ``"T"`` or None), from its own mapping: for an OME-Zarr image whose
-        declared order differs from the experiment mapping's, these are its
-        roles, not the experiment's. Required in manifest version 2; None in older
-        manifests.
+        The role of each of the file's axes (``"X"``, ``"Y"``, ``"Z"``, ``"C"``, ``"T"`` or
+        None), from its own mapping: for an OME-Zarr image whose declared order differs from
+        the experiment mapping's, these are its roles, not the experiment's. Required in
+        manifest version 2; None in older manifests.
     axes_from
         Where the roles come from: ``"header"`` (the metadata names every axis and
         the roles agree) or ``"confirmed"`` (a plain stack: the user confirmed them).
@@ -158,9 +158,10 @@ class FileEntry:
         TIFF file, and in manifests written before it was recorded (such an OME-Zarr
         image counts as changed: its fingerprint is not known to be complete).
     content_sha256
-        The SHA-256 of the file's content (an OME-Zarr folder: of its files' paths and
-        SHA-256s, :func:`.source_content.source_content_sha256`), in a ``verify_content``
-        run only; None otherwise.
+        The SHA-256 of the file's content (an OME-Zarr folder: of its files' paths and SHA-256s,
+        :func:`.source_content.source_content_sha256`) in a ``verify_content`` run; else None.
+    raw_channel_names
+        Header channel names, untrimmed; written only when not empty; not in :meth:`same_source`.
     """
 
     relative_path: str
@@ -180,6 +181,7 @@ class FileEntry:
     pixel_sizes: tuple[RecordedSize, ...] = ()
     inventory_sha256: str | None = None
     content_sha256: str | None = None
+    raw_channel_names: tuple[str | None, ...] = ()
 
     def same_source(self, other: FileEntry) -> bool:
         """Return True if ``other`` describes the same unchanged file, read the same way."""
@@ -555,6 +557,8 @@ def _file_dict(entry: FileEntry, *, strict: bool = True) -> dict:
         data[INVENTORY_KEY] = entry.inventory_sha256
     if entry.content_sha256 is not None:
         data[CONTENT_KEY] = entry.content_sha256
+    if entry.raw_channel_names:
+        data[RAW_NAMES_KEY] = list(entry.raw_channel_names)
     return data
 
 
@@ -588,6 +592,7 @@ def _file_entry(data: dict, *, strict: bool = True) -> FileEntry:
         pixel_sizes=sizes_from_data(data.get("pixel_sizes")),
         inventory_sha256=optional_text(data.get(INVENTORY_KEY)),
         content_sha256=optional_text(data.get(CONTENT_KEY)),
+        raw_channel_names=recorded_raw_names(data.get(RAW_NAMES_KEY), path),
     )
 
 

src/morphobricks_preprocessing/manifest_fields.py

@@ -14,6 +14,22 @@
 from .axes import AxisMapping
 from .image_names import position_id
 
+RAW_NAMES_KEY = "raw_channel_names"
+"""The file entry's key for the channel names as the header gave them (written when not empty)."""
+
+
+def recorded_raw_names(value: object, relative_path: str) -> tuple[str | None, ...]:
+    """Read a recorded ``raw_channel_names`` (a list of text or null), or refuse it.
+
+    Absent (a manifest written before it was recorded, or an image whose header names no
+    channel) reads as empty. A name is kept exactly as written, never trimmed.
+    """
+    if value is None:
+        return ()
+    if not isinstance(value, list) or any(n is not None and not isinstance(n, str) for n in value):
+        raise ValueError(f"the raw channel names of {relative_path!r} are not text or null")
+    return tuple(value)
+
 
 def recorded_position_id(value: object, relative_path: str, *, strict: bool = True) -> str | None:
     """Check the ``position_id`` of a manifest file entry against its image's name.

src/morphobricks_preprocessing/masks/files.py

@@ -10,11 +10,22 @@
 from collections.abc import Collection, Iterable, Mapping
 from pathlib import Path
 
-from ..discovery import DiscoveryResult, is_system_name, is_tiff_name
+from ..discovery import (
+    DiscoveryResult,
+    IdentityError,
+    check_identity_text,
+    is_system_name,
+    is_tiff_name,
+)
 from ..folder_rules import masks_folder_issues
 from ..image_names import position_id
 from ..links import LinkGuard
 from ..metadata import TiffMetadata
+from ..name_issues import (
+    invalid_mask_types_issue,
+    is_nfc,
+    non_nfc_mask_types_issue,
+)
 from ..validation import ValidationIssue, agree, error, has_errors, plural, warning
 from ..zarr_input import (
     NgffError,
@@ -35,6 +46,7 @@ def find_masks(
     mask_axes: Mapping[str, str] | None = None,
     output_folder: str | os.PathLike | None = None,
     bind: bool = False,
+    with_description: bool = False,
 ) -> MaskInventory:
     """Pair every image with its masks and check them from their headers, without pixels.
 
@@ -65,6 +77,12 @@ def find_masks(
         fingerprint with it (:attr:`.model.MaskFile.bound`, issue #160), so a run or the
         mask step refuses a mask that is no longer the one its header described; a mask
         that changed while its header was read is the error ``mask_changed_during_check``.
+    with_description
+        True when an experiment description is in use: a discovered mask type name that
+        breaks the Label rules (``NA``, `` cells``) is then the error
+        ``R_MASK_TYPE_NAME_INVALID``, and one that is not Unicode NFC the error
+        ``R_FOLDER_NOT_NFC`` (Spec experiment-v1 §6; never normalized or renamed). Without a
+        description, mask type names are accepted as before (owner decision Q1).
 
     Returns
     -------
@@ -85,6 +103,8 @@ def find_masks(
     found, unreadable = _label_masks(discovery.root, images)
     files.extend(found)
     issues.extend(_name_issues(files))
+    if with_description:
+        issues.extend(_description_name_issues(files))
     files.sort(key=lambda f: (f.image, f.mask_type))
     issues.extend(_duplicate_issues(files))
     stated = dict(mask_axes or {})
@@ -287,6 +307,19 @@ def _name_issues(files: list[MaskFile]) -> list[ValidationIssue]:
     ]
 
 
+def _description_name_issues(files: list[MaskFile]) -> list[ValidationIssue]:
+    """Refuse the discovered mask types that cannot be a ``source_type`` of a description."""
+    invalid, not_nfc = [], []
+    for name in sorted({f.mask_type for f in files}):
+        try:
+            check_identity_text(name, "mask type")
+        except IdentityError:
+            invalid.append(name)
+        if not is_nfc(name):
+            not_nfc.append(name)
+    return [*invalid_mask_types_issue(invalid), *non_nfc_mask_types_issue(not_nfc)]
+
+
 def _duplicate_issues(files: list[MaskFile]) -> list[ValidationIssue]:
     seen: dict[tuple[str, str], MaskFile] = {}
     duplicates = []

src/morphobricks_preprocessing/metadata.py

@@ -21,14 +21,14 @@
 from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
-from xml.etree import ElementTree
 
 import numpy as np
 import tifffile
 from morphobricks_normalization.dtypes import UnsupportedDtypeError, check_supported
 
 from .axis_names import OPEN_AXIS, named_letters
 from .discovery import DiscoveryResult
+from .ome_header import ome_details
 from .physical_sizes import PhysicalSizes, _tiff_sizes
 from .pixel_size_checks import pixel_size_issues
 from .size_checks import (
@@ -48,7 +48,6 @@
     ZarrUnavailableError,
     is_zarr_name,
     open_ngff,
-    read_channel_names,
 )
 
 logger = logging.getLogger(__name__)
@@ -93,6 +92,11 @@ class TiffMetadata:
     channel_names_trimmed
         True when leading or trailing whitespace was removed from a channel name
         (recorded in the run manifest).
+    raw_channel_names
+        The channel names exactly as the header gives them, untrimmed (a name that is not
+        text is None); empty when the header names no channel. The evidence an experiment
+        description's header names are compared with (Spec experiment-v1 §4.3); recorded in
+        the run manifest as read.
     significant_bits
         Bits per pixel that carry data (e.g. 12 for a 12-bit camera in 16-bit
         files), from OME metadata; None when not given. For information only.
@@ -132,6 +136,7 @@ class TiffMetadata:
     rgb_samples: bool = False
     channel_names_trimmed: bool = False
     axes_from_metadata: bool = True
+    raw_channel_names: tuple[str | None, ...] = ()
 
     def length(self, axis: str) -> int:
         """Return the length of the dimension labelled ``axis``, 1 if there is none."""
@@ -281,7 +286,7 @@ def inspect_tiff(path: str | os.PathLike, relative_path: str | None = None) -> T
                 axes=str(first.axes),
                 series_count=len(series),
                 page_count=len(tif.pages),
-                **_ome_details(tif),
+                **ome_details(tif),
                 physical_sizes=_tiff_sizes(tif),
                 rgb_samples=_rgb_samples(first),
                 axes_from_metadata=series_axes_from_metadata(first),
@@ -317,6 +322,7 @@ def inspect_ome_zarr(path: str | os.PathLike, relative_path: str | None = None)
             page_count=0,
             channel_names=image.channel_names,
             channel_names_trimmed=image.channel_names_trimmed,
+            raw_channel_names=image.raw_channel_names,
             kind=OME_ZARR,
             physical_sizes=image.physical_sizes,
         )
@@ -511,29 +517,6 @@ def check_metadata(files: Iterable[TiffMetadata]) -> list[ValidationIssue]:
     return issues
 
 
-def _ome_details(tif: tifffile.TiffFile) -> dict:
-    """Channel names and significant bits of the first OME image, if present."""
-    if not tif.is_ome:
-        return {}
-    try:
-        root = ElementTree.fromstring(tif.ome_metadata)
-    except (ElementTree.ParseError, TypeError, ValueError):
-        logger.debug("Cannot parse the OME metadata of %s", tif.filename, exc_info=True)
-        return {}
-    pixels = next((e for e in root.iter() if _local_name(e.tag) == "Pixels"), None)
-    if pixels is None:
-        return {}
-    names, trimmed = read_channel_names(
-        channel.get("Name") for channel in pixels if _local_name(channel.tag) == "Channel"
-    )
-    bits = pixels.get("SignificantBits")
-    return {
-        "channel_names": names,
-        "channel_names_trimmed": trimmed,
-        "significant_bits": int(bits) if bits and bits.isdigit() else None,
-    }
-
-
 def _rgb_samples(series: tifffile.TiffPageSeries) -> bool:
     """Return True if the series stores RGB colour: photometric RGB, three samples per pixel."""
     keyframe = series.keyframe
@@ -577,10 +560,6 @@ def series_axes_from_metadata(series: tifffile.TiffPageSeries) -> bool:
     return True
 
 
-def _local_name(tag: str) -> str:
-    return tag.rsplit("}", 1)[-1]
-
-
 def _check_channel_names(files: list[TiffMetadata]) -> list[ValidationIssue]:
     named = [f for f in files if f.channel_names]
     if len(named) < 2: