From 8f78f44387fa64936e184f2287d5b77315b44073 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:11:09 +0000 Subject: [PATCH 1/9] Skip the notebook modules in the frozen clock without the notebook extra (#262) tests/frozen_clock.py imported every module of the package, and the notebook modules raise ImportError without ipywidgets/matplotlib, so the frozen-clock tests failed on a plain dev install. A module whose import fails only because a notebook-extra package is missing (followed through the re-raised cause) is now skipped; any other import error is raised. The write test checks a core module's patch always and the notebook module's only with the extra installed. Regression tests: tests/test_frozen_clock_extras.py (blocked extra in a fresh interpreter; a missing required dependency is still raised). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- CHANGELOG.md | 11 +++++++ tests/frozen_clock.py | 45 ++++++++++++++++++++++++++-- tests/test_description_write.py | 6 ++-- tests/test_frozen_clock_extras.py | 49 +++++++++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 4 deletions(-) create mode 100644 tests/test_frozen_clock_extras.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e974a1a..8e835bb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -217,6 +217,17 @@ ADR 0001 GS1; contract §9). **API and CLI additions only.** wrong-case repeat codes are refused; `i99` is still refused with links given. The input's own checks and the CLI: `tests/test_lab_links.py`. +### SWEEP-PP: the remaining compatibility code removed + +The obsolete-code sweep of this repository (MorphoFeature `docs/restart/designs/obsolete-code-inventory-2026-11-10.md` +§1 and §12, Stage 2 plan §3.7; owner policy of 10 November 2026, Spec ADR 0002 AR10). What PP-C +already removed is not repeated here. **Breaking** where it says so. + +- **Tests: the frozen clock without the `notebook` extra** (#262). `tests/frozen_clock.py` + skips a module whose import fails only because ipywidgets or matplotlib is missing (any + other import error is raised), so a plain `pip install -e '.[dev]'` passes the frozen-clock + tests (`tests/test_frozen_clock_extras.py`). + ## 0.5.0 - 2026-11-09 Stage 1 of the experiment description (MorphoBricks-Spec experiment-v1, pinned at `1bed024`): diff --git a/tests/frozen_clock.py b/tests/frozen_clock.py index 6849937e..29efd131 100644 --- a/tests/frozen_clock.py +++ b/tests/frozen_clock.py @@ -5,28 +5,69 @@ :class:`datetime.date` and :class:`datetime.datetime` in the :mod:`datetime` module and in every module of the package that imported them by name, with subclasses whose ``today``, ``now`` and ``utcnow`` return the frozen day (at noon UTC) and count each read. + +The notebook modules need the ``notebook`` extra (ipywidgets, matplotlib). Without it they cannot +be imported, so they are left out (issue #262): such a module cannot run, so it cannot read the +clock either. Any other import failure is raised. """ from __future__ import annotations import datetime as dt import importlib +import importlib.util import pkgutil import sys import morphobricks_preprocessing PACKAGE = morphobricks_preprocessing.__name__ +NOTEBOOK_EXTRA = frozenset({"ipywidgets", "matplotlib"}) +"""The top-level packages of the ``notebook`` extra; a module needing one is skipped without it.""" def _import_package() -> list[object]: - """Import every module of the package (not ``__main__``) and return them all.""" + """Import every module of the package (not ``__main__``) and return them all. + + A module whose import fails only because a package of the ``notebook`` extra is missing is + skipped (issue #262); every other import error is raised. + """ for info in pkgutil.walk_packages(morphobricks_preprocessing.__path__, PACKAGE + "."): - if not info.name.endswith(".__main__"): + if info.name.endswith(".__main__"): + continue + try: importlib.import_module(info.name) + except ImportError as exc: + if not _needs_notebook_extra(exc): + raise return [m for name, m in sorted(sys.modules.items()) if name.split(".")[0] == PACKAGE] +def notebook_extra_installed() -> bool: + """True when every package of the ``notebook`` extra can be imported.""" + return all(_importable(name) for name in NOTEBOOK_EXTRA) + + +def _importable(name: str) -> bool: + try: + return importlib.util.find_spec(name) is not None + except (ImportError, ValueError): # blocked in sys.modules, or a broken entry + return False + + +def _needs_notebook_extra(exc: BaseException | None) -> bool: + """True when ``exc`` (or what it was raised from) is a missing ``notebook`` extra package. + + The notebook modules re-raise a missing ipywidgets or matplotlib with an install hint, so + the chain of causes is followed to the :class:`ModuleNotFoundError` itself. + """ + while exc is not None: + if isinstance(exc, ModuleNotFoundError): + return (exc.name or "").split(".")[0] in NOTEBOOK_EXTRA + exc = exc.__cause__ or exc.__context__ + return False + + class FrozenClock: """The day every clock read of the package returns, and how many reads there were. diff --git a/tests/test_description_write.py b/tests/test_description_write.py index 57b398ee..4316d202 100644 --- a/tests/test_description_write.py +++ b/tests/test_description_write.py @@ -14,7 +14,7 @@ import pytest -from frozen_clock import FrozenClock +from frozen_clock import FrozenClock, notebook_extra_installed from morphobricks_preprocessing.description import ( Description, DescriptionError, @@ -210,7 +210,9 @@ def test_a_frozen_clock_moved_by_a_day_writes_identical_bytes_for_an_unchanged_m ): """Conformance plan section 2, ``confirmed_on``: the writer never reads the clock.""" clock = FrozenClock(monkeypatch, datetime.date(2026, 11, 9)) - assert "morphobricks_preprocessing.notebook_check_save.date" in clock.patched + assert "morphobricks_preprocessing.description_reconcile.date" in clock.patched + if notebook_extra_installed(): # the notebook modules are skipped without it (#262) + assert "morphobricks_preprocessing.notebook_check_save.date" in clock.patched models = {name: _model(name) for name in VALID} clock.reads = 0 first = {name: canonical_bytes(model) for name, model in models.items()} diff --git a/tests/test_frozen_clock_extras.py b/tests/test_frozen_clock_extras.py new file mode 100644 index 00000000..133f8c7e --- /dev/null +++ b/tests/test_frozen_clock_extras.py @@ -0,0 +1,49 @@ +"""The frozen clock works without the ``notebook`` extra (issue #262). + +``tests/frozen_clock.py`` imports every module of the package. Without ipywidgets and +matplotlib the notebook modules cannot be imported; they are skipped, and any other import +failure is still raised. Run in a fresh interpreter where both packages are blocked. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +TESTS = Path(__file__).resolve().parent + +_BLOCKED = """ +import sys +sys.path.insert(0, {tests!r}) +for name in ("ipywidgets", "matplotlib"): + sys.modules[name] = None # import of either raises ModuleNotFoundError +import frozen_clock +modules = {{m.__name__ for m in frozen_clock._import_package()}} +assert "morphobricks_preprocessing.notebook_ui" not in modules, modules +assert "morphobricks_preprocessing.design_derive" in modules, modules +assert "morphobricks_preprocessing.run.pipeline" in modules, modules +print("ok") +""" + + +def test_the_notebook_modules_are_skipped_without_the_extra(): + code = _BLOCKED.format(tests=str(TESTS)) + done = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) + assert done.returncode == 0, done.stderr + assert done.stdout.strip() == "ok" + + +def test_another_missing_module_is_still_raised(monkeypatch): + import frozen_clock + + def missing(name, package=None): + raise ModuleNotFoundError("No module named 'tifffile'", name="tifffile") + + monkeypatch.setattr(frozen_clock.importlib, "import_module", missing) + try: + frozen_clock._import_package() + except ModuleNotFoundError as exc: + assert exc.name == "tifffile" + else: + raise AssertionError("a missing required dependency was skipped") From 80c0f0bf7a16b83cdeda474df79cd68b69bf48e8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:11:37 +0000 Subject: [PATCH 2/9] SWEEP-PP PP-07: remove the deprecated GroupSummary.plane_count alias The alias of channel_histogram_count (before #104) warned and served nothing current: no caller in the nine repositories. Its deprecation test becomes a check of the pooled count itself; AGENTS.md and the CHANGELOG say it is gone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- AGENTS.md | 4 ++-- CHANGELOG.md | 4 ++++ src/morphobricks_preprocessing/report.py | 16 ---------------- tests/test_report.py | 5 ++--- 4 files changed, 8 insertions(+), 21 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 24f47b59..eb6ce2bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -549,8 +549,8 @@ functions the tests use. `GroupSummary` per group and channel (`channel`, `xyc_plane_count`, `channel_histogram_count`), a `channel` column in the table and one plot panel per channel. Pooling every channel together (as with version 2) only with - `group_summaries(..., pool_channels=True)`, labelled `all channels (pooled)`; - `GroupSummary.plane_count` is a deprecated alias of `channel_histogram_count`. + `group_summaries(..., pool_channels=True)`, labelled `all channels (pooled)`. (The + deprecated alias `GroupSummary.plane_count` was removed in v0.5.1, SWEEP-PP.) - **`position_id`** is filled in every row (D02, plan §12 question 20). Earlier tables with it blank are filled on the next run without reading pixels. - **Reading:** labels are always read back as text (`NA`, `001`, `TRUE` stay as typed). diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e835bb0..ec4b90b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -228,6 +228,10 @@ already removed is not repeated here. **Breaking** where it says so. other import error is raised), so a plain `pip install -e '.[dev]'` passes the frozen-clock tests (`tests/test_frozen_clock_extras.py`). +- **Removed: `GroupSummary.plane_count`** (inventory PP-07), the deprecated alias of + `channel_histogram_count` from before #104; use `xyc_plane_count` (XYC planes) or + `channel_histogram_count` (channel histograms pooled). No caller in the nine repositories. + ## 0.5.0 - 2026-11-09 Stage 1 of the experiment description (MorphoBricks-Spec experiment-v1, pinned at `1bed024`): diff --git a/src/morphobricks_preprocessing/report.py b/src/morphobricks_preprocessing/report.py index 70782673..fc485934 100644 --- a/src/morphobricks_preprocessing/report.py +++ b/src/morphobricks_preprocessing/report.py @@ -7,7 +7,6 @@ from __future__ import annotations -import warnings from collections import Counter from collections.abc import Iterable, Sequence from dataclasses import dataclass @@ -354,21 +353,6 @@ class GroupSummary: median: float | None = None percentiles: dict[float, float] | None = None - @property - def plane_count(self) -> int: - """Deprecated name of ``channel_histogram_count`` (before issue #104). - - It counted each channel of an XYC plane as one plane; use ``xyc_plane_count`` for - planes and ``channel_histogram_count`` for histograms. - """ - warnings.warn( - "GroupSummary.plane_count is deprecated: use xyc_plane_count (XYC planes) or " - "channel_histogram_count (channel histograms pooled).", - DeprecationWarning, - stacklevel=2, - ) - return self.channel_histogram_count - def group_summaries( records: Sequence[PlaneRecord], diff --git a/tests/test_report.py b/tests/test_report.py index 40e52c17..b22d70db 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -250,10 +250,9 @@ def test_a_group_with_different_channel_counts_is_refused(): group_summaries(records, "condition") -def test_plane_count_is_a_deprecated_alias_of_the_histogram_count(): +def test_pooled_channels_count_each_channel_histogram(): (summary,) = group_summaries(channel_records((10, 1000)), "condition", pool_channels=True) - with pytest.warns(DeprecationWarning, match="xyc_plane_count"): - assert summary.plane_count == 4 + assert summary.channel_histogram_count == 2 * summary.xyc_plane_count def test_group_without_finite_pixels(tmp_path): From 28055c799c08e0fc3972ee5fa6a5b209ac5ec625 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:12:27 +0000 Subject: [PATCH 3/9] SWEEP-PP PP-08: remove source_fingerprint (owner answer 7) source_fingerprint returned only (size, mtime_ns) of source_signature and was kept "for callers"; none in the nine repositories. Removed from inventory, the manifest re-export and the package top level. The tests that used it as a helper take source_signature directly; the contract and AGENTS.md name source_signature. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- AGENTS.md | 2 +- CHANGELOG.md | 5 ++++ docs/contract.md | 6 ++--- src/morphobricks_preprocessing/__init__.py | 2 -- .../content_hash.py | 2 +- src/morphobricks_preprocessing/inventory.py | 23 +------------------ src/morphobricks_preprocessing/manifest.py | 1 - tests/test_inventory.py | 8 +++---- tests/test_manifest_v2.py | 6 ++--- tests/test_masks_v040.py | 8 +++---- 10 files changed, 22 insertions(+), 41 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index eb6ce2bf..cfb92f47 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -139,7 +139,7 @@ src/morphobricks_preprocessing/ axes.py AxisMapping (role per dimension), prefill, check_mapping, review_axes (one mapping applied to every file, missing axes as singletons only when confirmed) axis_review.py review_axes and check_mapping (moved from axes.py in PP-4) - inventory.py source_signature/source_fingerprint: a file's size and mtime, an OME-Zarr + inventory.py source_signature: a file's size and mtime, an OME-Zarr folder's total size, newest mtime and inventory digest (issue #86; PP-12); store_files (the files an OME-Zarr fingerprint covers) source_content.py verify_content: a source's content SHA-256 (TIFF bytes; OME-Zarr per-file diff --git a/CHANGELOG.md b/CHANGELOG.md index ec4b90b0..36ae2ca3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -232,6 +232,11 @@ already removed is not repeated here. **Breaking** where it says so. `channel_histogram_count` from before #104; use `xyc_plane_count` (XYC planes) or `channel_histogram_count` (channel histograms pooled). No caller in the nine repositories. +- **Removed: `source_fingerprint`** (inventory PP-08, owner answer 7) from `inventory`, its + re-export from `manifest` and the package's top level. It was kept "for callers" that compared + size and modification time only; no source caller in the nine repositories. Use + `inventory.source_signature` (size, modification time and the OME-Zarr inventory digest). + ## 0.5.0 - 2026-11-09 Stage 1 of the experiment description (MorphoBricks-Spec experiment-v1, pinned at `1bed024`): diff --git a/docs/contract.md b/docs/contract.md index b2b76fa5..f3a1f6c4 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -241,7 +241,7 @@ interpretation. | `files` | One entry per analysed file, sorted by path: `relative_path`, `size` (bytes), `mtime_ns`, the three labels, `axes`, `singleton_roles`, `plane_count` (its rows in the table: XYC planes), `channel_names` (list, null for unnamed channels; empty if none in metadata), `significant_bits` (null if not in metadata), `read_in_blocks` (true when the image's planes are over the memory limit and are read in blocks; information only, false when absent), `position_id` (the image's position identifier, §1: its name without the extension, never edited; required in version 2, where an entry without it, with null, or with another value than `position_id(relative_path)` is refused naming the entry, by the reader and the writer, round-2 audit A-m5; null when absent in a version 1 manifest), `channel_names_trimmed` (true when leading or trailing whitespace was removed from the image's channel names as they were read, §2; `channel_names` holds the trimmed names; false when absent), `roles` (the role of each of the file's `axes`, `X`, `Y`, `Z`, `C`, `T` or null, from the file's own mapping: an OME-Zarr image in another order than `axis_mapping` has its own roles, so later bricks read these rather than rebuild them from `axis_mapping`; null when absent, in manifests written before them; optional key, version 1). For an OME-Zarr image, `size` is the total size of its files, `mtime_ns` the time of its newest file, and `inventory_sha256` (v0.4.0, PP-12, issue #86; only for OME-Zarr images) the SHA-256 of its inventory: one line per file (its path relative to the image with `/`, a NUL, its size, a NUL, its `mtime_ns`, a newline), sorted by path, its root `labels` group left out; links to folders inside the image are followed, as the reader follows them, and links that reach one folder twice are refused. Adding, removing, renaming, resizing or touching any chunk or metadata file changes it; an OME-Zarr image recorded without it (v0.4.0rc3 and earlier) counts as changed (recomputed; `RunInputs` refuses it). Like a TIFF file's size and time it is metadata only (design v0.4 §1): a file rewritten with the same size and its time set back is not seen, only hashing the content would. **Version 2 (v0.4.0):** `roles` is required, with `axes_from` (required): `header` when the metadata names every axis and the roles agree, `confirmed` for a plain stack whose roles the user confirmed (§2); and `pixel_sizes`, one entry per role `X`, `Y`, `Z`, `T` with `value`, `unit` (OME-NGFF name) and `source`: `header` (the image's metadata) or `stated` (`calibration.csv`, §9.3) with a value, or `pixels` (stated as deliberately uncalibrated) with `value` and `unit` null; `value` and `source` null for an axis nothing gives a size (never guessed). A `header` or `stated` size without a value, or a value with no source or with `pixels`, is refused. The reader refuses a version 2 entry without `roles` or `axes_from`, and the writer refuses to write one. `content_sha256` (only in a `verify_content` run): the SHA-256 of the image's bytes, or for an OME-Zarr image of one line per file (its path relative to the image with `/`, a NUL, the file's SHA-256, a newline) over the files its inventory covers, in path order (`source_content.source_content_sha256`). | | `csv_sha256` | SHA-256 of the table written with this manifest. | | `experiment_design_sha256` | SHA-256 of the `experiment_design.csv` the run derived from its description and wrote with the table (§12), so later bricks can check that the design they read is the one the run wrote. | -| `masks` | The masks paired with the images (§15), when the run was given a masks folder or its OME-Zarr images have label images (always used): `folder` (absolute path of the masks folder, null when only OME-Zarr label images were used), `types` (sorted mask-type names) and `files` (one entry per image and mask type, sorted by image then type: `image`, `type`, `path`, `source` `folder` or `ome-zarr`, `axes` and `shape`: the mask's axes and lengths without its length-1 axes other than Y and X, `YX` for a 2D mask (`[1, 512]` for a mask one pixel high), with `Z` and/or `T` for a mask with depth slices or time points, e.g. `TYX` and `[50, 512, 512]`; §15). Absent when the run had no masks. Optional key: the table does not depend on the masks. `axes` and `shape` are absent in manifests written before them, whose masks are all 2D; a later brick must read `axes` before using a mask and never apply a 2D mask to several planes. **Version 2:** `axes` are the mask's axes as stored (its header's letters: `IYX` for a plain stack), and each mask entry also has `size` and `mtime_ns` (its fingerprint, as `source_fingerprint` gives it: the mask file, or the label image's folder), `roles` (one per axis of `axes`, what each is read as: `TYX` for that plain stack stated `--mask-axes Cells=TYX`), `axes_from` (`header`, or `stated` when the axes were stated for its mask type). The release candidates up to v0.4.0rc6 also wrote `shared` (true for a 2-D mask stated `YX` for its type: one plane shared by every plane of its image); the option was removed by the owner decision of 5 November 2026, the key is no longer written, and `RunInputs.open` refuses a manifest that records `shared: true` (stated or not), naming the masks and asking for one mask plane per time point and a new run. A mask is read as stored, never reordered. | +| `masks` | The masks paired with the images (§15), when the run was given a masks folder or its OME-Zarr images have label images (always used): `folder` (absolute path of the masks folder, null when only OME-Zarr label images were used), `types` (sorted mask-type names) and `files` (one entry per image and mask type, sorted by image then type: `image`, `type`, `path`, `source` `folder` or `ome-zarr`, `axes` and `shape`: the mask's axes and lengths without its length-1 axes other than Y and X, `YX` for a 2D mask (`[1, 512]` for a mask one pixel high), with `Z` and/or `T` for a mask with depth slices or time points, e.g. `TYX` and `[50, 512, 512]`; §15). Absent when the run had no masks. Optional key: the table does not depend on the masks. `axes` and `shape` are absent in manifests written before them, whose masks are all 2D; a later brick must read `axes` before using a mask and never apply a 2D mask to several planes. **Version 2:** `axes` are the mask's axes as stored (its header's letters: `IYX` for a plain stack), and each mask entry also has `size` and `mtime_ns` (its fingerprint, the size and modification time `source_signature` gives: the mask file, or the label image's folder), `roles` (one per axis of `axes`, what each is read as: `TYX` for that plain stack stated `--mask-axes Cells=TYX`), `axes_from` (`header`, or `stated` when the axes were stated for its mask type). The release candidates up to v0.4.0rc6 also wrote `shared` (true for a 2-D mask stated `YX` for its type: one plane shared by every plane of its image); the option was removed by the owner decision of 5 November 2026, the key is no longer written, and `RunInputs.open` refuses a manifest that records `shared: true` (stated or not), naming the masks and asking for one mask plane per time point and a new run. A mask is read as stored, never reordered. | | `image_type` | `fluorescence` or `rgb`, the image type the user confirmed (§13); null when the run was given none, or absent in manifests written before it. Optional key: the manifest stays version 1. Changing it recomputes nothing. | | `channels_from_rgb_samples` | `true` when the C role is the RGB samples axis of images stored as RGB colour (§13), `false` otherwise; absent in manifests written before it. Optional key. | | `intensity_checks` | The limits of the intensity checks run on the table (§14): `IntensityCheckSettings.to_dict()` of the normalization brick (`library_version`, `outlier_limit`, `saturation_limit`, `min_images`); absent in manifests written before it. Optional key. Changing the limits recomputes nothing. | @@ -599,7 +599,7 @@ masks/ masks folder, outside the experiment folder | Rule | Detail | |---|---| -| Layout | Either one subfolder per mask type, each mirroring the experiment's folders (`//.../.tif`), or the experiment's folders with the type after the image name (`/.../_.tif`, as segmentation tools often write it, e.g. MorphoRelate's JUMP-Target-2 conversion); the two may be mixed. Any number of mask types. The label images of an OME-Zarr image (its OME-NGFF `labels` group, with the `zarr` extra) are masks too, of the type named by the label image; since v0.4.0 they are always used when present, with or without a masks folder. Each label image has its own fingerprint in the manifest, and the `labels` group is left out of its image's fingerprint (`source_fingerprint`), so adding or changing a label image does not recompute the image's rows. | +| Layout | Either one subfolder per mask type, each mirroring the experiment's folders (`//.../.tif`), or the experiment's folders with the type after the image name (`/.../_.tif`, as segmentation tools often write it, e.g. MorphoRelate's JUMP-Target-2 conversion); the two may be mixed. Any number of mask types. The label images of an OME-Zarr image (its OME-NGFF `labels` group, with the `zarr` extra) are masks too, of the type named by the label image; since v0.4.0 they are always used when present, with or without a masks folder. Each label image has its own fingerprint in the manifest, and the `labels` group is left out of its image's fingerprint (`source_signature`), so adding or changing a label image does not recompute the image's rows. | | Pairing | A mask belongs to the image in the same leaf folder with the same `position_id` (§1), ignoring upper/lower case; its file is a TIFF (`.tif`, `.tiff`, optionally `.ome`, any case). Excluded images have no masks: their masks are reported apart (`mask_of_excluded_image`, warning) and not used. | | Names | A mask type is named by its folder, by the suffix after the image name, or by its label image. A file that could belong to two images (an image name that starts another's, `a_b_cells.tif` beside images `a` and `a_b`) or fit both layouts is the error `ambiguous_mask`. Two names differing only in case are the error `mask_names_differ_in_case`; one image with two masks of one type (two extensions, or a file and a label image) is the error `duplicate_mask`. | | Header checks | From the headers only, no pixels read (`check`, `run`, `preflight`): once its length-1 axes other than Y and X are removed (Y and X are kept whatever their length, so a mask one pixel high or wide, or a single pixel, is a 2D `YX` mask; a TIFF whose series has no Y or X but is one 2D page, such as a 1D array `tifffile` saved as `{"shape": [N]}` and reads as `X`, is that page, `YX` of 1 row; plan §12 question 44), a mask has Y and X and at most a depth axis Z and a time axis T, each named so in its own metadata (TIFF axes as `tifffile` reads them, e.g. an ImageJ hyperstack or OME-TIFF; OME-NGFF axes for a label image); any other or unnamed axis, such as a plain stack that could be depth or time, is `mask_not_2d` and is never guessed, unless the axes are stated for its mask type (`--mask-axes Cells=TYX`, `mask_axes={"Cells": "TYX"}`: one role, Y, X, Z or T, per axis of the header with length-1 axes other than Y and X left out; recorded as `axes_from: "stated"`; a statement that does not fit is `invalid_mask_axes`, one that contradicts an axis the metadata names is `mask_axes_contradict_header`, one for a type not found is `invalid_mask_axes`; a statement `YX` is `shared_mask_not_supported`: it was the option of one 2-D mask shared by every plane of its image, removed by the owner decision of 5 November 2026, and a 2-D mask needs no statement). Masks are never converted or reordered: a mask whose X is stored before its Y, or whose Y and X are its image's swapped, is `mask_layout`, with a `numpy.swapaxes` hint; the message names each mask with its header axes (`Cells/a.tif: header axes XY: X is stored before Y`). Unlike an image whose own header names X before Y (read in canonical order), a mask is never reordered, so `run` refuses it too (before PP-8b `run` dropped this error with the header size comparison and recorded the mask with `axes: ""`); a manifest never records a mask without axes. A mask has the Y and X sizes of its image (`mask_size_mismatch`; `check` and `preflight` compare with the Y and X the image's header names, and not for a plain stack, whose Y and X are not named; `run` compares with the image's Y and X under the confirmed roles, `mask_size_issues`, and leaves out the masks of an image with an axis error (§2), which is never read — before v0.4.0 `run` compared with the header's letters, so a swapped image gave a misleading size error) and integer or boolean pixels (`mask_pixel_type`; a mask holding a negative label or a label above 2,147,483,647, the int32 limit every brick shares (§16, decision D-b), raises an error when its pixels are read); the masks of one image have the same depth slices and time points (`mask_planes_differ`); a mask that cannot be opened, or holds several series, is `unreadable_mask`, as is an OME-Zarr image's `labels` group that cannot be read or does not list its label images (path `/labels`), so its mask types never disappear unreported. All five are errors. | @@ -732,7 +732,7 @@ instead of keeping copies. | Declared axes (removed) | `check_declared_axes`, `declared_axes_transposed` | **Removed in v0.4.0** (spec revision 4): they accepted a declaration with Y and X swapped and read it transposed, which v0.4 forbids. Bricks take no axis declarations; preprocessing's axis check (§2) compares the confirmed roles with the header. | | Mask files | `read_mask_file(path, z=, t=)` (`masks`) | Reads one plane of a TIFF mask or of an OME-Zarr label image (`.zarr/labels/`) given by its path, with `read_mask`'s rules (§15): its axes are its own metadata's, Y and X plus at most a Z and a T axis named so, length-1 axes other than Y and X left out (a mask one pixel high or wide is read as a 2D array, plan §12 question 44); any other or unnamed dimension (a plain stack, `IYX`; a `C` axis) is refused, the message giving the header's axes and shape and the dimension that is not named, never taken for depth or time; `z` and `t` are required for a mask with that axis; integer or boolean pixels; labels checked with `check_labels`. A `ValueError` names the file and the problem. | | Identity (D-e) | `FOV_KEY`, `check_identity_text(value, field)`, `check_plane_index(value, axis)`, `check_identity_case(keys)`, `MISSING_TEXT`, `IdentityError` (`discovery`) | `FOV_KEY` = (`condition`, `biological_repeat`, `technical_repeat`, `position_id`), the position key of §1. `check_identity_text` accepts a label or `position_id` that is text without leading or trailing whitespace and not `""`, `NA`, `nan`, `none`, `null`, `n/a` or `` in any case (`MISSING_TEXT`); `check_plane_index` accepts None or an integer of at least 0, a Python `int` or a NumPy integer (returned as a plain `int`), and refuses everything else (bool, float, `"1"`) without converting it. Both raise `IdentityError` (a `ValueError`) naming the field and the value. Discovery applies the text rule to every folder and image name (`invalid_name`) and refuses names of one level that differ only in case anywhere in the experiment (`case_conflict`, §1). `position_id` keeps its own scope: unique within its technical repeat, ignoring case (`duplicate_position_id`). `check_identity_case` applies the same case rule to identities a brick reads from a table, one key per row in `FOV_KEY` order (or the three labels): a label of one level spelled two ways anywhere in the run, or two `position_id` values of one technical repeat equal ignoring case, raise `IdentityError` naming both spellings; the same key may repeat (several planes of one position). | -| Sources of a run | `check_sources(manifest, experiment_root, relative_paths, image_axes=, identities=)`, `source_fingerprint`, `file_sha256` (`manifest`) | The checks a brick makes before using a run's table for some images (formerly CellPatcher `check_images` and MorphoFeature `histogram_fit`); from v0.4.0 file runs read through `RunInputs` (§9.1), which also checks masks. A version 1 manifest fails every image with `manifest_before_v040` alone (design v0.4 §6: every v1 run is redone). Otherwise each image must be in the manifest (`images_not_in_manifest`), still exist (`images_missing`), and be unchanged by `source_fingerprint`, its size and modification time, for an OME-Zarr folder its files' total size and newest time, so a rewritten chunk counts (`images_changed`). Its recorded `position_id` must be the one its name gives; with `identities` (`FOV_KEY` order), its labels and `position_id` must be the caller's (`identity_differs`; `position_id_not_recorded` for a manifest written before positions were recorded). With `image_axes`, the caller's axes must give each axis the role recorded for the file (`FileEntry.roles`, or the run's mapping resolved for the header in older manifests); an axis without a role (length 1) fits any letter (`axis_mapping_mismatch`). All errors, naming their images; the table's own hash is checked with `file_sha256` against `csv_sha256`. | +| Sources of a run | `check_sources(manifest, experiment_root, relative_paths, image_axes=, identities=)`, `source_signature` (`inventory`), `file_sha256` (`manifest`) | The checks a brick makes before using a run's table for some images (formerly CellPatcher `check_images` and MorphoFeature `histogram_fit`); from v0.4.0 file runs read through `RunInputs` (§9.1), which also checks masks. A version 1 manifest fails every image with `manifest_before_v040` alone (design v0.4 §6: every v1 run is redone). Otherwise each image must be in the manifest (`images_not_in_manifest`), still exist (`images_missing`), and be unchanged by `source_signature`, its size and modification time, for an OME-Zarr folder its files' total size and newest time, so a rewritten chunk counts (`images_changed`). Its recorded `position_id` must be the one its name gives; with `identities` (`FOV_KEY` order), its labels and `position_id` must be the caller's (`identity_differs`; `position_id_not_recorded` for a manifest written before positions were recorded). With `image_axes`, the caller's axes must give each axis the role recorded for the file (`FileEntry.roles`, or the run's mapping resolved for the header in older manifests); an axis without a role (length 1) fits any letter (`axis_mapping_mismatch`). All errors, naming their images; the table's own hash is checked with `file_sha256` against `csv_sha256`. | | Masks named as images (D-i) | `mask_name_issues(leaves, mask_names=())`, `images_named_as_masks(leaves, suffixes=MASK_NAME_SUFFIXES)`, `mask_folder_part(path)`, `MASK_NAME_SUFFIXES`, `MASK_FOLDER_WORDS` (`discovery`) | The screen MorphoRelate's `init` ran (`project_image_names`), now discovery's warnings `image_named_as_mask` and `mask_folder_in_experiment` (§1) for every brick. Names are compared in lower case across the whole experiment; the longest `position_id` owns a file, an image of the same folder first. `mask_names` adds the caller's expected mask names (`cells` for `f1_cells.tif`) to `MASK_NAME_SUFFIXES`; `discover_experiment` gives none. A folder counts as a masks folder when a word of its name (`CellLabels` is `Cell` and `Labels`) is in `MASK_FOLDER_WORDS`. `images_named_as_masks` returns the `(image, owner)` pairs of that detection for the caller's own name endings (`["_cells"]`), for a brick that acts on the files itself (MorphoRelate leaves out the images named with a declared mask name). | | Pixel sizes (D-a) | `same_pixel_size(a, b, roles=("X", "Y"))`, `pixel_size_groups(sizes, roles=)`, `PIXEL_SIZE_TOLERANCE` (`physical_sizes`) | Two pixel sizes are the same when `|a - b| <= 1e-3 * max(a, b)` on every compared axis, in micrometres; one rule for a declared size against a header and for two headers. `same_pixel_size` takes `PhysicalSizes` (converted with `micrometers`) or sizes already in micrometres, one per role, and returns None when either is unknown (no size, or a unit that cannot be converted), never True. The rule is not transitive (1.0 ≈ 1.0009 ≈ 1.0018, but 1.0 ≠ 1.0018), so there is no per-image grouping key: `pixel_size_groups` links images by single linkage, so images of two groups always differ while the extremes of one group may drift; unknown sizes are left out; the largest group first. A size that cannot be converted to a positive finite length in micrometres (`1e308 m`) is unknown, never infinite, and an infinite or NaN value is never the same as any other (issue #93; before, `inf` matched every size). The folder check's `different_pixel_sizes` uses both (§11). | | Experiment design | `read_confirmed_design(path, expected_sha256=)`, `ConfirmedDesign`, `DesignCheckError` (`design`) | The one reader of `experiment_design.csv` (§12) for a brick that pools biological repeats (formerly CellPatcher `load_experiment_design` and MorphoFeature `ExperimentDesign.read`). It takes the file as this brick writes it and fills nothing in: UTF-8 without a byte-order mark; the header exactly the §12 columns in order (a column named twice or without a name is refused naming it, as by the lenient reader, §12); one value per column in every row (an unclosed quote, a missing or extra value, or a blank line is refused); no value with outer whitespace; `control` and `confirmed` exactly `true` or `false`; counts written as whole numbers of at least 0; `schema_version` 1; `confirmed_on` empty or `YYYY-MM-DD`; `condition`, `biological_repeat` and `repeat_group` pass `check_identity_text` and do not differ only in case (`check_identity_case`); no folder twice, one `control` per condition, at least one row; every group confirmed with one folder per condition (`design_issues`); with `expected_sha256` (the manifest's `experiment_design_sha256`) the file must be that one. The SHA-256 is of the bytes parsed. `ConfirmedDesign` gives `path`, `sha256`, the `design`, read-only `repeat_groups` (`{(condition, biological_repeat): repeat_group}`, the normalization brick's input), `group_members()` (`{repeat_group: {condition: biological_repeat}}`), `conditions`, `controls` and `groups`. Refusals raise `DesignCheckError` (a `ValueError`) with its `issues`: `missing_experiment_design`, `unreadable_experiment_design`, `invalid_experiment_design`, `repeat_group_conflict`, `design_not_confirmed` or `changed_experiment_design`, each an error naming the file (or the group's folders). This brick's step 1 and `check` read the file being edited with the lenient `load_design(output_folder, repeats)` (§12), which returns `(design, changes)`: the saved design brought up to date with the folders, or the default design when there is no file. A saved file that cannot be read raises `DesignCheckError` with the one error `unreadable_experiment_design`, whose message is the same "The experiment design cannot be read: …" as here followed by the lenient reader's refusal, naming the line and column where it can (the `DesignFormatError` is its `__cause__`); no default design is returned and the file is not touched (owner decision of 31 October 2026, plan §12 question 43). **Breaking in v0.4.0:** up to v0.3.4 `load_design` returned a third value, `problem`, the reason as text, together with the default design. | diff --git a/src/morphobricks_preprocessing/__init__.py b/src/morphobricks_preprocessing/__init__.py index bc7a67e6..52305528 100644 --- a/src/morphobricks_preprocessing/__init__.py +++ b/src/morphobricks_preprocessing/__init__.py @@ -122,7 +122,6 @@ RunSettings, file_sha256, read_manifest, - source_fingerprint, write_manifest, ) from .mask_checks import mask_issues, mask_size_issues @@ -455,7 +454,6 @@ "sample_planes", "scan_masks", "series_axes_from_metadata", - "source_fingerprint", "space_unit", "to_micrometers", "write_calibration", diff --git a/src/morphobricks_preprocessing/content_hash.py b/src/morphobricks_preprocessing/content_hash.py index d61685e8..a440fab9 100644 --- a/src/morphobricks_preprocessing/content_hash.py +++ b/src/morphobricks_preprocessing/content_hash.py @@ -7,7 +7,7 @@ §1). An OME-Zarr copy is hashed as the SHA-256 of its files' relative paths and SHA-256s, in -path order. Its ``labels`` group is left out, as :func:`.inventory.source_fingerprint` +path order. Its ``labels`` group is left out, as :func:`.inventory.source_signature` leaves it out: label images are masks, recorded and checked one by one by the run, and a brick may add a label image to an OME-Zarr image without changing the image. A folder holding a link to a folder is not hashed (:class:`OSError`): the files behind the link diff --git a/src/morphobricks_preprocessing/inventory.py b/src/morphobricks_preprocessing/inventory.py index c42887cd..7a4d8fcf 100644 --- a/src/morphobricks_preprocessing/inventory.py +++ b/src/morphobricks_preprocessing/inventory.py @@ -14,7 +14,7 @@ Design v0.4 §1 keeps the fingerprint metadata only by default; a run that verifies content (``verify_content``, :mod:`.source_content`) also records and checks each source's SHA-256. -Moved from :mod:`.manifest` (PP-12), which re-exports :func:`source_fingerprint`. +Moved from :mod:`.manifest` (PP-12). """ from __future__ import annotations @@ -106,27 +106,6 @@ def store_files(path: Path) -> tuple[int, list[tuple[str, os.stat_result]]]: return newest, found -def source_fingerprint(path: str | os.PathLike) -> tuple[int, int]: - """Return the size and modification time of an image, without its inventory. - - Kept for callers that show or compare only these two numbers; whether an image - changed is decided with the inventory too (:func:`source_signature`, - :class:`.fingerprint.Fingerprint`). - - Parameters - ---------- - path - A TIFF file or an OME-Zarr folder. - - Returns - ------- - tuple - ``(size, mtime_ns)`` as :func:`source_signature` gives them. - """ - size, mtime_ns, _ = source_signature(path) - return size, mtime_ns - - def _raise(error: OSError) -> None: """Raise a listing error of :func:`os.walk`, which would otherwise skip the folder.""" raise error diff --git a/src/morphobricks_preprocessing/manifest.py b/src/morphobricks_preprocessing/manifest.py index 88360aa4..72ed7d7d 100644 --- a/src/morphobricks_preprocessing/manifest.py +++ b/src/morphobricks_preprocessing/manifest.py @@ -35,7 +35,6 @@ from .description_record import recorded_description from .image_names import position_id from .inventory import INVENTORY_KEY, source_signature -from .inventory import source_fingerprint as source_fingerprint # re-exported (moved, PP-12) from .lab_links import KEY as LAB_LINKS_KEY from .lab_links import recorded_lab_links from .manifest_fields import ( diff --git a/tests/test_inventory.py b/tests/test_inventory.py index ece8cbdd..b076211b 100644 --- a/tests/test_inventory.py +++ b/tests/test_inventory.py @@ -22,7 +22,7 @@ read_manifest, ) from morphobricks_preprocessing.fingerprint import Fingerprint -from morphobricks_preprocessing.inventory import source_fingerprint, source_signature +from morphobricks_preprocessing.inventory import source_signature from morphobricks_preprocessing.run_inputs import RunInputsError from zarr_tree import write_ngff @@ -68,9 +68,9 @@ def test_a_rewritten_chunk_below_the_newest_time_changes_the_fingerprint(tmp_pat # The issue's counterexample: same total size, same newest time. image = two_chunks(tmp_path / "a.ome.zarr") before = Fingerprint.of(image) - old_size, old_time = source_fingerprint(image) + old_size, old_time, _ = source_signature(image) swap_chunks(image) - assert source_fingerprint(image) == (old_size, old_time) # the old rule saw nothing + assert source_signature(image)[:2] == (old_size, old_time) # size and time saw nothing assert Fingerprint.of(image) != before @@ -82,7 +82,7 @@ def test_a_chunk_removed_and_another_added_changes_the_fingerprint(tmp_path): shutil.copy2(first, moved) first.unlink() os.utime(first.parent, ns=(3_000, 3_000)) - assert source_fingerprint(image) == (before.size, before.mtime_ns) + assert source_signature(image)[:2] == (before.size, before.mtime_ns) assert Fingerprint.of(image) != before diff --git a/tests/test_manifest_v2.py b/tests/test_manifest_v2.py index b4350eb7..8d0dddae 100644 --- a/tests/test_manifest_v2.py +++ b/tests/test_manifest_v2.py @@ -24,12 +24,12 @@ ) from morphobricks_preprocessing.axis_check import ConfirmedAxes from morphobricks_preprocessing.fingerprint import Fingerprint, RowKey, confirm_axes +from morphobricks_preprocessing.inventory import source_signature from morphobricks_preprocessing.manifest import ( MANIFEST_VERSION, FileEntry, ManifestFormatError, RunSettings, - source_fingerprint, write_manifest, ) from morphobricks_preprocessing.recorded_sizes import RecordedSize @@ -92,7 +92,7 @@ def test_masks_record_their_fingerprint_and_roles(tmp_path): mask = write_data(tmp_path / "masks/Cells/ctrl/a.tif", np.ones((6, 8), np.uint16), "YX") result = run_described(root, prefill_mapping("YX"), masks_folder=tmp_path / "masks") (recorded,) = read_manifest(result.manifest_path).masks["files"] - size, mtime_ns = source_fingerprint(mask) + size, mtime_ns, _ = source_signature(mask) assert recorded == { "image": "ctrl/a.tif", "type": "Cells", @@ -172,7 +172,7 @@ def test_the_row_key(): def test_fingerprint_of_a_file(tmp_path): path = write_data(tmp_path / "a.tif", stack(), "TYX") - assert Fingerprint.of(path) == Fingerprint(*source_fingerprint(path)) + assert Fingerprint.of(path) == Fingerprint(*source_signature(path)) # A plain stack's confirmation covers only its files (audit 2 B-1) --------------------------- diff --git a/tests/test_masks_v040.py b/tests/test_masks_v040.py index 14fcb605..0e186903 100644 --- a/tests/test_masks_v040.py +++ b/tests/test_masks_v040.py @@ -24,7 +24,7 @@ read_manifest, read_mask, ) -from morphobricks_preprocessing.manifest import source_fingerprint +from morphobricks_preprocessing.inventory import source_signature from morphobricks_preprocessing.masks.stated import mask_roles from tiff_tree import write_data, write_file, write_plain @@ -297,7 +297,7 @@ def test_label_images_are_always_used(tmp_path): recorded = read_manifest(result.manifest_path).masks assert recorded["folder"] is None and recorded["types"] == ["cells"] (entry,) = recorded["files"] - size, mtime_ns = source_fingerprint(root / "ctrl/a.ome.zarr/labels/cells") + size, mtime_ns, _ = source_signature(root / "ctrl/a.ome.zarr/labels/cells") assert (entry["size"], entry["mtime_ns"]) == (size, mtime_ns) @@ -305,10 +305,10 @@ def test_label_images_are_left_out_of_the_image_fingerprint(tmp_path): from zarr_tree import write_labels root, path = zarr_experiment(tmp_path) - before = source_fingerprint(path) + before = source_signature(path) first = run_described(root, prefill_mapping("YX")) write_labels(path, {"cells": labels((6, 8), 2), "nuclei": labels((6, 8))}, "yx") - assert source_fingerprint(path) == before + assert source_signature(path) == before again = run_described(root, prefill_mapping("YX"), output_folder=first.output_folder) assert again.read == () # the image's rows are reused data = json.loads(again.manifest_path.read_text(encoding="utf-8")) From 7068ec7c106d8162fbfd72c30c72d8d21cc0c1f0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:13:58 +0000 Subject: [PATCH 4/9] SWEEP-PP PP-06: read conversion record version 3 only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The writer writes version 3 only. Version 2 was still read (as a tree without a projection) and version 1 had a pointed message of its own. Now any version but 3 is refused with one message naming its version and the fix (convert the originals again into a new folder); nothing is converted. The record's and each entry's projection key is required, as version 3 always writes it. Tests: the version-2 read test becomes a refusal test; 1, 2, 4 and a missing version are refused; a record without its projection key is refused. Contract §9.1/§17, user guide and CHANGELOG updated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- CHANGELOG.md | 8 +++ docs/contract.md | 6 +-- docs/user_guide.md | 8 +-- .../conversion_record.py | 49 +++++++------------ tests/test_projection_record.py | 22 ++++----- 5 files changed, 44 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36ae2ca3..b9d4c871 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -237,6 +237,14 @@ already removed is not repeated here. **Breaking** where it says so. size and modification time only; no source caller in the nine repositories. Use `inventory.source_signature` (size, modification time and the OME-Zarr inventory digest). +- **Conversion record: version 3 only** (inventory PP-06). **Breaking.** A + `.morphobricks-conversion.json` of any other version (2 was still read as a tree without a + projection; 1 had its own message) is refused by `convert`, the run's `review_conversion` and + `RunInputs` with `conversion_record_unreadable`, naming its version and the fix: convert the + originals again into a new folder. The record's and each entry's `projection` key is required + (version 3 always writes it). Removed: `_READABLE_VERSIONS`, `_OLD_VERSIONS` and the "version 2 + cannot hold a projection" check. + ## 0.5.0 - 2026-11-09 Stage 1 of the experiment description (MorphoBricks-Spec experiment-v1, pinned at `1bed024`): diff --git a/docs/contract.md b/docs/contract.md index f3a1f6c4..34559414 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -270,7 +270,7 @@ masks_root=None)` (design v0.4 §6). It refuses (`RunInputsError`) a version 1 m `plane_histograms.csv`, `experiment_design.csv` or `calibration.csv` whose SHA-256 is not the manifest's (also a `calibration.csv` added since the run), the conversion record of a run on named-axis copies whose SHA-256 is not the manifest's `conversion_record_sha256` (changed -or gone since the run; §17), a conversion record of version 1 (no copy SHA-256: convert +or gone since the run; §17), a conversion record of another version than 3 (convert again), a copy of a recorded image whose fingerprint or SHA-256 is not the record's (or that the record does not list; PP-11c, issue #87), a manifest whose `projection` is not exactly what the record gives for the recorded images and masks (or that has none for projected copies, or a `projection` without a conversion record: projected copies are never read as plain images; issue #176), a run whose experiment (as recorded or given), named-axis copies or output folder is inside its masks folder, or whose masks folder @@ -755,8 +755,8 @@ exclude=, missing_as_singleton=, memory_limit=, progress=)` (`conversion`), the | Copies | Uncompressed OME-TIFF, BigTIFF when the pixels and 1 KiB per page (for its directory) would take more than 4 GiB less 32 MiB. Axes: every axis with a role in the source's order, then Y and X (`ZCYX`); an axis without a role (length 1) or missing (`missing_as_singleton`) is not added. Each plane is the plane the run reads under the confirmed roles (`iter_planes`), bit for bit: same pixel type (in native byte order), no rescaling, NaN and infinities kept; a Y/X swap and X before Y are written rows (Y) then columns (X). Planes over the memory limit are read in blocks, as the run reads them, and each block is written to its place: no plane is held whole when the run would not hold it. The copy's metadata names every axis, so `check` and `run` read it with `axes_from: "header"`. | | Pixel sizes | They follow the roles; a relabelled axis gets an unknown scale, never a guessed one (`copy_axes`). A size the source header gives is written only for the role the header names it for, and only when the user confirmed that role for that same axis; otherwise it is left out and the record says why. One exception (owner decision of 2 November 2026, plan §12 question 59): a Y or X relabelled as the header's other one of Y and X (a Y/X swap, X before Y) keeps the header's size when the header gives X and Y exactly equal, same value and unit, since nothing can be exchanged; otherwise both are unknown in the copy (state them in `calibration.csv`, §9.3, if known). A Y/X swap therefore never exchanges the sizes of Y and X. A z-step is never written as a time step. A plain stack's Y and X confirmed Y before X keep the header's sizes. Units are written as OME symbols (`µm`, `s`, ...). Channel names are written when the C axis keeps its role and every channel is named; significant bits as they are. | | Safe | The copy tree's lock (`.morphobricks-preprocessing.lock`, §9.2's rule; `--remove-stale-lock`) is held throughout; a second conversion is `output_locked` (a lock left by a conversion on this computer that is no longer running is called stale and kept, as in §9.2; cross-brick test F11). Each copy is written as `.partial`, the source's fingerprint checked before and after; once every copy is written they are moved into place, then the record is written (atomic, with the file-mode rule of §9). `not_enough_space` is refused before writing. Any failure or interruption removes the `.partial` files of the conversion, the copies already moved into place (each where nothing was) and the folders it made, so the folder is left as it was; a read or write failure is `conversion_failed`, anything else is raised again; `.partial` files of an interrupted conversion are written again. New files get the mode a plain `open()` gives. | -| Repeat | A copy already in the tree is reused when its source (fingerprint and axes), roles, copy axes, pixel sizes, unknown sizes (roles and reasons), channel names, significant bits, the copy's fingerprint and the copy's SHA-256 (read whole) are as recorded; a new image is added; when every copy is reused the record is kept as it was (same SHA-256). Anything else (a source changed, other roles, a copy changed, an image no longer in the experiment, another experiment's copy tree, a file at a new image's place that the record does not list) is `copy_differs`: nothing is overwritten. A conversion record that cannot be read, or of version 1, is `conversion_record_unreadable`. | -| Conversion record | `.morphobricks-conversion.json` in the copy tree's root (a dot-file, so discovery skips it), record version 3: `source_root`, `mapping`, `projection` (null, or `{method, z_range}`: the method and slice range of a tree of Z-projected copies, see "Projected copies" below), `missing_as_singleton`, `written_at`, `versions`, and per image `relative_path`, `action` (`converted` or `copied`), `reason` (the axis code, `excluded`, or null), `source` and `copy` fingerprints (`size`, `mtime_ns`), `copy_sha256` (the copy's SHA-256, computed in one pass after it is written; an OME-Zarr copy's is the SHA-256 of its files' relative paths and SHA-256s in path order, its `labels` group left out as for fingerprints: `content_hash`), `source_axes`, `roles`, `copy_axes`, `pixel_sizes` (by role, as §9), `unknown_sizes` (role: why), `channel_names` (as the copy holds them; `[]` for none), `significant_bits` (or null) and `projection` (null, or the projected copy's provenance, below). Sources keep the fingerprint rule (design v0.4 §1); the copies' SHA-256 is the owner's decision of 2 November 2026 (plan §12 question 59 (16)). Version 2 (PP-11c) added `copy_sha256`, `channel_names` and `significant_bits`; version 3 (issue #176) added the record-level `projection` and the entry's `projection`, and the action `projected`; a version 2 record is still read (a tree without a projection; it may not hold a projection field), and a version 3 record is refused by a reader that knows only version 2; a version 1 record (v0.4.0rc3 or earlier) is refused everywhere it is read (`convert`, the run's `review_conversion`, `RunInputs`) with a message to convert the originals again into a new folder: its copies cannot be checked. `read_conversion_record(folder)` reads it with its SHA-256. | +| Repeat | A copy already in the tree is reused when its source (fingerprint and axes), roles, copy axes, pixel sizes, unknown sizes (roles and reasons), channel names, significant bits, the copy's fingerprint and the copy's SHA-256 (read whole) are as recorded; a new image is added; when every copy is reused the record is kept as it was (same SHA-256). Anything else (a source changed, other roles, a copy changed, an image no longer in the experiment, another experiment's copy tree, a file at a new image's place that the record does not list) is `copy_differs`: nothing is overwritten. A conversion record that cannot be read, or of another version than 3, is `conversion_record_unreadable`. | +| Conversion record | `.morphobricks-conversion.json` in the copy tree's root (a dot-file, so discovery skips it), record version 3: `source_root`, `mapping`, `projection` (null, or `{method, z_range}`: the method and slice range of a tree of Z-projected copies, see "Projected copies" below), `missing_as_singleton`, `written_at`, `versions`, and per image `relative_path`, `action` (`converted` or `copied`), `reason` (the axis code, `excluded`, or null), `source` and `copy` fingerprints (`size`, `mtime_ns`), `copy_sha256` (the copy's SHA-256, computed in one pass after it is written; an OME-Zarr copy's is the SHA-256 of its files' relative paths and SHA-256s in path order, its `labels` group left out as for fingerprints: `content_hash`), `source_axes`, `roles`, `copy_axes`, `pixel_sizes` (by role, as §9), `unknown_sizes` (role: why), `channel_names` (as the copy holds them; `[]` for none), `significant_bits` (or null) and `projection` (null, or the projected copy's provenance, below). Sources keep the fingerprint rule (design v0.4 §1); the copies' SHA-256 is the owner's decision of 2 November 2026 (plan §12 question 59 (16)). Version 2 (PP-11c) added `copy_sha256`, `channel_names` and `significant_bits`; version 3 (issue #176) added the record-level `projection` and the entry's `projection`, and the action `projected`. Only version 3 is read (v0.5.1, SWEEP-PP: no backward compatibility, Spec ADR 0002 AR10): a record of any other version, 1 and 2 included, is refused everywhere it is read (`convert`, the run's `review_conversion`, `RunInputs`), never converted, with a message to convert the originals again into a new folder. `read_conversion_record(folder)` reads it with its SHA-256. | | Projected copies (issue #176, record version 3) | A tree made with a Z projection holds, for every image, a 2-D copy of its Z-stack made with the record-level `projection` `{method, z_range}` (`max`, `mean` or `sum`; there is no default method; `z_range` is `[start, stop]`, zero-based, stop excluded, the same for every image, or null for every slice). An entry has the action `projected` and a `projection` object with exactly: `source_relative_path`, `source_content_sha256` (the source's content as read while writing), `source_z_axis` (`{letter, index}` of the header's Z axis, or null for an image without one, which is one slice), `source_z_size`, `z_range` (the slices actually used), `method`, `source_dtype`, `output_dtype`, `dtype_rule`, `rounding` (`none`, `float64, one division` or `float64 addition, z ascending`), `nan_policy` (`propagate`), `dropped_sizes` (`{"Z": {value, unit, source}}`, or `{"Z": null}` when the source gave no Z size: the Z size is never written to the copy) and `significant_bits_dropped` (a reason, or null: kept for `max`, dropped for `sum` and `mean`). Excluded images stay plain copies (`copied`, reason `excluded`). The output type depends only on the input type, the method and the slice count (never on the data): `max` keeps the type; `sum` of `uint8`/`uint16` is `uint32`, of `int8`/`int16` is `int32`, of `uint32`/`int32` is `float64` holding the exact integer, of floats is `float64`; `mean` is `float64`; 64-bit integers are refused (`cannot_convert_pixel_type`). A sum or mean whose worst case (the type's extreme times the slices) could not be held exactly is refused (`projection_cannot_be_exact`), never wrapped or clipped. A NaN makes its pixel NaN, infinities are kept. The reader refuses an entry whose output type or rounding is not the table's for its method, a projected entry in a record without a projection (and the reverse), and a method or range that is not the record's. The reuse key holds the projection (everything of it except the content SHA-256, which the write computes), so another method or range into the same tree is `copy_differs`. Masks are never projected: `convert_experiment(..., projection=Projection(...), masks_folder=)` scans the masks folder from the headers only and refuses, before writing anything, masks of the images to project that have depth slices (`projection_masks_3d`) or more than Y and X with axes that are not named (`projection_masks_axes_unknown`); 2-D masks are left alone. The other refusals, all before anything is written: `nothing_to_project` (no image has a Z role), `projection_z_range_outside`, `projection_cannot_be_exact`, `cannot_convert_pixel_type`, `projection_over_memory_limit` (one input plane, the running plane and, when it is another type, the output plane must fit `memory_limit`) and `projection_zarr_not_supported` (OME-Zarr images are not projected yet). The copy's OME-XML holds an `Image/Description` with the method, slices, source, source SHA-256 and types, and a UUID derived from it, so one projection of one source is the same bytes; the source's content SHA-256 is taken before and after the projection and the source fingerprint before and after, and a source that changed is refused. A sum that overflows to infinity (`projection_overflow`) or adds an infinity to its opposite (`projection_undefined`) stops the conversion and is never written. Excluded images are copied byte for byte. | | The run on the copies | The user runs preprocessing on the copy tree with the same output folder (`--output`): the recorded experiment root becomes the copy tree and the report says so (`experiment_root_changed`, and `conversion` with the record, its SHA-256 and `source_root`; the text says "named-axis copies of ..."); every row is recomputed (the row key includes the read path, §9); the design, image type and calibration are kept (same relative paths). The manifest records `conversion_record_sha256`, and for projected copies the `projection` summary (§9); the report says "max projections of slices 1:4 (2 images) of " (and `conversion.projection` in `discovery_report.json`). Masks are paired as usual (the image has no Z now, so 2-D masks fit; a mask with depth slices stays `mask_planes_mismatch` and stops the run), and the run warns `masks_on_projected_images`, naming the method, for masks paired with projections, in the report and the manifest's `projection.warnings`. The run and `check` warn when a source changed or went missing after the conversion (`conversion_source_changed`) and refuse a copy that changed (its fingerprint or its SHA-256: each copy is read whole, so `check` and `run` on a copy tree cost one more read of the copies), went missing or was added since (`copy_changed_after_conversion`: an error since PP-12, owner decision of 2 November 2026; the run stops after the header checks, before any pixel is read, writing only its report, since `RunInputs` would refuse the run; the message says to convert the originals again into a new folder); an unreadable record stops the run (`conversion_record_unreadable`). `check` and `run` also read the OME `Image/Description` header (first page's description tag only, no pixels) of every analysed TIFF the record does not list as `projected`: a description that begins "MorphoBricks Z projection:" in a tree with no record, or whose record does not list the image as projected, is refused as `projection_record_missing` (owner decision of 8 November 2026; plain images are unaffected). `RunInputs` reads the copies like any images, and refuses a run whose conversion record changed or went missing since (`conversion_record_sha256`), is of version 1, or whose recorded images' copies no longer have the recorded fingerprint and SHA-256 (§9.1). | | Notebook (PP-11b) | Step 2's **Make named-axis copies (optional)** card, under the axis check, shown only when it reports images with `axes_yx_swapped`, `axes_contradict_header`, `axes_x_before_y` or `axes_not_named` for the roles being edited (design v0.4 §5: offered only for files with errors), and not with `folder_step(show_output=False)`. The roles are the user's: those confirmed in step 2, or the roles shown once the preview is checked and the card's own question is answered; any change of a role or of the missing-axes answer takes it back. A required folder for the copies (Browse…, or pasted) checked with the command's rule (`copy_folder_issues`); `preview_conversion` (`conversion_plan`, headers only, writes nothing) gives what will be converted and copied byte for byte (the command's `plan_conversion`), the space needed and the free space; then `convert_experiment` with a progress bar, under the copy tree's lock. Refusals use the existing cards (the lock card with **Remove the lock of a stopped run**, the rule of `--remove-stale-lock`). After the copies: the record's path, and **Check the copies in step 1**, which sets the experiment folder to the copy tree, keeps the output folder and checks again; the card says that the missing-axes answer must be given again and lists the copies whose relabelled sizes are unknown (`ConversionResult.unknown_sizes`), to state under **Pixel sizes**. | diff --git a/docs/user_guide.md b/docs/user_guide.md index f777549f..cb2d8485 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -894,9 +894,9 @@ so `check` and `run` on a folder of copies, and opening their run, take about on the copies longer. An original that changes while `convert` reads its metadata is refused (`source_changed_during_check`); convert again once nothing writes into the experiment. -A folder of copies made by v0.4.0rc3 or earlier has a conversion record of version 1, without the -copies' SHA-256: `convert`, `run` and the bricks refuse it (`conversion_record_unreadable`, -naming version 1). Convert the originals again into a new folder. +A folder of copies made before v0.5.1 may have a conversion record of version 1 or 2: `convert`, +`run` and the bricks read version 3 only and refuse it (`conversion_record_unreadable`, naming +its version); nothing is converted. Convert the originals again into a new folder. Running `convert` again with the same images and roles reuses the copies and only adds new images; when nothing is added the record is left as it was. Bricks reading the run on the @@ -1168,7 +1168,7 @@ continues. | `source_changed_during_check` | Error | `convert`: an original changed while its metadata was read, so the metadata may not describe the file that would be copied. Nothing was written; convert again when nothing writes into the experiment. | | `conversion_failed` | Error | `convert`: an image could not be copied (unreadable, changed while copied, or the folder cannot be written). The copies written by this conversion were removed; fix the cause and convert again, or exclude the image. | | `not_enough_space` | Error | `convert`: the copies would not fit on the drive. Free space or choose another drive. | -| `conversion_record_unreadable` | Error | The conversion record of a folder of named-axis copies (**.morphobricks-conversion.json**) cannot be read, or is of version 1 (made by v0.4.0rc3 or earlier, without the copies' SHA-256), so `run` stops and `convert` refuses the folder. Convert again into a new folder. | +| `conversion_record_unreadable` | Error | The conversion record of a folder of named-axis copies (**.morphobricks-conversion.json**) cannot be read, or is not of version 3 (versions 1 and 2 were made before v0.5.1), so `run` stops and `convert` refuses the folder. Convert again into a new folder. | | `conversion_source_changed` | Warning | An original changed or went missing after its named-axis copy was made; the copies hold the images as converted. If the originals were corrected, convert again into a new folder. | | `copy_changed_after_conversion` | Error | An image of the folder of named-axis copies changed (its size, time or SHA-256), went missing or was added after the conversion, so the conversion record does not describe it, and the bricks (`RunInputs`) would refuse a run on it. The run stops before it writes anything but the report: convert the originals again into a new folder, and run preprocessing on that one (same output folder). | | `source_changed_during_run` | Error | An image changed or went missing while the run was checking or reading it, so its rows would not describe it. No scientific outputs are published and sources are untouched; the discovery report is written. Wait until nothing writes to the experiment (an acquisition or a copy still running), then run again: the images already read resume from their staged rows. | diff --git a/src/morphobricks_preprocessing/conversion_record.py b/src/morphobricks_preprocessing/conversion_record.py index e2272474..ad39ed6e 100644 --- a/src/morphobricks_preprocessing/conversion_record.py +++ b/src/morphobricks_preprocessing/conversion_record.py @@ -9,13 +9,13 @@ :func:`review_conversion` warns when a source changed after the conversion, or a copy since it was written (its fingerprint or its SHA-256). -Record version 2 (PP-11c) added the copies' SHA-256, channel names and significant bits; -a version 1 record is refused with a message to convert again -(:class:`ConversionRecordError`), since its copies cannot be checked. Record version 3 -(issue #176) adds the record-level ``projection`` (the method and slice range of a tree of -Z-projected copies, or null) and the entry action ``"projected"`` with its -:class:`.projection_record.ProjectionEntry`; version 2 records are still read, as trees -without a projection. +Only record version 3 is written and read (issue #176): the copies' SHA-256, channel names +and significant bits (PP-11c), the record-level ``projection`` (the method and slice range of a +tree of Z-projected copies, or null) and the entry action ``"projected"`` with its +:class:`.projection_record.ProjectionEntry`. A record of any other version (1 and 2 were +written before v0.5.1) is refused with :class:`ConversionRecordError`, naming the fix: convert +the originals again into a new folder. Nothing is converted (no backward compatibility before +the first pipeline release, Spec ADR 0002 AR10; SWEEP-PP). """ from __future__ import annotations @@ -42,13 +42,8 @@ """The conversion record, in the root of the copy tree.""" RECORD_VERSION = "3" -"""Version 3 (issue #176): the record-level projection and projected entries. Version 2 -(PP-11c: each copy's SHA-256, channel names and significant bits) is still read.""" - -_READABLE_VERSIONS = ("2", RECORD_VERSION) - -_OLD_VERSIONS = ("1",) -"""Record versions written before the copies' SHA-256: refused, convert again.""" +"""The one version written and read (issue #176): the record-level projection and projected +entries, with each copy's SHA-256, channel names and significant bits (PP-11c).""" CONVERTED = "converted" """An image written as a named-axis OME-TIFF with the confirmed roles.""" @@ -199,7 +194,7 @@ def from_dict(cls, data: Mapping) -> CopyEntry: significant_bits=_bits(data["significant_bits"]), projection=( None - if data.get("projection") is None + if data["projection"] is None else ProjectionEntry.from_dict(data["projection"]) ), ) @@ -261,21 +256,17 @@ def from_dict(cls, data: Mapping) -> ConversionRecord: Raises ------ ConversionRecordError - For another record version (a version 1 record: convert again) or data that is - not a record. + For another record version (refused, never converted: convert the originals again) + or data that is not a record. """ - if isinstance(data, Mapping) and data.get("record_version") in _OLD_VERSIONS: + if not isinstance(data, Mapping) or "record_version" not in data: + raise ConversionRecordError(f"not a conversion record of version {RECORD_VERSION}") + if data["record_version"] != RECORD_VERSION: raise ConversionRecordError( - f"it is a record of version {data['record_version']}, written before the " - "copies' SHA-256 was recorded, so the copies cannot be checked; convert the " - "originals again into a new folder" + f"it is a record of version {data['record_version']}, and this version of " + f"preprocessing reads version {RECORD_VERSION} only (older records are not " + "converted); convert the originals again into a new folder" ) - if not isinstance(data, Mapping) or data.get("record_version") not in _READABLE_VERSIONS: - raise ConversionRecordError(f"not a conversion record of version {RECORD_VERSION}") - if data["record_version"] != RECORD_VERSION and ( - "projection" in data or any("projection" in f for f in data.get("files") or []) - ): - raise ConversionRecordError("a version 2 record cannot hold a projection") try: mapping = data["mapping"] return cls( @@ -286,9 +277,7 @@ def from_dict(cls, data: Mapping) -> ConversionRecord: versions={str(k): str(v) for k, v in (data.get("versions") or {}).items()}, written_at=str(data.get("written_at", "")), projection=( - None - if data.get("projection") is None - else projection_from_data(data["projection"]) + None if data["projection"] is None else projection_from_data(data["projection"]) ), ) except (KeyError, TypeError, ValueError) as exc: diff --git a/tests/test_projection_record.py b/tests/test_projection_record.py index f31598cc..0f40f01a 100644 --- a/tests/test_projection_record.py +++ b/tests/test_projection_record.py @@ -109,7 +109,7 @@ def test_a_record_without_a_projection_is_version_3_with_null(tmp_path): assert read_conversion_record(copies)[0].projection is None -def test_a_version_2_record_is_still_read(tmp_path): +def test_a_version_2_record_is_refused_naming_the_fix(tmp_path): exp, copies = swapped_experiment(tmp_path / "exp"), tmp_path / "copies" assert convert_experiment(exp, SWAPPED, copies).written path = copies / RECORD_FILENAME @@ -119,23 +119,21 @@ def test_a_version_2_record_is_still_read(tmp_path): for entry in data["files"]: del entry["projection"] path.write_text(json.dumps(data)) - old, _ = read_conversion_record(copies) - assert old.projection is None and old.files - assert all(entry.projection is None for entry in old.files) + with pytest.raises(ConversionRecordError, match="version 2,.*convert the originals again"): + read_conversion_record(copies) -def test_a_version_2_record_cannot_hold_a_projection(): - data = edited(lambda d: d.update(record_version="2")) - with pytest.raises(ConversionRecordError, match="version 2.*projection"): - ConversionRecord.from_dict(data) - - -@pytest.mark.parametrize("version", ["1", "4", None]) +@pytest.mark.parametrize("version", ["1", "2", "4", None]) def test_other_versions_are_refused(version): - with pytest.raises(ConversionRecordError): + with pytest.raises(ConversionRecordError, match="reads version 3 only"): ConversionRecord.from_dict(edited(lambda d: d.update(record_version=version))) +def test_a_record_without_its_projection_key_is_refused(): + with pytest.raises(ConversionRecordError, match="projection"): + ConversionRecord.from_dict(edited(lambda d: d.pop("projection"))) + + def test_the_record_is_written_and_read_from_the_copy_tree(tmp_path): rec = record() sha = write_record(rec, tmp_path) From 10bf92ef7c2d9d1c5b06a24d7cd59f461ca759ed Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:19:40 +0000 Subject: [PATCH 5/9] SWEEP-PP PP-05: read every manifest file entry strictly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only version 4 manifests are read (PP-C), so the tolerant reading of version-1 file entries served nothing: the strict flag of recorded_position_id and of the entry reader/writer, the defaults for keys the writer always writes (singleton_roles, channel_names, significant_bits, read_in_blocks, channel_names_trimmed, pixel_sizes, settings.histogram, settings.verify_content) and the version-1 branch of verify_content in to_dict. Such a manifest is now refused (ManifestFormatError). sizes_from_data refuses an absent list. check_sources drops position_id_not_recorded (an entry without a position is identity_differs) and the fallback that resolved roles from the image header for an entry without them (it fits no axes: axis_mapping_mismatch). Tests converted from "defaults when absent" to "refused when absent"; contract §9/§16 and the user guide's code table updated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- CHANGELOG.md | 13 +++ docs/contract.md | 6 +- docs/user_guide.md | 2 +- .../conversion_record.py | 2 +- src/morphobricks_preprocessing/manifest.py | 82 ++++++++----------- .../manifest_fields.py | 45 +++++----- .../recorded_sizes.py | 4 +- src/morphobricks_preprocessing/sources.py | 26 +----- tests/test_check_sources.py | 23 ++++-- tests/test_manifest.py | 15 ++-- tests/test_manifest_position_id.py | 3 +- 11 files changed, 102 insertions(+), 119 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9d4c871..17416b6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -245,6 +245,19 @@ already removed is not repeated here. **Breaking** where it says so. (version 3 always writes it). Removed: `_READABLE_VERSIONS`, `_OLD_VERSIONS` and the "version 2 cannot hold a projection" check. +- **Run manifest: nothing read leniently** (inventory PP-05, owner ruling 16). **Breaking** for a + hand-edited or truncated manifest only (version 4 always writes these keys). Removed: the + `strict` argument of `manifest_fields.recorded_position_id` (version-1 entries kept any value, + or None), the `strict` branches of the file-entry reader and writer, and the defaults for + absent keys: a file entry needs `position_id`, `roles`, `axes_from`, `pixel_sizes`, + `singleton_roles`, `channel_names`, `significant_bits`, `read_in_blocks` and + `channel_names_trimmed`, the settings `histogram` and `verify_content`; a manifest without + one is refused (`ManifestFormatError`). `recorded_sizes.sizes_from_data` refuses an absent + list (also in a conversion record). `check_sources` no longer has the + `position_id_not_recorded` issue (an entry without a position is `identity_differs`) and no + longer resolves roles from the image header for an entry without them (it fits no axes, + `axis_mapping_mismatch`). + ## 0.5.0 - 2026-11-09 Stage 1 of the experiment description (MorphoBricks-Spec experiment-v1, pinned at `1bed024`): diff --git a/docs/contract.md b/docs/contract.md index 34559414..12f12e9d 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -235,10 +235,10 @@ interpretation. | `written_at` | UTC time of writing, ISO 8601. A rerun whose manifest would differ from the earlier one only in `written_at` keeps the earlier time, so the manifest's bytes and SHA-256, which later bricks pin, stay the same (cross-brick test F10, 3 November 2026); the report still records the rerun. | | `experiment_root` | Absolute path of the experiment folder, for information. | | `dtype` | Pixel type of the experiment. | -| `settings` | `bin_count`, `memory_limit` (bytes), `missing_as_singleton`, `exclude` (sorted relative paths), `histogram` (`HistogramSettings.to_dict()` of the normalization brick: `library_version`, `histogram_range`, `robust_quantiles`, `min_central_bins`, §4; the default `minmax` range when absent, in manifests written before follow-up P6), `verify_content` (JSON boolean, below: `true` when the run verified content, `false` when it opted out, written by every version 2 and 3 manifest so that an opt-out is on record (#159); a manifest written before that has no key and reads as false, not verified). | +| `settings` | `bin_count`, `memory_limit` (bytes), `missing_as_singleton`, `exclude` (sorted relative paths), `histogram` (`HistogramSettings.to_dict()` of the normalization brick: `library_version`, `histogram_range`, `robust_quantiles`, `min_central_bins`, §4; required), `verify_content` (JSON boolean, below: `true` when the run verified content, `false` when it opted out, so that an opt-out is on record (#159); required). Since v0.5.1 (SWEEP-PP) no key the writer always writes is defaulted when absent: such a manifest is refused. | | `axis_mapping` | `axes` (file axis labels) and `roles` (`X`, `Y`, `Z`, `C`, `T` or null per axis). | | `labels` | One entry per leaf folder: `folder` (relative path, `.` for the root) and its three labels. | -| `files` | One entry per analysed file, sorted by path: `relative_path`, `size` (bytes), `mtime_ns`, the three labels, `axes`, `singleton_roles`, `plane_count` (its rows in the table: XYC planes), `channel_names` (list, null for unnamed channels; empty if none in metadata), `significant_bits` (null if not in metadata), `read_in_blocks` (true when the image's planes are over the memory limit and are read in blocks; information only, false when absent), `position_id` (the image's position identifier, §1: its name without the extension, never edited; required in version 2, where an entry without it, with null, or with another value than `position_id(relative_path)` is refused naming the entry, by the reader and the writer, round-2 audit A-m5; null when absent in a version 1 manifest), `channel_names_trimmed` (true when leading or trailing whitespace was removed from the image's channel names as they were read, §2; `channel_names` holds the trimmed names; false when absent), `roles` (the role of each of the file's `axes`, `X`, `Y`, `Z`, `C`, `T` or null, from the file's own mapping: an OME-Zarr image in another order than `axis_mapping` has its own roles, so later bricks read these rather than rebuild them from `axis_mapping`; null when absent, in manifests written before them; optional key, version 1). For an OME-Zarr image, `size` is the total size of its files, `mtime_ns` the time of its newest file, and `inventory_sha256` (v0.4.0, PP-12, issue #86; only for OME-Zarr images) the SHA-256 of its inventory: one line per file (its path relative to the image with `/`, a NUL, its size, a NUL, its `mtime_ns`, a newline), sorted by path, its root `labels` group left out; links to folders inside the image are followed, as the reader follows them, and links that reach one folder twice are refused. Adding, removing, renaming, resizing or touching any chunk or metadata file changes it; an OME-Zarr image recorded without it (v0.4.0rc3 and earlier) counts as changed (recomputed; `RunInputs` refuses it). Like a TIFF file's size and time it is metadata only (design v0.4 §1): a file rewritten with the same size and its time set back is not seen, only hashing the content would. **Version 2 (v0.4.0):** `roles` is required, with `axes_from` (required): `header` when the metadata names every axis and the roles agree, `confirmed` for a plain stack whose roles the user confirmed (§2); and `pixel_sizes`, one entry per role `X`, `Y`, `Z`, `T` with `value`, `unit` (OME-NGFF name) and `source`: `header` (the image's metadata) or `stated` (`calibration.csv`, §9.3) with a value, or `pixels` (stated as deliberately uncalibrated) with `value` and `unit` null; `value` and `source` null for an axis nothing gives a size (never guessed). A `header` or `stated` size without a value, or a value with no source or with `pixels`, is refused. The reader refuses a version 2 entry without `roles` or `axes_from`, and the writer refuses to write one. `content_sha256` (only in a `verify_content` run): the SHA-256 of the image's bytes, or for an OME-Zarr image of one line per file (its path relative to the image with `/`, a NUL, the file's SHA-256, a newline) over the files its inventory covers, in path order (`source_content.source_content_sha256`). | +| `files` | One entry per analysed file, sorted by path: `relative_path`, `size` (bytes), `mtime_ns`, the three labels, `axes`, `singleton_roles`, `plane_count` (its rows in the table: XYC planes), `channel_names` (list, null for unnamed channels; empty if none in metadata), `significant_bits` (null if not in metadata), `read_in_blocks` (true when the image's planes are over the memory limit and are read in blocks; information only; required), `position_id` (the image's position identifier, §1: its name without the extension, never edited; required: an entry without it, with null, or with another value than `position_id(relative_path)` is refused naming the entry, by the reader and the writer, round-2 audit A-m5), `channel_names_trimmed` (true when leading or trailing whitespace was removed from the image's channel names as they were read, §2; `channel_names` holds the trimmed names; required), `roles` (the role of each of the file's `axes`, `X`, `Y`, `Z`, `C`, `T` or null, from the file's own mapping: an OME-Zarr image in another order than `axis_mapping` has its own roles, so later bricks read these rather than rebuild them from `axis_mapping`; required). For an OME-Zarr image, `size` is the total size of its files, `mtime_ns` the time of its newest file, and `inventory_sha256` (v0.4.0, PP-12, issue #86; only for OME-Zarr images) the SHA-256 of its inventory: one line per file (its path relative to the image with `/`, a NUL, its size, a NUL, its `mtime_ns`, a newline), sorted by path, its root `labels` group left out; links to folders inside the image are followed, as the reader follows them, and links that reach one folder twice are refused. Adding, removing, renaming, resizing or touching any chunk or metadata file changes it; an OME-Zarr image recorded without it (v0.4.0rc3 and earlier) counts as changed (recomputed; `RunInputs` refuses it). Like a TIFF file's size and time it is metadata only (design v0.4 §1): a file rewritten with the same size and its time set back is not seen, only hashing the content would. **Version 2 (v0.4.0):** `roles` is required, with `axes_from` (required): `header` when the metadata names every axis and the roles agree, `confirmed` for a plain stack whose roles the user confirmed (§2); and `pixel_sizes`, one entry per role `X`, `Y`, `Z`, `T` with `value`, `unit` (OME-NGFF name) and `source`: `header` (the image's metadata) or `stated` (`calibration.csv`, §9.3) with a value, or `pixels` (stated as deliberately uncalibrated) with `value` and `unit` null; `value` and `source` null for an axis nothing gives a size (never guessed). A `header` or `stated` size without a value, or a value with no source or with `pixels`, is refused. The reader refuses an entry without `roles`, `axes_from` or `pixel_sizes`, and the writer refuses to write one (no older manifest is read leniently, SWEEP-PP). `content_sha256` (only in a `verify_content` run): the SHA-256 of the image's bytes, or for an OME-Zarr image of one line per file (its path relative to the image with `/`, a NUL, the file's SHA-256, a newline) over the files its inventory covers, in path order (`source_content.source_content_sha256`). | | `csv_sha256` | SHA-256 of the table written with this manifest. | | `experiment_design_sha256` | SHA-256 of the `experiment_design.csv` the run derived from its description and wrote with the table (§12), so later bricks can check that the design they read is the one the run wrote. | | `masks` | The masks paired with the images (§15), when the run was given a masks folder or its OME-Zarr images have label images (always used): `folder` (absolute path of the masks folder, null when only OME-Zarr label images were used), `types` (sorted mask-type names) and `files` (one entry per image and mask type, sorted by image then type: `image`, `type`, `path`, `source` `folder` or `ome-zarr`, `axes` and `shape`: the mask's axes and lengths without its length-1 axes other than Y and X, `YX` for a 2D mask (`[1, 512]` for a mask one pixel high), with `Z` and/or `T` for a mask with depth slices or time points, e.g. `TYX` and `[50, 512, 512]`; §15). Absent when the run had no masks. Optional key: the table does not depend on the masks. `axes` and `shape` are absent in manifests written before them, whose masks are all 2D; a later brick must read `axes` before using a mask and never apply a 2D mask to several planes. **Version 2:** `axes` are the mask's axes as stored (its header's letters: `IYX` for a plain stack), and each mask entry also has `size` and `mtime_ns` (its fingerprint, the size and modification time `source_signature` gives: the mask file, or the label image's folder), `roles` (one per axis of `axes`, what each is read as: `TYX` for that plain stack stated `--mask-axes Cells=TYX`), `axes_from` (`header`, or `stated` when the axes were stated for its mask type). The release candidates up to v0.4.0rc6 also wrote `shared` (true for a 2-D mask stated `YX` for its type: one plane shared by every plane of its image); the option was removed by the owner decision of 5 November 2026, the key is no longer written, and `RunInputs.open` refuses a manifest that records `shared: true` (stated or not), naming the masks and asking for one mask plane per time point and a new run. A mask is read as stored, never reordered. | @@ -732,7 +732,7 @@ instead of keeping copies. | Declared axes (removed) | `check_declared_axes`, `declared_axes_transposed` | **Removed in v0.4.0** (spec revision 4): they accepted a declaration with Y and X swapped and read it transposed, which v0.4 forbids. Bricks take no axis declarations; preprocessing's axis check (§2) compares the confirmed roles with the header. | | Mask files | `read_mask_file(path, z=, t=)` (`masks`) | Reads one plane of a TIFF mask or of an OME-Zarr label image (`.zarr/labels/`) given by its path, with `read_mask`'s rules (§15): its axes are its own metadata's, Y and X plus at most a Z and a T axis named so, length-1 axes other than Y and X left out (a mask one pixel high or wide is read as a 2D array, plan §12 question 44); any other or unnamed dimension (a plain stack, `IYX`; a `C` axis) is refused, the message giving the header's axes and shape and the dimension that is not named, never taken for depth or time; `z` and `t` are required for a mask with that axis; integer or boolean pixels; labels checked with `check_labels`. A `ValueError` names the file and the problem. | | Identity (D-e) | `FOV_KEY`, `check_identity_text(value, field)`, `check_plane_index(value, axis)`, `check_identity_case(keys)`, `MISSING_TEXT`, `IdentityError` (`discovery`) | `FOV_KEY` = (`condition`, `biological_repeat`, `technical_repeat`, `position_id`), the position key of §1. `check_identity_text` accepts a label or `position_id` that is text without leading or trailing whitespace and not `""`, `NA`, `nan`, `none`, `null`, `n/a` or `` in any case (`MISSING_TEXT`); `check_plane_index` accepts None or an integer of at least 0, a Python `int` or a NumPy integer (returned as a plain `int`), and refuses everything else (bool, float, `"1"`) without converting it. Both raise `IdentityError` (a `ValueError`) naming the field and the value. Discovery applies the text rule to every folder and image name (`invalid_name`) and refuses names of one level that differ only in case anywhere in the experiment (`case_conflict`, §1). `position_id` keeps its own scope: unique within its technical repeat, ignoring case (`duplicate_position_id`). `check_identity_case` applies the same case rule to identities a brick reads from a table, one key per row in `FOV_KEY` order (or the three labels): a label of one level spelled two ways anywhere in the run, or two `position_id` values of one technical repeat equal ignoring case, raise `IdentityError` naming both spellings; the same key may repeat (several planes of one position). | -| Sources of a run | `check_sources(manifest, experiment_root, relative_paths, image_axes=, identities=)`, `source_signature` (`inventory`), `file_sha256` (`manifest`) | The checks a brick makes before using a run's table for some images (formerly CellPatcher `check_images` and MorphoFeature `histogram_fit`); from v0.4.0 file runs read through `RunInputs` (§9.1), which also checks masks. A version 1 manifest fails every image with `manifest_before_v040` alone (design v0.4 §6: every v1 run is redone). Otherwise each image must be in the manifest (`images_not_in_manifest`), still exist (`images_missing`), and be unchanged by `source_signature`, its size and modification time, for an OME-Zarr folder its files' total size and newest time, so a rewritten chunk counts (`images_changed`). Its recorded `position_id` must be the one its name gives; with `identities` (`FOV_KEY` order), its labels and `position_id` must be the caller's (`identity_differs`; `position_id_not_recorded` for a manifest written before positions were recorded). With `image_axes`, the caller's axes must give each axis the role recorded for the file (`FileEntry.roles`, or the run's mapping resolved for the header in older manifests); an axis without a role (length 1) fits any letter (`axis_mapping_mismatch`). All errors, naming their images; the table's own hash is checked with `file_sha256` against `csv_sha256`. | +| Sources of a run | `check_sources(manifest, experiment_root, relative_paths, image_axes=, identities=)`, `source_signature` (`inventory`), `file_sha256` (`manifest`) | The checks a brick makes before using a run's table for some images (formerly CellPatcher `check_images` and MorphoFeature `histogram_fit`); from v0.4.0 file runs read through `RunInputs` (§9.1), which also checks masks. Only a current manifest is read at all (§9, version 4). Each image must be in the manifest (`images_not_in_manifest`), still exist (`images_missing`), and be unchanged by `source_signature`, its size and modification time, for an OME-Zarr folder its files' total size and newest time, so a rewritten chunk counts (`images_changed`). Its recorded `position_id` must be the one its name gives; with `identities` (`FOV_KEY` order), its labels and `position_id` must be the caller's (`identity_differs`, also for an entry without a `position_id`). With `image_axes`, the caller's axes must give each axis the role recorded for the file (`FileEntry.roles`; an entry without roles fits no axes, nothing is resolved from the header in their place); an axis without a role (length 1) fits any letter (`axis_mapping_mismatch`). All errors, naming their images; the table's own hash is checked with `file_sha256` against `csv_sha256`. | | Masks named as images (D-i) | `mask_name_issues(leaves, mask_names=())`, `images_named_as_masks(leaves, suffixes=MASK_NAME_SUFFIXES)`, `mask_folder_part(path)`, `MASK_NAME_SUFFIXES`, `MASK_FOLDER_WORDS` (`discovery`) | The screen MorphoRelate's `init` ran (`project_image_names`), now discovery's warnings `image_named_as_mask` and `mask_folder_in_experiment` (§1) for every brick. Names are compared in lower case across the whole experiment; the longest `position_id` owns a file, an image of the same folder first. `mask_names` adds the caller's expected mask names (`cells` for `f1_cells.tif`) to `MASK_NAME_SUFFIXES`; `discover_experiment` gives none. A folder counts as a masks folder when a word of its name (`CellLabels` is `Cell` and `Labels`) is in `MASK_FOLDER_WORDS`. `images_named_as_masks` returns the `(image, owner)` pairs of that detection for the caller's own name endings (`["_cells"]`), for a brick that acts on the files itself (MorphoRelate leaves out the images named with a declared mask name). | | Pixel sizes (D-a) | `same_pixel_size(a, b, roles=("X", "Y"))`, `pixel_size_groups(sizes, roles=)`, `PIXEL_SIZE_TOLERANCE` (`physical_sizes`) | Two pixel sizes are the same when `|a - b| <= 1e-3 * max(a, b)` on every compared axis, in micrometres; one rule for a declared size against a header and for two headers. `same_pixel_size` takes `PhysicalSizes` (converted with `micrometers`) or sizes already in micrometres, one per role, and returns None when either is unknown (no size, or a unit that cannot be converted), never True. The rule is not transitive (1.0 ≈ 1.0009 ≈ 1.0018, but 1.0 ≠ 1.0018), so there is no per-image grouping key: `pixel_size_groups` links images by single linkage, so images of two groups always differ while the extremes of one group may drift; unknown sizes are left out; the largest group first. A size that cannot be converted to a positive finite length in micrometres (`1e308 m`) is unknown, never infinite, and an infinite or NaN value is never the same as any other (issue #93; before, `inf` matched every size). The folder check's `different_pixel_sizes` uses both (§11). | | Experiment design | `read_confirmed_design(path, expected_sha256=)`, `ConfirmedDesign`, `DesignCheckError` (`design`) | The one reader of `experiment_design.csv` (§12) for a brick that pools biological repeats (formerly CellPatcher `load_experiment_design` and MorphoFeature `ExperimentDesign.read`). It takes the file as this brick writes it and fills nothing in: UTF-8 without a byte-order mark; the header exactly the §12 columns in order (a column named twice or without a name is refused naming it, as by the lenient reader, §12); one value per column in every row (an unclosed quote, a missing or extra value, or a blank line is refused); no value with outer whitespace; `control` and `confirmed` exactly `true` or `false`; counts written as whole numbers of at least 0; `schema_version` 1; `confirmed_on` empty or `YYYY-MM-DD`; `condition`, `biological_repeat` and `repeat_group` pass `check_identity_text` and do not differ only in case (`check_identity_case`); no folder twice, one `control` per condition, at least one row; every group confirmed with one folder per condition (`design_issues`); with `expected_sha256` (the manifest's `experiment_design_sha256`) the file must be that one. The SHA-256 is of the bytes parsed. `ConfirmedDesign` gives `path`, `sha256`, the `design`, read-only `repeat_groups` (`{(condition, biological_repeat): repeat_group}`, the normalization brick's input), `group_members()` (`{repeat_group: {condition: biological_repeat}}`), `conditions`, `controls` and `groups`. Refusals raise `DesignCheckError` (a `ValueError`) with its `issues`: `missing_experiment_design`, `unreadable_experiment_design`, `invalid_experiment_design`, `repeat_group_conflict`, `design_not_confirmed` or `changed_experiment_design`, each an error naming the file (or the group's folders). This brick's step 1 and `check` read the file being edited with the lenient `load_design(output_folder, repeats)` (§12), which returns `(design, changes)`: the saved design brought up to date with the folders, or the default design when there is no file. A saved file that cannot be read raises `DesignCheckError` with the one error `unreadable_experiment_design`, whose message is the same "The experiment design cannot be read: …" as here followed by the lenient reader's refusal, naming the line and column where it can (the `DesignFormatError` is its `__cause__`); no default design is returned and the file is not touched (owner decision of 31 October 2026, plan §12 question 43). **Breaking in v0.4.0:** up to v0.3.4 `load_design` returned a third value, `problem`, the reason as text, together with the default design. | diff --git a/docs/user_guide.md b/docs/user_guide.md index cb2d8485..4c98bd8a 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -1237,7 +1237,7 @@ continues. | `mask_missing` | Warning | Some images have no mask of a kind that other images have; their objects of that kind cannot be measured. Add the masks, or exclude the images. | | `ambiguous_mask` | Error | A mask's name could belong to two images (one image name starts another's) or fit both layouts. Rename the masks or the images so each names one image and one kind. | | `mask_without_image`, `mask_outside_type_folder`, `mask_of_excluded_image` | Warning | Files in the masks folder that match no image of the experiment (check the folder and file names), other files directly in the masks folder, or masks of excluded images. They are not used. | -| `manifest_before_v040`, `images_not_in_manifest`, `images_missing`, `images_changed`, `position_id_not_recorded`, `identity_differs`, `axis_mapping_mismatch` | Error (other bricks) | A later brick checked its images against this brick's run (`check_sources`): a run written before v0.4.0 (redo it), an image the run did not analyse, no longer there, changed since, recorded with other labels or position (or by a version that did not record positions), or read with axes the run gave other roles. Run this brick again on the experiment as it is, or use the run's labels and axis roles. | +| `images_not_in_manifest`, `images_missing`, `images_changed`, `identity_differs`, `axis_mapping_mismatch` | Error (other bricks) | A later brick checked its images against this brick's run (`check_sources`): an image the run did not analyse, no longer there, changed since, recorded with other labels or position (or with none), or read with axes the run gave other roles. Run this brick again on the experiment as it is, or use the run's labels and axis roles. | | `UnrecordedFilesError` | Error (other bricks) | The experiment holds images or masks that the preprocessing run neither recorded nor excluded, for example added after the run. Later bricks never look for images themselves, so they refuse rather than leave them out: run this brick again on the experiment as it is (with its masks folder, `--masks`, for masks), or exclude the images. Files the run ignores (non-image files, system files, masks matching no image) do not count. A run holding OME-Zarr images also needs the `zarr` extra here (`pip install "morphobricks-preprocessing[zarr]"`), so that their label images can be checked; an OME-Zarr image whose `labels` group cannot be read is refused too. | | `zarr_not_installed` | Error | OME-Zarr images need the `zarr` extra. Install it, or exclude the OME-Zarr images. | diff --git a/src/morphobricks_preprocessing/conversion_record.py b/src/morphobricks_preprocessing/conversion_record.py index ad39ed6e..02a3894e 100644 --- a/src/morphobricks_preprocessing/conversion_record.py +++ b/src/morphobricks_preprocessing/conversion_record.py @@ -187,7 +187,7 @@ def from_dict(cls, data: Mapping) -> CopyEntry: source_axes=str(data.get("source_axes", "")), roles=None if roles is None else tuple(None if r is None else str(r) for r in roles), copy_axes=None if data.get("copy_axes") is None else str(data["copy_axes"]), - pixel_sizes=sizes_from_data(data.get("pixel_sizes")), + pixel_sizes=sizes_from_data(data["pixel_sizes"]), unknown_sizes={str(k): str(v) for k, v in (data.get("unknown_sizes") or {}).items()}, copy_sha256=str(data["copy_sha256"]), channel_names=tuple(None if n is None else str(n) for n in data["channel_names"]), diff --git a/src/morphobricks_preprocessing/manifest.py b/src/morphobricks_preprocessing/manifest.py index 72ed7d7d..0bc5e0a4 100644 --- a/src/morphobricks_preprocessing/manifest.py +++ b/src/morphobricks_preprocessing/manifest.py @@ -86,13 +86,11 @@ class RunSettings: Relative paths of images the user excluded from the run, sorted. histogram The histogram range of histograms that are not exact, with the limits of - the range check (the normalization brick's ``HistogramSettings``). - Manifests written before it was recorded read as the default, the - minimum to maximum. + the range check (the normalization brick's ``HistogramSettings``); always recorded. verify_content Strict reuse (CLI ``--verify-content``): each source's content SHA-256 is recorded (:attr:`FileEntry.content_sha256`) and must match before its staged or earlier rows - are reused. False (the default, and in manifests without it): the fingerprint only. + are reused. False: the fingerprint only. Always recorded. """ bin_count: int @@ -139,20 +137,21 @@ class FileEntry: For information only: the rows are the same either way. position_id The image's position identifier, its name without the extension - (:func:`~morphobricks_preprocessing.image_names.position_id`). Required in manifest - version 2, and equal to that rule (round-2 audit A-m5); None in older manifests. + (:func:`~morphobricks_preprocessing.image_names.position_id`). Required when written + and read, and equal to that rule (round-2 audit A-m5); None only before the entry is + complete. roles The role of each of the file's axes (``"X"``, ``"Y"``, ``"Z"``, ``"C"``, ``"T"`` or None), from its own mapping: for an OME-Zarr image whose declared order differs from - the experiment mapping's, these are its roles, not the experiment's. Required in - manifest version 2; None in older manifests. + the experiment mapping's, these are its roles, not the experiment's. Required when + written and read. axes_from Where the roles come from: ``"header"`` (the metadata names every axis and the roles agree) or ``"confirmed"`` (a plain stack: the user confirmed them). - Required in manifest version 2; None in older manifests. + Required when written and read. pixel_sizes The size of one step along X, Y, Z and T, each with its unit and source - (:class:`.recorded_sizes.RecordedSize`); empty in older manifests. + (:class:`.recorded_sizes.RecordedSize`); always written. inventory_sha256 For an OME-Zarr folder, the SHA-256 of its inventory (each file's relative path, size and modification time, :func:`.inventory.source_signature`); None for a @@ -328,25 +327,15 @@ def to_dict(self) -> dict: ), "exclude": list(self.settings.exclude), "histogram": self.settings.histogram.to_dict(), - # Written by every version 2 and 3 manifest (#159): a run that opted out of - # content hashing says so. Manifests written before it have no key and read - # as not verified; version 1 is unchanged. - **( - { - "verify_content": recorded_flag( - self.settings.verify_content, "verify_content" - ) - } - if self.manifest_version != "1" - else {} - ), + # Always written (#159): a run that opted out of content hashing says so. + "verify_content": recorded_flag(self.settings.verify_content, "verify_content"), }, "axis_mapping": {"axes": self.mapping.axes, "roles": list(self.mapping.roles)}, "labels": [ dict(zip(_LABEL_KEYS, (folder, *labels), strict=True)) for folder, labels in self.leaves.items() ], - "files": [_file_dict(f, strict=self.manifest_version != "1") for f in self.files], + "files": [_file_dict(f) for f in self.files], "csv_sha256": self.csv_sha256, "experiment_design_sha256": self.experiment_design_sha256, "image_type": self.image_type, @@ -390,15 +379,11 @@ def from_dict(cls, data: dict) -> RunManifest: memory_limit=int(settings["memory_limit"]), missing_as_singleton=recorded_flag(settings["missing_as_singleton"]), exclude=tuple(str(p) for p in settings["exclude"]), - histogram=( - HistogramSettings() - if settings.get("histogram") is None - else HistogramSettings.from_dict(settings["histogram"]) - ), - verify_content=recorded_flag(settings.get("verify_content", False)), + histogram=HistogramSettings.from_dict(settings["histogram"]), + verify_content=recorded_flag(settings["verify_content"], "verify_content"), ), mapping=AxisMapping(str(mapping["axes"]), tuple(mapping["roles"])), - files=tuple(_file_entry(f, strict=version != "1") for f in data["files"]), + files=tuple(_file_entry(f) for f in data["files"]), csv_sha256=str(data["csv_sha256"]), versions={str(k): str(v) for k, v in data.get("versions", {}).items()}, written_at=str(data.get("written_at", "")), @@ -520,14 +505,14 @@ def read_manifest(path: str | os.PathLike) -> RunManifest: return RunManifest.from_dict(data) -def _file_dict(entry: FileEntry, *, strict: bool = True) -> dict: - """Return a file entry as manifest data; ``strict`` (version 2) requires roles and source. +def _file_dict(entry: FileEntry) -> dict: + """Return a file entry as manifest data; its roles, their source and position are required. The writer refuses what the reader would refuse, so a written manifest always reads back. """ - if strict and (entry.roles is None or entry.axes_from not in AXES_FROM): + if entry.roles is None or entry.axes_from not in AXES_FROM: raise ValueError(f"the entry of {entry.relative_path!r} has no roles or axes_from") - recorded_position_id(entry.position_id, entry.relative_path, strict=strict) + recorded_position_id(entry.position_id, entry.relative_path) data = { "relative_path": entry.relative_path, "size": entry.size, @@ -558,13 +543,16 @@ def _file_dict(entry: FileEntry, *, strict: bool = True) -> dict: return data -def _file_entry(data: dict, *, strict: bool = True) -> FileEntry: - """Read a file entry; ``strict`` (manifest version 2) requires roles, source and position. +def _file_entry(data: dict) -> FileEntry: + """Read a file entry; every key the writer always writes is required, none defaulted. - The ``position_id`` must be the one the entry's name gives (round-2 audit A-m5). + The roles, their source (``axes_from``), the pixel sizes and the ``position_id`` (the one + the entry's name gives, round-2 audit A-m5) are required. Only ``inventory_sha256``, + ``content_sha256`` and ``raw_channel_names`` may be absent: the writer leaves them out when + they do not apply. """ - bits, path = data.get("significant_bits"), str(data["relative_path"]) - if strict and (data.get("roles") is None or data.get("axes_from") not in AXES_FROM): + bits, path = data["significant_bits"], str(data["relative_path"]) + if data.get("roles") is None or data.get("axes_from") not in AXES_FROM: raise ValueError(f"the entry of {path!r} has no roles or axes_from") return FileEntry( relative_path=path, @@ -576,16 +564,16 @@ def _file_entry(data: dict, *, strict: bool = True) -> FileEntry: str(data["technical_repeat"]), ), axes=str(data["axes"]), - singleton_roles=tuple(str(r) for r in data.get("singleton_roles", [])), + singleton_roles=tuple(str(r) for r in data["singleton_roles"]), plane_count=int(data["plane_count"]), - channel_names=tuple(None if n is None else str(n) for n in data.get("channel_names", [])), + channel_names=tuple(None if n is None else str(n) for n in data["channel_names"]), significant_bits=None if bits is None else int(bits), - read_in_blocks=recorded_flag(data.get("read_in_blocks", False)), - position_id=recorded_position_id(data.get("position_id"), path, strict=strict), - channel_names_trimmed=recorded_flag(data.get("channel_names_trimmed", False)), - roles=recorded_roles(data.get("roles"), str(data["axes"])), - axes_from=optional_text(data.get("axes_from")), - pixel_sizes=sizes_from_data(data.get("pixel_sizes")), + read_in_blocks=recorded_flag(data["read_in_blocks"], "read_in_blocks"), + position_id=recorded_position_id(data.get("position_id"), path), + channel_names_trimmed=recorded_flag(data["channel_names_trimmed"], "channel_names_trimmed"), + roles=recorded_roles(data["roles"], str(data["axes"])), + axes_from=str(data["axes_from"]), + pixel_sizes=sizes_from_data(data["pixel_sizes"]), inventory_sha256=optional_text(data.get(INVENTORY_KEY)), content_sha256=optional_text(data.get(CONTENT_KEY)), raw_channel_names=recorded_raw_names(data.get(RAW_NAMES_KEY), path), diff --git a/src/morphobricks_preprocessing/manifest_fields.py b/src/morphobricks_preprocessing/manifest_fields.py index 9e3b6ab2..f2f0df97 100644 --- a/src/morphobricks_preprocessing/manifest_fields.py +++ b/src/morphobricks_preprocessing/manifest_fields.py @@ -2,10 +2,12 @@ A file entry's ``position_id`` is not free text: it is the image's name without its extension (:func:`.image_names.position_id`, MorphoBricks D02 and D85), never parsed or -edited. Manifest version 2 records it for every image, so a version 2 entry without it, or -with another one, is refused rather than read as an empty or a different position (round-2 -audit A-m5: ``RunInputs`` turned a missing one into ``""``). Also the check of an entry's -axis roles, and of its flags, optional texts and masks entry, moved from :mod:`.manifest`. +edited. Every manifest this package reads (version 4) records it for every image, so an entry +without it, or with another one, is refused rather than read as an empty or a different position +(round-2 audit A-m5: ``RunInputs`` turned a missing one into ``""``). Nothing is read leniently +for an older manifest: older versions are refused whole (:mod:`.manifest_versions`). Also the +check of an entry's axis roles, and of its flags, optional texts and masks entry, moved from +:mod:`.manifest`. Pure: no I/O. """ @@ -31,7 +33,7 @@ def recorded_raw_names(value: object, relative_path: str) -> tuple[str | None, . return tuple(value) -def recorded_position_id(value: object, relative_path: str, *, strict: bool = True) -> str | None: +def recorded_position_id(value: object, relative_path: str) -> str: """Check the ``position_id`` of a manifest file entry against its image's name. Parameters @@ -40,29 +42,24 @@ def recorded_position_id(value: object, relative_path: str, *, strict: bool = Tr The recorded ``position_id``. relative_path The entry's ``relative_path``, whose name gives the position identifier. - strict - True for manifest version 2, which records it for every image. False for version 1 - (read for reporting only): any value is kept as text, and None when absent. Returns ------- - str or None - The position identifier; None only when not ``strict`` and none is recorded. + str + The position identifier. Raises ------ ValueError - When ``strict`` and the value is missing, null or not text, or is not - ``position_id(relative_path)``, or the path's name is only an extension; the - message names the entry and the identifier its name gives. + When the value is missing, null or not text, or is not ``position_id(relative_path)``, + or the path's name is only an extension; the message names the entry and the + identifier its name gives. """ - if not strict: - return None if value is None else str(value) expected = position_id(relative_path) if not isinstance(value, str): raise ValueError( - f"the entry of {relative_path!r} has no position_id ({value!r}); manifest version 2 " - f"records one for every image: {expected!r}, its name without the extension" + f"the entry of {relative_path!r} has no position_id ({value!r}); a manifest records " + f"one for every image: {expected!r}, its name without the extension" ) if value != expected: raise ValueError( @@ -73,28 +70,26 @@ def recorded_position_id(value: object, relative_path: str, *, strict: bool = Tr return value -def recorded_roles(value: object, axes: str) -> tuple[str | None, ...] | None: - """Check a file's recorded axis roles: one role or null per axis (absent in older manifests). +def recorded_roles(value: object, axes: str) -> tuple[str | None, ...]: + """Check a file's recorded axis roles: one role or null per axis, always recorded. Parameters ---------- value - The recorded ``roles``: a list with one role or null per axis, or None. + The recorded ``roles``: a list with one role or null per axis. axes The file's axis letters. Returns ------- - tuple or None - The roles; None when none are recorded. + tuple + The roles. Raises ------ ValueError - If they are not one role or null per axis, or a role is unknown. + If they are absent, not one role or null per axis, or a role is unknown. """ - if value is None: - return None if not isinstance(value, list) or len(value) != len(axes): raise ValueError(f"the roles {value!r} do not fit the axes {axes!r}") roles = tuple(None if r is None else str(r) for r in value) diff --git a/src/morphobricks_preprocessing/recorded_sizes.py b/src/morphobricks_preprocessing/recorded_sizes.py index 8879823a..a85c4a2b 100644 --- a/src/morphobricks_preprocessing/recorded_sizes.py +++ b/src/morphobricks_preprocessing/recorded_sizes.py @@ -122,15 +122,13 @@ def sizes_to_data(sizes: Iterable[RecordedSize]) -> list[dict]: def sizes_from_data(data: object) -> tuple[RecordedSize, ...]: - """Read recorded sizes (absent in v1 manifests, which gives none). + """Read recorded sizes: a list, one entry per role (always written; never absent). Raises ------ ValueError If the data is not a list of sizes. """ - if data is None: - return () if not isinstance(data, list): raise ValueError(f"the pixel sizes {data!r} are not a list") return tuple(RecordedSize.from_dict(item) for item in data) diff --git a/src/morphobricks_preprocessing/sources.py b/src/morphobricks_preprocessing/sources.py index 7c9d24a3..0b99e984 100644 --- a/src/morphobricks_preprocessing/sources.py +++ b/src/morphobricks_preprocessing/sources.py @@ -10,11 +10,9 @@ from collections.abc import Iterable, Mapping, Sequence from pathlib import Path -from .axes import file_mapping_for from .image_names import position_id from .inventory import source_signature from .manifest import RunManifest -from .metadata import inspect_image from .validation import ValidationIssue, error, plural @@ -50,8 +48,7 @@ def check_sources( The axes the caller reads each image with, one letter per dimension (``"ZYX"``), by path; every path must have one. A dimension the run gave no role (length 1) fits any letter; the others must have the run's role. The roles are the file's - own as recorded (:attr:`FileEntry.roles`), or for manifests written before they - were recorded, the run's mapping resolved for the image's header. + own as recorded (:attr:`FileEntry.roles`); none recorded fits no axes. identities The caller's ``(condition, biological_repeat, technical_repeat, position_id)`` of each image (:data:`.discovery.FOV_KEY` order), by path; every path must have one. @@ -60,9 +57,8 @@ def check_sources( ------- list of ValidationIssue Errors, each naming its images: ``images_not_in_manifest``, ``images_missing`` - (no longer on disk), ``images_changed``, ``position_id_not_recorded`` (a manifest - from before ``position_id`` was recorded, when ``identities`` are checked), - ``identity_differs`` and ``axis_mapping_mismatch``. Empty when all images pass. + (no longer on disk), ``images_changed``, ``identity_differs`` and + ``axis_mapping_mismatch``. Empty when all images pass. Raises ------ @@ -93,27 +89,18 @@ def report(code: str, path: str, detail: str = "") -> None: if fingerprint != (entry.size, entry.mtime_ns, entry.inventory_sha256): report("images_changed", path) recorded = (*entry.labels, entry.position_id) - if entry.position_id is not None and entry.position_id != position_id(path): + if entry.position_id != position_id(path): report( "identity_differs", path, f"recorded position_id {entry.position_id!r}, its name gives {position_id(path)!r}", ) - elif identities is not None and entry.position_id is None: - report("position_id_not_recorded", path) elif identities is not None and tuple(identities[path]) != recorded: report( "identity_differs", path, f"given {tuple(identities[path])}, recorded {recorded}" ) if image_axes is not None: roles = entry.roles - if roles is None: - mapping = file_mapping_for( - inspect_image(root / path, path), - manifest.mapping, - missing_as_singleton=manifest.settings.missing_as_singleton, - ) - roles = None if mapping is None else mapping.roles axes = str(image_axes[path]).upper() if not _roles_agree(axes, roles): described = "none fit" if roles is None else "".join(r or "-" for r in roles) @@ -157,11 +144,6 @@ def _roles_agree(axes: str, roles: Sequence[str | None] | None) -> bool: "time differs from the manifest, so their rows may not describe them.", _RERUN + "the images as they were.", ), - "position_id_not_recorded": ( - "The manifest does not record the position of {n}; it was written by a version " - "of the preprocessing brick before positions were recorded.", - "Run the preprocessing brick again on this experiment.", - ), "identity_differs": ( "The labels or position of {n} differ from those the preprocessing run recorded.", "Use the labels the preprocessing run confirmed, or run it again with the folders as " diff --git a/tests/test_check_sources.py b/tests/test_check_sources.py index c75741e6..07cfea28 100644 --- a/tests/test_check_sources.py +++ b/tests/test_check_sources.py @@ -113,12 +113,12 @@ def test_labels_and_position_must_be_the_runs(run): assert "('ctrl', 'rep2', 'single', 'b')" in issues[0].message -def test_a_manifest_without_positions_cannot_confirm_them(run): +def test_an_entry_without_a_position_is_refused(run): root, manifest = run old = replace(manifest, files=tuple(replace(f, position_id=None) for f in manifest.files)) issues = check_sources(old, root, IMAGES, identities={p: identity(p) for p in IMAGES}) - assert codes(issues) == {"position_id_not_recorded": tuple(sorted(IMAGES))} - assert check_sources(old, root, IMAGES) == [] # nothing to compare without identities + assert codes(issues) == {"identity_differs": tuple(sorted(IMAGES))} + assert codes(check_sources(old, root, IMAGES)) == {"identity_differs": tuple(sorted(IMAGES))} def test_a_recorded_position_that_is_not_the_name_is_refused(run): @@ -131,13 +131,9 @@ def test_a_recorded_position_that_is_not_the_name_is_refused(run): assert found == {"identity_differs": ("ctrl/rep1/a.tif",)} -@pytest.mark.parametrize("roles_recorded", [True, False]) -def test_axes_must_have_the_confirmed_roles(run, roles_recorded): - # CellPatcher's check_images rule; manifests written before per-file roles fall back on - # the run's mapping resolved for the image's header. +def test_axes_must_have_the_confirmed_roles(run): + # CellPatcher's check_images rule, against each file's recorded roles. root, manifest = run - if not roles_recorded: - manifest = replace(manifest, files=tuple(replace(f, roles=None) for f in manifest.files)) axes = dict.fromkeys(IMAGES, "tyx") assert check_sources(manifest, root, IMAGES, image_axes=axes) == [] axes["drug/rep1/a.tif"] = "ZYX" @@ -147,6 +143,15 @@ def test_axes_must_have_the_confirmed_roles(run, roles_recorded): assert "drug/rep1/a.tif: given ZYX, preprocessing TYX" in issues[0].message +def test_an_entry_without_roles_fits_no_axes(run): + # Nothing is resolved from the header in their place (no older manifest is read). + root, manifest = run + old = replace(manifest, files=tuple(replace(f, roles=None) for f in manifest.files)) + issues = check_sources(old, root, IMAGES, image_axes=dict.fromkeys(IMAGES, "TYX")) + assert codes(issues) == {"axis_mapping_mismatch": tuple(sorted(IMAGES))} + assert "preprocessing none fit" in issues[0].message + + def test_an_axis_without_a_role_fits_any_letter(run): root, manifest = run files = tuple(replace(f, roles=(None, "Y", "X")) for f in manifest.files) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index d373b128..226869d4 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -129,15 +129,16 @@ def test_same_source_ignores_informational_fields(): assert entry().same_source(entry(read_in_blocks=True)) -def test_read_in_blocks_round_trips_and_defaults_to_false(tmp_path): +def test_read_in_blocks_round_trips_and_is_required(tmp_path): original = manifest(files=(entry(read_in_blocks=True), entry("cond/rep/b.tif"))) path = write_manifest(original, tmp_path / "m.json") assert read_manifest(path) == original data = json.loads(path.read_bytes()) assert [f["read_in_blocks"] for f in data["files"]] == [True, False] - del data["files"][0]["read_in_blocks"] # written before planes could be read in blocks + del data["files"][0]["read_in_blocks"] # always written: never defaulted when absent path.write_text(json.dumps(data), encoding="utf-8") - assert not read_manifest(path).files[0].read_in_blocks + with pytest.raises(ManifestFormatError, match="read_in_blocks"): + read_manifest(path) def test_position_id_round_trips(tmp_path): @@ -175,7 +176,7 @@ def test_intensity_check_limits_round_trip_and_are_none_when_absent(tmp_path): assert read_manifest(path).intensity_checks is None -def test_histogram_settings_and_narrow_ranges_round_trip_and_default_when_absent(tmp_path): +def test_histogram_settings_and_narrow_ranges_round_trip(tmp_path): import numpy as np from morphobricks_normalization.ranges import HistogramSettings, RangeCheck @@ -201,10 +202,10 @@ def test_histogram_settings_and_narrow_ranges_round_trip_and_default_when_absent data = json.loads(path.read_bytes()) assert data["settings"]["histogram"]["histogram_range"] == "robust" assert data["narrow_histogram_ranges"][0]["robust_limits"] == [61.5, 140.25] - del data["settings"]["histogram"], data["narrow_histogram_ranges"] # written before P6 + del data["settings"]["histogram"] # always written: never the default when absent path.write_text(json.dumps(data), encoding="utf-8") - read = read_manifest(path) - assert read.settings.histogram == HistogramSettings() and read.narrow_ranges == () + with pytest.raises(ManifestFormatError, match="histogram"): + read_manifest(path) def test_file_entry_for_reads_size_and_time(tmp_path): diff --git a/tests/test_manifest_position_id.py b/tests/test_manifest_position_id.py index 517c7dd1..4cb02bf5 100644 --- a/tests/test_manifest_position_id.py +++ b/tests/test_manifest_position_id.py @@ -109,7 +109,8 @@ def test_a_name_that_is_only_an_extension_is_refused(): with pytest.raises(ValueError, match="only an extension"): recorded_position_id(".tif", "ctrl/.tif") assert recorded_position_id("img.01", "ctrl/img.01.ome.tif") == "img.01" - assert recorded_position_id(None, "ctrl/a.tif", strict=False) is None + with pytest.raises(ValueError, match="no position_id"): + recorded_position_id(None, "ctrl/a.tif") def test_a_run_after_a_manifest_without_positions_reads_every_image_again(run): From 86c846e1586aed2767acac3c7e4648bf1da567bc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:20:14 +0000 Subject: [PATCH 6/9] SWEEP-PP PP-10: remove the v0.3 to v0.4 migration guide No run of v0.3 or v0.4 is read any more (manifest version 4 only), so the migration guide described a path that no longer exists. README, user guide and CHANGELOG no longer point to it; the user guide's upgrade note says what an older run gets now. The rest of PP-10 (parity tests, test_removed_in_v040.py, released_code.py, fetch-depth) was already removed in PP-C, and the histogram goldens were renamed there. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- CHANGELOG.md | 10 +- README.md | 6 +- docs/migration-0.3-to-0.4.md | 404 ----------------------------------- docs/user_guide.md | 8 +- 4 files changed, 14 insertions(+), 414 deletions(-) delete mode 100644 docs/migration-0.3-to-0.4.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 17416b6d..2d8db432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,8 @@ All notable changes to `morphobricks-preprocessing`. Numbers are pull requests (PR) and issues of [CellMigrationLab/morphobricks-preprocessing](https://github.com/CellMigrationLab/morphobricks-preprocessing). The design is `docs/restart/preprocessing-design-v0.4.md` in MorphoFeature, on branch `claude/restart-doc` ([commit ff19de8](https://github.com/CellMigrationLab/MorphoFeature/blob/ff19de8938d5ed8c4143cff563715551ee874a05/docs/restart/preprocessing-design-v0.4.md); not on `main`). The decisions -are in [docs/plan.md](docs/plan.md) §12. Upgrading from v0.3: see the -[migration guide](docs/migration-0.3-to-0.4.md). +are in [docs/plan.md](docs/plan.md) §12. (The v0.3 to v0.4 migration guide was removed in +v0.5.1: no run before v0.5.1 is read.) ## Unreleased (v0.5.1) @@ -258,6 +258,12 @@ already removed is not repeated here. **Breaking** where it says so. longer resolves roles from the image header for an entry without them (it fits no axes, `axis_mapping_mismatch`). +- **Docs: `docs/migration-0.3-to-0.4.md` removed** (inventory PP-10): no run of v0.3 or v0.4 is + read any more. README, user guide and this file no longer point to it. (The rest of PP-10, the + v0.4.0 parity tests, `test_removed_in_v040.py`, `released_code.py` and the CI's full-history + fetch, went in PP-C; `test_schema_v2_equivalence.py` was renamed `test_histogram_goldens.py` + there.) + ## 0.5.0 - 2026-11-09 Stage 1 of the experiment description (MorphoBricks-Spec experiment-v1, pinned at `1bed024`): diff --git a/README.md b/README.md index 5919ab74..6fc24470 100644 --- a/README.md +++ b/README.md @@ -50,9 +50,9 @@ removed in v0.4.0). Optional (the `zarr` extra): OME-Zarr images as input (`zarr_input`). **New users: start with the [user guide](docs/user_guide.md).** -**Upgrading from v0.3:** v0.4.0 is a breaking release. Every v0.3 run must be run again, and -bricks read runs through `RunInputs`. See the [migration guide](docs/migration-0.3-to-0.4.md) -and the [CHANGELOG](CHANGELOG.md). +**Upgrading:** there is no backward compatibility before the first pipeline release (Spec +ADR 0002 AR10). Every run needs a published experiment description, and a run written before +v0.5.1 is refused until it is run again; nothing is converted. See the [CHANGELOG](CHANGELOG.md). ## Installation diff --git a/docs/migration-0.3-to-0.4.md b/docs/migration-0.3-to-0.4.md deleted file mode 100644 index 1a2d7219..00000000 --- a/docs/migration-0.3-to-0.4.md +++ /dev/null @@ -1,404 +0,0 @@ -# Migrating from v0.3.4 to v0.4.0 - -v0.4.0 makes preprocessing the one place where an experiment's files, axes, masks and pixel -sizes are checked and recorded. Every later brick reads that record through `RunInputs` -instead of discovering, pairing or calibrating files itself. The rule behind every change is -the same: **nothing is guessed, nothing is silent, and nothing is reused when its identity is -uncertain.** - -This guide lists every breaking change from v0.3.4. Each item gives **what changed**, **why** -(the scientific rule) and **what to do**. The first section is for people who run -preprocessing (notebook, command line or Python). The second is for brick authors. The full -rules are in the [contract](contract.md) and the [user guide](user_guide.md). The decisions -are in the [plan](plan.md), §12 questions 45 to 74. The complete list of changes is in the -[CHANGELOG](../CHANGELOG.md). - -**In short, for users:** install v0.4.0 and run preprocessing again on every experiment, -into the same output folder. The first run recomputes every row. It asks again for the axis -roles of plain stacks, and it stops on the new errors listed below. Fix them, then run the -later bricks on the new run. - -## For users - -### 1. Every v0.3 run must be run again (manifest v1 → v2) - -- **What changed.** `run_manifest.json` is version 2 (contract §9). Each image records its - `roles`, where they come from (`axes_from`: `header` or `confirmed`) and its pixel sizes by - role, with a unit and a source. Each mask records its `size`, `mtime_ns`, `roles` and - `axes_from`. The manifest also records the SHA-256 - of `calibration.csv` and, for a run on named-axis copies, of the conversion record. -- **Why.** A v1 manifest does not say which axis is which, or whether the user or the header - decided it. A row computed under unknown roles cannot be trusted as the same row. -- **What to do.** Run preprocessing again into the same output folder. A v1 manifest gives - **no reusable row and no recorded input**: the run recomputes every row, and you give the - axis roles, masks and exclusions again. The report says that the earlier manifest was - version 1. The later bricks refuse a v1 run (`RunInputs.open` raises `RunInputsError`). - -### 2. Axis checks: swaps and contradictions are errors; plain stacks are confirmed per file - -- **What changed.** - - Confirmed roles that swap the Y and X the header names (`axes_yx_swapped`), contradict - the header (`axes_contradict_header`) or put X before Y in a plain stack - (`axes_x_before_y`) are **errors**. Such an image is never read transposed or under a - contradicted declaration. - - A plain stack (the header names no axes, `axes_not_named`, a warning) takes the roles you - confirm. The confirmation covers only the files it was given for, at their recorded - fingerprint. A new or changed plain stack is `plain_stack_not_confirmed` until you - confirm it again (CLI `--axes`). - - An image whose own header names X before Y (for example ImageJ `ZXY`) is still read, in - the canonical Y-before-X order, because its metadata names the axes. - - **Z-stacks are refused** (`z_stack_not_supported`, owner decision of 5 November 2026): - MorphoBricks v1 analyses 2-D planes. Every image with more than one depth slice is named, - from its header's Z axis and again from the confirmed roles (a plain stack given a Z - role). A Z axis of one slice is accepted. v0.3 analysed every depth slice as a plane. - Project each Z-stack first (for example a maximum-intensity projection in Fiji) and use - the projected image; v0.4.0 has no projection step. Time-lapses are still analysed. -- **Why.** A guessed or contradicted axis gives wrong pixels with no error. The roles must - come from the header or from an explicit answer. -- **What to do.** Correct the roles to match the header. If the header itself is wrong, make - named-axis copies (§9 below) and run preprocessing on them. - -### 3. Recorded inputs, modes and the output folder's lock - -- **What changed.** - - `force_rebuild=True` / `--force-rebuild` are **removed**. They are replaced by - `mode="restart"` (the default: reuse the rows of unchanged files) and `mode="full"` - (recompute every row and check). Both keep your confirmations: design, axis and - mask-axis statements, stated calibration and image type. - - One output folder belongs to one experiment. A later run takes every input you do not - give from the earlier v2 manifest: experiment folder, axis roles, exclusions, masks - folder and mask axes, image type, design, bins and histogram settings, intensity limits - and `missing_as_singleton`. `run --output OUT` alone reruns the recorded experiment - (without one: `experiment_not_recorded`). A value you give that differs is used, and the - report names the change and its effect. Masks are dropped only with `--no-masks` - (`masks_folder=False`). Exclusions are cleared only with `--no-exclude` (`exclude=[]`). - - A run holds an exclusive lock file, `.morphobricks-preprocessing.lock`, in the output - folder. A second run is refused with `output_locked`, naming the computer, process ID and - start time. A stale lock is removed only when you ask: `--remove-stale-lock` (every - command), `remove_stale_lock()`, or the notebook's **Remove the lock of a stopped run**. - The lock of a process still running on the same computer is never removed. On Google - Drive the lock cannot guard a folder across computers or sessions, so use one session per - output folder. -- **Why.** A silent change of input, or two runs writing one folder, gives a table that - describes neither run. -- **What to do.** Replace `force_rebuild=True` with `mode="full"` and `--force-rebuild` with - `--mode full`. Use one output folder per experiment. Read the report's `input_changes` - after a run. - -### 4. The report, written on every run - -- **What changed.** Every run that holds the lock writes `discovery_report.json` and - `discovery_report.txt`, also when it stops. The report has `status` (`stopped` or - `written`), the SHA-256 of the manifest it describes, and what the run reused, recomputed, - resumed and replaced, with the reasons. `check` writes the same report (`status: - "stopped"`, `step: "check"`) and, when the output folder holds no design, a prefilled - `experiment_design.csv` (nothing confirmed). -- **Why.** A run that stops must still say why, and what it kept. -- **What to do.** Read the report after every run. Bricks never read it. - -### 5. Pixel sizes: `calibration.csv` and the `calibrate` command - -- **What changed.** For each image, the X, Y and Z sizes come from the header, or from a - statement in the output folder's `calibration.csv` (`target,axis,size,unit`; target `.`, - a folder or an image; a length unit or `pixels` for deliberately uncalibrated). The most - specific statement applies. The run reads this file and never writes it. You write it with - `morphobricks-preprocessing calibrate`, `write_calibration()`, the notebook's **Pixel - sizes** card or by hand. Each axis is recorded with its source (`header`, `stated` or - `pixels`). A Z or T size is recorded only for an image that has that axis. A size that is - not a positive finite length in micrometres is refused when stated, and unknown (with - `invalid_physical_calibration`) when it comes from a header. So is an ImageJ unit without - resolution tags (v0.4.0rc5); `read_physical_sizes` gives a TIFF without pages no sizes - with the reason in `invalid` (in `check` and `run` it is the error `unreadable_tiff`). Since v0.4.0rc5 a length is - converted to micrometres from its shortest decimal, so one length written in any unit is - one double (`108.37 nm` is `0.10837 µm`): **a size in micrometres computed from another - unit may differ by 1 ULP** (the last bit) from v0.4.0rc4. Manifests are unchanged (each - size is recorded in its own unit), and the 0.1 % comparison is not affected. -- **Why.** Pixel sizes belong to the experiment, not to each brick's configuration. A size - nobody gave is uncalibrated, never a default. -- **What to do.** Move the pixel sizes you gave to bricks (run files, FOV-table columns, - configs) into `calibration.csv`. Changing it recomputes no row; the next run records the - new sizes. `RunInputs` refuses a `calibration.csv` changed, added or removed since the run. - -### 6. Experiment design: the `design` command and stricter readers - -- **What changed.** - - New command `design --output OUT`. It prints the design and what is still unconfirmed, - and changes it without editing the CSV: `--repeat-group CONDITION/REPEAT=GROUP`, - `--control` / `--no-control CONDITION`, `--confirm GROUP`, `--confirm-all`. Python: - `edit_design` with `DesignEdit`. Nothing is confirmed unless you ask. - - `design_not_confirmed` stops a `restart` that has new unconfirmed repeats. - - Every design reader refuses a header that names a column twice or has a column without - a name (`DesignFormatError`). - - Technical-repeat folders whose images are all excluded are not counted. - - In the notebook, changing the output folder reloads that folder's design; nothing is - carried over. - - `load_design` returns `(design, changes)` and raises `DesignCheckError` for a file it - cannot read (in v0.3.4 it returned a third value, `problem`). -- **Why.** Pooling biological repeats is a scientific claim. It must be an explicit, recorded - answer for the output folder it belongs to. -- **What to do.** Confirm the design with `design` or the notebook. Remove duplicate - columns from hand-edited files. Update callers of `load_design` to unpack two values. - -### 7. Masks - -- **What changed.** - - `image_named_as_mask` is an **error**: an image named like a mask of another image - (`_mask`, `_masks`, `_cp_masks`, `_labels`, `_label`, `_seg`, `_segmentation`) is not - analysed as an image. `_cp_masks` and `_masks` are new mask suffixes. - - Masks are checked **as stored** and never converted or reordered. A mask whose header, - or stated axes, put X before Y is refused (`mask_layout`, with a `numpy.swapaxes` hint). - - A plain-stack mask needs its axes stated once per mask type: `--mask-axes Cells=TYX`, or - **Mask axes** in the notebook. - - A 2-D mask of a time-lapse is refused (`mask_planes_mismatch`): the masks of an image - with T > 1 need the same T, one mask plane per time point, whatever a 2-D mask's - metadata names (owner decision of 5 November 2026). The release candidates' stated - shared 2-D mask (`--mask-axes Cells=YX`, YX under the notebook's Mask axes) is removed: - a `YX` statement is the error `shared_mask_not_supported`, a `YX` statement recorded by - an earlier run is not applied again, and `RunInputs.open` refuses a run folder that - recorded a mask `shared: true`: save one mask plane per time point and run - preprocessing again. - - OME-Zarr `labels/` are always used as masks when present. `find_masks(ome_zarr_labels=)` - is **removed**. - - The notebook has a **Masks folder** field and a **Describe the masks** button (the - `masks` step). - - `same_objects` in `mask_relationships.json` is a one-to-one matching (report version 2). - - A child lying exactly as much in two or more parents has **no parent** (it went to - the parent with the lower label before, so the counts depended on the numbering): - it is not `inside` and is counted as `ambiguous_parent`, so `inside` and - `parents_with_N_children` may be lower than with v0.4.0rc4 (report version 3, - v0.4.0rc5). - - The masks folder may hold only masks: an experiment, named-axis copies or the output - folder inside it is refused (`experiment_inside_masks`, `copies_inside_masks`, - `output_inside_masks`, before anything is written), and so is a `convert` folder - inside it (`copy_folder_overlaps_masks`). `RunInputs.open` refuses a recorded run whose - folders break this rule. -- **Why.** A mask read under the wrong axes describes other objects. Objects must be joined - by their recorded pairing, never by guessing a layout. -- **What to do.** Rename or exclude images named like masks. Save masks with their axes - named (or state them per type). Save X-before-Y masks again with `numpy.swapaxes`. Remove - `ome_zarr_labels=` from `find_masks` calls. Move an experiment or copies out of the - masks folder, and run again a run recorded with them. - -### 8. OME-Zarr input - -- **What changed.** - - An OME-NGFF 0.1 or 0.2 store is `unreadable_ome_zarr`: its metadata never name the - axes, and the order its specification fixes is not assumed. A 0.3, 0.4 or 0.5 store - without `axes` is unreadable too. - - Every OME-Zarr image is matched by the roles its axes declare. A confirmed role that is - not the declared one is `ome_zarr_axes_mismatch` (OME-Zarr images are never converted: - correct the roles). - - A malformed scale gives unknown sizes with `invalid_physical_calibration` (a warning). - - An OME-Zarr image's fingerprint adds the SHA-256 of its file inventory (each file's path, - size and time). **Images recorded by v0.4.0rc3 or earlier have no inventory: they count - as changed and are recomputed, and `RunInputs` refuses such a run.** - - Links inside a store follow the same containment rule as the experiment (§10 below). -- **Why.** The axes and the scale must come from the metadata. A chunk changed inside a store - must not leave an old row in place. -- **What to do.** Re-save OME-NGFF 0.1 and 0.2 stores as OME-NGFF 0.4 or later with their - axes. Run preprocessing again after upgrading, also from an rc. - -### 9. Conversion: `convert_tiffs` removed, `convert` added - -- **What changed.** - - **Removed:** `convert_tiffs` and the `zarr_export` module (OME-Zarr copies in - `/ome_zarr`): `ConversionSettings`, `ConversionEstimate`, `estimate_conversion`, - `OmeZarrWriter`, `ome_zarr_relative_path`; the manifest's `convert_tiffs`, - `conversion` and `ome_zarr_path` (ignored when an old manifest holds them); the CLI - `--convert-tiff`, `--chunk-size`, `--compression`, `--compression-level` and - `--pyramid-levels`; the notebook's "(Optional) Image conversion" card; and - `notebook_options.PYRAMID_OPTIONS` (nothing replaces it). - - **Added:** `convert EXPERIMENT --to FOLDER --axes ROLES` (`convert_experiment`; the - notebook's **Make named-axis copies** card). It is only for images with axis errors or - unnamed axes. They are written as named-axis OME-TIFF with your confirmed roles, pixel for - pixel. Every other image is copied byte for byte. Masks are never converted. - - The copies keep the file names and folders, so `source_relative_path`, `plane_id` and the - FOV key stay the same. A relabelled axis gets an unknown size, never a guessed one. The - only exception: a Y/X swap keeps the size when X and Y are exactly equal. - - The copies are written as `.partial` files and moved into place together. The conversion - record, `.morphobricks-conversion.json` (version 2), is written last, with the source and - copy fingerprints and each copy's SHA-256. A version 1 record (v0.4.0rc3 or earlier) is - refused: convert again into a new folder. -- **Why.** The run reads every image through its own readers, so the OME-Zarr copies served - no brick. A file whose header is wrong must be fixed in a new, recorded copy, never read - transposed. -- **What to do.** Stop passing the removed options. The old `/ome_zarr` folder is not - touched; delete it yourself if you no longer need it. For images with axis errors, run - `convert`, then run preprocessing on the copy folder with the **same output folder**: the - design, image type and calibration are kept, and every row is recomputed. A changed copy - stops the run (`copy_changed_after_conversion`). - -### 10. Links, the `--log` file and exit codes - -- **What changed.** - - A symbolic link (or a Windows junction) in the experiment or masks folder is followed - only when its real path stays inside that folder. Otherwise it is `link_outside_folder`. - One file reached by two paths, hard links included, is `linked_twice`. A loop is - `link_loop`. - - **Mount points below the experiment or masks folder are refused** (v0.4.0rc5): a folder - or file listed in Linux's `/proc/self/mountinfo`, or a folder, image or mask on another - device than the folder selected, is the error `mounted_folder`, in discovery, `check`, - `run`, `convert` and `RunInputs`. A mount made after the walk started is not seen, and - without a mount table (macOS, BSD, Windows) neither is a mounted single file nor a - mount of the same device. A file with more than one hard link is the warning `hard_linked_file`. - - `--log` is checked before the file is opened. It is refused inside the experiment, the - masks folder or the folder of copies, and as or inside a protected output file (the - table, manifest, reports, design, `calibration.csv`, `mask_relationships.json`, lock, - conversion record, `.publish`, `.staging`) or the `--design` file. - - Every command exits with status **1 when it prints an error** and 0 otherwise. Warnings - never change the status, and options that cannot be parsed exit with 2. `check` and - `design` exited with 0 on some errors before. -- **Why.** A link that leaves the experiment brings in images that are not part of it, and - one file seen twice is counted twice. A log must never overwrite a result. Scripts must be - able to trust the exit status. -- **What to do.** Copy images into the experiment rather than linking to them or mounting - them from elsewhere. Write logs next to the output folder. In scripts, check the exit status. - -### 11. Staging, resume and crash-consistent publication - -- **What changed.** - - Each image's rows are staged in the output folder's `.staging` as soon as the image is - read. A run that was interrupted is resumed by the next run: staged rows are reused only - when their row key, pixel type, schema version and package versions still match and the - image still has its fingerprint. Anything else is discarded and reported. - - Publication is crash-consistent. The new files are written into `.publish`, then one - `os.replace` of `run_manifest.json` commits them, then they are moved into place. The - next holder of the lock finishes or discards an interrupted publication. The v0.3 - backup-and-restore files are gone. - - An image that changes during the run is `source_changed_during_run`: no scientific - outputs are published and sources are untouched; the discovery report is written. - - A run replaces or removes only `plane_histograms.csv`, `run_manifest.json`, - `experiment_design.csv` (with a design), the reports, `mask_relationships.json`, - `.publish` and `.staging`. It never touches `calibration.csv`, source images or masks. - - New output files get your umask's permissions; existing files keep theirs. - - A source is identified by its fingerprint (size and modification time; for OME-Zarr the - inventory of its files), which misses a rewrite of the same size with the time set - back. The opt-in `--verify-content` (notebook: **Verify each image's content**, - `RunMode(verify_content=True)`, v0.4.0rc5) records each source's content SHA-256 and - reuses rows only for the same content, and hashes every source again right before the - commit; `RunInputs` checks it at `open`. Turning it on or off recomputes every row; a - default run's manifest is unchanged (version 2). - - **A strict run's manifest is version 3** (v0.4.0rc5): v0.4.0rc4 and earlier refuse it - (`ManifestFormatError`), so a reader that does not verify content never opens a strict - run as a fingerprint-only one. A brick that checks `manifest_version == "2"` should - accept `"3"` too, or go through `RunInputs`. - - The output folder is flushed to disk right after the manifest's rename (POSIX). -- **Why.** A reader must see the old run or the new one, never a mix. A row is reused only - when it is certainly the same row. -- **What to do.** Nothing, except: read the outputs through `RunInputs` (bricks), or check - `csv_sha256` before pairing files by hand. Files written `0600` by earlier versions keep - that mode until you change it. - -### 12. Other changes users see - -- Mixed pixel types stay an error. MorphoRelate projects with mixed dtypes must be saved - again in one type. -- `--memory-limit` must be a whole number of bytes above 0 (`1.5GiB` is accepted, `1.5` - is refused). -- `different_sizes` and `different_plane_counts` compare the images by their confirmed roles - once the roles are confirmed. -- `run` needs `--axes` only when some images are plain stacks; images that all name their - axes are run with those roles. A rerun that changes nothing keeps `run_manifest.json` - byte-identical (its `written_at` too). `convert` takes `--output`, the output folder of - your runs on the originals, for the next commands it prints (cross-brick test, plan §12 - question 73). -- The notebook runs on Google Colab and in JupyterLab. It installs the pinned release - (`RELEASE_TAG`), never `main`. - -## For brick authors - -### 13. Read runs only through `RunInputs` - -Bricks read a preprocessing run through -`RunInputs.open(output_folder, experiment_root=None, masks_root=None)` (contract §9.1). They -take no axis or calibration arguments for file runs, never transpose, and never call -`find_masks`, `discover_experiment` or `position_id` for file runs. For in-memory arrays a -brick keeps only a canonical layout argument (Y before X, no swaps). Without a clean run a -brick refuses and gives the preprocessing command; there are no standalone exceptions. - -**`RunInputs` gives:** - -- `fovs()`: `Fov` records with the FOV key, `plane_id`s, axes, roles, `axes_from`, channel - names, dtype, `significant_bits` and the Z, T and C counts. They come from the manifest and - the table and are never parsed from names. -- `pixel_sizes(fov)`: the value, unit and source per role, or unknown. -- `read_plane(fov, z, t)`: a `(C, Y, X)` array of the stored values. `read_image(fov)`: a lazy - `(T, Z, C, Y, X)` `LazyImage`. Both are in canonical order. `z` and `t` must be integers - (a `plane_id` is a `TypeError`). -- `mask_types()` and `read_mask(fov, type, z=None, t=None)`: the recorded pairing only, read - as stored with `check_labels` (labels 0 to int32 max). A run that recorded a 2-D mask - shared by every plane (`shared: true`, release candidates only) is refused at `open`. A - `z` or `t` outside the image is an `IndexError`, for - `read_plane` too (audit I5). -- A header size in an unsupported unit (OME `pixel`, `inch`) or without a unit is unknown - in `pixel_sizes` (audit I3). `to_micrometers` takes NumPy scalars (audit m4). The - warnings `partial_pixel_size` and `non_square_pixels` name the images you will refuse - for length features (audit m5). `read_plane`'s memory limit counts only reading (I2). -- `histograms()`, `design()` (the strict `read_confirmed_design`, checked against the - manifest's SHA-256) and `image_type()`. -- A version 2 manifest **requires** each file entry's `position_id`, equal to the image's - name without its extension (`position_id(relative_path)`); a missing, null or edited one - is a `ManifestFormatError` (v0.4.0rc5, audit A-m5). Every v0.4.0 writer wrote it, so runs - of rc1 to rc4 still open; a hand-edited manifest may not. -- `mask_relationships.json` is report version 3 (v0.4.0rc5): a child lying exactly as much - in two or more parents has no parent and is counted as `ambiguous_parent`, never - `inside`; `inside` and `parents_with_N_children` no longer depend on label numbers. -- `RunInputs` is picklable for worker processes. `morphobricks_preprocessing.testing. - make_clean_run(folder, ...)` writes a small clean run for tests. - -**`RunInputs.open` refuses** (`RunInputsError`), naming the cause and the fix: - -- a version 1 manifest; -- a table, design or `calibration.csv` whose SHA-256 is not the manifest's (also a - `calibration.csv` added since the run); -- an image or mask changed or gone since the run (every read checks the fingerprint again); -- a mask recorded without axes (a v0.4.0rc1 run); -- an OME-Zarr image recorded without its inventory (v0.4.0rc3 or earlier); -- **files the run did not record** (`UnrecordedFilesError`, with `files.images` and - `files.masks`): images discovery would analyse, and masks it would pair with a recorded - image, that the run neither recorded nor excluded. There is no opt-out. The fix is to run - preprocessing again, or exclude the images; -- a `masks_root` given for a run that recorded no masks folder; -- a run on named-axis copies whose conversion record changed or disappeared - (`conversion_record_sha256`), whose record is version 1, or whose copies do not match the - record's fingerprints and SHA-256. The copies are read once, at `open`, to check them; -- a run holding OME-Zarr images when the `zarr` extra is missing (install - `morphobricks-preprocessing[zarr]`), and a recorded OME-Zarr image whose `labels` group - cannot be read; -- links that leave the experiment or masks folder, as discovery refuses them. - -`unrecorded_files(manifest, experiment_root, masks_root, output_folder=None)` gives the same -list without raising. - -### 14. Removed APIs - -| Removed | Use instead | -|---|---| -| `convert_tiffs=`, `zarr_export` (`ConversionSettings`, `ConversionEstimate`, `estimate_conversion`, `OmeZarrWriter`, `ome_zarr_relative_path`) | nothing (images are read directly); `convert_experiment` for axis errors | -| The stand-ins of the code moved to the normalization brick: `morphobricks_preprocessing.{csv_io,dtypes,histogram,normalization,pooling,records}` and their top-level names (`PlaneRecord`, `PlaneHistogram`, `compute_plane_histogram`, `compute_block_histogram`, `read_plane_histograms`, `write_plane_histograms`, `group_records`, `PooledHistogram`, `PooledStatistics`, `pooled_statistics`, `NormalizationParameters`, `percentile_range`, `min_max`, `z_score`, `robust_z_score`, `default_bin_count`, `make_plane_id`, `SINGLE`, `OUTPUT_FILENAME`, `NoValidPixelsError`, `UnsupportedDtypeError`) | import them from `morphobricks_normalization` | -| `check_declared_axes`, `declared_axes_transposed` | nothing. They accepted a Y/X swap and read it transposed, which v0.4 forbids. Read through `RunInputs` | -| `find_masks(..., ome_zarr_labels=)` | omit it: label images are always used | -| `run_experiment(force_rebuild=)`, CLI `--force-rebuild` | `mode="full"`, `--mode full` | -| `notebook_options.PYRAMID_OPTIONS` | nothing | -| CLI `--convert-tiff`, `--chunk-size`, `--compression`, `--compression-level`, `--pyramid-levels` | nothing | -| Pickled `PhysicalSizes` from v0.3 | not supported (owner decision, design v0.4 change log, 1 November 2026). The class gained `invalid` (issue #94). A v0.3.4 pickle loads today with `invalid` None, but this is not guaranteed. Do not keep pickles across versions: read the metadata again, or use `RunInputs.pixel_sizes` | - -Superseded in the bricks (each brick's own migration PR): CellPatcher `image_axes`/`mask_axes` -for file runs, `image_read_transposed`, its own mask pairing and name parsing; MorphoFeature's -run-file axes and calibration, FOV-table pixel-size columns and `inputs.read_transposed`; -MorphoRelate's own mask pairing and calibration config; fiberfeatures' `spacing_cols`; and -the brick-side swap refusals. - -### 15. Caches keyed on the run - -- A `restart` that changes nothing leaves `plane_histograms.csv` byte-identical, so a brick - cache keyed on its SHA-256 (the manifest's `csv_sha256`) survives. -- Key caches on the manifest's SHA-256 and the file fingerprints, never on paths alone. -- After upgrading from v0.3 or from a release candidate up to v0.4.0rc3, the table is - recomputed. Caches keyed on the old table must not be reused. - -### 16. The `zarr` extra - -Install `morphobricks-preprocessing[zarr]` in every brick that may open a run holding -OME-Zarr images. Without it, `RunInputs.open` refuses such a run and names the extra. It -never skips the images or their label images. diff --git a/docs/user_guide.md b/docs/user_guide.md index 4c98bd8a..f8ecacc8 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -1084,11 +1084,9 @@ Run the notebook or `run` again after adding, changing or removing images: - changing only the image type reads nothing again: the histograms are the same for both types, and only the manifest records the new type. -**After upgrading from v0.3** (or from v0.4.0rc3 or earlier with OME-Zarr images): the -first run recomputes every row. A v0.3 manifest (version 1) gives no reusable row and no -recorded input, so give the axis roles, masks and exclusions again; an OME-Zarr image -recorded without its file inventory counts as changed. Later bricks refuse such older runs -until they are run again. The [migration guide](migration-0.3-to-0.4.md) lists every change. +**After upgrading from a version before v0.5.1:** its run manifest is not read (only version 4 +is), so the first run recomputes every row and reuses no recorded input: give the axis roles, +masks and exclusions again. Later bricks refuse such older runs until they are run again. Run the brick again after adding images or masks before using a later brick: later bricks read the images and masks this brick's run recorded, and refuse an experiment holding images From ef9ad9881d8ddbf258061ea60582f30a50b2cb7f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 14:10:08 +0000 Subject: [PATCH 7/9] SWEEP-PP PP-05 review: name the missing manifest key and the fix With the defaults for always-written keys removed, a manifest missing one was refused as "incomplete or malformed: 'histogram'" (a bare KeyError text, no fix). It now says which key is absent, that every version 4 manifest records it, and to run preprocessing again. The version check no longer defaults settings.verify_content to false either. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- src/morphobricks_preprocessing/manifest.py | 8 +++++- .../manifest_versions.py | 2 +- tests/test_manifest.py | 25 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/morphobricks_preprocessing/manifest.py b/src/morphobricks_preprocessing/manifest.py index 0bc5e0a4..7bcc61d3 100644 --- a/src/morphobricks_preprocessing/manifest.py +++ b/src/morphobricks_preprocessing/manifest.py @@ -48,6 +48,7 @@ ) from .manifest_versions import ( CONTENT_KEY, + RERUN, UnsupportedVersionError, check_version, recorded_version, @@ -420,7 +421,12 @@ def from_dict(cls, data: dict) -> RunManifest: raise except UnsupportedVersionError as exc: # a later version's manifest: plainly refused raise ManifestFormatError(str(exc)) from exc - except (KeyError, TypeError, ValueError, AttributeError) as exc: + except KeyError as exc: # a key every current manifest writes: never defaulted (PP-05) + raise ManifestFormatError( + f"The manifest is incomplete: it has no {exc.args[0]!r}, which every version " + f"{MANIFEST_VERSION} manifest records. {RERUN}" + ) from exc + except (TypeError, ValueError, AttributeError) as exc: raise ManifestFormatError(f"The manifest is incomplete or malformed: {exc}") from exc diff --git a/src/morphobricks_preprocessing/manifest_versions.py b/src/morphobricks_preprocessing/manifest_versions.py index 0f9d42a4..87c1550d 100644 --- a/src/morphobricks_preprocessing/manifest_versions.py +++ b/src/morphobricks_preprocessing/manifest_versions.py @@ -110,7 +110,7 @@ def recorded_version(data: dict) -> str: files = data["files"] check_version( data["manifest_version"], - recorded_flag(data["settings"].get("verify_content", False)), + recorded_flag(data["settings"]["verify_content"], "verify_content"), [(entry.get("relative_path"), entry.get(CONTENT_KEY)) for entry in files], data.get(DESCRIPTION_KEY) is not None, ) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 226869d4..7e156216 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -332,3 +332,28 @@ def test_file_entry_for_records_the_files_own_roles(tmp_path): write_tiff(tmp_path / "a.tif") mapping = prefill_mapping("ZYX").with_role(0, None) assert file_entry_for(tmp_path, "a.tif", ("c", "b", "t"), mapping).roles == (None, "Y", "X") + + +@pytest.mark.parametrize( + "where, key", + [ + ("settings", "histogram"), + ("settings", "verify_content"), + ("file", "singleton_roles"), + ("file", "channel_names"), + ("file", "significant_bits"), + ("file", "channel_names_trimmed"), + ("file", "pixel_sizes"), + ], +) +def test_a_missing_always_written_key_is_refused_naming_it_and_the_fix(tmp_path, where, key): + # PP-05: nothing is defaulted; the refusal names the key and the fix, not a bare KeyError. + path = write_manifest(manifest(), tmp_path / "m.json") + data = json.loads(path.read_bytes()) + del (data["settings"] if where == "settings" else data["files"][0])[key] + path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(ManifestFormatError) as refused: + read_manifest(path) + message = str(refused.value) + assert f"has no {key!r}" in message + assert "Run preprocessing again" in message From e500159af86f8a8086c45b8855b403b2f2df37cb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 14:11:41 +0000 Subject: [PATCH 8/9] SWEEP-PP PP-06 review: show a refused record version exactly A record_version of the number 3 (not the text "3") was refused as "it is a record of version 3, and this version of preprocessing reads version 3 only". The version is now shown as recorded (repr), and a version 3 record missing a key it always holds names the key and the fix instead of a bare KeyError text. RunInputs.open's docstring no longer mentions version 1 records only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- .../conversion_record.py | 16 +++++++++---- src/morphobricks_preprocessing/run_inputs.py | 2 +- tests/test_projection_record.py | 23 +++++++++++++++++-- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/morphobricks_preprocessing/conversion_record.py b/src/morphobricks_preprocessing/conversion_record.py index 02a3894e..b2bf7c03 100644 --- a/src/morphobricks_preprocessing/conversion_record.py +++ b/src/morphobricks_preprocessing/conversion_record.py @@ -45,6 +45,9 @@ """The one version written and read (issue #176): the record-level projection and projected entries, with each copy's SHA-256, channel names and significant bits (PP-11c).""" +_CONVERT_AGAIN = "convert the originals again into a new folder" +"""The fix a refused record names.""" + CONVERTED = "converted" """An image written as a named-axis OME-TIFF with the confirmed roles.""" @@ -263,9 +266,9 @@ def from_dict(cls, data: Mapping) -> ConversionRecord: raise ConversionRecordError(f"not a conversion record of version {RECORD_VERSION}") if data["record_version"] != RECORD_VERSION: raise ConversionRecordError( - f"it is a record of version {data['record_version']}, and this version of " - f"preprocessing reads version {RECORD_VERSION} only (older records are not " - "converted); convert the originals again into a new folder" + f"its record_version {data['record_version']!r} is not {RECORD_VERSION!r}: this " + f"version of preprocessing reads record version {RECORD_VERSION!r} only (older " + f"records are not converted); {_CONVERT_AGAIN}" ) try: mapping = data["mapping"] @@ -280,7 +283,12 @@ def from_dict(cls, data: Mapping) -> ConversionRecord: None if data["projection"] is None else projection_from_data(data["projection"]) ), ) - except (KeyError, TypeError, ValueError) as exc: + except KeyError as exc: # a key every version 3 record writes: never defaulted (PP-06) + raise ConversionRecordError( + f"the record has no {exc.args[0]!r}, which every version {RECORD_VERSION} record " + f"holds; {_CONVERT_AGAIN}" + ) from exc + except (TypeError, ValueError) as exc: raise ConversionRecordError(f"not a valid conversion record: {exc}") from exc def entry(self, relative_path: str) -> CopyEntry | None: diff --git a/src/morphobricks_preprocessing/run_inputs.py b/src/morphobricks_preprocessing/run_inputs.py index a290a62f..616a6865 100644 --- a/src/morphobricks_preprocessing/run_inputs.py +++ b/src/morphobricks_preprocessing/run_inputs.py @@ -211,7 +211,7 @@ def open( inside the masks folder or a masks folder inside the experiment (:mod:`.folder_rules`), a table, design, calibration or conversion record (of a run on named-axis copies) changed or gone since the run, a named-axis copy whose - SHA-256 is not its record's (or a record of version 1, without them), + SHA-256 is not its record's (or a record of another version than 3), an image or mask changed or gone, a mask recorded without axes, OME-Zarr images without the ``zarr`` extra (``morphobricks-preprocessing[zarr]``), or an OME-Zarr image whose ``labels`` group cannot be read. diff --git a/tests/test_projection_record.py b/tests/test_projection_record.py index 0f40f01a..0c9f450b 100644 --- a/tests/test_projection_record.py +++ b/tests/test_projection_record.py @@ -119,13 +119,13 @@ def test_a_version_2_record_is_refused_naming_the_fix(tmp_path): for entry in data["files"]: del entry["projection"] path.write_text(json.dumps(data)) - with pytest.raises(ConversionRecordError, match="version 2,.*convert the originals again"): + with pytest.raises(ConversionRecordError, match="record_version '2'.*convert the originals"): read_conversion_record(copies) @pytest.mark.parametrize("version", ["1", "2", "4", None]) def test_other_versions_are_refused(version): - with pytest.raises(ConversionRecordError, match="reads version 3 only"): + with pytest.raises(ConversionRecordError, match="reads record version '3' only"): ConversionRecord.from_dict(edited(lambda d: d.update(record_version=version))) @@ -241,3 +241,22 @@ def test_an_excluded_image_stays_a_plain_copy_in_a_projected_tree(): ConversionRecord( "/exp", AxisMapping("ZYX", tuple("ZYX")), False, (plain,), projection=Projection("max") ) + + +@pytest.mark.parametrize("version, shown", [(3, "3"), (None, "None"), ("2", "'2'")]) +def test_a_refused_version_is_shown_exactly_with_the_fix(version, shown): + # PP-06: the number 3 is not the text "3"; the message must not read "version 3 ... reads + # version 3 only". + with pytest.raises(ConversionRecordError) as refused: + ConversionRecord.from_dict(edited(lambda d: d.update(record_version=version))) + message = str(refused.value) + assert f"record_version {shown}" in message and "'3' only" in message + assert "convert the originals again into a new folder" in message + + +def test_a_missing_key_is_refused_naming_it_and_the_fix(): + with pytest.raises(ConversionRecordError) as refused: + ConversionRecord.from_dict(edited(lambda d: d.pop("projection"))) + message = str(refused.value) + assert "has no 'projection'" in message + assert "convert the originals again into a new folder" in message From 98b2c00a4e567a41262a5aa01e6f630394042334 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 14:23:52 +0000 Subject: [PATCH 9/9] SWEEP-PP PP-06 review: keep "version N" in a refused record's message The previous review commit quoted every refused version ('1'), which dropped the "version 1" wording that convert's conversion_record_unreadable message is tested for (test_convert_integrity). A text version is shown as before; only a version that is not text (the number 3, null) is refused as "record_version 3 is not text". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- src/morphobricks_preprocessing/conversion_record.py | 11 ++++++++--- tests/test_projection_record.py | 10 +++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/morphobricks_preprocessing/conversion_record.py b/src/morphobricks_preprocessing/conversion_record.py index b2bf7c03..e6fe6839 100644 --- a/src/morphobricks_preprocessing/conversion_record.py +++ b/src/morphobricks_preprocessing/conversion_record.py @@ -265,10 +265,15 @@ def from_dict(cls, data: Mapping) -> ConversionRecord: if not isinstance(data, Mapping) or "record_version" not in data: raise ConversionRecordError(f"not a conversion record of version {RECORD_VERSION}") if data["record_version"] != RECORD_VERSION: + version = data["record_version"] + if not isinstance(version, str): # the number 3 is not the version "3" + raise ConversionRecordError( + f"its record_version {version!r} is not text: this version of preprocessing " + f'reads version "{RECORD_VERSION}" only; {_CONVERT_AGAIN}' + ) raise ConversionRecordError( - f"its record_version {data['record_version']!r} is not {RECORD_VERSION!r}: this " - f"version of preprocessing reads record version {RECORD_VERSION!r} only (older " - f"records are not converted); {_CONVERT_AGAIN}" + f"it is a record of version {version}, and this version of preprocessing reads " + f"version {RECORD_VERSION} only (older records are not converted); {_CONVERT_AGAIN}" ) try: mapping = data["mapping"] diff --git a/tests/test_projection_record.py b/tests/test_projection_record.py index 0c9f450b..217a39de 100644 --- a/tests/test_projection_record.py +++ b/tests/test_projection_record.py @@ -119,13 +119,13 @@ def test_a_version_2_record_is_refused_naming_the_fix(tmp_path): for entry in data["files"]: del entry["projection"] path.write_text(json.dumps(data)) - with pytest.raises(ConversionRecordError, match="record_version '2'.*convert the originals"): + with pytest.raises(ConversionRecordError, match="version 2,.*convert the originals again"): read_conversion_record(copies) @pytest.mark.parametrize("version", ["1", "2", "4", None]) def test_other_versions_are_refused(version): - with pytest.raises(ConversionRecordError, match="reads record version '3' only"): + with pytest.raises(ConversionRecordError, match="reads version .?3.? only"): ConversionRecord.from_dict(edited(lambda d: d.update(record_version=version))) @@ -243,14 +243,14 @@ def test_an_excluded_image_stays_a_plain_copy_in_a_projected_tree(): ) -@pytest.mark.parametrize("version, shown", [(3, "3"), (None, "None"), ("2", "'2'")]) -def test_a_refused_version_is_shown_exactly_with_the_fix(version, shown): +@pytest.mark.parametrize("version", [3, None]) +def test_a_version_that_is_not_text_is_shown_exactly_with_the_fix(version): # PP-06: the number 3 is not the text "3"; the message must not read "version 3 ... reads # version 3 only". with pytest.raises(ConversionRecordError) as refused: ConversionRecord.from_dict(edited(lambda d: d.update(record_version=version))) message = str(refused.value) - assert f"record_version {shown}" in message and "'3' only" in message + assert f"record_version {version!r} is not text" in message assert "convert the originals again into a new folder" in message