From f4eec0a00bbc1448223207f994470c3d2a3062b5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 06:47:52 +0000 Subject: [PATCH 1/5] Make axis problems errors: axes_not_named, axes_yx_swapped, axes_contradict_header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit review_axes now compares every TIFF's confirmed roles with the axes its own header names (header_axis_issue / axis_disagreements on named_axes). A plain stack whose metadata does not name an axis longer than 1, roles that swap the named Y and X, and roles that otherwise contradict the header are errors that name every file with its header and confirmed axes and recommend saving the images as OME-Zarr. Nothing is read transposed or under a contradicted declaration; OME-Zarr images take their NGFF roles and never fail. Masks are compared with their image's Y and X by the confirmed roles (mask_size_issues; find_masks compared with the header's letters), and the mask size and plane checks skip images with an axis error (B1). The notebook marks each role that disagrees with the header before confirmation and shows "Axes checked" on success; check and run print the same. check_declared_axes and declared_axes_transposed are deprecated (removed in v0.5.0). Contract §2/§8/§15/§16, user guide, plan §12 Q45. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WX6rdMV1r5jUPUR94kxV5a --- AGENTS.md | 13 +- docs/contract.md | 9 +- docs/plan.md | 19 + docs/user_guide.md | 25 +- notebooks/MorphoBricks_Preprocessing.ipynb | 10 +- src/morphobricks_preprocessing/__init__.py | 6 + src/morphobricks_preprocessing/axes.py | 225 +++++++++- src/morphobricks_preprocessing/cli.py | 19 +- src/morphobricks_preprocessing/masks.py | 76 +++- src/morphobricks_preprocessing/notebook_ui.py | 58 ++- src/morphobricks_preprocessing/report.py | 19 +- src/morphobricks_preprocessing/run.py | 37 +- tests/test_axes.py | 19 +- tests/test_axis_check.py | 390 ++++++++++++++++++ tests/test_cli.py | 7 +- tests/test_header_axes.py | 16 + tests/test_notebook_ui.py | 6 +- tests/test_planes.py | 8 +- tests/test_run.py | 5 +- tests/test_zarr_export.py | 1 - 20 files changed, 918 insertions(+), 50 deletions(-) create mode 100644 tests/test_axis_check.py diff --git a/AGENTS.md b/AGENTS.md index 65f5dc42..2cb03754 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,7 +86,9 @@ src/morphobricks_preprocessing/ validation.py ValidationIssue (severity, code, message, paths, action), error()/warning() builders axes.py AxisMapping (role per dimension), prefill, check_mapping, review_axes (one mapping applied to every file, missing axes as singletons only when confirmed), - check_declared_axes (a brick's declared axes against the header, contract §16) + header_axis_issue/axis_disagreements (the axis check: confirmed roles + against the header's named axes, contract §2), check_declared_axes and + declared_axes_transposed (deprecated in v0.4.0, removed in v0.5.0) planes.py plane-by-plane reader (iter_planes, read_plane, read_sample_plane), table rows as XYC planes whose channels are read one at a time (iter_xyc_planes, XycPlane), memory estimate, @@ -225,6 +227,15 @@ functions the tests use. - X/Y/Z/C/T keep their role; - `S` (RGB samples) becomes C when there is no other C; - everything else (`I`, `Q`, …) is left unassigned: **never guess** depth vs time. +- **Axis check (v0.4.0, plan §12 question 45):** every TIFF's confirmed roles must agree + with the axes its header names (`axes.header_axis_issue`, position by position, on + `metadata.named_axes`). Otherwise one of three errors naming the files with their header + and confirmed axes: `axes_not_named` (plain stack), `axes_yx_swapped`, + `axes_contradict_header`. Such an image is **never read** (not transposed, not under a + contradicted declaration); the remedy is an OME-Zarr copy with the confirmed axes. + `AxisReview.axis_error_files` lists them; mask sizes and planes are checked by the + confirmed roles and skip those images (`run.mask_issues`). OME-Zarr images never have + these errors. - **Position-like axes** (`R` tile, `M` mosaic, `J` column, `K` row) cannot get a role. If they hold more than one plane: `unsupported_axis` error. - **Missing axes:** a file lacking some of the mapping's axes (e.g. `ZYX` next to `ZCYX`) is diff --git a/docs/contract.md b/docs/contract.md index ff3fc068..46e97211 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -45,6 +45,7 @@ channel order; an image without a C axis has one channel. | Rule | Detail | |---|---| | Prefill | Axis labels from metadata: `X`, `Y`, `Z`, `C`, `T` keep their role; the RGB samples axis `S` becomes C when there is no other `C`. Every other label is left unassigned for the user to choose. | +| Axis check (v0.4.0) | Every TIFF's confirmed roles are compared with the axes its own header names (`named_axes`, §16), position by position (`header_axis_issue`, `axis_disagreements`): a named axis keeps its letter as role, except that the RGB samples `S` may be C when the header has no C, and a length-1 axis may have no role. Any other result is one of three **errors**, each naming every file with its header axes and confirmed roles, asking the user to check them and recommending saving the images as OME-Zarr with the confirmed axes (then running preprocessing on the copies): `axes_not_named` (the metadata does not name an axis longer than 1: a plain stack `IYX`, `QYX`, `QQYX`, or shaped metadata with a placeholder `QYX`; a plain 2D `YX` page and RGB `YXS` are named, decision D-d), `axes_yx_swapped` (the named Y given role X and the named X role Y) and `axes_contradict_header` (any other role for a named axis: `ZYX` confirmed `TYX`, a named angle or lambda axis given a role; a named axis longer than 1 without a role is `unassigned_axis`, a position axis `unsupported_axis`). An image with one of them is never read: not transposed, not under a contradicted declaration, not with roles its metadata does not give. OME-Zarr images take their roles from their OME-NGFF axes and never have these errors. Since one mapping covers each TIFF arrangement, all files of one arrangement pass or fail together. When the roles agree with every header, the notebook and `run` say so ("Axes checked"). Before v0.4.0 such images were read: plain stacks with the user's roles, swaps transposed by the plane reader, contradictions under the declaration. | | Unsupported axes | Position-like axes (tile `R`, mosaic `M`, row `K`, column `J`) cannot have a role. If they hold more than one plane, the file is an error. | | Singleton axes | An axis of length 1 needs no role; its index is always 0. | | Missing axes | A file lacking axes the mapping has is read with those axes as length 1 only when the user confirmed this and only one arrangement fits; its coordinates for them are 0. | @@ -120,7 +121,7 @@ method is an offset and a scale, `(pixels − offset) / scale`. | Severity | Situations | Effect | |---|---|---| -| Error | Experiment design not confirmed (`design_not_confirmed`) or a repeat group holding two folders of one condition (`repeat_group_conflict`), when a design is used (§12); image type RGB for images that are not three channels of an integer type (`rgb_not_possible`, §13); mixed dtypes; images with different numbers of channels (`different_channel_counts`, §2) or different channel names at the same channel number (`channel_names_differ`, §2); unreadable TIFF (header or pixel data) or OME-Zarr folder in a leaf folder that is not excluded; OME-Zarr images, or converting TIFF images to OME-Zarr, without the `zarr` extra; OME-Zarr images whose declared axes do not fit the experiment's roles; every image excluded; more than one image series in a file; unsupported sample format or axis; missing/duplicate X or Y, duplicate roles, unassigned non-singleton axis, inconsistent mapping; plane that cannot be read within the memory limit, not even in blocks (for example a plane stored compressed in one piece); folder named `single` in another case, or beside other folders (§1); folders of one level differing only in case, anywhere in the experiment (`case_conflict`); folder or image names with outer whitespace or reading as a missing value (`invalid_name`, §1); images in one leaf folder with the same position identifier (§1); folder below the technical-repeat level; images in a folder that also holds subfolders (`tiff_outside_leaf`, §1); output folder inside the experiment; no usable planes; when masks are given, a masks folder missing or inside the experiment, mask names differing only in case, two masks of one type for an image, a mask whose name could belong to two images or types, or a mask that is unreadable, has axes other than Y, X and named Z/T (`mask_not_2d`), is of another size than its image, is not of integers or booleans, has other depth slices or time points than the image's other masks or, in `run`, than the image (§15) | Processing is blocked | +| Error | Experiment design not confirmed (`design_not_confirmed`) or a repeat group holding two folders of one condition (`repeat_group_conflict`), when a design is used (§12); image type RGB for images that are not three channels of an integer type (`rgb_not_possible`, §13); mixed dtypes; images with different numbers of channels (`different_channel_counts`, §2) or different channel names at the same channel number (`channel_names_differ`, §2); unreadable TIFF (header or pixel data) or OME-Zarr folder in a leaf folder that is not excluded; OME-Zarr images, or converting TIFF images to OME-Zarr, without the `zarr` extra; OME-Zarr images whose declared axes do not fit the experiment's roles; every image excluded; more than one image series in a file; unsupported sample format or axis; missing/duplicate X or Y, duplicate roles, unassigned non-singleton axis, inconsistent mapping; an image whose metadata does not name its axes, or whose confirmed roles swap Y and X or contradict its header (`axes_not_named`, `axes_yx_swapped`, `axes_contradict_header`, §2); plane that cannot be read within the memory limit, not even in blocks (for example a plane stored compressed in one piece); folder named `single` in another case, or beside other folders (§1); folders of one level differing only in case, anywhere in the experiment (`case_conflict`); folder or image names with outer whitespace or reading as a missing value (`invalid_name`, §1); images in one leaf folder with the same position identifier (§1); folder below the technical-repeat level; images in a folder that also holds subfolders (`tiff_outside_leaf`, §1); output folder inside the experiment; no usable planes; when masks are given, a masks folder missing or inside the experiment, mask names differing only in case, two masks of one type for an image, a mask whose name could belong to two images or types, or a mask that is unreadable, has axes other than Y, X and named Z/T (`mask_not_2d`), is of another size than its image (in `run`, its image's Y and X under the confirmed roles; not reported for an image with an axis error), is not of integers or booleans, has other depth slices or time points than the image's other masks or, in `run`, than the image (§15) | Processing is blocked | | Warning | Images named as masks of other images, or folders named like a masks folder holding them (`image_named_as_mask`, `mask_folder_in_experiment`, §1); intensity checks of the written table (`image_intensity_outlier`, `repeat_intensity_shift`, `saturated_image`, `no_intensity_spread`, §14), reported apart from the run's issues; files that are not images; empty folders; images excluded by the user, or excluded paths that match no image; non-finite pixels; different image widths or heights; different numbers of depth slices, time points or other planes; physical pixel sizes that differ or are known for only some images; unbalanced numbers of images; biological repeats missing from some conditions (§11); planes without finite pixels while others remain; channels whose central intensity range covers too few histogram bins (`narrow_histogram_range`, §4); planes over the memory limit that are read in blocks; images without a mask of a type other images have, masks matching no analysed image, files outside the mask-type folders (§15) | Processing continues; the item is reported | Every message states what happened in plain language, lists the affected paths, and @@ -315,8 +316,8 @@ masks/ masks folder, outside the experiment folder | Layout | Either one subfolder per mask type, each mirroring the experiment's folders (`//.../.tif`), or the experiment's folders with the type after the image name (`/.../_.tif`, as segmentation tools often write it, e.g. MorphoRelate's JUMP-Target-2 conversion); the two may be mixed. Any number of mask types. The label images of an OME-Zarr image (its OME-NGFF `labels` group, with the `zarr` extra) are masks too, of the type named by the label image; they are used whenever masks are asked for, with or without a masks folder. | | Pairing | A mask belongs to the image in the same leaf folder with the same `position_id` (§1), ignoring upper/lower case; its file is a TIFF (`.tif`, `.tiff`, optionally `.ome`, any case). Excluded images have no masks. | | Names | A mask type is named by its folder, by the suffix after the image name, or by its label image. A file that could belong to two images (an image name that starts another's, `a_b_cells.tif` beside images `a` and `a_b`) or fit both layouts is the error `ambiguous_mask`. Two names differing only in case are the error `mask_names_differ_in_case`; one image with two masks of one type (two extensions, or a file and a label image) is the error `duplicate_mask`. | -| Header checks | From the headers only, no pixels read (`check`, `run`, `preflight`): once its length-1 axes other than Y and X are removed (Y and X are kept whatever their length, so a mask one pixel high or wide, or a single pixel, is a 2D `YX` mask; a TIFF whose series has no Y or X but is one 2D page, such as a 1D array `tifffile` saved as `{"shape": [N]}` and reads as `X`, is that page, `YX` of 1 row; plan §12 question 44), a mask has Y and X and at most a depth axis Z and a time axis T, each named so in its own metadata (TIFF axes as `tifffile` reads them, e.g. an ImageJ hyperstack or OME-TIFF; OME-NGFF axes for a label image); any other or unnamed axis, such as a plain stack that could be depth or time, is `mask_not_2d` and is never guessed. A mask has the Y and X sizes of its image (`mask_size_mismatch`) and integer or boolean pixels (`mask_pixel_type`; a mask holding a negative label or a label above 2,147,483,647, the int32 limit every brick shares (§16, decision D-b), raises an error when its pixels are read); the masks of one image have the same depth slices and time points (`mask_planes_differ`); a mask that cannot be opened, or holds several series, is `unreadable_mask`, as is an OME-Zarr image's `labels` group that cannot be read or does not list its label images (path `/labels`), so its mask types never disappear unreported. All five are errors. | -| Planes | A mask belongs to the image's planes (issue #36): its Z and T lengths (1 for an axis it lacks) must equal the image's under the confirmed axis roles, checked by `run` (`mask_planes_mismatch`, error; `mask_plane_issues`). So a 2D mask belongs only to an image with one depth slice and one time point, and is refused for a Z-stack or a time-lapse, even as a projection (a decision for the owner if wanted later); a mask's plane at index `i` along Z or T belongs to the image's rows with `z = i` or `t = i`. `read_mask(..., z=, t=)` reads one plane and requires `z` and `t` for a mask with those axes. `check` and the `masks` command have no confirmed roles and do not compare with the image. | +| Header checks | From the headers only, no pixels read (`check`, `run`, `preflight`): once its length-1 axes other than Y and X are removed (Y and X are kept whatever their length, so a mask one pixel high or wide, or a single pixel, is a 2D `YX` mask; a TIFF whose series has no Y or X but is one 2D page, such as a 1D array `tifffile` saved as `{"shape": [N]}` and reads as `X`, is that page, `YX` of 1 row; plan §12 question 44), a mask has Y and X and at most a depth axis Z and a time axis T, each named so in its own metadata (TIFF axes as `tifffile` reads them, e.g. an ImageJ hyperstack or OME-TIFF; OME-NGFF axes for a label image); any other or unnamed axis, such as a plain stack that could be depth or time, is `mask_not_2d` and is never guessed. A mask has the Y and X sizes of its image (`mask_size_mismatch`; `check` and `preflight` compare with the Y and X the image's header names, and not for a plain stack, whose Y and X are not named; `run` compares with the image's Y and X under the confirmed roles, `mask_size_issues`, and leaves out the masks of an image with an axis error (§2), which is never read — before v0.4.0 `run` compared with the header's letters, so a swapped image gave a misleading size error) and integer or boolean pixels (`mask_pixel_type`; a mask holding a negative label or a label above 2,147,483,647, the int32 limit every brick shares (§16, decision D-b), raises an error when its pixels are read); the masks of one image have the same depth slices and time points (`mask_planes_differ`); a mask that cannot be opened, or holds several series, is `unreadable_mask`, as is an OME-Zarr image's `labels` group that cannot be read or does not list its label images (path `/labels`), so its mask types never disappear unreported. All five are errors. | +| Planes | A mask belongs to the image's planes (issue #36): its Z and T lengths (1 for an axis it lacks) must equal the image's under the confirmed axis roles, checked by `run` (`mask_planes_mismatch`, error; `mask_plane_issues`; not for an image with an axis error, §2). So a 2D mask belongs only to an image with one depth slice and one time point, and is refused for a Z-stack or a time-lapse, even as a projection (a decision for the owner if wanted later); a mask's plane at index `i` along Z or T belongs to the image's rows with `z = i` or `t = i`. `read_mask(..., z=, t=)` reads one plane and requires `z` and `t` for a mask with those axes. `check` and the `masks` command have no confirmed roles and do not compare with the image. | | Folder | The masks folder must exist (`masks_folder_not_found`) and be outside the experiment folder (`masks_inside_experiment`), where its folders would be read as labels. Both errors. | | Warnings | `mask_missing` (images without a mask of a type that other images have: their objects of that type cannot be measured), `mask_without_image` (files matching no analysed image), `mask_outside_type_folder` (files directly in the masks folder that are not TIFF masks). | | Run | `run_experiment(..., masks_folder=...)` stops on a mask error and records the masks in the manifest's `masks` key (§9). Without masks, the run and its outputs are unchanged. | @@ -407,7 +408,7 @@ instead of keeping copies. | Plain-TIFF resolution (D-c) | `read_physical_sizes` | A TIFF's resolution tags give a pixel size only with OME or ImageJ metadata and a unit; a plain TIFF (scanner or image-editor dpi) is uncalibrated, never converted from inches or centimetres. | | Labels (D-b) | `check_labels(pixels, name)`, `LARGEST_LABEL` (`masks`) | A mask holds whole-number labels from 0 (background) to `LARGEST_LABEL` = 2,147,483,647: MorphoBricks labels are int32 in every brick. A negative label, a label above the limit, or a pixel type that is not integer or boolean is refused with a `ValueError` naming the mask and the value. `read_mask`, `piece_counts`, `pair_counts` and `scan_masks` call it. Since D86 the limit is int32; before, this brick refused only labels above 2^63 − 1 (int64), so a `uint32` or wider mask with a label of 3,000,000,000 is now refused. | | Header axes (D-d) | `header_axes(meta)`, `named_axes(meta)`, `series_axes_from_metadata(series)`, `PLACEHOLDER_AXES`, `OPEN_AXIS` (`metadata`) | The axes an image's own metadata names, as `tifffile` reads them with length-1 axes left out (`tifffile` keeps Y and X even of length 1, so an image one pixel high is `YX`): OME, ImageJ and `tifffile` "shaped" metadata (`{"axes": "CYX"}`, decision D-d), a plain 2D `YX` (or `YXS` RGB) TIFF, and an OME-Zarr image's OME-NGFF axes. None when an axis is a placeholder (`I`: a plain stack of pages; `Q`: shaped without axes, or an OME-NGFF axis of another type), or the image does not open: the file does not say which dimension is depth, time or channel. `named_axes` gives the rule position by position, with `?` (`OPEN_AXIS`) for a dimension the metadata does not name: a placeholder, and, in a plain stack of pages without axis metadata (`IYX`, `QYX`, `QQYX`; `TiffMetadata.axes_from_metadata` False, from `series_axes_from_metadata`; True, the strict reading, for a header built by hand), the `Y` and `X` too, which only say where each page's rows and columns are stored (a (Y, X, C) array written without metadata reads as `QYX`). `header_axes` gives axes exactly when nothing is open, and `check_declared_axes` then accepts those axes, them with `Y` and `X` swapped, and (without a C axis) `S` declared C; nothing else. | -| Declared axes | `check_declared_axes(meta, declared)`, `declared_axes_transposed(meta, declared)` (`axes`) | Axes a brick or user declares for an image or mask (`image_axes="CYX"`) may name what the header leaves open, never contradict it: one letter per header axis (any case), roles X, Y, Z, C, T each at most once, with `Y` and `X` declared when the header has them (a plain stack too); an axis the metadata names (`named_axes`) keeps its letter, except that the named `Y` and `X` may be declared swapped (header `CYX` declared `CXY`, `ZYX` declared `ZXY`): the image is read transposed, and `declared_axes_transposed` returns True so the caller can record it (owner decision of 31 October 2026; it raises `ValueError` for a declaration that does not fit); any other disagreement is refused; a dimension it leaves open may be declared any role, so a plain `QYX` stack may be declared `YXC` as well as `CYX` (since v0.3.4; before, its positional `Y` and `X` were binding and a placeholder could not be declared X or Y), while an ImageJ `ZYX` declared `CYX`, an OME `CYX` declared `YXC` or a shaped `CYX` declared `YXC` are refused; the RGB samples `S` may be declared C when the header has no C; any other header axis (tile `R`, ...) cannot be declared. Otherwise the error `declared_axes_mismatch` naming the image and both axes (`unreadable_tiff` or `unreadable_ome_zarr` when the header does not open). | +| Declared axes | `check_declared_axes(meta, declared)`, `declared_axes_transposed(meta, declared)` (`axes`) | **Deprecated in v0.4.0, removed in v0.5.0** (they warn with a `DeprecationWarning`): bricks take no axis declarations and read the files of a preprocessing run whose axes were checked (§2); nothing is read transposed. Until then: axes a brick or user declares for an image or mask (`image_axes="CYX"`) may name what the header leaves open, never contradict it: one letter per header axis (any case), roles X, Y, Z, C, T each at most once, with `Y` and `X` declared when the header has them (a plain stack too); an axis the metadata names (`named_axes`) keeps its letter, except that the named `Y` and `X` may be declared swapped (header `CYX` declared `CXY`, `ZYX` declared `ZXY`): the image is read transposed, and `declared_axes_transposed` returns True so the caller can record it (owner decision of 31 October 2026; it raises `ValueError` for a declaration that does not fit); any other disagreement is refused; a dimension it leaves open may be declared any role, so a plain `QYX` stack may be declared `YXC` as well as `CYX` (since v0.3.4; before, its positional `Y` and `X` were binding and a placeholder could not be declared X or Y), while an ImageJ `ZYX` declared `CYX`, an OME `CYX` declared `YXC` or a shaped `CYX` declared `YXC` are refused; the RGB samples `S` may be declared C when the header has no C; any other header axis (tile `R`, ...) cannot be declared. Otherwise the error `declared_axes_mismatch` naming the image and both axes (`unreadable_tiff` or `unreadable_ome_zarr` when the header does not open). | | Mask files | `read_mask_file(path, z=, t=)` (`masks`) | Reads one plane of a TIFF mask or of an OME-Zarr label image (`.zarr/labels/`) given by its path, with `read_mask`'s rules (§15): its axes are its own metadata's, Y and X plus at most a Z and a T axis named so, length-1 axes other than Y and X left out (a mask one pixel high or wide is read as a 2D array, plan §12 question 44); any other or unnamed dimension (a plain stack, `IYX`; a `C` axis) is refused, the message giving the header's axes and shape and the dimension that is not named, never taken for depth or time; `z` and `t` are required for a mask with that axis; integer or boolean pixels; labels checked with `check_labels`. A `ValueError` names the file and the problem. | | Identity (D-e) | `FOV_KEY`, `check_identity_text(value, field)`, `check_plane_index(value, axis)`, `check_identity_case(keys)`, `MISSING_TEXT`, `IdentityError` (`discovery`) | `FOV_KEY` = (`condition`, `biological_repeat`, `technical_repeat`, `position_id`), the position key of §1. `check_identity_text` accepts a label or `position_id` that is text without leading or trailing whitespace and not `""`, `NA`, `nan`, `none`, `null`, `n/a` or `` in any case (`MISSING_TEXT`); `check_plane_index` accepts None or an integer of at least 0, a Python `int` or a NumPy integer (returned as a plain `int`), and refuses everything else (bool, float, `"1"`) without converting it. Both raise `IdentityError` (a `ValueError`) naming the field and the value. Discovery applies the text rule to every folder and image name (`invalid_name`) and refuses names of one level that differ only in case anywhere in the experiment (`case_conflict`, §1). `position_id` keeps its own scope: unique within its technical repeat, ignoring case (`duplicate_position_id`). `check_identity_case` applies the same case rule to identities a brick reads from a table, one key per row in `FOV_KEY` order (or the three labels): a label of one level spelled two ways anywhere in the run, or two `position_id` values of one technical repeat equal ignoring case, raise `IdentityError` naming both spellings; the same key may repeat (several planes of one position). | | Sources of a run | `check_sources(manifest, experiment_root, relative_paths, image_axes=, identities=)`, `source_fingerprint`, `file_sha256` (`manifest`) | The checks a brick makes before using a run's table for some images (formerly CellPatcher `check_images` and MorphoFeature `histogram_fit`). Each image must be in the manifest (`images_not_in_manifest`), still exist (`images_missing`), and be unchanged by `source_fingerprint`, its size and modification time, for an OME-Zarr folder its files' total size and newest time, so a rewritten chunk counts (`images_changed`). Its recorded `position_id` must be the one its name gives; with `identities` (`FOV_KEY` order), its labels and `position_id` must be the caller's (`identity_differs`; `position_id_not_recorded` for a manifest written before positions were recorded). With `image_axes`, the caller's axes must give each axis the role recorded for the file (`FileEntry.roles`, or the run's mapping resolved for the header in older manifests); an axis without a role (length 1) fits any letter (`axis_mapping_mismatch`). All errors, naming their images; the table's own hash is checked with `file_sha256` against `csv_sha256`. | diff --git a/docs/plan.md b/docs/plan.md index 2b3448d8..0f7a37e6 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -1035,3 +1035,22 @@ Raised during review; to be decided before the milestones that need them: 1D array keeps the header axes `X` (`header_axes`, `check_declared_axes`), so the folder check suggests a mapping without Y and the run refuses it with `missing_xy_axis`; reading it as one row too would change the shared header-axes rule (§16).* +45. How are the axes checked, now that every brick reads preprocessing's run (MorphoBricks + axes redesign, design note of 1 November 2026)? Until now a confirmed mapping was never + compared with the axes the header names: a plain stack was read with the user's roles, + a Y/X swap read transposed and a contradiction (header `ZYX` confirmed `TYX`) read + under the declaration, with no trace but the manifest's `roles`. + *Decided (1 November 2026, owner's final design and preprocessing sequence): any axis + problem is an error, and preprocessing is the only place axes are checked. `review_axes` + compares each TIFF's confirmed roles with its header (`header_axis_issue`, built from + the declared-axes rule): `axes_not_named` (a plain stack whose metadata does not name + an axis longer than 1), `axes_yx_swapped` and `axes_contradict_header`, each naming the + files with their header and confirmed axes, asking the user to check them and + recommending saving them as OME-Zarr. Nothing is read transposed or under a + contradicted declaration. When the axes are correct, the notebook and the CLI say so. + The notebook marks each role that disagrees with the header before confirmation. + Masks are compared with their image's Y and X by the confirmed roles (they were + compared with the header's letters), and not at all for an image with an axis error. + `check_declared_axes` and `declared_axes_transposed` are deprecated, to be removed in + v0.5.0. Breaking for the next release (v0.4.0): experiments of plain stacks, or run + with roles that swap or contradict the header, no longer run as they are.* diff --git a/docs/user_guide.md b/docs/user_guide.md index 7f532ecd..3e8ae5d2 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -143,6 +143,9 @@ that measure objects; the optional `masks` command (§5) also tells how they rel ## 3. Supported images - **TIFF files:** `.tif` and `.tiff` (any case), including OME-TIFF and ImageJ hyperstacks. + Their metadata must name every axis with more than one plane, and the axis roles must + agree with it (§4 step 2). A plain stack of pages without axis metadata, or a file whose + axes you know better than its header, is saved as OME-Zarr with the right axes first. - **OME-Zarr images:** OME-NGFF 0.4 and 0.5 (needs the `zarr` extra; 0.5 needs Python 3.11 or newer). Their axes (time, channel, z, y, x) come from their metadata. Only the full-resolution level of a single image is read; a folder holding several images (a @@ -238,8 +241,17 @@ results or comparing another grouping. channels, a message says so and step 3 stays locked. Each picture should be a normal image of your sample; stripes or a smear mean X or Y is on the wrong dimension. Brightness is stretched for display only. + - The roles are checked against the axes each image's metadata names. A role that + disagrees with the header, or an axis the metadata does not name (a plain stack of + pages), is marked in amber under its dropdown, and the messages give the error + (`axes_not_named`, `axes_yx_swapped` or `axes_contradict_header`, §8) naming the + images with their header axes and your roles. Such images are never read: check the + axes, and if your roles are right, save the images as OME-Zarr with them and run + preprocessing on the copies. - If the preview looks right, press **Yes, the preview looks right**, then click - **Confirm axis roles**. Changing a role afterwards asks you to check the preview again. + **Confirm axis roles**. When the roles agree with every header, a green card says + **Axes checked: they are correct for every image**. Changing a role afterwards asks + you to check the preview again. The confirmed roles are locked until you click **Change axis roles**. - If some images lack a dimension the others have (often a single channel), a yellow box lists them; press **Yes, they have a single plane there** if that is the case. @@ -375,6 +387,12 @@ morphobricks-preprocessing run /data/exp1 --axes ZCYX `T`, or `-` for a length-1 dimension without a role. Roles are never taken from metadata without being stated. +Both commands check the axes (§4, step 2): `check` with the suggested roles, which are the +headers' own, so only images whose metadata does not name their axes fail there +(`axes_not_named`); `run` with the roles given in `--axes`. They print **Axes checked: +every image names its axes in its metadata, and the roles agree with them** when all is +well, or the errors naming the images, and stop (exit status 1). + `--image-type fluorescence` or `--image-type rgb` gives the image type (§4, step 2). Without it, `run` uses the type suggested from the image headers and prints it, with why it was suggested and how to choose the other one; `check` prints the suggestion too. `rgb` @@ -555,7 +573,10 @@ continues. | `inconsistent_dimensions`, `inconsistent_mapping` | Error | Images have their dimensions in a different order. Export them the same way or process separately. | | `missing_axes` | Error | Some images lack a dimension. Confirm they have one plane along it (step 2), or export them again. | | `ambiguous_missing_axes` | Error | It is unclear which dimension an image lacks. Export it again. | -| `declared_axes_mismatch` | Error (other bricks) | The axes a brick was told for an image or mask contradict the axes in its metadata (for example a depth stack declared as channels). Declare the axes the metadata gives, or export the image again with the right axes. | +| `axes_not_named` | Error | The metadata of some images does not name their axes (a plain stack of pages, `IYX` or `QYX`): which dimension is depth, time or channel cannot be checked, so they are not read. The message names each image with its axes and the roles you confirmed. Check these axes; if they are right, save the images as OME-Zarr with them, then run preprocessing on the copies. | +| `axes_yx_swapped` | Error | The roles you confirmed give the Y and X the image header names the other way round. The images are not read transposed. Check the axes: if the header is right, confirm its axes; if your roles are right, save the images as OME-Zarr with them and run preprocessing on the copies. | +| `axes_contradict_header` | Error | The roles you confirmed contradict the axes the image header names (for example the header says depth, Z, and you chose time, T). The images are not read under a contradicted declaration. Check the axes as for `axes_yx_swapped`. | +| `declared_axes_mismatch` | Error (other bricks; deprecated, removed in v0.5.0) | The axes a brick was told for an image or mask contradict the axes in its metadata (for example a depth stack declared as channels). Declare the axes the metadata gives, or export the image again with the right axes. | | `missing_xy_axis`, `duplicate_axis_role` | Error | X/Y missing or a role used twice. Correct the axis roles. | | `unassigned_axis` | Error | A dimension with several planes has no role. Choose Z, C or T. | | `unsupported_axis` | Error | A position or tile dimension. Export each position separately. | diff --git a/notebooks/MorphoBricks_Preprocessing.ipynb b/notebooks/MorphoBricks_Preprocessing.ipynb index 82dda79b..55e4c979 100644 --- a/notebooks/MorphoBricks_Preprocessing.ipynb +++ b/notebooks/MorphoBricks_Preprocessing.ipynb @@ -292,11 +292,14 @@ "Run the cell below, then:\n", "\n", "1. Check the suggested role in each box. The line under each box says what the choice means.\n", + " An amber line marks a role that disagrees with the axes the image header names, or an axis\n", + " the metadata does not name (a plain stack of pages): such images are not read.\n", "2. Click **Show preview** and move the sliders. Each picture should look like a normal image of your\n", " sample. Stripes or a smeared picture mean X or Y is on the wrong dimension.\n", "3. If the preview looks right, press **Yes, the preview looks right** in the yellow box under the\n", " picture. Changing a role afterwards asks you to check the preview again.\n", - "4. Press **Confirm axis roles**. The roles are locked until you press **Change axis roles**.\n", + "4. Press **Confirm axis roles**. A green card says **Axes checked** when the roles agree with every\n", + " image header. The roles are locked until you press **Change axis roles**.\n", "5. Under **Image type**, check the suggestion: **Fluorescence** (each channel is a separate stain\n", " or marker) or **RGB** (colour images, such as brightfield or histology stains). RGB is suggested\n", " when the images are stored as RGB colour. If the suggestion is right, press\n", @@ -467,6 +470,11 @@ "\n", "**\"No role is chosen for the axis\"**: a dimension with several planes needs Z, C or T in step 2.\n", "\n", + "**\"… do not name their axes in their metadata\"**, **\"… swap the Y and X their header names\"**\n", + "or **\"… contradict their header\"**: the images are not read. Check the axes in step 2. If the\n", + "header is right, choose its roles; if your roles are right, save the images as OME-Zarr with them\n", + "and run preprocessing on the copies.\n", + "\n", "**\"The images have different numbers of channels\"**: each row of the table holds every channel of\n", "a plane, so all images need the same channels. Process the others as a separate experiment, or\n", "exclude them; if a dimension got the role C by mistake, correct it in step 2.\n", diff --git a/src/morphobricks_preprocessing/__init__.py b/src/morphobricks_preprocessing/__init__.py index 4c3ab833..b5c57d19 100644 --- a/src/morphobricks_preprocessing/__init__.py +++ b/src/morphobricks_preprocessing/__init__.py @@ -12,6 +12,7 @@ # deprecated stand-ins for one version (see _moved). from . import _moved, csv_io, dtypes, histogram, normalization, pooling, records # noqa: F401 from .axes import ( + AXIS_ERROR_CODES, AxisMapping, AxisReview, axis_length_ranges, @@ -20,6 +21,7 @@ declared_axes_transposed, declared_mapping, file_mapping_for, + header_axis_issue, prefill_experiment_mapping, prefill_mapping, reference_axes, @@ -121,6 +123,7 @@ check_labels, find_masks, mask_plane_issues, + mask_size_issues, read_mask, read_mask_file, ) @@ -208,6 +211,7 @@ def __dir__() -> list[str]: __version__ = "0.0.0+unknown" __all__ = [ + "AXIS_ERROR_CODES", "CONSISTENCY_CODES", "DEFAULT_MEMORY_LIMIT", "DEFAULT_THRESHOLDS", @@ -324,6 +328,7 @@ def __dir__() -> list[str]: "group_summaries", "has_errors", "header_axes", + "header_axis_issue", "image_type_issues", "inspect_experiment", "inspect_image", @@ -338,6 +343,7 @@ def __dir__() -> list[str]: "images_named_as_masks", "mask_name_issues", "mask_plane_issues", + "mask_size_issues", "mask_thresholds", "min_max", "named_axes", diff --git a/src/morphobricks_preprocessing/axes.py b/src/morphobricks_preprocessing/axes.py index 8cf71c5b..c1f8e49f 100644 --- a/src/morphobricks_preprocessing/axes.py +++ b/src/morphobricks_preprocessing/axes.py @@ -5,7 +5,10 @@ experiment. :func:`review_axes` checks the confirmed mapping against the rules of ``docs/contract.md`` §2 and resolves it for every file, including files that lack a dimension the others have (the ``missing_axes`` case of -:mod:`~morphobricks_preprocessing.metadata`). +:mod:`~morphobricks_preprocessing.metadata`). It also checks every TIFF's confirmed roles +against the axes its header names (:func:`header_axis_issue`): an image whose metadata +does not name its axes, or whose confirmed roles swap or contradict them, is an error and +is never read; the remedy is to save it as OME-Zarr with the confirmed axes. Nothing here reads pixels; the preview plane is read by :func:`~morphobricks_preprocessing.planes.read_sample_plane`. @@ -13,6 +16,7 @@ from __future__ import annotations +import warnings from collections import Counter from collections.abc import Iterable, Mapping from dataclasses import dataclass, field @@ -64,6 +68,20 @@ {"missing_axes", "inconsistent_dimensions", "different_channel_counts"} ) +AXES_NOT_NAMED = "axes_not_named" +"""A TIFF whose metadata does not name an axis longer than 1 (a plain stack).""" + +AXES_YX_SWAPPED = "axes_yx_swapped" +"""Confirmed roles that give the header's named Y and X the other way round.""" + +AXES_CONTRADICT_HEADER = "axes_contradict_header" +"""Confirmed roles that give a named axis another role than the header's.""" + +AXIS_ERROR_CODES = (AXES_NOT_NAMED, AXES_YX_SWAPPED, AXES_CONTRADICT_HEADER) +"""The three errors of an image whose confirmed axes cannot be checked against, or do not +agree with, the axes its header names (contract §2). Such an image is never read: it is +saved again as OME-Zarr with the confirmed axes, and preprocessing runs on the copy.""" + @dataclass(frozen=True) class AxisMapping: @@ -209,6 +227,26 @@ def ok(self) -> bool: """True if no issue blocks processing.""" return not has_errors(self.issues) + @property + def axis_errors(self) -> tuple[ValidationIssue, ...]: + """The ``axes_not_named``, ``axes_yx_swapped`` and ``axes_contradict_header`` errors.""" + return tuple(i for i in self.issues if i.code in AXIS_ERROR_CODES) + + @property + def axis_error_files(self) -> tuple[str, ...]: + """Images with one of the three axis errors (:data:`AXIS_ERROR_CODES`), sorted.""" + return tuple(sorted({path for issue in self.axis_errors for path in issue.paths})) + + @property + def only_axis_errors(self) -> bool: + """True if there are axis errors and no other error. + + The confirmed mapping then fits every file, so the images can be saved again as + OME-Zarr with it. + """ + errors = [i for i in self.issues if i.is_error] + return bool(errors) and all(i.code in AXIS_ERROR_CODES for i in errors) + def combined_issues(self, metadata: MetadataReport) -> tuple[ValidationIssue, ...]: """Merge the metadata issues with this review's, errors first. @@ -244,6 +282,11 @@ def prefill_mapping(axes: str) -> AxisMapping: def check_declared_axes(meta: TiffMetadata, declared: str) -> ValidationIssue | None: """Check the axes a brick or user declares for an image against its header. + .. deprecated:: 0.4.0 + Bricks no longer take axis declarations: they read the files of a preprocessing + run whose axes were checked (:func:`review_axes`, contract §2). This function will + be removed in v0.5.0. + One rule for every brick that is told an image's (or mask's) axes, for example ``image_axes="CYX"`` (MorphoBricks D86): a declaration can name what the header leaves open, never contradict it. Only what the metadata names binds @@ -280,6 +323,11 @@ def check_declared_axes(meta: TiffMetadata, declared: str) -> ValidationIssue | axes (or ``unreadable_tiff`` / ``unreadable_ome_zarr`` when its header could not be read); None when the declaration fits the header. """ + _deprecated("check_declared_axes") + return _check_declared_axes(meta, declared) + + +def _check_declared_axes(meta: TiffMetadata, declared: str) -> ValidationIssue | None: if not meta.readable: code = "unreadable_ome_zarr" if meta.is_ome_zarr else "unreadable_tiff" return error( @@ -305,6 +353,10 @@ def check_declared_axes(meta: TiffMetadata, declared: str) -> ValidationIssue | def declared_axes_transposed(meta: TiffMetadata, declared: str) -> bool: """Return True when a declaration fits the header only with its named Y and X swapped. + .. deprecated:: 0.4.0 + Nothing is read transposed any more: a confirmed Y/X swap is the error + ``axes_yx_swapped`` (:func:`review_axes`). This function will be removed in v0.5.0. + :func:`check_declared_axes` accepts a declaration that differs from the axes the metadata names only by swapping Y and X (header ``CYX`` declared ``CXY``): the image is read transposed. A brick that accepts such a declaration records it with this function. @@ -329,7 +381,8 @@ def declared_axes_transposed(meta: TiffMetadata, declared: str) -> bool: When the declaration does not fit the header at all (:func:`check_declared_axes` gives an error), with that error's message. """ - issue = check_declared_axes(meta, declared) + _deprecated("declared_axes_transposed") + issue = _check_declared_axes(meta, declared) if issue is not None: raise ValueError(issue.message) named = named_axes(meta) @@ -363,6 +416,151 @@ def _declared_axes_problem(header: str, named: str, declared: str) -> str | None return None +def _deprecated(name: str) -> None: + warnings.warn( + f"morphobricks_preprocessing.{name} is deprecated and will be removed in v0.5.0: " + "bricks read the files of a preprocessing run whose axes were checked, and take no " + "axis declarations.", + DeprecationWarning, + stacklevel=3, + ) + + +def header_axis_issue(meta: TiffMetadata, mapping: AxisMapping) -> str | None: + """Compare an image's confirmed roles with the axes its header names (contract §2). + + The one rule of the axis check: a TIFF is read only when its metadata names every axis + longer than 1 and the confirmed roles agree with those names. Position by position + (:func:`.metadata.named_axes`): + + - an axis the metadata leaves open (a placeholder ``I``/``Q``, and the ``Y`` and ``X`` + of a plain stack of pages) with more than one plane: ``axes_not_named``; an open + axis of length 1 is no problem; + - a named axis with its own letter as role, the RGB samples ``S`` as C when the header + has no C, and a length-1 axis without a role agree; + - the named Y given role X and the named X role Y: ``axes_yx_swapped``; + - any other role for a named axis: ``axes_contradict_header``. That includes a Z read + as T, and a named axis that is not a role (angle ``A``, lambda ``E``, ...) given one. + A named axis longer than 1 without a role is :func:`check_mapping`'s + ``unassigned_axis``, and a position axis (``R``, ``M``, ``J``, ``K``) its + ``unsupported_axis``, so they are left to those. + + OME-Zarr images name their axes in their OME-NGFF metadata, which gives their roles + (:func:`declared_mapping`): they never have one of these problems. + + Parameters + ---------- + meta + The image's header. + mapping + The image's own mapping (:attr:`AxisReview.file_mappings`, :func:`file_mapping_for`). + + Returns + ------- + str or None + ``"axes_not_named"``, ``"axes_contradict_header"`` or ``"axes_yx_swapped"`` (the + first of these when several apply), or None when the roles agree with the header. + """ + found = set(axis_disagreements(meta, mapping).values()) + # A file that cannot be checked is that first; a contradiction is more than a swap. + for code in (AXES_NOT_NAMED, AXES_CONTRADICT_HEADER, AXES_YX_SWAPPED): + if code in found: + return code + return None + + +def axis_disagreements(meta: TiffMetadata, mapping: AxisMapping) -> dict[int, str]: + """Return, for each dimension whose role disagrees with the header, the axis error code. + + The position-by-position rule of :func:`header_axis_issue`, for showing which axis is + wrong (the notebook marks its role dropdown before the roles are confirmed). + + Parameters + ---------- + meta + The image's header. + mapping + The image's own mapping. + + Returns + ------- + dict + ``{dimension: code}``, with ``"axes_not_named"``, ``"axes_yx_swapped"`` or + ``"axes_contradict_header"``; empty for an OME-Zarr image, an unreadable one, or + roles that agree with the header. + """ + if not meta.readable or meta.is_ome_zarr or len(mapping.roles) != len(meta.axes): + return {} + found: dict[int, str] = {} + named = named_axes(meta) + for index, (letter, role, length) in enumerate( + zip(named, mapping.roles, meta.shape, strict=True) + ): + if letter == OPEN_AXIS: + if length > 1: + found[index] = AXES_NOT_NAMED + continue + if role == letter or (letter == "S" and role == "C" and "C" not in meta.axes): + continue + if role is None or letter in POSITION_AXES: + continue + found[index] = ( + AXES_YX_SWAPPED if letter in "YX" and role in "YX" else AXES_CONTRADICT_HEADER + ) + return found + + +def _axis_issues(found: Mapping[str, list[tuple[TiffMetadata, AxisMapping]]]) -> list: + """Build one error per axis problem, naming every file with its header and roles.""" + issues = [] + for code in AXIS_ERROR_CODES: + files = found.get(code) + if not files: + continue + groups: dict[tuple[str, str], list[str]] = {} + for meta, mapping in files: + groups.setdefault((meta.axes, mapping.roles_string), []).append(meta.relative_path) + details = "; ".join( + f"header {axes}, confirmed {roles}: {_some(paths)}" + for (axes, roles), paths in sorted(groups.items()) + ) + n = plural(len(files), "image") + if code == AXES_NOT_NAMED: + message = ( + f"{n} do not name their axes in their metadata ({details}). Which dimension " + "is depth, time or channel cannot be checked, so they are not read." + ) + action = _SAVE_AS_OME_ZARR + elif code == AXES_YX_SWAPPED: + message = ( + f"The axes confirmed for {n} swap the Y and X their header names ({details}). " + "They are not read transposed." + ) + action = _SAVE_AS_OME_ZARR + _OR_HEADER + else: + message = ( + f"The axes confirmed for {n} contradict their header ({details}). They are " + "not read under a contradicted declaration." + ) + action = _SAVE_AS_OME_ZARR + _OR_HEADER + paths = sorted(meta.relative_path for meta, _ in files) + issues.append(error(code, message, paths, action)) + return issues + + +_SAVE_AS_OME_ZARR = ( + "Check these axes. If the confirmed axes are right, save the images as OME-Zarr with " + "them, so that every axis is named, then run preprocessing on the copies." +) +_OR_HEADER = " If the header is right, confirm the header's axes instead." + + +def _some(paths: list[str], shown: int = 3) -> str: + paths = sorted(paths) + text = ", ".join(paths[:shown]) + return text if len(paths) <= shown else f"{text} and {len(paths) - shown} more" + + def reference_axes(files: Iterable[TiffMetadata]) -> str | None: """Return the arrangement the experiment mapping is defined on. @@ -608,8 +806,13 @@ def review_axes( from :func:`check_mapping`, files that lack dimensions without confirmation (``missing_axes``) or in an ambiguous way (``ambiguous_missing_axes``), files the mapping does not fit - (``inconsistent_mapping``), and files with different numbers of channels - along the C axis (``different_channel_counts``; 1 without one). + (``inconsistent_mapping``), files with different numbers of channels + along the C axis (``different_channel_counts``; 1 without one), and the + axis check of every TIFF against its header (:func:`header_axis_issue`): + ``axes_not_named``, ``axes_yx_swapped`` and ``axes_contradict_header``, + each naming its files with their header axes and confirmed roles. These + three are errors: such an image is never read transposed or under a + contradicted declaration. """ if isinstance(files, MetadataReport): files = files.files @@ -708,6 +911,20 @@ def review_axes( } ) ) + # Each file's confirmed roles against the axes its own header names. Files the mapping + # does not fit already have their error; OME-Zarr images take their roles from their + # metadata. + found: dict[str, list[tuple[TiffMetadata, AxisMapping]]] = {} + for f in readable: + resolved = file_mappings.get(f.relative_path) + if resolved is None: + resolved = mapping.for_axes(f.axes) if f.relative_path in unconfirmed else None + if resolved is None: + continue + code = header_axis_issue(f, resolved) + if code is not None: + found.setdefault(code, []).append((f, resolved)) + issues.extend(_axis_issues(found)) return AxisReview(mapping=mapping, file_mappings=file_mappings, issues=tuple(issues)) diff --git a/src/morphobricks_preprocessing/cli.py b/src/morphobricks_preprocessing/cli.py index 7c7f9a49..a2128eb7 100644 --- a/src/morphobricks_preprocessing/cli.py +++ b/src/morphobricks_preprocessing/cli.py @@ -57,6 +57,7 @@ ) from .planes import DEFAULT_MEMORY_LIMIT, format_bytes from .report import ( + axis_check_summary, balance_summary, design_summary, format_issues, @@ -363,20 +364,28 @@ def _check(args: argparse.Namespace) -> int: "Experiment design: not confirmed yet. 'run' writes experiment_design.csv to " "the output folder for you to confirm." ) + axis_errors = () if pre.suggested_mapping is not None: + review = review_axes(pre.metadata, pre.suggested_mapping) + # The suggested roles are the header's own, so only an image that does not name its + # axes can fail the axis check here; 'run' checks the roles given with --axes. + axis_errors = review.axis_errors + print() + print(axis_check_summary(review)) + if axis_errors: + print() + print(format_issues(axis_errors)) print() print( f"Suggested option for 'run': --axes {pre.suggested_mapping.roles_string} " f"(axes in the files: {pre.suggested_mapping.axes})" ) - type_review = review_image_type( - pre.metadata, review_axes(pre.metadata, pre.suggested_mapping) - ) + type_review = review_image_type(pre.metadata, review) print( f"Suggested image type: {type_review.suggested}, because {type_review.reason}. " "'run' uses it unless you give --image-type." ) - return 0 if pre.ok else 1 + return 0 if pre.ok and not axis_errors else 1 def _masks(args: argparse.Namespace) -> int: @@ -476,6 +485,8 @@ def _run(args: argparse.Namespace) -> int: print(mapping_summary(mapping, pre.metadata.files)) print() review = review_axes(pre.metadata, mapping, missing_as_singleton=args.missing_as_singleton) + print(axis_check_summary(review)) + print() type_review = review_image_type(pre.metadata, review) image_type = args.image_type or type_review.suggested print(f"Image type: {image_type}") diff --git a/src/morphobricks_preprocessing/masks.py b/src/morphobricks_preprocessing/masks.py index 1e7be877..26f28879 100644 --- a/src/morphobricks_preprocessing/masks.py +++ b/src/morphobricks_preprocessing/masks.py @@ -41,7 +41,7 @@ from .axes import AxisMapping from .discovery import DiscoveryResult, is_system_name, is_tiff_name, position_id -from .metadata import TiffMetadata +from .metadata import TiffMetadata, named_axes from .validation import ValidationIssue, error, has_errors, plural, warning from .zarr_input import ( NgffError, @@ -666,7 +666,8 @@ def _header_issues( files[index] = MaskFile( mask.image, mask.mask_type, mask.path, mask.source, shape, dtype, axes ) - if _yx(axes, shape) != _image_yx(images[mask.image]): + expected = _image_yx(images[mask.image]) + if expected is not None and _yx(axes, shape) != expected: problems.setdefault("mask_size_mismatch", []).append(mask.path) if dtype.kind not in "biu": problems.setdefault("mask_pixel_type", []).append(mask.path) @@ -780,6 +781,8 @@ def mask_plane_issues( inventory: MaskInventory, images: Iterable[TiffMetadata], file_mappings: Mapping[str, AxisMapping], + *, + skip: Iterable[str] = (), ) -> list[ValidationIssue]: """Check that every mask has the depth slices and time points of its image (issue #36). @@ -796,6 +799,8 @@ def mask_plane_issues( file_mappings Each image's own axis mapping (:attr:`.axes.AxisReview.file_mappings`); images without one are left to the axis review's errors. + skip + Images not to check: those with an axis error, as in :func:`mask_size_issues`. Returns ------- @@ -803,10 +808,11 @@ def mask_plane_issues( The error ``mask_planes_mismatch`` naming the masks that do not fit, or nothing. """ headers = {meta.relative_path: meta for meta in images} + skip = set(skip) mismatched = [] for mask in inventory.files: mapping = file_mappings.get(mask.image) - if not mask.axes or mapping is None or mask.image not in headers: + if not mask.axes or mapping is None or mask.image not in headers or mask.image in skip: continue shape = headers[mask.image].shape planes = {} @@ -832,10 +838,68 @@ def mask_plane_issues( def _image_yx(meta: TiffMetadata) -> tuple[int, int] | None: - """Return the image's Y and X sizes from its header, or None when they are not known.""" - if not meta.readable or "Y" not in meta.axes or "X" not in meta.axes: + """Return the Y and X sizes the image's header names, or None when it does not name them. + + Used by :func:`find_masks`, before the axis roles are confirmed. A plain stack's ``Y`` + and ``X`` are only where its pages' rows and columns are stored + (:func:`.metadata.named_axes`), so its masks are compared by the confirmed roles only + (:func:`mask_size_issues`). + """ + named = named_axes(meta) + if "Y" not in named or "X" not in named: return None - return (meta.shape[meta.axes.index("Y")], meta.shape[meta.axes.index("X")]) + return (meta.shape[named.index("Y")], meta.shape[named.index("X")]) + + +def mask_size_issues( + inventory: MaskInventory, + images: Iterable[TiffMetadata], + file_mappings: Mapping[str, AxisMapping], + *, + skip: Iterable[str] = (), +) -> list[ValidationIssue]: + """Check that every mask has the Y and X sizes its image has under the confirmed roles. + + The image's Y and X are the dimensions its confirmed roles call Y and X, not its + header's letters. :func:`.run.run_experiment` uses this in place of :func:`find_masks`'s + header comparison, which it supersedes. + + Parameters + ---------- + inventory + The masks found by :func:`find_masks`. + images + Header information of the analysed images. + file_mappings + Each image's own axis mapping (:attr:`.axes.AxisReview.file_mappings`); images + without one are left to the axis review's errors. + skip + Images not to check: those with an axis error (:attr:`.axes.AxisReview.axis_error_files`), + which are never read, so a mask size would only be a misleading second message. Their + masks are checked when the images are saved as OME-Zarr. + + Returns + ------- + list of ValidationIssue + The error ``mask_size_mismatch`` naming the masks that do not fit, or nothing. + """ + headers = {meta.relative_path: meta for meta in images} + skip = set(skip) + mismatched = [] + for mask in inventory.files: + mapping = file_mappings.get(mask.image) + meta = headers.get(mask.image) + if not mask.axes or mapping is None or meta is None or mask.image in skip: + continue + y, x = mapping.index_of("Y"), mapping.index_of("X") + if y is None or x is None: + continue + if _yx(mask.axes, mask.shape) != (meta.shape[y], meta.shape[x]): + mismatched.append(mask.path) + if not mismatched: + return [] + message, action = _TEXTS["mask_size_mismatch"] + return [error("mask_size_mismatch", message, sorted(mismatched), action)] def _missing_issues( diff --git a/src/morphobricks_preprocessing/notebook_ui.py b/src/morphobricks_preprocessing/notebook_ui.py index 6e11ad6f..4e2bc6f4 100644 --- a/src/morphobricks_preprocessing/notebook_ui.py +++ b/src/morphobricks_preprocessing/notebook_ui.py @@ -64,6 +64,7 @@ from .axes import ( AxisMapping, AxisReview, + axis_disagreements, axis_length_ranges, axis_name, file_mapping_for, @@ -1378,10 +1379,13 @@ def _mapping_changed(self) -> None: self.axis_status.value = ui.status("error", str(exc)) return ranges = axis_length_ranges(self.mapping, self.preflight.metadata.files) - for note, role, (low, high) in zip( - self.role_notes, self.mapping.roles, ranges, strict=True + marks = header_marks(self.mapping, self.preflight.metadata.files) + for index, (note, role, (low, high)) in enumerate( + zip(self.role_notes, self.mapping.roles, ranges, strict=True) ): - note.value = f"
{role_effect(role, low, high)}
" + note.value = ( + f"
{role_effect(role, low, high)}
{marks.get(index, '')}" + ) self.plane_summary.value = plane_summary_html(self.mapping, ranges) review = review_axes( self.preflight.metadata, @@ -1394,7 +1398,8 @@ def _mapping_changed(self) -> None: else: self.axis_status.value = ui.status( "ok", - "These roles are valid for every image.", + "These roles are valid for every image, and agree with the axes their " + "metadata names.", ) self.axis_next.value = ui.next_steps( "Check the preview in Step 2, then confirm the axis roles.", @@ -1517,7 +1522,15 @@ def _toggle_confirmation(self) -> None: ) if r is not None ] - self.confirmed_status.value = ui.status("ok", "Axis roles confirmed.", roles) + self.confirmed_status.value = ui.status( + "ok", + "Axes checked: they are correct for every image.", + [ + "Every image names its axes in its metadata, and the confirmed roles " + "agree with them. Nothing needs to be saved again.", + *roles, + ], + ) self._open_image_type() self._refresh_axis_next() else: @@ -3020,6 +3033,41 @@ def labels_html(pre: Preflight) -> str: ) +def header_marks(mapping: AxisMapping, files: Iterable) -> dict[int, str]: + """Return a warning (HTML) for each role that disagrees with the axes the headers name. + + Shown under the role dropdowns before the roles are confirmed, from the TIFF images + with the mapping's arrangement (:func:`.axes.axis_disagreements`); empty when the + roles agree with every header. + """ + found: dict[int, set[str]] = {} + letters: dict[int, set[str]] = {} + for f in files: + if not f.readable or f.is_ome_zarr or f.axes != mapping.axes: + continue + for index, code in axis_disagreements(f, mapping).items(): + found.setdefault(index, set()).add(code) + letters.setdefault(index, set()).add(f.axes[index]) + marks = {} + for index, codes in found.items(): + if "axes_not_named" in codes: + text = ( + "The image metadata does not name this axis, so it cannot be checked: these " + "images are not read until they are saved as OME-Zarr with named axes." + ) + else: + named = ", ".join(axis_name(letter) for letter in sorted(letters[index])) + what = ( + "swaps the Y and X it names" if codes == {"axes_yx_swapped"} else "contradicts it" + ) + text = ( + f"The image header names this axis {html.escape(named)}; this role {what}. " + "Choose the header's role, or save the images as OME-Zarr with these axes." + ) + marks[index] = f"
⚠ {text}
" + return marks + + def role_effect(role: str | None, low: int, high: int) -> str: """One plain sentence on what a role means for an axis of this length.""" length = f"{low}" if low == high else f"{low} to {high}" diff --git a/src/morphobricks_preprocessing/report.py b/src/morphobricks_preprocessing/report.py index 61e75be0..8406defa 100644 --- a/src/morphobricks_preprocessing/report.py +++ b/src/morphobricks_preprocessing/report.py @@ -14,7 +14,7 @@ from morphobricks_normalization.pooling import NoValidPixelsError, PooledHistogram from morphobricks_normalization.records import LEVELS, PlaneRecord, group_records -from .axes import AxisMapping, axis_length_ranges, axis_name +from .axes import AxisMapping, AxisReview, axis_length_ranges, axis_name from .dataset import BalanceTable from .design import ExperimentDesign from .discovery import DiscoveryResult @@ -145,6 +145,23 @@ def mapping_summary(mapping: AxisMapping, files: Iterable = ()) -> str: return "\n".join(lines) +def axis_check_summary(review: AxisReview) -> str: + """Say in one line whether the axis check passed (contract §2), for the CLI. + + The success message names what was checked; with axis errors it says how many images + are not read and that the errors are listed (they are printed with the other issues). + """ + if review.axis_errors: + n = plural(len(review.axis_error_files), "image") + return ( + f"Axes check: {n} cannot be read with these axes (see the errors below). " + "Check their axes; if they are right, save the images as OME-Zarr with them." + ) + return ( + "Axes checked: every image names its axes in its metadata, and the roles agree with them." + ) + + def preflight_summary(pre: Preflight) -> str: """Summarise pass 1: what will be analysed, what is skipped, pixel type and axes.""" discovery, metadata = pre.discovery, pre.metadata diff --git a/src/morphobricks_preprocessing/run.py b/src/morphobricks_preprocessing/run.py index dcffc0ee..110f8cf1 100644 --- a/src/morphobricks_preprocessing/run.py +++ b/src/morphobricks_preprocessing/run.py @@ -44,7 +44,13 @@ from morphobricks_normalization.ranges import HistogramSettings, range_check_issues from morphobricks_normalization.records import PlaneRecord -from .axes import AxisMapping, file_mapping_for, prefill_experiment_mapping, review_axes +from .axes import ( + AxisMapping, + AxisReview, + file_mapping_for, + prefill_experiment_mapping, + review_axes, +) from .dataset import DatasetReport, check_dataset from .design import ( DESIGN_FILENAME, @@ -69,7 +75,7 @@ read_manifest, write_manifest, ) -from .masks import MaskInventory, find_masks, mask_plane_issues +from .masks import MaskInventory, find_masks, mask_plane_issues, mask_size_issues from .metadata import MetadataReport, TiffMetadata, inspect_images from .planes import ( DEFAULT_MEMORY_LIMIT, @@ -461,12 +467,7 @@ def run_experiment( *image_type_issues(image_type, type_review), *design_problems, *dataset.issues, - *(pre.masks.issues if pre.masks is not None else ()), - *( - mask_plane_issues(pre.masks, pre.metadata.files, review.file_mappings) - if pre.masks is not None - else () - ), + *(mask_issues(pre.masks, pre.metadata, review) if pre.masks is not None else ()), ] dtype = pre.metadata.dtype bins = None @@ -709,6 +710,26 @@ def stop(**counts) -> RunResult: ) +def mask_issues( + masks: MaskInventory, metadata: MetadataReport, review: AxisReview +) -> list[ValidationIssue]: + """Return the mask issues of a run, with the masks' sizes and planes by the confirmed roles. + + :func:`.masks.find_masks` compares a mask's Y and X with those its image's header + names; with confirmed roles that comparison is replaced by + :func:`.masks.mask_size_issues`, and the planes are checked by + :func:`.masks.mask_plane_issues`. Images with an axis error are never read, so their + masks are left out of both: a size or plane message would only repeat the axis error + in another form. + """ + skip = review.axis_error_files + return [ + *(i for i in masks.issues if i.code != "mask_size_mismatch"), + *mask_size_issues(masks, metadata.files, review.file_mappings, skip=skip), + *mask_plane_issues(masks, metadata.files, review.file_mappings, skip=skip), + ] + + class _OutputRollbackError(RuntimeError): """Raised when an output update fails and its earlier files cannot be restored.""" diff --git a/tests/test_axes.py b/tests/test_axes.py index f7c0d9d5..e60dac90 100644 --- a/tests/test_axes.py +++ b/tests/test_axes.py @@ -256,10 +256,12 @@ def test_channel_count_follows_the_confirmed_channel_axis(): report = MetadataReport(tuple(files), tuple(check_metadata(files))) assert "different_channel_counts" not in codes(report.issues) as_time = review_axes(report, prefill_mapping("IYX").with_role(0, "T")) - assert as_time.ok + # A plain stack never names its axes, so it is an error whatever the roles (v0.4.0). + assert codes(as_time.issues) == ["axes_not_named"] as_channel = review_axes(report, prefill_mapping("IYX").with_role(0, "C")) assert codes(as_channel.combined_issues(report)) == [ "different_channel_counts", + "axes_not_named", "different_plane_counts", ] @@ -303,8 +305,9 @@ def test_differently_arranged_file_is_an_inconsistent_mapping(): def test_review_includes_the_mapping_rules(): files = [meta("a.tif", (5, 8, 8), "IYX")] review = review_axes(files, prefill_mapping("IYX")) - assert not review.ok and codes(review.issues) == ["unassigned_axis"] - assert review_axes(files, prefill_mapping("IYX").with_role(0, "T")).ok + assert not review.ok and codes(review.issues) == ["unassigned_axis", "axes_not_named"] + as_time = review_axes(files, prefill_mapping("IYX").with_role(0, "T")) + assert codes(as_time.issues) == ["axes_not_named"] def test_review_from_metadata_report_replaces_missing_axes(tmp_path): @@ -328,8 +331,10 @@ def test_review_from_metadata_report_replaces_missing_axes(tmp_path): "unreadable_tiff", "different_channel_counts", ] + # The channel count is the confirmed roles' (C read as T gives one channel), but a named + # C read as T contradicts the header (v0.4.0). as_time = review_axes(report, mapping.with_role(1, "T"), missing_as_singleton=True) - assert codes(as_time.combined_issues(report)) == ["unreadable_tiff"] + assert codes(as_time.combined_issues(report)) == ["unreadable_tiff", "axes_contradict_header"] def test_combined_issues_put_errors_first(tmp_path): @@ -337,7 +342,11 @@ def test_combined_issues_put_errors_first(tmp_path): write_tiff(tmp_path / "b.tif", (3, 8, 9), axes="QYX") report = inspect_images(tmp_path, ["a.tif", "b.tif"]) review = review_axes(report, prefill_mapping("QYX")) - assert codes(review.combined_issues(report)) == ["unassigned_axis", "different_sizes"] + assert codes(review.combined_issues(report)) == [ + "unassigned_axis", + "axes_not_named", + "different_sizes", + ] def test_roles_string_round_trips(): diff --git a/tests/test_axis_check.py b/tests/test_axis_check.py new file mode 100644 index 00000000..a387be6c --- /dev/null +++ b/tests/test_axis_check.py @@ -0,0 +1,390 @@ +"""The axis check (contract §2, v0.4.0): any axis problem is an error, never read. + +An image whose metadata does not name its axes (``axes_not_named``), or whose confirmed +roles swap Y and X (``axes_yx_swapped``) or otherwise contradict its header +(``axes_contradict_header``), is an error naming the file with its header and confirmed +axes. Before v0.4.0 these were read silently: a plain stack with the user's roles, a swap +transposed, a contradiction under the declaration. +""" + +import numpy as np +import pytest +import tifffile + +from morphobricks_preprocessing import ( + AXIS_ERROR_CODES, + AxisMapping, + TiffMetadata, + find_masks, + header_axis_issue, + inspect_image, + mask_size_issues, + prefill_mapping, + preflight, + review_axes, + run_experiment, +) +from morphobricks_preprocessing.axes import axis_disagreements +from morphobricks_preprocessing.cli import main +from morphobricks_preprocessing.design import default_design, write_experiment_design +from morphobricks_preprocessing.discovery import default_output_folder +from tiff_tree import write_data + + +def imagej(path, data, axes): + path.parent.mkdir(parents=True, exist_ok=True) + tifffile.imwrite(path, data, imagej=True, metadata={"axes": axes}) + return path + + +def ome(path, data, axes): + path.parent.mkdir(parents=True, exist_ok=True) + tifffile.imwrite(path, data, ome=True, metadata={"axes": axes}, photometric="minisblack") + return path + + +def plain(path, data): + """A plain stack of pages without axis metadata (``IYX``).""" + path.parent.mkdir(parents=True, exist_ok=True) + tifffile.imwrite(path, data, photometric="minisblack", metadata=None) + return path + + +def data(shape, dtype=np.uint8): + return (np.arange(np.prod(shape)) % 251).reshape(shape).astype(dtype) + + +def codes(issues): + return [i.code for i in issues] + + +def one(issues, code): + (found,) = [i for i in issues if i.code == code] + return found + + +# The rule, position by position --------------------------------------------------------- + + +WRITERS = { + # ImageJ hyperstacks of two channels (three would be stored as RGB). + "imagej": lambda p, a: imagej(p, data((2 if a[0] == "C" else 3, 6, 8)), a), + "ome": lambda p, a: ome(p, data((3, 6, 8)), a), + "shaped": lambda p, a: write_data(p, data((3, 6, 8)), a), +} + + +@pytest.mark.parametrize("kind", sorted(WRITERS)) +@pytest.mark.parametrize( + ("header", "roles", "expected"), + [ + ("ZYX", "ZYX", None), + ("ZYX", "ZXY", "axes_yx_swapped"), + ("ZYX", "TYX", "axes_contradict_header"), + ("ZYX", "CYX", "axes_contradict_header"), + ("ZYX", "TXY", "axes_contradict_header"), # a contradiction is more than a swap + ("CYX", "CYX", None), + ("CYX", "CXY", "axes_yx_swapped"), + ("CYX", "ZYX", "axes_contradict_header"), + ("TYX", "ZYX", "axes_contradict_header"), + ], +) +def test_named_headers(tmp_path, kind, header, roles, expected): + meta = inspect_image(WRITERS[kind](tmp_path / "a.tif", header)) + assert meta.axes == header + mapping = AxisMapping(header, roles) + assert header_axis_issue(meta, mapping) == expected + review = review_axes([meta], mapping) + if expected is None: + assert review.ok and not review.axis_errors + else: + assert codes(review.issues) == [expected] + assert review.axis_error_files == ("a.tif",) + + +@pytest.mark.parametrize("roles", ["TYX", "ZYX", "CYX", "TXY"]) +@pytest.mark.parametrize("writer", ["plain-IYX", "shaped-QYX"]) +def test_plain_stacks_never_name_their_axes(tmp_path, writer, roles): + path = tmp_path / "a.tif" + if writer == "plain-IYX": + plain(path, data((3, 6, 8))) + else: + write_data(path, data((3, 6, 8))) # metadata={}: shaped without axes + meta = inspect_image(path) + assert meta.axes in ("IYX", "QYX") + review = review_axes([meta], AxisMapping(meta.axes, roles)) + found = one(review.issues, "axes_not_named") + assert found.is_error and found.paths == ("a.tif",) + assert f"header {meta.axes}, confirmed {roles}: a.tif" in found.message + assert "do not name their axes in their metadata" in found.message + assert "save the images as OME-Zarr" in found.action + + +def test_shaped_axes_with_a_placeholder_are_not_named(tmp_path): + meta = inspect_image(write_data(tmp_path / "a.tif", data((3, 6, 8)), "QYX")) + assert meta.axes_from_metadata and meta.axes == "QYX" + assert header_axis_issue(meta, AxisMapping("QYX", "ZYX")) == "axes_not_named" + # Only the placeholder is open: Y and X are named. + assert axis_disagreements(meta, AxisMapping("QYX", "ZYX")) == {0: "axes_not_named"} + + +def test_a_plain_2d_page_names_its_axes(tmp_path): + meta = inspect_image(plain(tmp_path / "a.tif", data((6, 8)))) + assert meta.axes == "YX" + assert review_axes([meta], prefill_mapping("YX")).ok + # Decision D-d: the rows and columns of a single page are its Y and X. + assert header_axis_issue(meta, AxisMapping("YX", "XY")) == "axes_yx_swapped" + + +def test_rgb_samples_as_channels_agree(tmp_path): + path = tmp_path / "a.tif" + tifffile.imwrite(path, data((6, 8, 3)), photometric="rgb") + meta = inspect_image(path) + assert meta.axes == "YXS" + review = review_axes([meta], prefill_mapping("YXS")) + assert review.ok and review.mapping.roles == ("Y", "X", "C") + assert header_axis_issue(meta, AxisMapping("YXS", "XYC")) == "axes_yx_swapped" + assert header_axis_issue(meta, AxisMapping("YXS", "YXZ")) == "axes_contradict_header" + + +def hand(axes, shape, from_metadata=True): + return TiffMetadata( + "a.tif", True, shape, np.dtype("uint8"), axes, 1, 1, axes_from_metadata=from_metadata + ) + + +def test_length_1_axes_without_a_role_are_not_contradictions(): + assert header_axis_issue(hand("TZYX", (1, 3, 6, 8)), AxisMapping("TZYX", "-ZYX")) is None + assert header_axis_issue(hand("QZYX", (1, 3, 6, 8)), AxisMapping("QZYX", "-ZYX")) is None + # A length-1 axis given another role is still a contradiction of its header. + found = header_axis_issue(hand("TZYX", (1, 3, 6, 8)), AxisMapping("TZYX", "ZTYX")) + assert found == "axes_contradict_header" + + +def test_named_axes_without_a_role_are_contradicted_when_given_one(): + # Lambda (E) is named but is not a role: giving it one contradicts the header. + meta = hand("EYX", (4, 6, 8)) + assert header_axis_issue(meta, AxisMapping("EYX", "CYX")) == "axes_contradict_header" + # Without a role it is check_mapping's unassigned_axis, not a second error. + review = review_axes([meta], AxisMapping("EYX", "-YX")) + assert codes(review.issues) == ["unassigned_axis"] + + +def test_position_axes_are_left_to_unsupported_axis(): + review = review_axes([hand("RYX", (3, 6, 8))], AxisMapping("RYX", "ZYX")) + assert "unsupported_axis" in codes(review.issues) + assert not review.axis_errors + + +def test_ome_zarr_images_never_have_an_axis_error(tmp_path): + pytest.importorskip("zarr") + from zarr_tree import write_ngff + + write_ngff(tmp_path / "a.ome.zarr", data((3, 6, 8)), axes="zyx") + meta = inspect_image(tmp_path / "a.ome.zarr", "a.ome.zarr") + assert meta.is_ome_zarr + assert header_axis_issue(meta, AxisMapping("ZYX", "ZYX")) is None + assert header_axis_issue(meta, AxisMapping("ZYX", "TXY")) is None + assert review_axes([meta], prefill_mapping("ZYX")).ok + + +def test_every_file_is_named_with_its_header_and_confirmed_axes(tmp_path): + files = [hand("CYX", (2, 6, 8)) for _ in range(5)] + files = [ + TiffMetadata(f"c/r/img_{i}.tif", *(f.readable, f.shape, f.dtype, f.axes, 1, 1)) + for i, f in enumerate(files) + ] + review = review_axes(files, AxisMapping("CYX", "CXY")) + found = one(review.issues, "axes_yx_swapped") + assert found.paths == tuple(f"c/r/img_{i}.tif" for i in range(5)) + assert "The axes confirmed for 5 images swap the Y and X their header names" in found.message + assert "header CYX, confirmed CXY: c/r/img_0.tif, c/r/img_1.tif, c/r/img_2.tif and 2 more" in ( + found.message + ) + assert "They are not read transposed." in found.message + assert "confirm the header's axes instead" in found.action + + +def test_the_three_codes(): + assert AXIS_ERROR_CODES == ("axes_not_named", "axes_yx_swapped", "axes_contradict_header") + + +# The run stops, reading nothing -------------------------------------------------------- + + +@pytest.fixture +def reads(monkeypatch): + import morphobricks_preprocessing.run as run_module + + seen = [] + original = run_module.iter_xyc_planes + + def spy(path, *args, **kwargs): + seen.append(str(path)) + return original(path, *args, **kwargs) + + monkeypatch.setattr(run_module, "iter_xyc_planes", spy) + return seen + + +@pytest.mark.parametrize( + ("write", "axes", "roles", "code"), + [ + (lambda p: write_data(p, data((2, 6, 8)), "CYX"), "CYX", "CXY", "axes_yx_swapped"), + (lambda p: imagej(p, data((3, 6, 8)), "ZYX"), "ZYX", "TYX", "axes_contradict_header"), + (lambda p: plain(p, data((3, 6, 8))), "IYX", "ZYX", "axes_not_named"), + ], +) +def test_run_stops_naming_every_file(tmp_path, reads, write, axes, roles, code): + root = tmp_path / "exp" + for rel in ["ctrl/r1/a.tif", "ctrl/r1/b.tif", "drug/r1/a.tif"]: + write(root / rel) + result = run_experiment(root, AxisMapping(axes, roles)) + assert not result.written and reads == [] + found = one(result.issues, code) + assert found.paths == ("ctrl/r1/a.tif", "ctrl/r1/b.tif", "drug/r1/a.tif") + assert [i.code for i in result.issues if i.is_error] == [code] + assert not default_output_folder(root).exists() + + +def test_a_clean_run_is_unchanged(tmp_path): + root = tmp_path / "exp" + write_data(root / "ctrl/a.tif", data((2, 6, 8)), "CYX") + result = run_experiment(root, prefill_mapping("CYX")) + assert result.written and not [i for i in result.issues if i.code in AXIS_ERROR_CODES] + + +# Masks: sizes by the confirmed roles (B1) ------------------------------------------------ + + +def masks_tree(tmp_path, image, mask_shape, mask_axes=None): + root = tmp_path / "exp" + image(root / "ctrl/a.tif") + write_data(tmp_path / "masks/Cells/ctrl/a.tif", np.ones(mask_shape, np.uint16), mask_axes) + return root, tmp_path / "masks" + + +def test_a_swapped_image_gets_no_spurious_mask_size_error(tmp_path): + # Header CYX of 6 x 8; confirmed CXY, so by the roles the image is 8 high and 6 wide. + root, masks = masks_tree(tmp_path, lambda p: write_data(p, data((2, 6, 8)), "CYX"), (8, 6)) + result = run_experiment(root, AxisMapping("CYX", "CXY"), masks_folder=masks) + assert [i.code for i in result.issues if i.is_error] == ["axes_yx_swapped"] + + +def test_a_relabelled_stack_gets_no_spurious_mask_plane_error(tmp_path): + root, masks = masks_tree( + tmp_path, lambda p: imagej(p, data((3, 6, 8)), "ZYX"), (3, 6, 8), "ZYX" + ) + result = run_experiment(root, AxisMapping("ZYX", "TYX"), masks_folder=masks) + assert [i.code for i in result.issues if i.is_error] == ["axes_contradict_header"] + + +def test_mask_sizes_are_compared_by_the_confirmed_roles(tmp_path): + root, masks = masks_tree(tmp_path, lambda p: write_data(p, data((2, 6, 8)), "CYX"), (6, 8)) + pre = preflight(root, masks_folder=masks) + files = pre.metadata.files + review = review_axes(files, prefill_mapping("CYX")) + assert mask_size_issues(pre.masks, files, review.file_mappings) == [] + # By roles that swap Y and X the image is 8 x 6, so the 6 x 8 mask does not fit... + swapped = review_axes(files, AxisMapping("CYX", "CXY")) + found = mask_size_issues(pre.masks, files, swapped.file_mappings) + assert codes(found) == ["mask_size_mismatch"] + # ...but the image is never read, so the run leaves its masks to the conversion. + assert mask_size_issues(pre.masks, files, swapped.file_mappings, skip=["ctrl/a.tif"]) == [] + + +def test_a_mask_of_the_wrong_size_is_still_an_error(tmp_path): + root, masks = masks_tree(tmp_path, lambda p: write_data(p, data((2, 6, 8)), "CYX"), (5, 5)) + result = run_experiment(root, prefill_mapping("CYX"), masks_folder=masks) + assert [i.code for i in result.issues if i.is_error] == ["mask_size_mismatch"] + assert one(result.issues, "mask_size_mismatch").paths == ("Cells/ctrl/a.tif",) + + +def test_plain_stack_masks_are_not_compared_before_the_roles(tmp_path): + root, masks = masks_tree(tmp_path, lambda p: plain(p, data((3, 6, 8))), (8, 6)) + pre = preflight(root, masks_folder=masks) + # The plain stack's Y and X are only where its pages' rows and columns are stored. + assert "mask_size_mismatch" not in codes(pre.masks.issues) + inventory = find_masks(masks, pre.discovery, pre.metadata.files) + assert "mask_size_mismatch" not in codes(inventory.issues) + + +# CLI and notebook ------------------------------------------------------------------------ + + +def confirm(root): + design = default_design(preflight(root).dataset.repeats) + for group in design.groups: + design = design.with_confirmed(group, True, "2026-11-01") + output = default_output_folder(root) + output.mkdir(parents=True, exist_ok=True) + write_experiment_design(design, output / "experiment_design.csv") + + +def test_cli_check_reports_plain_stacks(tmp_path, capsys): + root = tmp_path / "exp" + plain(root / "ctrl/a.tif", data((3, 6, 8))) + assert main(["check", str(root)]) == 1 + out = capsys.readouterr().out + assert "Axes check: 1 image cannot be read with these axes" in out + assert "do not name their axes in their metadata (header IYX, confirmed -YX" in out + + +def test_cli_check_and_run_say_the_axes_are_correct(tmp_path, capsys): + root = tmp_path / "exp" + write_data(root / "ctrl/a.tif", data((2, 6, 8)), "CYX") + assert main(["check", str(root)]) == 0 + assert "Axes checked: every image names its axes in its metadata" in capsys.readouterr().out + confirm(root) + assert main(["run", str(root), "--axes", "CYX"]) == 0 + assert "Axes checked: every image names its axes" in capsys.readouterr().out + + +def test_cli_run_names_the_swapped_files(tmp_path, capsys): + root = tmp_path / "exp" + write_data(root / "ctrl/a.tif", data((2, 6, 8)), "CYX") + confirm(root) + assert main(["run", str(root), "--axes", "CXY"]) == 1 + out = capsys.readouterr().out + assert "header CYX, confirmed CXY: ctrl/a.tif" in out + assert "Nothing was written" in out + + +def test_notebook_marks_roles_that_disagree_with_the_header(tmp_path): + pytest.importorskip("ipywidgets") + pytest.importorskip("matplotlib") + from morphobricks_preprocessing.notebook_ui import PreprocessingNotebook + + root = tmp_path / "exp" + write_data(root / "ctrl/a.tif", data((2, 6, 8)), "CYX") + nb = PreprocessingNotebook(str(root)) + nb.axis_step() + nb.check_button.click() + assert "⚠" not in "".join(note.value for note in nb.role_notes) + nb.role_dropdowns[1].value = "X" + nb.role_dropdowns[2].value = "Y" + assert "The image header names this axis Y (height)" in nb.role_notes[1].value + assert "swaps the Y and X it names" in nb.role_notes[1].value + assert "⚠" not in nb.role_notes[0].value + assert "axes_yx_swapped" in nb.axis_status.value or "swap the Y and X" in nb.axis_status.value + nb.preview_button.click() + nb.preview_checked.button.click() + assert nb.confirm_button.disabled + nb.role_dropdowns[1].value = "Y" + nb.role_dropdowns[2].value = "X" + nb.preview_checked.button.click() + nb.confirm_button.click() + assert "Axes checked: they are correct for every image" in nb.confirmed_status.value + + +def test_notebook_marks_axes_a_plain_stack_does_not_name(tmp_path): + pytest.importorskip("ipywidgets") + pytest.importorskip("matplotlib") + from morphobricks_preprocessing.notebook_ui import header_marks + + root = tmp_path / "exp" + plain(root / "ctrl/a.tif", data((3, 6, 8))) + files = preflight(root).metadata.files + marks = header_marks(AxisMapping("IYX", "TYX"), files) + assert sorted(marks) == [0, 1, 2] + assert "does not name this axis" in marks[0] diff --git a/tests/test_cli.py b/tests/test_cli.py index 474f1be1..573d1c86 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -118,10 +118,11 @@ def test_rgb_is_refused_for_images_that_cannot_be_rgb(experiment, capsys): out = capsys.readouterr().out assert "Image type: fluorescence" in out assert "Give --image-type rgb if these are rgb images." in out - assert main(["run", str(experiment), "--axes", "TCYX", "--image-type", "rgb"]) == 0 + assert main(["run", str(experiment), "--axes", "ZCYX", "--image-type", "rgb"]) == 0 capsys.readouterr() assert main(["run", str(experiment), "--axes", "ZYCX", "--image-type", "rgb"]) == 1 out = capsys.readouterr().out + assert "contradict their header" in out assert "RGB images need exactly three colour channels" in out assert "8 channels of type uint8" in out and "Nothing was written" in out @@ -172,7 +173,7 @@ def test_invalid_check_limits_are_refused(experiment, capsys): def test_run_options_reach_the_run(experiment, tmp_path, capsys): out_dir = tmp_path / "results" log = tmp_path / "run.log" - args = ["run", str(experiment), "--axes", "TCYX", "--output", str(out_dir)] + args = ["run", str(experiment), "--axes", "ZCYX", "--output", str(out_dir)] args += ["--bins", "16", "--memory-limit", "256MiB", "--exclude", "drug/rep1/a.tif"] args += ["--log", str(log)] confirm_design(experiment, out_dir) @@ -181,7 +182,7 @@ def test_run_options_reach_the_run(experiment, tmp_path, capsys): assert manifest.settings.bin_count == 16 assert manifest.settings.memory_limit == 256 * 2**20 assert manifest.settings.exclude == ("drug/rep1/a.tif",) - assert manifest.mapping.roles == ("T", "C", "Y", "X") + assert manifest.mapping.roles == ("Z", "C", "Y", "X") assert "Wrote" in log.read_text(encoding="utf-8") capsys.readouterr() assert main(args + ["--force-rebuild"]) == 0 diff --git a/tests/test_header_axes.py b/tests/test_header_axes.py index aadf94f9..37b8be96 100644 --- a/tests/test_header_axes.py +++ b/tests/test_header_axes.py @@ -21,6 +21,22 @@ ) from tiff_tree import write_data +# check_declared_axes and declared_axes_transposed are deprecated in v0.4.0 (removed in +# v0.5.0); their rules stay tested until then, and the warning has its own test below. +pytestmark = pytest.mark.filterwarnings( + "ignore:morphobricks_preprocessing.(check_declared_axes|declared_axes_transposed) is " + "deprecated:DeprecationWarning" +) + + +@pytest.mark.filterwarnings("error") +def test_declared_axes_checks_are_deprecated(tmp_path): + meta = inspect_image(write_data(tmp_path / "a.tif", np.zeros((2, 4, 5), np.uint8), "CYX")) + with pytest.warns(DeprecationWarning, match="check_declared_axes is deprecated.*v0.5.0"): + assert check_declared_axes(meta, "CYX") is None + with pytest.warns(DeprecationWarning, match="declared_axes_transposed is deprecated"): + assert declared_axes_transposed(meta, "CXY") + def imagej(path, data, axes): path.parent.mkdir(parents=True, exist_ok=True) diff --git a/tests/test_notebook_ui.py b/tests/test_notebook_ui.py index c1acff32..62762972 100644 --- a/tests/test_notebook_ui.py +++ b/tests/test_notebook_ui.py @@ -526,11 +526,11 @@ def test_confirm_needs_valid_roles_and_a_checked_preview(experiment): assert "needs a role" in nb.role_notes[1].value assert "No role is chosen for the axis C (channel)" in nb.axis_status.value assert nb.confirm_button.disabled - nb.role_dropdowns[1].value = "T" + nb.role_dropdowns[1].value = "C" assert nb.confirm_button.disabled # new roles: the preview must be checked again nb.preview_checked.button.click() assert not nb.confirm_button.disabled - assert nb.mapping.roles == ("Z", "T", "Y", "X") + assert nb.mapping.roles == ("Z", "C", "Y", "X") def test_confirmations_are_buttons_that_can_be_undone(experiment): @@ -566,7 +566,7 @@ def test_confirmation_locks_and_unlocks(experiment): assert nb.confirmed_mapping == nb.mapping assert all(d.disabled for d in nb.role_dropdowns) assert nb.confirm_button.description == " Change axis roles" - assert "Axis roles confirmed" in nb.confirmed_status.value + assert "Axes checked: they are correct for every image" in nb.confirmed_status.value assert "Z (depth) → Z" in nb.confirmed_status.value nb.confirm_button.click() assert nb.confirmed_mapping is None diff --git a/tests/test_planes.py b/tests/test_planes.py index e33a4900..5de1ccb9 100644 --- a/tests/test_planes.py +++ b/tests/test_planes.py @@ -110,7 +110,13 @@ def test_planes_match_the_source(tmp_path, page_reads, shape, axes, kwargs, role mapping = prefill_mapping(info.axes) if roles is not None: mapping = AxisMapping(info.axes, roles) - assert review_axes([info], mapping).ok + review = review_axes([info], mapping) + if axes is None: + # A plain stack is never analysed (axes_not_named); the reader still reads it with + # the confirmed roles to save it as OME-Zarr. + assert [i.code for i in review.issues] == ["axes_not_named"] + else: + assert review.ok planes = list(iter_planes(path, mapping)) expected = expected_planes(data, mapping.roles) diff --git a/tests/test_run.py b/tests/test_run.py index 8da66280..9d727e72 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -483,7 +483,10 @@ def test_changed_bin_count_rebuilds(experiment, reads): def test_changed_axis_mapping_rebuilds(experiment): run_experiment(experiment, ZYX) - result = run_experiment(experiment, AxisMapping("ZYX", "TYX")) + # The same images saved again as time-lapses: their headers now name T. + for seed, rel in enumerate(IMAGES): + write_data(experiment / rel, stack(seed), "TYX") + result = run_experiment(experiment, AxisMapping("TYX", "TYX")) assert result.rebuild_reason == "the axis mapping changed" records = read_plane_histograms(result.csv_path) assert {r.t for r in records} == {0, 1} and {r.z for r in records} == {0} diff --git a/tests/test_zarr_export.py b/tests/test_zarr_export.py index c9e44952..3ba12f93 100644 --- a/tests/test_zarr_export.py +++ b/tests/test_zarr_export.py @@ -267,7 +267,6 @@ def test_missing_zarr_is_a_clear_error(tmp_path, monkeypatch): ), pytest.param((3, 6, 7), "ZXY", {}, None, ("Z", "Y", "X"), id="x-before-y"), pytest.param((1, 3, 6, 7), "QZYX", {}, None, ("Z", "Y", "X"), id="unassigned-singleton"), - pytest.param((4, 6, 7), None, {}, "TYX", ("T", "Y", "X"), id="pages-as-time"), ] From 23182c0228075de6a7c87f2f67a5e59ffbd3ca32 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 07:15:48 +0000 Subject: [PATCH 2/5] Read plain stacks with the confirmed roles; name the cause of axis errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revised design after the audit (owner decision of 1 November 2026): - axes_not_named is a warning, not an error. A plain stack is read with the roles the user confirms, and AxisReview.axes_from records each file as "confirmed" (plain stack) or "header". The warning recommends named-axis copies before segmenting in other tools. - axes_yx_swapped and axes_contradict_header stay errors. They now name the likely cause (likely_axis_cause: tifffile's CYX default for 3-D stacks saved with imagej=True or ome=True, an ImageJ slice stack read as Z, or rows and columns exchanged) and give two fixes: correct the axes, or convert to named-axis copies. - The notebook shows the three codes as one "Axis check" card. A role on an unnamed axis is marked as "recorded as yours" rather than as an error. The success message stays and names the confirmed plain stacks. - AXIS_ERROR_CODES now holds the two errors; AXIS_CHECK_CODES holds all three. - B1 is kept: mask sizes are compared by the confirmed roles, which still matters for plain stacks. Docs: contract §2/§8, user guide, plan §12 Q45, AGENTS.md, notebook text. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WX6rdMV1r5jUPUR94kxV5a --- AGENTS.md | 8 +- docs/contract.md | 6 +- docs/plan.md | 41 ++-- docs/user_guide.md | 36 ++-- notebooks/MorphoBricks_Preprocessing.ipynb | 13 +- src/morphobricks_preprocessing/__init__.py | 2 + src/morphobricks_preprocessing/axes.py | 178 +++++++++++++----- src/morphobricks_preprocessing/cli.py | 8 +- src/morphobricks_preprocessing/notebook_ui.py | 89 ++++++--- src/morphobricks_preprocessing/report.py | 17 +- tests/test_axes.py | 10 +- tests/test_axis_check.py | 67 +++++-- tests/test_planes.py | 9 +- tests/test_zarr_export.py | 1 + 14 files changed, 332 insertions(+), 153 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2cb03754..1a8b90fb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -230,9 +230,11 @@ functions the tests use. - **Axis check (v0.4.0, plan §12 question 45):** every TIFF's confirmed roles must agree with the axes its header names (`axes.header_axis_issue`, position by position, on `metadata.named_axes`). Otherwise one of three errors naming the files with their header - and confirmed axes: `axes_not_named` (plain stack), `axes_yx_swapped`, - `axes_contradict_header`. Such an image is **never read** (not transposed, not under a - contradicted declaration); the remedy is an OME-Zarr copy with the confirmed axes. + and confirmed axes: the errors `axes_yx_swapped` and `axes_contradict_header` (with the + likely cause; such an image is **never read**, not transposed, not under a contradicted + declaration; fix: correct the axes, or named-axis copies) and the warning + `axes_not_named` (a plain stack, read with the confirmed roles, `axes_from: + "confirmed"`). `AxisReview.axis_error_files` lists them; mask sizes and planes are checked by the confirmed roles and skip those images (`run.mask_issues`). OME-Zarr images never have these errors. diff --git a/docs/contract.md b/docs/contract.md index 46e97211..96dfa5ef 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -45,7 +45,7 @@ channel order; an image without a C axis has one channel. | Rule | Detail | |---|---| | Prefill | Axis labels from metadata: `X`, `Y`, `Z`, `C`, `T` keep their role; the RGB samples axis `S` becomes C when there is no other `C`. Every other label is left unassigned for the user to choose. | -| Axis check (v0.4.0) | Every TIFF's confirmed roles are compared with the axes its own header names (`named_axes`, §16), position by position (`header_axis_issue`, `axis_disagreements`): a named axis keeps its letter as role, except that the RGB samples `S` may be C when the header has no C, and a length-1 axis may have no role. Any other result is one of three **errors**, each naming every file with its header axes and confirmed roles, asking the user to check them and recommending saving the images as OME-Zarr with the confirmed axes (then running preprocessing on the copies): `axes_not_named` (the metadata does not name an axis longer than 1: a plain stack `IYX`, `QYX`, `QQYX`, or shaped metadata with a placeholder `QYX`; a plain 2D `YX` page and RGB `YXS` are named, decision D-d), `axes_yx_swapped` (the named Y given role X and the named X role Y) and `axes_contradict_header` (any other role for a named axis: `ZYX` confirmed `TYX`, a named angle or lambda axis given a role; a named axis longer than 1 without a role is `unassigned_axis`, a position axis `unsupported_axis`). An image with one of them is never read: not transposed, not under a contradicted declaration, not with roles its metadata does not give. OME-Zarr images take their roles from their OME-NGFF axes and never have these errors. Since one mapping covers each TIFF arrangement, all files of one arrangement pass or fail together. When the roles agree with every header, the notebook and `run` say so ("Axes checked"). Before v0.4.0 such images were read: plain stacks with the user's roles, swaps transposed by the plane reader, contradictions under the declaration. | +| Axis check (v0.4.0) | Every TIFF's confirmed roles are compared with the axes its own header names (`named_axes`, §16), position by position (`header_axis_issue`, `axis_disagreements`): a named axis keeps its letter as role, except that the RGB samples `S` may be C when the header has no C, and a length-1 axis may have no role. Confirmed roles that give the named Y and X the other way round are the **error** `axes_yx_swapped`; any other role for a named axis (`ZYX` confirmed `TYX`, a named angle or lambda axis given a role) is the **error** `axes_contradict_header` (a named axis longer than 1 without a role is `unassigned_axis`, a position axis `unsupported_axis`). Each error names every file with its header axes and confirmed roles and the likely cause (`likely_axis_cause`: tifffile labels a 3-D stack saved with `imagej=True` or `ome=True` `CYX`; an ImageJ stack of slices reads as Z; rows and columns exchanged), and offers two fixes: correct the axes, or save named-axis copies (`convert`, §17) and run preprocessing on them. Such an image is never read: not transposed, not under a contradicted declaration. An axis the metadata does not name (a plain stack `IYX`, `QYX`, `QQYX`, or shaped metadata with a placeholder `QYX`; a plain 2D `YX` page and RGB `YXS` are named, decision D-d) takes the role the user confirmed: the image is read with it, with the **warning** `axes_not_named` recommending named copies before segmenting in other tools, and recorded as `axes_from: "confirmed"` (`AxisReview.axes_from`; `"header"` for every other image). OME-Zarr images take their roles from their OME-NGFF axes and never have these issues. Since one mapping covers each TIFF arrangement, all files of one arrangement pass or fail together. The notebook shows the three codes as one card; when the roles agree with every header, the notebook and `run` say so ("Axes checked"). Before v0.4.0 a swap was read transposed and a contradiction under the declaration, silently. | | Unsupported axes | Position-like axes (tile `R`, mosaic `M`, row `K`, column `J`) cannot have a role. If they hold more than one plane, the file is an error. | | Singleton axes | An axis of length 1 needs no role; its index is always 0. | | Missing axes | A file lacking axes the mapping has is read with those axes as length 1 only when the user confirmed this and only one arrangement fits; its coordinates for them are 0. | @@ -121,8 +121,8 @@ method is an offset and a scale, `(pixels − offset) / scale`. | Severity | Situations | Effect | |---|---|---| -| Error | Experiment design not confirmed (`design_not_confirmed`) or a repeat group holding two folders of one condition (`repeat_group_conflict`), when a design is used (§12); image type RGB for images that are not three channels of an integer type (`rgb_not_possible`, §13); mixed dtypes; images with different numbers of channels (`different_channel_counts`, §2) or different channel names at the same channel number (`channel_names_differ`, §2); unreadable TIFF (header or pixel data) or OME-Zarr folder in a leaf folder that is not excluded; OME-Zarr images, or converting TIFF images to OME-Zarr, without the `zarr` extra; OME-Zarr images whose declared axes do not fit the experiment's roles; every image excluded; more than one image series in a file; unsupported sample format or axis; missing/duplicate X or Y, duplicate roles, unassigned non-singleton axis, inconsistent mapping; an image whose metadata does not name its axes, or whose confirmed roles swap Y and X or contradict its header (`axes_not_named`, `axes_yx_swapped`, `axes_contradict_header`, §2); plane that cannot be read within the memory limit, not even in blocks (for example a plane stored compressed in one piece); folder named `single` in another case, or beside other folders (§1); folders of one level differing only in case, anywhere in the experiment (`case_conflict`); folder or image names with outer whitespace or reading as a missing value (`invalid_name`, §1); images in one leaf folder with the same position identifier (§1); folder below the technical-repeat level; images in a folder that also holds subfolders (`tiff_outside_leaf`, §1); output folder inside the experiment; no usable planes; when masks are given, a masks folder missing or inside the experiment, mask names differing only in case, two masks of one type for an image, a mask whose name could belong to two images or types, or a mask that is unreadable, has axes other than Y, X and named Z/T (`mask_not_2d`), is of another size than its image (in `run`, its image's Y and X under the confirmed roles; not reported for an image with an axis error), is not of integers or booleans, has other depth slices or time points than the image's other masks or, in `run`, than the image (§15) | Processing is blocked | -| Warning | Images named as masks of other images, or folders named like a masks folder holding them (`image_named_as_mask`, `mask_folder_in_experiment`, §1); intensity checks of the written table (`image_intensity_outlier`, `repeat_intensity_shift`, `saturated_image`, `no_intensity_spread`, §14), reported apart from the run's issues; files that are not images; empty folders; images excluded by the user, or excluded paths that match no image; non-finite pixels; different image widths or heights; different numbers of depth slices, time points or other planes; physical pixel sizes that differ or are known for only some images; unbalanced numbers of images; biological repeats missing from some conditions (§11); planes without finite pixels while others remain; channels whose central intensity range covers too few histogram bins (`narrow_histogram_range`, §4); planes over the memory limit that are read in blocks; images without a mask of a type other images have, masks matching no analysed image, files outside the mask-type folders (§15) | Processing continues; the item is reported | +| Error | Experiment design not confirmed (`design_not_confirmed`) or a repeat group holding two folders of one condition (`repeat_group_conflict`), when a design is used (§12); image type RGB for images that are not three channels of an integer type (`rgb_not_possible`, §13); mixed dtypes; images with different numbers of channels (`different_channel_counts`, §2) or different channel names at the same channel number (`channel_names_differ`, §2); unreadable TIFF (header or pixel data) or OME-Zarr folder in a leaf folder that is not excluded; OME-Zarr images, or converting TIFF images to OME-Zarr, without the `zarr` extra; OME-Zarr images whose declared axes do not fit the experiment's roles; every image excluded; more than one image series in a file; unsupported sample format or axis; missing/duplicate X or Y, duplicate roles, unassigned non-singleton axis, inconsistent mapping; an image whose confirmed roles swap Y and X or contradict its header (`axes_yx_swapped`, `axes_contradict_header`, §2); plane that cannot be read within the memory limit, not even in blocks (for example a plane stored compressed in one piece); folder named `single` in another case, or beside other folders (§1); folders of one level differing only in case, anywhere in the experiment (`case_conflict`); folder or image names with outer whitespace or reading as a missing value (`invalid_name`, §1); images in one leaf folder with the same position identifier (§1); folder below the technical-repeat level; images in a folder that also holds subfolders (`tiff_outside_leaf`, §1); output folder inside the experiment; no usable planes; when masks are given, a masks folder missing or inside the experiment, mask names differing only in case, two masks of one type for an image, a mask whose name could belong to two images or types, or a mask that is unreadable, has axes other than Y, X and named Z/T (`mask_not_2d`), is of another size than its image (in `run`, its image's Y and X under the confirmed roles; not reported for an image with an axis error), is not of integers or booleans, has other depth slices or time points than the image's other masks or, in `run`, than the image (§15) | Processing is blocked | +| Warning | Images whose metadata does not name their axes, read with the confirmed roles (`axes_not_named`, §2); images named as masks of other images, or folders named like a masks folder holding them (`image_named_as_mask`, `mask_folder_in_experiment`, §1); intensity checks of the written table (`image_intensity_outlier`, `repeat_intensity_shift`, `saturated_image`, `no_intensity_spread`, §14), reported apart from the run's issues; files that are not images; empty folders; images excluded by the user, or excluded paths that match no image; non-finite pixels; different image widths or heights; different numbers of depth slices, time points or other planes; physical pixel sizes that differ or are known for only some images; unbalanced numbers of images; biological repeats missing from some conditions (§11); planes without finite pixels while others remain; channels whose central intensity range covers too few histogram bins (`narrow_histogram_range`, §4); planes over the memory limit that are read in blocks; images without a mask of a type other images have, masks matching no analysed image, files outside the mask-type folders (§15) | Processing continues; the item is reported | Every message states what happened in plain language, lists the affected paths, and suggests an action. Technical details go to the diagnostic log. diff --git a/docs/plan.md b/docs/plan.md index 0f7a37e6..76be60be 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -1036,21 +1036,26 @@ Raised during review; to be decided before the milestones that need them: check suggests a mapping without Y and the run refuses it with `missing_xy_axis`; reading it as one row too would change the shared header-axes rule (§16).* 45. How are the axes checked, now that every brick reads preprocessing's run (MorphoBricks - axes redesign, design note of 1 November 2026)? Until now a confirmed mapping was never - compared with the axes the header names: a plain stack was read with the user's roles, - a Y/X swap read transposed and a contradiction (header `ZYX` confirmed `TYX`) read - under the declaration, with no trace but the manifest's `roles`. - *Decided (1 November 2026, owner's final design and preprocessing sequence): any axis - problem is an error, and preprocessing is the only place axes are checked. `review_axes` - compares each TIFF's confirmed roles with its header (`header_axis_issue`, built from - the declared-axes rule): `axes_not_named` (a plain stack whose metadata does not name - an axis longer than 1), `axes_yx_swapped` and `axes_contradict_header`, each naming the - files with their header and confirmed axes, asking the user to check them and - recommending saving them as OME-Zarr. Nothing is read transposed or under a - contradicted declaration. When the axes are correct, the notebook and the CLI say so. - The notebook marks each role that disagrees with the header before confirmation. - Masks are compared with their image's Y and X by the confirmed roles (they were - compared with the header's letters), and not at all for an image with an axis error. - `check_declared_axes` and `declared_axes_transposed` are deprecated, to be removed in - v0.5.0. Breaking for the next release (v0.4.0): experiments of plain stacks, or run - with roles that swap or contradict the header, no longer run as they are.* + axes redesign, design note and audit of 1 November 2026)? Until now a confirmed + mapping was never compared with the axes the header names: a Y/X swap was read + transposed and a contradiction (header `ZYX` confirmed `TYX`) read under the + declaration, with no trace but the manifest's `roles`. + *Decided (1 November 2026, owner's final design after the audit): preprocessing is the + only place axes are checked. `review_axes` compares each TIFF's confirmed roles with + its header (`header_axis_issue`, built from the declared-axes rule). Roles that swap + the named Y and X (`axes_yx_swapped`) or otherwise contradict them + (`axes_contradict_header`) are errors naming the files with their header and + confirmed axes and the likely cause (tifffile's `CYX` default, an ImageJ slice stack), + with two fixes: correct the axes, or convert (named-axis copies). Nothing is read + transposed or under a contradicted declaration. A plain stack whose metadata does not + name its axes is read with the roles the user confirms, with the warning + `axes_not_named` recommending named copies, recorded per file as `axes_from: + "confirmed"`. The notebook shows the three codes as one card and marks each role that + disagrees with the header before confirmation; when the axes are correct, the notebook + and the CLI say so. Masks are compared with their image's Y and X by the confirmed + roles (they were compared with the header's letters; the role-based comparison still + matters for plain stacks, whose positional Y and X need not be the roles), and not at + all for an image with an axis error. `check_declared_axes` and + `declared_axes_transposed` are deprecated, to be removed in v0.5.0. Breaking for + v0.4.0: experiments run with roles that swap or contradict the header no longer run + as they are.* diff --git a/docs/user_guide.md b/docs/user_guide.md index 3e8ae5d2..1a3ac6f8 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -143,9 +143,9 @@ that measure objects; the optional `masks` command (§5) also tells how they rel ## 3. Supported images - **TIFF files:** `.tif` and `.tiff` (any case), including OME-TIFF and ImageJ hyperstacks. - Their metadata must name every axis with more than one plane, and the axis roles must - agree with it (§4 step 2). A plain stack of pages without axis metadata, or a file whose - axes you know better than its header, is saved as OME-Zarr with the right axes first. + The axis roles must agree with the axes their metadata names (§4 step 2); a file whose + axes you know better than its header is saved as a named-axis copy first. A plain stack + of pages without axis metadata is read with the roles you confirm (a warning). - **OME-Zarr images:** OME-NGFF 0.4 and 0.5 (needs the `zarr` extra; 0.5 needs Python 3.11 or newer). Their axes (time, channel, z, y, x) come from their metadata. Only the full-resolution level of a single image is read; a folder holding several images (a @@ -241,13 +241,14 @@ results or comparing another grouping. channels, a message says so and step 3 stays locked. Each picture should be a normal image of your sample; stripes or a smear mean X or Y is on the wrong dimension. Brightness is stretched for display only. - - The roles are checked against the axes each image's metadata names. A role that - disagrees with the header, or an axis the metadata does not name (a plain stack of - pages), is marked in amber under its dropdown, and the messages give the error - (`axes_not_named`, `axes_yx_swapped` or `axes_contradict_header`, §8) naming the - images with their header axes and your roles. Such images are never read: check the - axes, and if your roles are right, save the images as OME-Zarr with them and run - preprocessing on the copies. + - The roles are checked against the axes each image's metadata names, in one **Axis + check** card. A role that disagrees with the header is marked in amber under its + dropdown; the card gives the error (`axes_yx_swapped` or `axes_contradict_header`, + §8) naming the images with their header axes, your roles and the likely cause. Such + images are never read: correct the axes, or save named-axis copies with your roles and + run preprocessing on the copies. An axis the metadata does not name (a plain stack of + pages) takes the role you confirm: the card warns (`axes_not_named`) and the run + records the roles as confirmed by you. - If the preview looks right, press **Yes, the preview looks right**, then click **Confirm axis roles**. When the roles agree with every header, a green card says **Axes checked: they are correct for every image**. Changing a role afterwards asks @@ -388,10 +389,11 @@ morphobricks-preprocessing run /data/exp1 --axes ZCYX without being stated. Both commands check the axes (§4, step 2): `check` with the suggested roles, which are the -headers' own, so only images whose metadata does not name their axes fail there -(`axes_not_named`); `run` with the roles given in `--axes`. They print **Axes checked: -every image names its axes in its metadata, and the roles agree with them** when all is -well, or the errors naming the images, and stop (exit status 1). +headers' own, so only the `axes_not_named` warning of plain stacks can appear there; `run` +with the roles given in `--axes`. They print **Axes checked: every image names its axes in +its metadata, and the roles agree with them** when all is well (and name the plain stacks +whose roles are recorded as confirmed), or the errors naming the images, the likely cause +and the two fixes, and `run` stops (exit status 1). `--image-type fluorescence` or `--image-type rgb` gives the image type (§4, step 2). Without it, `run` uses the type suggested from the image headers and prints it, with why it @@ -573,9 +575,9 @@ continues. | `inconsistent_dimensions`, `inconsistent_mapping` | Error | Images have their dimensions in a different order. Export them the same way or process separately. | | `missing_axes` | Error | Some images lack a dimension. Confirm they have one plane along it (step 2), or export them again. | | `ambiguous_missing_axes` | Error | It is unclear which dimension an image lacks. Export it again. | -| `axes_not_named` | Error | The metadata of some images does not name their axes (a plain stack of pages, `IYX` or `QYX`): which dimension is depth, time or channel cannot be checked, so they are not read. The message names each image with its axes and the roles you confirmed. Check these axes; if they are right, save the images as OME-Zarr with them, then run preprocessing on the copies. | -| `axes_yx_swapped` | Error | The roles you confirmed give the Y and X the image header names the other way round. The images are not read transposed. Check the axes: if the header is right, confirm its axes; if your roles are right, save the images as OME-Zarr with them and run preprocessing on the copies. | -| `axes_contradict_header` | Error | The roles you confirmed contradict the axes the image header names (for example the header says depth, Z, and you chose time, T). The images are not read under a contradicted declaration. Check the axes as for `axes_yx_swapped`. | +| `axes_not_named` | Warning | The metadata of some images does not name their axes (a plain stack of pages, `IYX` or `QYX`), so which dimension is depth, time or channel comes only from the roles you confirm. The images are read with them, and the run manifest records them as confirmed by you. Check the roles with the preview. Before segmenting these images in other tools, consider saving named-axis copies (OME-TIFF), so every tool reads the same axes. | +| `axes_yx_swapped` | Error | The roles you confirmed give the Y and X the image header names the other way round, so the images are not read (never transposed). The message names the likely cause. Two fixes: correct the axes (if the header is right, choose its roles), or convert (save named-axis copies with your roles with `morphobricks-preprocessing convert`, then run preprocessing on the copies). | +| `axes_contradict_header` | Error | The roles you confirmed contradict the axes the image header names (for example the header says channels, C, and you chose time, T). The images are not read under a contradicted declaration. The message names the likely cause, often the tool that saved the files: tifffile labels a 3-D stack saved with `imagej=True` or `ome=True` as `CYX`, and an ImageJ stack of slices reads as depth even for a time-lapse. The same two fixes as `axes_yx_swapped`. | | `declared_axes_mismatch` | Error (other bricks; deprecated, removed in v0.5.0) | The axes a brick was told for an image or mask contradict the axes in its metadata (for example a depth stack declared as channels). Declare the axes the metadata gives, or export the image again with the right axes. | | `missing_xy_axis`, `duplicate_axis_role` | Error | X/Y missing or a role used twice. Correct the axis roles. | | `unassigned_axis` | Error | A dimension with several planes has no role. Choose Z, C or T. | diff --git a/notebooks/MorphoBricks_Preprocessing.ipynb b/notebooks/MorphoBricks_Preprocessing.ipynb index 55e4c979..f7a9ec26 100644 --- a/notebooks/MorphoBricks_Preprocessing.ipynb +++ b/notebooks/MorphoBricks_Preprocessing.ipynb @@ -292,8 +292,8 @@ "Run the cell below, then:\n", "\n", "1. Check the suggested role in each box. The line under each box says what the choice means.\n", - " An amber line marks a role that disagrees with the axes the image header names, or an axis\n", - " the metadata does not name (a plain stack of pages): such images are not read.\n", + " An amber line marks a role that disagrees with the axes the image header names: such images\n", + " are not read. An axis the metadata does not name (a plain stack) takes the role you confirm.\n", "2. Click **Show preview** and move the sliders. Each picture should look like a normal image of your\n", " sample. Stripes or a smeared picture mean X or Y is on the wrong dimension.\n", "3. If the preview looks right, press **Yes, the preview looks right** in the yellow box under the\n", @@ -470,10 +470,11 @@ "\n", "**\"No role is chosen for the axis\"**: a dimension with several planes needs Z, C or T in step 2.\n", "\n", - "**\"… do not name their axes in their metadata\"**, **\"… swap the Y and X their header names\"**\n", - "or **\"… contradict their header\"**: the images are not read. Check the axes in step 2. If the\n", - "header is right, choose its roles; if your roles are right, save the images as OME-Zarr with them\n", - "and run preprocessing on the copies.\n", + "**\"… swap the Y and X their header names\"** or **\"… contradict their header\"**: the images\n", + "are not read; the message names the likely cause. Correct the axes in step 2 (choose the\n", + "header's roles if it is right), or save named-axis copies with your roles and run preprocessing on\n", + "the copies. **\"… do not name their axes in their metadata\"** is a warning: check the roles with\n", + "the preview; they are recorded as confirmed by you.\n", "\n", "**\"The images have different numbers of channels\"**: each row of the table holds every channel of\n", "a plane, so all images need the same channels. Process the others as a separate experiment, or\n", diff --git a/src/morphobricks_preprocessing/__init__.py b/src/morphobricks_preprocessing/__init__.py index b5c57d19..7dd8777b 100644 --- a/src/morphobricks_preprocessing/__init__.py +++ b/src/morphobricks_preprocessing/__init__.py @@ -12,6 +12,7 @@ # deprecated stand-ins for one version (see _moved). from . import _moved, csv_io, dtypes, histogram, normalization, pooling, records # noqa: F401 from .axes import ( + AXIS_CHECK_CODES, AXIS_ERROR_CODES, AxisMapping, AxisReview, @@ -211,6 +212,7 @@ def __dir__() -> list[str]: __version__ = "0.0.0+unknown" __all__ = [ + "AXIS_CHECK_CODES", "AXIS_ERROR_CODES", "CONSISTENCY_CODES", "DEFAULT_MEMORY_LIMIT", diff --git a/src/morphobricks_preprocessing/axes.py b/src/morphobricks_preprocessing/axes.py index c1f8e49f..e4b4b75a 100644 --- a/src/morphobricks_preprocessing/axes.py +++ b/src/morphobricks_preprocessing/axes.py @@ -6,9 +6,10 @@ ``docs/contract.md`` §2 and resolves it for every file, including files that lack a dimension the others have (the ``missing_axes`` case of :mod:`~morphobricks_preprocessing.metadata`). It also checks every TIFF's confirmed roles -against the axes its header names (:func:`header_axis_issue`): an image whose metadata -does not name its axes, or whose confirmed roles swap or contradict them, is an error and -is never read; the remedy is to save it as OME-Zarr with the confirmed axes. +against the axes its header names (:func:`header_axis_issue`): confirmed roles that swap or +contradict the named axes are an error, and such an image is never read (the user corrects +the roles, or saves named-axis copies); an image whose metadata does not name its axes (a +plain stack) is read with the confirmed roles, with a warning, and recorded as confirmed. Nothing here reads pixels; the preview plane is read by :func:`~morphobricks_preprocessing.planes.read_sample_plane`. @@ -29,7 +30,7 @@ channel_count_issues, named_axes, ) -from .validation import ValidationIssue, error, has_errors, plural +from .validation import ValidationIssue, error, has_errors, plural, warning ROLES = ("X", "Y", "Z", "C", "T") """Supported axis roles.""" @@ -69,18 +70,28 @@ ) AXES_NOT_NAMED = "axes_not_named" -"""A TIFF whose metadata does not name an axis longer than 1 (a plain stack).""" +"""A warning: a TIFF whose metadata does not name an axis longer than 1 (a plain stack). It +is read with the roles the user confirmed, recorded per file as ``axes_from: "confirmed"``.""" AXES_YX_SWAPPED = "axes_yx_swapped" -"""Confirmed roles that give the header's named Y and X the other way round.""" +"""An error: confirmed roles that give the header's named Y and X the other way round.""" AXES_CONTRADICT_HEADER = "axes_contradict_header" -"""Confirmed roles that give a named axis another role than the header's.""" +"""An error: confirmed roles that give a named axis another role than the header's.""" -AXIS_ERROR_CODES = (AXES_NOT_NAMED, AXES_YX_SWAPPED, AXES_CONTRADICT_HEADER) -"""The three errors of an image whose confirmed axes cannot be checked against, or do not -agree with, the axes its header names (contract §2). Such an image is never read: it is -saved again as OME-Zarr with the confirmed axes, and preprocessing runs on the copy.""" +AXIS_ERROR_CODES = (AXES_YX_SWAPPED, AXES_CONTRADICT_HEADER) +"""The errors of an image whose confirmed roles disagree with the axes its header names +(contract §2). Such an image is never read, not transposed and not under a contradicted +declaration: the user corrects the roles, or saves the image as a named-axis copy.""" + +AXIS_CHECK_CODES = (AXES_NOT_NAMED, *AXIS_ERROR_CODES) +"""Every code of the axis check, shown together as one card.""" + +AXES_FROM_HEADER = "header" +"""Every axis of the image is named in its metadata (or OME-NGFF), and the roles agree.""" + +AXES_FROM_CONFIRMED = "confirmed" +"""Some axis of the image is named only by the user's confirmation (a plain stack).""" @dataclass(frozen=True) @@ -212,15 +223,22 @@ class AxisReview: Mapping for each readable file that fits, keyed by relative path. Files lacking dimensions have :attr:`AxisMapping.singleton_roles`. issues - Errors about the mapping and about files it does not fit. + Errors about the mapping and about files it does not fit, and the axis check's + ``axes_not_named`` warning. + axes_from + For each file of ``file_mappings``, where its roles come from: ``"header"`` when its + metadata names every axis longer than 1 (and they agree), ``"confirmed"`` when + some axis is named only by the user's confirmation (a plain stack). """ mapping: AxisMapping file_mappings: Mapping[str, AxisMapping] = field(default_factory=dict) issues: tuple[ValidationIssue, ...] = () + axes_from: Mapping[str, str] = field(default_factory=dict) def __post_init__(self) -> None: object.__setattr__(self, "file_mappings", MappingProxyType(dict(self.file_mappings))) + object.__setattr__(self, "axes_from", MappingProxyType(dict(self.axes_from))) @property def ok(self) -> bool: @@ -229,9 +247,19 @@ def ok(self) -> bool: @property def axis_errors(self) -> tuple[ValidationIssue, ...]: - """The ``axes_not_named``, ``axes_yx_swapped`` and ``axes_contradict_header`` errors.""" + """The ``axes_yx_swapped`` and ``axes_contradict_header`` errors.""" return tuple(i for i in self.issues if i.code in AXIS_ERROR_CODES) + @property + def axis_check(self) -> tuple[ValidationIssue, ...]: + """Every issue of the axis check, the ``axes_not_named`` warning included.""" + return tuple(i for i in self.issues if i.code in AXIS_CHECK_CODES) + + @property + def confirmed_files(self) -> tuple[str, ...]: + """Images whose roles are named only by the user's confirmation, sorted.""" + return tuple(sorted(p for p, how in self.axes_from.items() if how == AXES_FROM_CONFIRMED)) + @property def axis_error_files(self) -> tuple[str, ...]: """Images with one of the three axis errors (:data:`AXIS_ERROR_CODES`), sorted.""" @@ -242,7 +270,7 @@ def only_axis_errors(self) -> bool: """True if there are axis errors and no other error. The confirmed mapping then fits every file, so the images can be saved again as - OME-Zarr with it. + named-axis copies with it. """ errors = [i for i in self.issues if i.is_error] return bool(errors) and all(i.code in AXIS_ERROR_CODES for i in errors) @@ -429,13 +457,14 @@ def _deprecated(name: str) -> None: def header_axis_issue(meta: TiffMetadata, mapping: AxisMapping) -> str | None: """Compare an image's confirmed roles with the axes its header names (contract §2). - The one rule of the axis check: a TIFF is read only when its metadata names every axis - longer than 1 and the confirmed roles agree with those names. Position by position - (:func:`.metadata.named_axes`): + The one rule of the axis check: a TIFF is read only when the confirmed roles agree with + the axes its metadata names; an axis it does not name takes the role the user confirmed, + with a warning. Position by position (:func:`.metadata.named_axes`): - an axis the metadata leaves open (a placeholder ``I``/``Q``, and the ``Y`` and ``X`` - of a plain stack of pages) with more than one plane: ``axes_not_named``; an open - axis of length 1 is no problem; + of a plain stack of pages) with more than one plane: ``axes_not_named``, a warning + (the file is read with the confirmed role, recorded as ``axes_from: "confirmed"``); + an open axis of length 1 is no problem; - a named axis with its own letter as role, the RGB samples ``S`` as C when the header has no C, and a length-1 axis without a role agree; - the named Y given role X and the named X role Y: ``axes_yx_swapped``; @@ -458,12 +487,12 @@ def header_axis_issue(meta: TiffMetadata, mapping: AxisMapping) -> str | None: Returns ------- str or None - ``"axes_not_named"``, ``"axes_contradict_header"`` or ``"axes_yx_swapped"`` (the + ``"axes_contradict_header"``, ``"axes_yx_swapped"`` or ``"axes_not_named"`` (the first of these when several apply), or None when the roles agree with the header. """ found = set(axis_disagreements(meta, mapping).values()) - # A file that cannot be checked is that first; a contradiction is more than a swap. - for code in (AXES_NOT_NAMED, AXES_CONTRADICT_HEADER, AXES_YX_SWAPPED): + # Errors first (a contradiction is more than a swap); then the plain stack's warning. + for code in (AXES_CONTRADICT_HEADER, AXES_YX_SWAPPED, AXES_NOT_NAMED): if code in found: return code return None @@ -511,48 +540,91 @@ def axis_disagreements(meta: TiffMetadata, mapping: AxisMapping) -> dict[int, st def _axis_issues(found: Mapping[str, list[tuple[TiffMetadata, AxisMapping]]]) -> list: - """Build one error per axis problem, naming every file with its header and roles.""" + """Build one issue per axis-check code, naming every file with its header and roles. + + The errors name the likely cause for each arrangement (:func:`likely_axis_cause`) and the + two fixes: correct the roles, or save named-axis copies with them. + """ issues = [] - for code in AXIS_ERROR_CODES: + for code in AXIS_CHECK_CODES: files = found.get(code) if not files: continue - groups: dict[tuple[str, str], list[str]] = {} + groups: dict[tuple[str, str], list[tuple[TiffMetadata, AxisMapping]]] = {} for meta, mapping in files: - groups.setdefault((meta.axes, mapping.roles_string), []).append(meta.relative_path) - details = "; ".join( - f"header {axes}, confirmed {roles}: {_some(paths)}" - for (axes, roles), paths in sorted(groups.items()) - ) + groups.setdefault((meta.axes, mapping.roles_string), []).append((meta, mapping)) + parts = [] + for (axes, roles), members in sorted(groups.items()): + text = ( + f"header {axes}, confirmed {roles}: {_some([m.relative_path for m, _ in members])}" + ) + if code != AXES_NOT_NAMED: + text += f"; likely cause: {likely_axis_cause(*members[0], code)}" + parts.append(text) + details = "; ".join(parts) if code == AXES_NOT_NAMED else " | ".join(parts) n = plural(len(files), "image") + paths = sorted(meta.relative_path for meta, _ in files) if code == AXES_NOT_NAMED: - message = ( - f"{n} do not name their axes in their metadata ({details}). Which dimension " - "is depth, time or channel cannot be checked, so they are not read." + issues.append( + warning( + code, + f"{n} do not name their axes in their metadata ({details}). They are read " + "with the roles you confirmed, recorded in the run manifest as confirmed " + "by you.", + paths, + "Check the roles with the preview. Before segmenting these images in " + "other tools, consider saving named-axis copies (OME-TIFF), so every tool " + "reads the same axes.", + ) ) - action = _SAVE_AS_OME_ZARR - elif code == AXES_YX_SWAPPED: + continue + if code == AXES_YX_SWAPPED: message = ( f"The axes confirmed for {n} swap the Y and X their header names ({details}). " "They are not read transposed." ) - action = _SAVE_AS_OME_ZARR + _OR_HEADER else: message = ( f"The axes confirmed for {n} contradict their header ({details}). They are " "not read under a contradicted declaration." ) - action = _SAVE_AS_OME_ZARR + _OR_HEADER - paths = sorted(meta.relative_path for meta, _ in files) - issues.append(error(code, message, paths, action)) + issues.append(error(code, message, paths, _TWO_FIXES)) return issues -_SAVE_AS_OME_ZARR = ( - "Check these axes. If the confirmed axes are right, save the images as OME-Zarr with " - "them, so that every axis is named, then run preprocessing on the copies." +def likely_axis_cause(meta: TiffMetadata, mapping: AxisMapping, code: str) -> str: + """Name the likely cause of a header the confirmed roles contradict, in plain words. + + Most contradictions are headers written by the saving tool, not user mistakes: tifffile + labels any 3-D array saved with ``imagej=True`` or ``ome=True`` ``CYX``, so a depth stack + or time-lapse saved that way reads as channels; an ImageJ stack of slices reads as depth + even for a time-lapse. + """ + pairs = { + (letter, role) + for letter, role in zip(named_axes(meta), mapping.roles, strict=True) + if role is not None and letter not in (role, OPEN_AXIS) + } + if code == AXES_YX_SWAPPED or pairs <= {("Y", "X"), ("X", "Y")}: + return ( + "the image was saved with its rows and columns exchanged (for example a " + "transposed array), or the confirmed X and Y are the wrong way round" + ) + if any(letter == "C" and role in ("Z", "T") for letter, role in pairs): + return ( + "tifffile labels a 3-D stack saved with imagej=True or ome=True as CYX (channels) " + "by default, so a depth stack or time-lapse saved that way reads as channels" + ) + if ("Z", "T") in pairs: + return "an ImageJ stack of slices reads as depth (Z) even when it is a time-lapse" + return "the tool that saved the file wrote a default or wrong axis order in its metadata" + + +_TWO_FIXES = ( + "Two fixes. Correct the axes: if the header is right, choose its roles. Or convert: if " + "your roles are right, save these images as named-axis copies (OME-TIFF) with them " + "(morphobricks-preprocessing convert), then run preprocessing on the copies." ) -_OR_HEADER = " If the header is right, confirm the header's axes instead." def _some(paths: list[str], shown: int = 3) -> str: @@ -808,11 +880,12 @@ def review_axes( (``ambiguous_missing_axes``), files the mapping does not fit (``inconsistent_mapping``), files with different numbers of channels along the C axis (``different_channel_counts``; 1 without one), and the - axis check of every TIFF against its header (:func:`header_axis_issue`): - ``axes_not_named``, ``axes_yx_swapped`` and ``axes_contradict_header``, - each naming its files with their header axes and confirmed roles. These - three are errors: such an image is never read transposed or under a - contradicted declaration. + axis check of every TIFF against its header (:func:`header_axis_issue`), + each naming its files with their header axes and confirmed roles: the + errors ``axes_yx_swapped`` and ``axes_contradict_header`` (with the likely + cause; such an image is never read transposed or under a contradicted + declaration) and the warning ``axes_not_named`` (a plain stack, read with + the confirmed roles; :attr:`AxisReview.axes_from` records it). """ if isinstance(files, MetadataReport): files = files.files @@ -915,17 +988,24 @@ def review_axes( # does not fit already have their error; OME-Zarr images take their roles from their # metadata. found: dict[str, list[tuple[TiffMetadata, AxisMapping]]] = {} + axes_from: dict[str, str] = {} for f in readable: resolved = file_mappings.get(f.relative_path) if resolved is None: resolved = mapping.for_axes(f.axes) if f.relative_path in unconfirmed else None if resolved is None: continue + codes = set(axis_disagreements(f, resolved).values()) code = header_axis_issue(f, resolved) if code is not None: found.setdefault(code, []).append((f, resolved)) + if f.relative_path in file_mappings: + confirmed = AXES_NOT_NAMED in codes + axes_from[f.relative_path] = AXES_FROM_CONFIRMED if confirmed else AXES_FROM_HEADER issues.extend(_axis_issues(found)) - return AxisReview(mapping=mapping, file_mappings=file_mappings, issues=tuple(issues)) + return AxisReview( + mapping=mapping, file_mappings=file_mappings, issues=tuple(issues), axes_from=axes_from + ) def channel_count(f: TiffMetadata, mapping: AxisMapping) -> int: diff --git a/src/morphobricks_preprocessing/cli.py b/src/morphobricks_preprocessing/cli.py index a2128eb7..86ffaabf 100644 --- a/src/morphobricks_preprocessing/cli.py +++ b/src/morphobricks_preprocessing/cli.py @@ -367,14 +367,14 @@ def _check(args: argparse.Namespace) -> int: axis_errors = () if pre.suggested_mapping is not None: review = review_axes(pre.metadata, pre.suggested_mapping) - # The suggested roles are the header's own, so only an image that does not name its - # axes can fail the axis check here; 'run' checks the roles given with --axes. + # The suggested roles are the header's own, so they never contradict it; 'run' + # checks the roles given with --axes. axis_errors = review.axis_errors print() print(axis_check_summary(review)) - if axis_errors: + if review.axis_check: print() - print(format_issues(axis_errors)) + print(format_issues(review.axis_check)) print() print( f"Suggested option for 'run': --axes {pre.suggested_mapping.roles_string} " diff --git a/src/morphobricks_preprocessing/notebook_ui.py b/src/morphobricks_preprocessing/notebook_ui.py index 4e2bc6f4..20f22849 100644 --- a/src/morphobricks_preprocessing/notebook_ui.py +++ b/src/morphobricks_preprocessing/notebook_ui.py @@ -62,6 +62,7 @@ from . import notebook_style as ui from .axes import ( + AXIS_CHECK_CODES, AxisMapping, AxisReview, axis_disagreements, @@ -1393,7 +1394,10 @@ def _mapping_changed(self) -> None: missing_as_singleton=self.missing_as_singleton.value, ) if review.issues: - self.axis_status.value = issues_html(review.issues) + others = [i for i in review.issues if i.code not in AXIS_CHECK_CODES] + self.axis_status.value = axis_check_html(review) + ( + issues_html(others) if others else "" + ) self.axis_next.value = "" else: self.axis_status.value = ui.status( @@ -1522,14 +1526,29 @@ def _toggle_confirmation(self) -> None: ) if r is not None ] + review = review_axes( + self.preflight.metadata, + self.confirmed_mapping, + missing_as_singleton=self.missing_as_singleton.value, + ) + confirmed = review.confirmed_files + lines = [ + "The confirmed roles agree with every axis the image metadata names." + if confirmed + else "Every image names its axes in its metadata, and the confirmed roles " + "agree with them.", + *( + [ + f"{plural(len(confirmed), 'image')} do not name their axes: their " + "roles are recorded in the run manifest as confirmed by you." + ] + if confirmed + else [] + ), + *roles, + ] self.confirmed_status.value = ui.status( - "ok", - "Axes checked: they are correct for every image.", - [ - "Every image names its axes in its metadata, and the confirmed roles " - "agree with them. Nothing needs to be saved again.", - *roles, - ], + "ok", "Axes checked: they are correct for every image.", lines ) self._open_image_type() self._refresh_axis_next() @@ -3033,6 +3052,34 @@ def labels_html(pre: Preflight) -> str: ) +def axis_check_html(review: AxisReview) -> str: + """Show the axis check (contract §2) as one card: errors, plain stacks, or success. + + The three codes (``axes_yx_swapped``, ``axes_contradict_header``, ``axes_not_named``) + share one card; each group of files is listed with its header and confirmed axes, the + errors with the likely cause and the two fixes. The codes stay in small print, for logs. + """ + errors, notes = review.axis_errors, [i for i in review.axis_check if not i.is_error] + if not errors and not notes: + return "" + lines = [] + for issue in (*errors, *notes): + lines.append( + f"{html.escape(issue.message)}{ui.path_list(list(issue.paths), limit=5)}" + f"
{html.escape(issue.action)} " + f"({html.escape(issue.code)})
" + ) + if errors: + n = plural(len(review.axis_error_files), "image") + return ui.status( + "error", f"Axis check: {n} cannot be read with these roles.", lines, raw=True + ) + n = plural(len(review.confirmed_files), "image") + return ui.status( + "warn", f"Axis check: {n} do not name their axes; your roles are recorded.", lines, raw=True + ) + + def header_marks(mapping: AxisMapping, files: Iterable) -> dict[int, str]: """Return a warning (HTML) for each role that disagrees with the axes the headers name. @@ -3050,20 +3097,20 @@ def header_marks(mapping: AxisMapping, files: Iterable) -> dict[int, str]: letters.setdefault(index, set()).add(f.axes[index]) marks = {} for index, codes in found.items(): - if "axes_not_named" in codes: - text = ( - "The image metadata does not name this axis, so it cannot be checked: these " - "images are not read until they are saved as OME-Zarr with named axes." - ) - else: - named = ", ".join(axis_name(letter) for letter in sorted(letters[index])) - what = ( - "swaps the Y and X it names" if codes == {"axes_yx_swapped"} else "contradicts it" - ) - text = ( - f"The image header names this axis {html.escape(named)}; this role {what}. " - "Choose the header's role, or save the images as OME-Zarr with these axes." + if codes == {"axes_not_named"}: + marks[index] = ( + "
ⓘ The image metadata does not name " + "this axis: the role you confirm is recorded as yours. Check it with the " + "preview.
" ) + continue + codes = codes - {"axes_not_named"} + named = ", ".join(axis_name(letter) for letter in sorted(letters[index]) if letter) + what = "swaps the Y and X it names" if codes == {"axes_yx_swapped"} else "contradicts it" + text = ( + f"The image header names this axis {html.escape(named)}; this role {what}. " + "Choose the header's role, or save named-axis copies with these axes." + ) marks[index] = f"
⚠ {text}
" return marks diff --git a/src/morphobricks_preprocessing/report.py b/src/morphobricks_preprocessing/report.py index 8406defa..efe157a7 100644 --- a/src/morphobricks_preprocessing/report.py +++ b/src/morphobricks_preprocessing/report.py @@ -146,16 +146,23 @@ def mapping_summary(mapping: AxisMapping, files: Iterable = ()) -> str: def axis_check_summary(review: AxisReview) -> str: - """Say in one line whether the axis check passed (contract §2), for the CLI. + """Say in one line how the axis check went (contract §2), for the CLI. - The success message names what was checked; with axis errors it says how many images - are not read and that the errors are listed (they are printed with the other issues). + The success message names what was checked. With axis errors it says how many images + cannot be read and that the errors (with the likely cause and the two fixes) are listed; + with plain stacks, that their roles are recorded as confirmed. """ if review.axis_errors: n = plural(len(review.axis_error_files), "image") return ( - f"Axes check: {n} cannot be read with these axes (see the errors below). " - "Check their axes; if they are right, save the images as OME-Zarr with them." + f"Axes check: {n} cannot be read with these roles (see the errors below): correct " + "the axes, or save named-axis copies with 'convert'." + ) + if review.confirmed_files: + n = plural(len(review.confirmed_files), "image") + return ( + f"Axes checked: the roles agree with every header; {n} do not name their axes, " + "so their roles are recorded as confirmed by you (see the warning)." ) return ( "Axes checked: every image names its axes in its metadata, and the roles agree with them." diff --git a/tests/test_axes.py b/tests/test_axes.py index e60dac90..1372a302 100644 --- a/tests/test_axes.py +++ b/tests/test_axes.py @@ -256,13 +256,13 @@ def test_channel_count_follows_the_confirmed_channel_axis(): report = MetadataReport(tuple(files), tuple(check_metadata(files))) assert "different_channel_counts" not in codes(report.issues) as_time = review_axes(report, prefill_mapping("IYX").with_role(0, "T")) - # A plain stack never names its axes, so it is an error whatever the roles (v0.4.0). - assert codes(as_time.issues) == ["axes_not_named"] + # A plain stack never names its axes: read with the confirmed roles, with a warning. + assert as_time.ok and codes(as_time.issues) == ["axes_not_named"] as_channel = review_axes(report, prefill_mapping("IYX").with_role(0, "C")) assert codes(as_channel.combined_issues(report)) == [ "different_channel_counts", - "axes_not_named", "different_plane_counts", + "axes_not_named", ] @@ -307,7 +307,7 @@ def test_review_includes_the_mapping_rules(): review = review_axes(files, prefill_mapping("IYX")) assert not review.ok and codes(review.issues) == ["unassigned_axis", "axes_not_named"] as_time = review_axes(files, prefill_mapping("IYX").with_role(0, "T")) - assert codes(as_time.issues) == ["axes_not_named"] + assert as_time.ok and codes(as_time.issues) == ["axes_not_named"] def test_review_from_metadata_report_replaces_missing_axes(tmp_path): @@ -344,8 +344,8 @@ def test_combined_issues_put_errors_first(tmp_path): review = review_axes(report, prefill_mapping("QYX")) assert codes(review.combined_issues(report)) == [ "unassigned_axis", - "axes_not_named", "different_sizes", + "axes_not_named", ] diff --git a/tests/test_axis_check.py b/tests/test_axis_check.py index a387be6c..eefd85f0 100644 --- a/tests/test_axis_check.py +++ b/tests/test_axis_check.py @@ -1,10 +1,11 @@ -"""The axis check (contract §2, v0.4.0): any axis problem is an error, never read. - -An image whose metadata does not name its axes (``axes_not_named``), or whose confirmed -roles swap Y and X (``axes_yx_swapped``) or otherwise contradict its header -(``axes_contradict_header``), is an error naming the file with its header and confirmed -axes. Before v0.4.0 these were read silently: a plain stack with the user's roles, a swap -transposed, a contradiction under the declaration. +"""The axis check (contract §2, v0.4.0). + +Confirmed roles that swap Y and X (``axes_yx_swapped``) or otherwise contradict the header +(``axes_contradict_header``) are errors naming the file with its header and confirmed axes +and the likely cause; such an image is never read. Before v0.4.0 a swap was read transposed +and a contradiction under the declaration, silently. A plain stack whose metadata does not +name its axes is read with the confirmed roles, with the ``axes_not_named`` warning, and +recorded as confirmed (``AxisReview.axes_from``). """ import numpy as np @@ -12,6 +13,7 @@ import tifffile from morphobricks_preprocessing import ( + AXIS_CHECK_CODES, AXIS_ERROR_CODES, AxisMapping, TiffMetadata, @@ -114,10 +116,12 @@ def test_plain_stacks_never_name_their_axes(tmp_path, writer, roles): assert meta.axes in ("IYX", "QYX") review = review_axes([meta], AxisMapping(meta.axes, roles)) found = one(review.issues, "axes_not_named") - assert found.is_error and found.paths == ("a.tif",) + assert not found.is_error and found.paths == ("a.tif",) + assert review.ok and review.axes_from == {"a.tif": "confirmed"} + assert review.confirmed_files == ("a.tif",) assert f"header {meta.axes}, confirmed {roles}: a.tif" in found.message - assert "do not name their axes in their metadata" in found.message - assert "save the images as OME-Zarr" in found.action + assert "read with the roles you confirmed" in found.message + assert "named-axis copies (OME-TIFF)" in found.action def test_shaped_axes_with_a_placeholder_are_not_named(tmp_path): @@ -202,11 +206,35 @@ def test_every_file_is_named_with_its_header_and_confirmed_axes(tmp_path): found.message ) assert "They are not read transposed." in found.message - assert "confirm the header's axes instead" in found.action + assert "likely cause: the image was saved with its rows and columns exchanged" in ( + found.message + ) + assert "Correct the axes" in found.action and "Or convert" in found.action + + +def test_the_likely_cause_of_tifffiles_cyx_default_is_named(tmp_path): + # tifffile labels any 3-D stack saved with imagej=True or ome=True CYX. + meta = inspect_image(ome(tmp_path / "a.tif", data((5, 6, 8)), "CYX")) + review = review_axes([meta], AxisMapping("CYX", "TYX")) + found = one(review.issues, "axes_contradict_header") + assert "likely cause: tifffile labels a 3-D stack saved with imagej=True or ome=True" in ( + found.message + ) + relabel = review_axes( + [inspect_image(imagej(tmp_path / "b.tif", data((3, 6, 8)), "ZYX"))], + AxisMapping("ZYX", "TYX"), + ) + assert "an ImageJ stack of slices reads as depth" in relabel.issues[0].message -def test_the_three_codes(): - assert AXIS_ERROR_CODES == ("axes_not_named", "axes_yx_swapped", "axes_contradict_header") +def test_the_codes(): + assert AXIS_ERROR_CODES == ("axes_yx_swapped", "axes_contradict_header") + assert AXIS_CHECK_CODES == ("axes_not_named", "axes_yx_swapped", "axes_contradict_header") + + +def test_named_images_are_recorded_as_from_the_header(tmp_path): + meta = inspect_image(write_data(tmp_path / "a.tif", data((2, 6, 8)), "CYX")) + assert review_axes([meta], prefill_mapping("CYX")).axes_from == {"a.tif": "header"} # The run stops, reading nothing -------------------------------------------------------- @@ -232,7 +260,6 @@ def spy(path, *args, **kwargs): [ (lambda p: write_data(p, data((2, 6, 8)), "CYX"), "CYX", "CXY", "axes_yx_swapped"), (lambda p: imagej(p, data((3, 6, 8)), "ZYX"), "ZYX", "TYX", "axes_contradict_header"), - (lambda p: plain(p, data((3, 6, 8))), "IYX", "ZYX", "axes_not_named"), ], ) def test_run_stops_naming_every_file(tmp_path, reads, write, axes, roles, code): @@ -247,6 +274,13 @@ def test_run_stops_naming_every_file(tmp_path, reads, write, axes, roles, code): assert not default_output_folder(root).exists() +def test_a_plain_stack_runs_with_a_warning(tmp_path): + root = tmp_path / "exp" + plain(root / "ctrl/a.tif", data((3, 6, 8))) + result = run_experiment(root, AxisMapping("IYX", "ZYX")) + assert result.written and one(result.issues, "axes_not_named").paths == ("ctrl/a.tif",) + + def test_a_clean_run_is_unchanged(tmp_path): root = tmp_path / "exp" write_data(root / "ctrl/a.tif", data((2, 6, 8)), "CYX") @@ -324,9 +358,10 @@ def confirm(root): def test_cli_check_reports_plain_stacks(tmp_path, capsys): root = tmp_path / "exp" plain(root / "ctrl/a.tif", data((3, 6, 8))) - assert main(["check", str(root)]) == 1 + # A warning: 'run' reads the stack with the roles given in --axes. + assert main(["check", str(root)]) == 0 out = capsys.readouterr().out - assert "Axes check: 1 image cannot be read with these axes" in out + assert "1 image do not name their axes, so their roles are recorded as confirmed" in out assert "do not name their axes in their metadata (header IYX, confirmed -YX" in out diff --git a/tests/test_planes.py b/tests/test_planes.py index 5de1ccb9..1a553a10 100644 --- a/tests/test_planes.py +++ b/tests/test_planes.py @@ -111,12 +111,9 @@ def test_planes_match_the_source(tmp_path, page_reads, shape, axes, kwargs, role if roles is not None: mapping = AxisMapping(info.axes, roles) review = review_axes([info], mapping) - if axes is None: - # A plain stack is never analysed (axes_not_named); the reader still reads it with - # the confirmed roles to save it as OME-Zarr. - assert [i.code for i in review.issues] == ["axes_not_named"] - else: - assert review.ok + assert review.ok + # A plain stack is read with the confirmed roles, with the axes_not_named warning. + assert [i.code for i in review.issues] == (["axes_not_named"] if axes is None else []) planes = list(iter_planes(path, mapping)) expected = expected_planes(data, mapping.roles) diff --git a/tests/test_zarr_export.py b/tests/test_zarr_export.py index 3ba12f93..c9e44952 100644 --- a/tests/test_zarr_export.py +++ b/tests/test_zarr_export.py @@ -267,6 +267,7 @@ def test_missing_zarr_is_a_clear_error(tmp_path, monkeypatch): ), pytest.param((3, 6, 7), "ZXY", {}, None, ("Z", "Y", "X"), id="x-before-y"), pytest.param((1, 3, 6, 7), "QZYX", {}, None, ("Z", "Y", "X"), id="unassigned-singleton"), + pytest.param((4, 6, 7), None, {}, "TYX", ("T", "Y", "X"), id="pages-as-time"), ] From 9209b4b962a44797433f4a0731c10f8ae11234c1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 11:22:05 +0000 Subject: [PATCH 3/5] Move the axis check, its notebook card and role-based mask checks into modules The code quality rules for v0.4 are focused modules, no growth of oversized files, and NumPy docstrings on every function. So: - axis_check.py: the axis-check codes, header_axis_issue, axis_disagreements, likely_axis_cause, axis_check_issues and check_header_axes (axes.review_axes calls it). - notebook_axes.py: axis_check_html and header_marks. - mask_checks.py: mask_size_issues and mask_issues (moved from masks.py and run.py). No behaviour change. The names stay exported from the package. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WX6rdMV1r5jUPUR94kxV5a --- AGENTS.md | 10 +- src/morphobricks_preprocessing/__init__.py | 15 +- src/morphobricks_preprocessing/axes.py | 241 +------------- src/morphobricks_preprocessing/axis_check.py | 300 ++++++++++++++++++ src/morphobricks_preprocessing/mask_checks.py | 102 ++++++ src/morphobricks_preprocessing/masks.py | 51 --- .../notebook_axes.py | 99 ++++++ src/morphobricks_preprocessing/notebook_ui.py | 67 +--- src/morphobricks_preprocessing/run.py | 24 +- tests/test_axis_check.py | 4 +- 10 files changed, 542 insertions(+), 371 deletions(-) create mode 100644 src/morphobricks_preprocessing/axis_check.py create mode 100644 src/morphobricks_preprocessing/mask_checks.py create mode 100644 src/morphobricks_preprocessing/notebook_axes.py diff --git a/AGENTS.md b/AGENTS.md index 1a8b90fb..1144a093 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,9 +86,10 @@ src/morphobricks_preprocessing/ validation.py ValidationIssue (severity, code, message, paths, action), error()/warning() builders axes.py AxisMapping (role per dimension), prefill, check_mapping, review_axes (one mapping applied to every file, missing axes as singletons only when confirmed), - header_axis_issue/axis_disagreements (the axis check: confirmed roles - against the header's named axes, contract §2), check_declared_axes and - declared_axes_transposed (deprecated in v0.4.0, removed in v0.5.0) + check_declared_axes and declared_axes_transposed (deprecated in v0.4.0, + removed in v0.5.0) + axis_check.py the axis check (contract §2): header_axis_issue, axis_disagreements, + likely_axis_cause, check_header_axes; AXIS_ERROR_CODES, AXIS_CHECK_CODES planes.py plane-by-plane reader (iter_planes, read_plane, read_sample_plane), table rows as XYC planes whose channels are read one at a time (iter_xyc_planes, XycPlane), memory estimate, @@ -109,6 +110,8 @@ src/morphobricks_preprocessing/ experiment's folders, mixed; OME-Zarr label images), header-only checks (2D, image YX, integer/bool), MaskInventory (to_manifest), read_mask, read_mask_file (a mask by its path), check_labels (int32 labels, contract §16) + mask_checks.py mask checks with the confirmed roles: mask_size_issues (Y/X by role), + mask_issues (a run's mask issues; images with axis errors skipped) mask_scan.py the optional mask step (Q32): piece_counts (scipy.ndimage), pair_counts, scan_masks -> MaskReport (kinds, relations, direct-parent hierarchy, evidence), write_mask_report (mask_relationships.json); DEFAULT_THRESHOLDS, @@ -121,6 +124,7 @@ src/morphobricks_preprocessing/ block by block), ConversionSettings (chunk size, compression, pyramid levels), estimate_conversion notebook_style.py notebook look and feel: cards, status cards, tiles, tables, ConfirmToggle, Browse dialogs + notebook_axes.py the notebook's axis-check card and the marks under the role dropdowns notebook_ui.py PreprocessingNotebook: the four notebook steps (needs the `notebook` extra); folder_step/axis_step take title, purpose, intro and show_next (and show_output and show_design for step 1) so other bricks can embed them diff --git a/src/morphobricks_preprocessing/__init__.py b/src/morphobricks_preprocessing/__init__.py index 7dd8777b..4f76d2ae 100644 --- a/src/morphobricks_preprocessing/__init__.py +++ b/src/morphobricks_preprocessing/__init__.py @@ -12,8 +12,6 @@ # deprecated stand-ins for one version (see _moved). from . import _moved, csv_io, dtypes, histogram, normalization, pooling, records # noqa: F401 from .axes import ( - AXIS_CHECK_CODES, - AXIS_ERROR_CODES, AxisMapping, AxisReview, axis_length_ranges, @@ -22,12 +20,18 @@ declared_axes_transposed, declared_mapping, file_mapping_for, - header_axis_issue, prefill_experiment_mapping, prefill_mapping, reference_axes, review_axes, ) +from .axis_check import ( + AXIS_CHECK_CODES, + AXIS_ERROR_CODES, + axis_disagreements, + header_axis_issue, + likely_axis_cause, +) from .dataset import ( CONSISTENCY_CODES, BalanceCell, @@ -105,6 +109,7 @@ source_fingerprint, write_manifest, ) +from .mask_checks import mask_issues, mask_size_issues from .mask_scan import ( DEFAULT_THRESHOLDS, MASK_REPORT_FILENAME, @@ -124,7 +129,6 @@ check_labels, find_masks, mask_plane_issues, - mask_size_issues, read_mask, read_mask_file, ) @@ -295,6 +299,7 @@ def __dir__() -> list[str]: "ValidationIssue", "XycPlane", "ZarrUnavailableError", + "axis_disagreements", "axis_length_ranges", "balance_table", "biological_repeats", @@ -340,9 +345,11 @@ def __dir__() -> list[str]: "is_zarr_store", "iter_planes", "iter_xyc_planes", + "likely_axis_cause", "load_design", "make_plane_id", "images_named_as_masks", + "mask_issues", "mask_name_issues", "mask_plane_issues", "mask_size_issues", diff --git a/src/morphobricks_preprocessing/axes.py b/src/morphobricks_preprocessing/axes.py index e4b4b75a..e4e53425 100644 --- a/src/morphobricks_preprocessing/axes.py +++ b/src/morphobricks_preprocessing/axes.py @@ -23,6 +23,13 @@ from dataclasses import dataclass, field from types import MappingProxyType +from .axis_check import ( + AXES_FROM_CONFIRMED, + AXIS_CHECK_CODES, + AXIS_ERROR_CODES, + POSITION_AXES, + check_header_axes, +) from .metadata import ( OPEN_AXIS, MetadataReport, @@ -30,7 +37,7 @@ channel_count_issues, named_axes, ) -from .validation import ValidationIssue, error, has_errors, plural, warning +from .validation import ValidationIssue, error, has_errors, plural ROLES = ("X", "Y", "Z", "C", "T") """Supported axis roles.""" @@ -38,10 +45,6 @@ PLANE_ROLES = ("Z", "C", "T") """Roles that index planes; X and Y span each plane.""" -# tifffile labels for positions in a sample (tile, mosaic, row, column). They -# identify where an image was taken, not a dimension of one image, so they -# cannot be given a role in this milestone. -POSITION_AXES = frozenset("RMKJ") _AXIS_NAMES = { "X": "width", @@ -69,30 +72,6 @@ {"missing_axes", "inconsistent_dimensions", "different_channel_counts"} ) -AXES_NOT_NAMED = "axes_not_named" -"""A warning: a TIFF whose metadata does not name an axis longer than 1 (a plain stack). It -is read with the roles the user confirmed, recorded per file as ``axes_from: "confirmed"``.""" - -AXES_YX_SWAPPED = "axes_yx_swapped" -"""An error: confirmed roles that give the header's named Y and X the other way round.""" - -AXES_CONTRADICT_HEADER = "axes_contradict_header" -"""An error: confirmed roles that give a named axis another role than the header's.""" - -AXIS_ERROR_CODES = (AXES_YX_SWAPPED, AXES_CONTRADICT_HEADER) -"""The errors of an image whose confirmed roles disagree with the axes its header names -(contract §2). Such an image is never read, not transposed and not under a contradicted -declaration: the user corrects the roles, or saves the image as a named-axis copy.""" - -AXIS_CHECK_CODES = (AXES_NOT_NAMED, *AXIS_ERROR_CODES) -"""Every code of the axis check, shown together as one card.""" - -AXES_FROM_HEADER = "header" -"""Every axis of the image is named in its metadata (or OME-NGFF), and the roles agree.""" - -AXES_FROM_CONFIRMED = "confirmed" -"""Some axis of the image is named only by the user's confirmation (a plain stack).""" - @dataclass(frozen=True) class AxisMapping: @@ -454,185 +433,6 @@ def _deprecated(name: str) -> None: ) -def header_axis_issue(meta: TiffMetadata, mapping: AxisMapping) -> str | None: - """Compare an image's confirmed roles with the axes its header names (contract §2). - - The one rule of the axis check: a TIFF is read only when the confirmed roles agree with - the axes its metadata names; an axis it does not name takes the role the user confirmed, - with a warning. Position by position (:func:`.metadata.named_axes`): - - - an axis the metadata leaves open (a placeholder ``I``/``Q``, and the ``Y`` and ``X`` - of a plain stack of pages) with more than one plane: ``axes_not_named``, a warning - (the file is read with the confirmed role, recorded as ``axes_from: "confirmed"``); - an open axis of length 1 is no problem; - - a named axis with its own letter as role, the RGB samples ``S`` as C when the header - has no C, and a length-1 axis without a role agree; - - the named Y given role X and the named X role Y: ``axes_yx_swapped``; - - any other role for a named axis: ``axes_contradict_header``. That includes a Z read - as T, and a named axis that is not a role (angle ``A``, lambda ``E``, ...) given one. - A named axis longer than 1 without a role is :func:`check_mapping`'s - ``unassigned_axis``, and a position axis (``R``, ``M``, ``J``, ``K``) its - ``unsupported_axis``, so they are left to those. - - OME-Zarr images name their axes in their OME-NGFF metadata, which gives their roles - (:func:`declared_mapping`): they never have one of these problems. - - Parameters - ---------- - meta - The image's header. - mapping - The image's own mapping (:attr:`AxisReview.file_mappings`, :func:`file_mapping_for`). - - Returns - ------- - str or None - ``"axes_contradict_header"``, ``"axes_yx_swapped"`` or ``"axes_not_named"`` (the - first of these when several apply), or None when the roles agree with the header. - """ - found = set(axis_disagreements(meta, mapping).values()) - # Errors first (a contradiction is more than a swap); then the plain stack's warning. - for code in (AXES_CONTRADICT_HEADER, AXES_YX_SWAPPED, AXES_NOT_NAMED): - if code in found: - return code - return None - - -def axis_disagreements(meta: TiffMetadata, mapping: AxisMapping) -> dict[int, str]: - """Return, for each dimension whose role disagrees with the header, the axis error code. - - The position-by-position rule of :func:`header_axis_issue`, for showing which axis is - wrong (the notebook marks its role dropdown before the roles are confirmed). - - Parameters - ---------- - meta - The image's header. - mapping - The image's own mapping. - - Returns - ------- - dict - ``{dimension: code}``, with ``"axes_not_named"``, ``"axes_yx_swapped"`` or - ``"axes_contradict_header"``; empty for an OME-Zarr image, an unreadable one, or - roles that agree with the header. - """ - if not meta.readable or meta.is_ome_zarr or len(mapping.roles) != len(meta.axes): - return {} - found: dict[int, str] = {} - named = named_axes(meta) - for index, (letter, role, length) in enumerate( - zip(named, mapping.roles, meta.shape, strict=True) - ): - if letter == OPEN_AXIS: - if length > 1: - found[index] = AXES_NOT_NAMED - continue - if role == letter or (letter == "S" and role == "C" and "C" not in meta.axes): - continue - if role is None or letter in POSITION_AXES: - continue - found[index] = ( - AXES_YX_SWAPPED if letter in "YX" and role in "YX" else AXES_CONTRADICT_HEADER - ) - return found - - -def _axis_issues(found: Mapping[str, list[tuple[TiffMetadata, AxisMapping]]]) -> list: - """Build one issue per axis-check code, naming every file with its header and roles. - - The errors name the likely cause for each arrangement (:func:`likely_axis_cause`) and the - two fixes: correct the roles, or save named-axis copies with them. - """ - issues = [] - for code in AXIS_CHECK_CODES: - files = found.get(code) - if not files: - continue - groups: dict[tuple[str, str], list[tuple[TiffMetadata, AxisMapping]]] = {} - for meta, mapping in files: - groups.setdefault((meta.axes, mapping.roles_string), []).append((meta, mapping)) - parts = [] - for (axes, roles), members in sorted(groups.items()): - text = ( - f"header {axes}, confirmed {roles}: {_some([m.relative_path for m, _ in members])}" - ) - if code != AXES_NOT_NAMED: - text += f"; likely cause: {likely_axis_cause(*members[0], code)}" - parts.append(text) - details = "; ".join(parts) if code == AXES_NOT_NAMED else " | ".join(parts) - n = plural(len(files), "image") - paths = sorted(meta.relative_path for meta, _ in files) - if code == AXES_NOT_NAMED: - issues.append( - warning( - code, - f"{n} do not name their axes in their metadata ({details}). They are read " - "with the roles you confirmed, recorded in the run manifest as confirmed " - "by you.", - paths, - "Check the roles with the preview. Before segmenting these images in " - "other tools, consider saving named-axis copies (OME-TIFF), so every tool " - "reads the same axes.", - ) - ) - continue - if code == AXES_YX_SWAPPED: - message = ( - f"The axes confirmed for {n} swap the Y and X their header names ({details}). " - "They are not read transposed." - ) - else: - message = ( - f"The axes confirmed for {n} contradict their header ({details}). They are " - "not read under a contradicted declaration." - ) - issues.append(error(code, message, paths, _TWO_FIXES)) - return issues - - -def likely_axis_cause(meta: TiffMetadata, mapping: AxisMapping, code: str) -> str: - """Name the likely cause of a header the confirmed roles contradict, in plain words. - - Most contradictions are headers written by the saving tool, not user mistakes: tifffile - labels any 3-D array saved with ``imagej=True`` or ``ome=True`` ``CYX``, so a depth stack - or time-lapse saved that way reads as channels; an ImageJ stack of slices reads as depth - even for a time-lapse. - """ - pairs = { - (letter, role) - for letter, role in zip(named_axes(meta), mapping.roles, strict=True) - if role is not None and letter not in (role, OPEN_AXIS) - } - if code == AXES_YX_SWAPPED or pairs <= {("Y", "X"), ("X", "Y")}: - return ( - "the image was saved with its rows and columns exchanged (for example a " - "transposed array), or the confirmed X and Y are the wrong way round" - ) - if any(letter == "C" and role in ("Z", "T") for letter, role in pairs): - return ( - "tifffile labels a 3-D stack saved with imagej=True or ome=True as CYX (channels) " - "by default, so a depth stack or time-lapse saved that way reads as channels" - ) - if ("Z", "T") in pairs: - return "an ImageJ stack of slices reads as depth (Z) even when it is a time-lapse" - return "the tool that saved the file wrote a default or wrong axis order in its metadata" - - -_TWO_FIXES = ( - "Two fixes. Correct the axes: if the header is right, choose its roles. Or convert: if " - "your roles are right, save these images as named-axis copies (OME-TIFF) with them " - "(morphobricks-preprocessing convert), then run preprocessing on the copies." -) - - -def _some(paths: list[str], shown: int = 3) -> str: - paths = sorted(paths) - text = ", ".join(paths[:shown]) - return text if len(paths) <= shown else f"{text} and {len(paths) - shown} more" - - def reference_axes(files: Iterable[TiffMetadata]) -> str | None: """Return the arrangement the experiment mapping is defined on. @@ -987,22 +787,15 @@ def review_axes( # Each file's confirmed roles against the axes its own header names. Files the mapping # does not fit already have their error; OME-Zarr images take their roles from their # metadata. - found: dict[str, list[tuple[TiffMetadata, AxisMapping]]] = {} - axes_from: dict[str, str] = {} - for f in readable: - resolved = file_mappings.get(f.relative_path) - if resolved is None: - resolved = mapping.for_axes(f.axes) if f.relative_path in unconfirmed else None - if resolved is None: - continue - codes = set(axis_disagreements(f, resolved).values()) - code = header_axis_issue(f, resolved) - if code is not None: - found.setdefault(code, []).append((f, resolved)) - if f.relative_path in file_mappings: - confirmed = AXES_NOT_NAMED in codes - axes_from[f.relative_path] = AXES_FROM_CONFIRMED if confirmed else AXES_FROM_HEADER - issues.extend(_axis_issues(found)) + checked = [ + (f, file_mappings.get(f.relative_path) or mapping.for_axes(f.axes)) + for f in readable + if f.relative_path in file_mappings or f.relative_path in unconfirmed + ] + axis_issues, axes_from = check_header_axes( + [(f, m) for f, m in checked if m is not None], analysed=file_mappings + ) + issues.extend(axis_issues) return AxisReview( mapping=mapping, file_mappings=file_mappings, issues=tuple(issues), axes_from=axes_from ) diff --git a/src/morphobricks_preprocessing/axis_check.py b/src/morphobricks_preprocessing/axis_check.py new file mode 100644 index 00000000..f89dd1ef --- /dev/null +++ b/src/morphobricks_preprocessing/axis_check.py @@ -0,0 +1,300 @@ +"""The axis check: confirmed roles against the axes an image's header names (contract §2). + +Every TIFF's confirmed roles are compared, position by position, with the axes its metadata +names (:func:`.metadata.named_axes`). Roles that swap the named Y and X, or give a named +axis another role, are errors: such an image is never read, not transposed and not under a +contradicted declaration. An axis the metadata does not name (a plain stack) takes the +role the user confirmed, with a warning, and is recorded as ``axes_from: "confirmed"``. +:func:`.axes.review_axes` runs this check for every file. + +Nothing here reads pixels. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from typing import TYPE_CHECKING + +from .metadata import OPEN_AXIS, TiffMetadata, named_axes +from .validation import ValidationIssue, error, plural, warning + +if TYPE_CHECKING: # the mapping class lives in .axes, which imports this module + from .axes import AxisMapping + +POSITION_AXES = frozenset("RMKJ") +"""``tifffile`` letters for positions in a sample (tile, mosaic, row, column): they identify +where an image was taken, not a dimension of one image, so they cannot be given a role.""" + +AXES_NOT_NAMED = "axes_not_named" +"""A warning: a TIFF whose metadata does not name an axis longer than 1 (a plain stack). It +is read with the roles the user confirmed, recorded per file as ``axes_from: "confirmed"``.""" + +AXES_YX_SWAPPED = "axes_yx_swapped" +"""An error: confirmed roles that give the header's named Y and X the other way round.""" + +AXES_CONTRADICT_HEADER = "axes_contradict_header" +"""An error: confirmed roles that give a named axis another role than the header's.""" + +AXIS_ERROR_CODES = (AXES_YX_SWAPPED, AXES_CONTRADICT_HEADER) +"""The errors of an image whose confirmed roles disagree with the axes its header names +(contract §2). Such an image is never read, not transposed and not under a contradicted +declaration: the user corrects the roles, or saves the image as a named-axis copy.""" + +AXIS_CHECK_CODES = (AXES_NOT_NAMED, *AXIS_ERROR_CODES) +"""Every code of the axis check, shown together as one card.""" + +AXES_FROM_HEADER = "header" +"""Every axis of the image is named in its metadata (or OME-NGFF), and the roles agree.""" + +AXES_FROM_CONFIRMED = "confirmed" +"""Some axis of the image is named only by the user's confirmation (a plain stack).""" + + +def header_axis_issue(meta: TiffMetadata, mapping: AxisMapping) -> str | None: + """Compare an image's confirmed roles with the axes its header names (contract §2). + + The one rule of the axis check: a TIFF is read only when the confirmed roles agree with + the axes its metadata names; an axis it does not name takes the role the user confirmed, + with a warning. Position by position (:func:`.metadata.named_axes`): + + - an axis the metadata leaves open (a placeholder ``I``/``Q``, and the ``Y`` and ``X`` + of a plain stack of pages) with more than one plane: ``axes_not_named``, a warning + (the file is read with the confirmed role, recorded as ``axes_from: "confirmed"``); + an open axis of length 1 is no problem; + - a named axis with its own letter as role, the RGB samples ``S`` as C when the header + has no C, and a length-1 axis without a role agree; + - the named Y given role X and the named X role Y: ``axes_yx_swapped``; + - any other role for a named axis: ``axes_contradict_header``. That includes a Z read + as T, and a named axis that is not a role (angle ``A``, lambda ``E``, ...) given one. + A named axis longer than 1 without a role is :func:`check_mapping`'s + ``unassigned_axis``, and a position axis (``R``, ``M``, ``J``, ``K``) its + ``unsupported_axis``, so they are left to those. + + OME-Zarr images name their axes in their OME-NGFF metadata, which gives their roles + (:func:`declared_mapping`): they never have one of these problems. + + Parameters + ---------- + meta + The image's header. + mapping + The image's own mapping (:attr:`AxisReview.file_mappings`, :func:`file_mapping_for`). + + Returns + ------- + str or None + ``"axes_contradict_header"``, ``"axes_yx_swapped"`` or ``"axes_not_named"`` (the + first of these when several apply), or None when the roles agree with the header. + """ + found = set(axis_disagreements(meta, mapping).values()) + # Errors first (a contradiction is more than a swap); then the plain stack's warning. + for code in (AXES_CONTRADICT_HEADER, AXES_YX_SWAPPED, AXES_NOT_NAMED): + if code in found: + return code + return None + + +def axis_disagreements(meta: TiffMetadata, mapping: AxisMapping) -> dict[int, str]: + """Return, for each dimension whose role disagrees with the header, the axis error code. + + The position-by-position rule of :func:`header_axis_issue`, for showing which axis is + wrong (the notebook marks its role dropdown before the roles are confirmed). + + Parameters + ---------- + meta + The image's header. + mapping + The image's own mapping. + + Returns + ------- + dict + ``{dimension: code}``, with ``"axes_not_named"``, ``"axes_yx_swapped"`` or + ``"axes_contradict_header"``; empty for an OME-Zarr image, an unreadable one, or + roles that agree with the header. + """ + if not meta.readable or meta.is_ome_zarr or len(mapping.roles) != len(meta.axes): + return {} + found: dict[int, str] = {} + named = named_axes(meta) + for index, (letter, role, length) in enumerate( + zip(named, mapping.roles, meta.shape, strict=True) + ): + if letter == OPEN_AXIS: + if length > 1: + found[index] = AXES_NOT_NAMED + continue + if role == letter or (letter == "S" and role == "C" and "C" not in meta.axes): + continue + if role is None or letter in POSITION_AXES: + continue + found[index] = ( + AXES_YX_SWAPPED if letter in "YX" and role in "YX" else AXES_CONTRADICT_HEADER + ) + return found + + +def axis_check_issues( + found: Mapping[str, list[tuple[TiffMetadata, AxisMapping]]], +) -> list[ValidationIssue]: + """Build one issue per axis-check code, naming every file with its header and roles. + + The errors name the likely cause for each arrangement (:func:`likely_axis_cause`) and the + two fixes: correct the roles, or save named-axis copies with them. + + Parameters + ---------- + found + The files with each code, as ``(header, own mapping)`` pairs. + + Returns + ------- + list of ValidationIssue + The ``axes_not_named`` warning and the two errors, in :data:`AXIS_CHECK_CODES` order. + """ + issues = [] + for code in AXIS_CHECK_CODES: + files = found.get(code) + if not files: + continue + groups: dict[tuple[str, str], list[tuple[TiffMetadata, AxisMapping]]] = {} + for meta, mapping in files: + groups.setdefault((meta.axes, mapping.roles_string), []).append((meta, mapping)) + parts = [] + for (axes, roles), members in sorted(groups.items()): + text = ( + f"header {axes}, confirmed {roles}: {_some([m.relative_path for m, _ in members])}" + ) + if code != AXES_NOT_NAMED: + text += f"; likely cause: {likely_axis_cause(*members[0], code)}" + parts.append(text) + details = "; ".join(parts) if code == AXES_NOT_NAMED else " | ".join(parts) + n = plural(len(files), "image") + paths = sorted(meta.relative_path for meta, _ in files) + if code == AXES_NOT_NAMED: + issues.append( + warning( + code, + f"{n} do not name their axes in their metadata ({details}). They are read " + "with the roles you confirmed, recorded in the run manifest as confirmed " + "by you.", + paths, + "Check the roles with the preview. Before segmenting these images in " + "other tools, consider saving named-axis copies (OME-TIFF), so every tool " + "reads the same axes.", + ) + ) + continue + if code == AXES_YX_SWAPPED: + message = ( + f"The axes confirmed for {n} swap the Y and X their header names ({details}). " + "They are not read transposed." + ) + else: + message = ( + f"The axes confirmed for {n} contradict their header ({details}). They are " + "not read under a contradicted declaration." + ) + issues.append(error(code, message, paths, _TWO_FIXES)) + return issues + + +def likely_axis_cause(meta: TiffMetadata, mapping: AxisMapping, code: str) -> str: + """Name the likely cause of a header the confirmed roles contradict, in plain words. + + Most contradictions are headers written by the saving tool, not user mistakes: tifffile + labels any 3-D array saved with ``imagej=True`` or ``ome=True`` ``CYX``, so a depth stack + or time-lapse saved that way reads as channels; an ImageJ stack of slices reads as depth + even for a time-lapse. + + Parameters + ---------- + meta + The image's header. + mapping + Its confirmed mapping. + code + ``"axes_yx_swapped"`` or ``"axes_contradict_header"``. + + Returns + ------- + str + The cause, as a phrase that follows "likely cause:". + """ + pairs = { + (letter, role) + for letter, role in zip(named_axes(meta), mapping.roles, strict=True) + if role is not None and letter not in (role, OPEN_AXIS) + } + if code == AXES_YX_SWAPPED or pairs <= {("Y", "X"), ("X", "Y")}: + return ( + "the image was saved with its rows and columns exchanged (for example a " + "transposed array), or the confirmed X and Y are the wrong way round" + ) + if any(letter == "C" and role in ("Z", "T") for letter, role in pairs): + return ( + "tifffile labels a 3-D stack saved with imagej=True or ome=True as CYX (channels) " + "by default, so a depth stack or time-lapse saved that way reads as channels" + ) + if ("Z", "T") in pairs: + return "an ImageJ stack of slices reads as depth (Z) even when it is a time-lapse" + return "the tool that saved the file wrote a default or wrong axis order in its metadata" + + +_TWO_FIXES = ( + "Two fixes. Correct the axes: if the header is right, choose its roles. Or convert: if " + "your roles are right, save these images as named-axis copies (OME-TIFF) with them " + "(morphobricks-preprocessing convert), then run preprocessing on the copies." +) + + +def _some(paths: Iterable[str], shown: int = 3) -> str: + """Name up to ``shown`` paths, sorted, then how many more there are. + + Parameters + ---------- + paths + The paths. + shown + How many to name. + + Returns + ------- + str + For example ``"a.tif, b.tif, c.tif and 2 more"``. + """ + paths = sorted(paths) + text = ", ".join(paths[:shown]) + return text if len(paths) <= shown else f"{text} and {len(paths) - shown} more" + + +def check_header_axes( + files: Iterable[tuple[TiffMetadata, AxisMapping]], analysed: Mapping[str, AxisMapping] +) -> tuple[list[ValidationIssue], dict[str, str]]: + """Run the axis check on every file and say where each analysed file's roles come from. + + Parameters + ---------- + files + Each file's header with its own mapping (files the mapping does not fit left out). + analysed + The files that are read (:attr:`.axes.AxisReview.file_mappings`); only these get an + ``axes_from`` entry. + + Returns + ------- + tuple + The axis-check issues (:func:`axis_check_issues`), and ``{path: axes_from}`` with + ``"header"`` or ``"confirmed"``. + """ + found: dict[str, list[tuple[TiffMetadata, AxisMapping]]] = {} + axes_from: dict[str, str] = {} + for meta, mapping in files: + code = header_axis_issue(meta, mapping) + if code is not None: + found.setdefault(code, []).append((meta, mapping)) + if meta.relative_path in analysed: + unnamed = AXES_NOT_NAMED in axis_disagreements(meta, mapping).values() + axes_from[meta.relative_path] = AXES_FROM_CONFIRMED if unnamed else AXES_FROM_HEADER + return axis_check_issues(found), axes_from diff --git a/src/morphobricks_preprocessing/mask_checks.py b/src/morphobricks_preprocessing/mask_checks.py new file mode 100644 index 00000000..bb91310f --- /dev/null +++ b/src/morphobricks_preprocessing/mask_checks.py @@ -0,0 +1,102 @@ +"""Mask checks that need the confirmed axis roles (contract §15). + +:func:`.masks.find_masks` checks masks from their headers before any role is confirmed. +With the confirmed roles, a mask's Y and X are compared with its image's Y and X *by role* +(:func:`mask_size_issues`) and its planes with the image's (:func:`.masks.mask_plane_issues`); +:func:`mask_issues` gives a run's mask issues. Masks are never converted or reordered; an +image with an axis error is never read, so its masks are not compared. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping + +from .axes import AxisMapping, AxisReview +from .masks import _TEXTS, MaskInventory, _yx, mask_plane_issues +from .metadata import MetadataReport, TiffMetadata +from .validation import ValidationIssue, error + + +def mask_size_issues( + inventory: MaskInventory, + images: Iterable[TiffMetadata], + file_mappings: Mapping[str, AxisMapping], + *, + skip: Iterable[str] = (), +) -> list[ValidationIssue]: + """Check that every mask has the Y and X sizes its image has under the confirmed roles. + + The image's Y and X are the dimensions its confirmed roles call Y and X, not its + header's letters. :func:`.run.run_experiment` uses this in place of :func:`find_masks`'s + header comparison, which it supersedes. + + Parameters + ---------- + inventory + The masks found by :func:`find_masks`. + images + Header information of the analysed images. + file_mappings + Each image's own axis mapping (:attr:`.axes.AxisReview.file_mappings`); images + without one are left to the axis review's errors. + skip + Images not to check: those with an axis error (:attr:`.axes.AxisReview.axis_error_files`), + which are never read, so a mask size would only be a misleading second message. Their + masks are checked when the images are saved as OME-Zarr. + + Returns + ------- + list of ValidationIssue + The error ``mask_size_mismatch`` naming the masks that do not fit, or nothing. + """ + headers = {meta.relative_path: meta for meta in images} + skip = set(skip) + mismatched = [] + for mask in inventory.files: + mapping = file_mappings.get(mask.image) + meta = headers.get(mask.image) + if not mask.axes or mapping is None or meta is None or mask.image in skip: + continue + y, x = mapping.index_of("Y"), mapping.index_of("X") + if y is None or x is None: + continue + if _yx(mask.axes, mask.shape) != (meta.shape[y], meta.shape[x]): + mismatched.append(mask.path) + if not mismatched: + return [] + message, action = _TEXTS["mask_size_mismatch"] + return [error("mask_size_mismatch", message, sorted(mismatched), action)] + + +def mask_issues( + masks: MaskInventory, metadata: MetadataReport, review: AxisReview +) -> list[ValidationIssue]: + """Return the mask issues of a run, with the masks' sizes and planes by the confirmed roles. + + :func:`.masks.find_masks` compares a mask's Y and X with those its image's header + names; with confirmed roles that comparison is replaced by + :func:`mask_size_issues`, and the planes are checked by + :func:`.masks.mask_plane_issues`. Images with an axis error are never read, so their + masks are left out of both: a size or plane message would only repeat the axis error + in another form. + + Parameters + ---------- + masks + The masks found by :func:`.masks.find_masks`. + metadata + The image headers. + review + The axis review of the confirmed mapping. + + Returns + ------- + list of ValidationIssue + The inventory's issues, with the size issues by role, and the plane issues. + """ + skip = review.axis_error_files + return [ + *(i for i in masks.issues if i.code != "mask_size_mismatch"), + *mask_size_issues(masks, metadata.files, review.file_mappings, skip=skip), + *mask_plane_issues(masks, metadata.files, review.file_mappings, skip=skip), + ] diff --git a/src/morphobricks_preprocessing/masks.py b/src/morphobricks_preprocessing/masks.py index 26f28879..2dda8d13 100644 --- a/src/morphobricks_preprocessing/masks.py +++ b/src/morphobricks_preprocessing/masks.py @@ -851,57 +851,6 @@ def _image_yx(meta: TiffMetadata) -> tuple[int, int] | None: return (meta.shape[named.index("Y")], meta.shape[named.index("X")]) -def mask_size_issues( - inventory: MaskInventory, - images: Iterable[TiffMetadata], - file_mappings: Mapping[str, AxisMapping], - *, - skip: Iterable[str] = (), -) -> list[ValidationIssue]: - """Check that every mask has the Y and X sizes its image has under the confirmed roles. - - The image's Y and X are the dimensions its confirmed roles call Y and X, not its - header's letters. :func:`.run.run_experiment` uses this in place of :func:`find_masks`'s - header comparison, which it supersedes. - - Parameters - ---------- - inventory - The masks found by :func:`find_masks`. - images - Header information of the analysed images. - file_mappings - Each image's own axis mapping (:attr:`.axes.AxisReview.file_mappings`); images - without one are left to the axis review's errors. - skip - Images not to check: those with an axis error (:attr:`.axes.AxisReview.axis_error_files`), - which are never read, so a mask size would only be a misleading second message. Their - masks are checked when the images are saved as OME-Zarr. - - Returns - ------- - list of ValidationIssue - The error ``mask_size_mismatch`` naming the masks that do not fit, or nothing. - """ - headers = {meta.relative_path: meta for meta in images} - skip = set(skip) - mismatched = [] - for mask in inventory.files: - mapping = file_mappings.get(mask.image) - meta = headers.get(mask.image) - if not mask.axes or mapping is None or meta is None or mask.image in skip: - continue - y, x = mapping.index_of("Y"), mapping.index_of("X") - if y is None or x is None: - continue - if _yx(mask.axes, mask.shape) != (meta.shape[y], meta.shape[x]): - mismatched.append(mask.path) - if not mismatched: - return [] - message, action = _TEXTS["mask_size_mismatch"] - return [error("mask_size_mismatch", message, sorted(mismatched), action)] - - def _missing_issues( types: tuple[str, ...], files: list[MaskFile], images: Mapping[str, TiffMetadata] ) -> list[ValidationIssue]: diff --git a/src/morphobricks_preprocessing/notebook_axes.py b/src/morphobricks_preprocessing/notebook_axes.py new file mode 100644 index 00000000..a671ddd7 --- /dev/null +++ b/src/morphobricks_preprocessing/notebook_axes.py @@ -0,0 +1,99 @@ +"""The axis check in the notebook: one card, and marks under the role dropdowns. + +HTML only, from the axis review (:mod:`.axis_check`); the widgets live in +:mod:`.notebook_ui` (the ``notebook`` extra). +""" + +from __future__ import annotations + +import html +from collections.abc import Iterable + +from . import notebook_style as ui +from .axes import AxisMapping, AxisReview, axis_name +from .axis_check import axis_disagreements +from .validation import plural + + +def axis_check_html(review: AxisReview) -> str: + """Show the axis check (contract §2) as one card: errors or plain stacks. + + The three codes (``axes_yx_swapped``, ``axes_contradict_header``, ``axes_not_named``) + share one card; each group of files is listed with its header and confirmed axes, the + errors with the likely cause and the two fixes. The codes stay in small print, for logs. + + Parameters + ---------- + review + The axis review of the roles being edited or confirmed. + + Returns + ------- + str + The card's HTML; empty when the axis check found nothing. + """ + errors, notes = review.axis_errors, [i for i in review.axis_check if not i.is_error] + if not errors and not notes: + return "" + lines = [] + for issue in (*errors, *notes): + lines.append( + f"{html.escape(issue.message)}{ui.path_list(list(issue.paths), limit=5)}" + f"
{html.escape(issue.action)} " + f"({html.escape(issue.code)})
" + ) + if errors: + n = plural(len(review.axis_error_files), "image") + return ui.status( + "error", f"Axis check: {n} cannot be read with these roles.", lines, raw=True + ) + n = plural(len(review.confirmed_files), "image") + return ui.status( + "warn", f"Axis check: {n} do not name their axes; your roles are recorded.", lines, raw=True + ) + + +def header_marks(mapping: AxisMapping, files: Iterable) -> dict[int, str]: + """Return a warning (HTML) for each role that disagrees with the axes the headers name. + + Shown under the role dropdowns before the roles are confirmed, from the TIFF images + with the mapping's arrangement (:func:`.axis_check.axis_disagreements`). + + Parameters + ---------- + mapping + The roles being edited. + files + The image headers. + + Returns + ------- + dict + ``{dimension: HTML}``; empty when the roles agree with every header. + """ + found: dict[int, set[str]] = {} + letters: dict[int, set[str]] = {} + for f in files: + if not f.readable or f.is_ome_zarr or f.axes != mapping.axes: + continue + for index, code in axis_disagreements(f, mapping).items(): + found.setdefault(index, set()).add(code) + letters.setdefault(index, set()).add(f.axes[index]) + marks = {} + for index, codes in found.items(): + if codes == {"axes_not_named"}: + marks[index] = ( + "
ⓘ The image metadata does not name " + "this axis: the role you confirm is recorded as yours. Check it with the " + "preview.
" + ) + continue + codes = codes - {"axes_not_named"} + named = ", ".join(axis_name(letter) for letter in sorted(letters[index]) if letter) + what = "swaps the Y and X it names" if codes == {"axes_yx_swapped"} else "contradicts it" + text = ( + f"The image header names this axis {html.escape(named)}; this role {what}. " + "Choose the header's role, or save named-axis copies with these axes." + ) + marks[index] = f"
⚠ {text}
" + return marks diff --git a/src/morphobricks_preprocessing/notebook_ui.py b/src/morphobricks_preprocessing/notebook_ui.py index 20f22849..a14a96da 100644 --- a/src/morphobricks_preprocessing/notebook_ui.py +++ b/src/morphobricks_preprocessing/notebook_ui.py @@ -62,15 +62,14 @@ from . import notebook_style as ui from .axes import ( - AXIS_CHECK_CODES, AxisMapping, AxisReview, - axis_disagreements, axis_length_ranges, axis_name, file_mapping_for, review_axes, ) +from .axis_check import AXIS_CHECK_CODES from .dataset import CONSISTENCY_CODES, BalanceTable, DatasetReport, check_dataset from .design import ( DESIGN_FILENAME, @@ -96,6 +95,7 @@ repeat_distributions, ) from .manifest import MANIFEST_FILENAME, ManifestFormatError, read_manifest +from .notebook_axes import axis_check_html, header_marks from .planes import ( DEFAULT_MEMORY_LIMIT, Plane, @@ -3052,69 +3052,6 @@ def labels_html(pre: Preflight) -> str: ) -def axis_check_html(review: AxisReview) -> str: - """Show the axis check (contract §2) as one card: errors, plain stacks, or success. - - The three codes (``axes_yx_swapped``, ``axes_contradict_header``, ``axes_not_named``) - share one card; each group of files is listed with its header and confirmed axes, the - errors with the likely cause and the two fixes. The codes stay in small print, for logs. - """ - errors, notes = review.axis_errors, [i for i in review.axis_check if not i.is_error] - if not errors and not notes: - return "" - lines = [] - for issue in (*errors, *notes): - lines.append( - f"{html.escape(issue.message)}{ui.path_list(list(issue.paths), limit=5)}" - f"
{html.escape(issue.action)} " - f"({html.escape(issue.code)})
" - ) - if errors: - n = plural(len(review.axis_error_files), "image") - return ui.status( - "error", f"Axis check: {n} cannot be read with these roles.", lines, raw=True - ) - n = plural(len(review.confirmed_files), "image") - return ui.status( - "warn", f"Axis check: {n} do not name their axes; your roles are recorded.", lines, raw=True - ) - - -def header_marks(mapping: AxisMapping, files: Iterable) -> dict[int, str]: - """Return a warning (HTML) for each role that disagrees with the axes the headers name. - - Shown under the role dropdowns before the roles are confirmed, from the TIFF images - with the mapping's arrangement (:func:`.axes.axis_disagreements`); empty when the - roles agree with every header. - """ - found: dict[int, set[str]] = {} - letters: dict[int, set[str]] = {} - for f in files: - if not f.readable or f.is_ome_zarr or f.axes != mapping.axes: - continue - for index, code in axis_disagreements(f, mapping).items(): - found.setdefault(index, set()).add(code) - letters.setdefault(index, set()).add(f.axes[index]) - marks = {} - for index, codes in found.items(): - if codes == {"axes_not_named"}: - marks[index] = ( - "
ⓘ The image metadata does not name " - "this axis: the role you confirm is recorded as yours. Check it with the " - "preview.
" - ) - continue - codes = codes - {"axes_not_named"} - named = ", ".join(axis_name(letter) for letter in sorted(letters[index]) if letter) - what = "swaps the Y and X it names" if codes == {"axes_yx_swapped"} else "contradicts it" - text = ( - f"The image header names this axis {html.escape(named)}; this role {what}. " - "Choose the header's role, or save named-axis copies with these axes." - ) - marks[index] = f"
⚠ {text}
" - return marks - - def role_effect(role: str | None, low: int, high: int) -> str: """One plain sentence on what a role means for an axis of this length.""" length = f"{low}" if low == high else f"{low} to {high}" diff --git a/src/morphobricks_preprocessing/run.py b/src/morphobricks_preprocessing/run.py index 110f8cf1..2ff3a4ca 100644 --- a/src/morphobricks_preprocessing/run.py +++ b/src/morphobricks_preprocessing/run.py @@ -46,7 +46,6 @@ from .axes import ( AxisMapping, - AxisReview, file_mapping_for, prefill_experiment_mapping, review_axes, @@ -75,7 +74,8 @@ read_manifest, write_manifest, ) -from .masks import MaskInventory, find_masks, mask_plane_issues, mask_size_issues +from .mask_checks import mask_issues +from .masks import MaskInventory, find_masks from .metadata import MetadataReport, TiffMetadata, inspect_images from .planes import ( DEFAULT_MEMORY_LIMIT, @@ -710,26 +710,6 @@ def stop(**counts) -> RunResult: ) -def mask_issues( - masks: MaskInventory, metadata: MetadataReport, review: AxisReview -) -> list[ValidationIssue]: - """Return the mask issues of a run, with the masks' sizes and planes by the confirmed roles. - - :func:`.masks.find_masks` compares a mask's Y and X with those its image's header - names; with confirmed roles that comparison is replaced by - :func:`.masks.mask_size_issues`, and the planes are checked by - :func:`.masks.mask_plane_issues`. Images with an axis error are never read, so their - masks are left out of both: a size or plane message would only repeat the axis error - in another form. - """ - skip = review.axis_error_files - return [ - *(i for i in masks.issues if i.code != "mask_size_mismatch"), - *mask_size_issues(masks, metadata.files, review.file_mappings, skip=skip), - *mask_plane_issues(masks, metadata.files, review.file_mappings, skip=skip), - ] - - class _OutputRollbackError(RuntimeError): """Raised when an output update fails and its earlier files cannot be restored.""" diff --git a/tests/test_axis_check.py b/tests/test_axis_check.py index eefd85f0..e349efda 100644 --- a/tests/test_axis_check.py +++ b/tests/test_axis_check.py @@ -26,7 +26,7 @@ review_axes, run_experiment, ) -from morphobricks_preprocessing.axes import axis_disagreements +from morphobricks_preprocessing.axis_check import axis_disagreements from morphobricks_preprocessing.cli import main from morphobricks_preprocessing.design import default_design, write_experiment_design from morphobricks_preprocessing.discovery import default_output_folder @@ -415,7 +415,7 @@ def test_notebook_marks_roles_that_disagree_with_the_header(tmp_path): def test_notebook_marks_axes_a_plain_stack_does_not_name(tmp_path): pytest.importorskip("ipywidgets") pytest.importorskip("matplotlib") - from morphobricks_preprocessing.notebook_ui import header_marks + from morphobricks_preprocessing.notebook_axes import header_marks root = tmp_path / "exp" plain(root / "ctrl/a.tif", data((3, 6, 8))) From 5f9b01559e14d9b7bed69164873e366236969d9a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 11:38:04 +0000 Subject: [PATCH 4/5] Name no missing convert command or manifest field in the axis messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The axis errors told users to run `morphobricks-preprocessing convert`, a command that does not exist yet (conversion is design v0.4 §5, PP-11), and the contract cited a §17 that does not exist. The plain-stack warning, the notebook and the user guide said the run manifest records the roles as confirmed, but manifest version 1 has no axes_from field (manifest v2 is PP-4). The messages now name only what exists; tests check both. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WX6rdMV1r5jUPUR94kxV5a --- docs/contract.md | 2 +- docs/user_guide.md | 4 ++-- src/morphobricks_preprocessing/axis_check.py | 7 +++---- src/morphobricks_preprocessing/mask_checks.py | 2 +- src/morphobricks_preprocessing/notebook_ui.py | 2 +- src/morphobricks_preprocessing/report.py | 2 +- tests/test_axis_check.py | 4 ++++ 7 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/contract.md b/docs/contract.md index 96dfa5ef..8ba7c28d 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -45,7 +45,7 @@ channel order; an image without a C axis has one channel. | Rule | Detail | |---|---| | Prefill | Axis labels from metadata: `X`, `Y`, `Z`, `C`, `T` keep their role; the RGB samples axis `S` becomes C when there is no other `C`. Every other label is left unassigned for the user to choose. | -| Axis check (v0.4.0) | Every TIFF's confirmed roles are compared with the axes its own header names (`named_axes`, §16), position by position (`header_axis_issue`, `axis_disagreements`): a named axis keeps its letter as role, except that the RGB samples `S` may be C when the header has no C, and a length-1 axis may have no role. Confirmed roles that give the named Y and X the other way round are the **error** `axes_yx_swapped`; any other role for a named axis (`ZYX` confirmed `TYX`, a named angle or lambda axis given a role) is the **error** `axes_contradict_header` (a named axis longer than 1 without a role is `unassigned_axis`, a position axis `unsupported_axis`). Each error names every file with its header axes and confirmed roles and the likely cause (`likely_axis_cause`: tifffile labels a 3-D stack saved with `imagej=True` or `ome=True` `CYX`; an ImageJ stack of slices reads as Z; rows and columns exchanged), and offers two fixes: correct the axes, or save named-axis copies (`convert`, §17) and run preprocessing on them. Such an image is never read: not transposed, not under a contradicted declaration. An axis the metadata does not name (a plain stack `IYX`, `QYX`, `QQYX`, or shaped metadata with a placeholder `QYX`; a plain 2D `YX` page and RGB `YXS` are named, decision D-d) takes the role the user confirmed: the image is read with it, with the **warning** `axes_not_named` recommending named copies before segmenting in other tools, and recorded as `axes_from: "confirmed"` (`AxisReview.axes_from`; `"header"` for every other image). OME-Zarr images take their roles from their OME-NGFF axes and never have these issues. Since one mapping covers each TIFF arrangement, all files of one arrangement pass or fail together. The notebook shows the three codes as one card; when the roles agree with every header, the notebook and `run` say so ("Axes checked"). Before v0.4.0 a swap was read transposed and a contradiction under the declaration, silently. | +| Axis check (v0.4.0) | Every TIFF's confirmed roles are compared with the axes its own header names (`named_axes`, §16), position by position (`header_axis_issue`, `axis_disagreements`): a named axis keeps its letter as role, except that the RGB samples `S` may be C when the header has no C, and a length-1 axis may have no role. Confirmed roles that give the named Y and X the other way round are the **error** `axes_yx_swapped`; any other role for a named axis (`ZYX` confirmed `TYX`, a named angle or lambda axis given a role) is the **error** `axes_contradict_header` (a named axis longer than 1 without a role is `unassigned_axis`, a position axis `unsupported_axis`). Each error names every file with its header axes and confirmed roles and the likely cause (`likely_axis_cause`: tifffile labels a 3-D stack saved with `imagej=True` or `ome=True` `CYX`; an ImageJ stack of slices reads as Z; rows and columns exchanged), and offers two fixes: correct the axes, or save named-axis copies and run preprocessing on them. Such an image is never read: not transposed, not under a contradicted declaration. An axis the metadata does not name (a plain stack `IYX`, `QYX`, `QQYX`, or shaped metadata with a placeholder `QYX`; a plain 2D `YX` page and RGB `YXS` are named, decision D-d) takes the role the user confirmed: the image is read with it, with the **warning** `axes_not_named` recommending named copies before segmenting in other tools, and recorded as `axes_from: "confirmed"` (`AxisReview.axes_from`; `"header"` for every other image). OME-Zarr images take their roles from their OME-NGFF axes and never have these issues. Since one mapping covers each TIFF arrangement, all files of one arrangement pass or fail together. The notebook shows the three codes as one card; when the roles agree with every header, the notebook and `run` say so ("Axes checked"). Before v0.4.0 a swap was read transposed and a contradiction under the declaration, silently. | | Unsupported axes | Position-like axes (tile `R`, mosaic `M`, row `K`, column `J`) cannot have a role. If they hold more than one plane, the file is an error. | | Singleton axes | An axis of length 1 needs no role; its index is always 0. | | Missing axes | A file lacking axes the mapping has is read with those axes as length 1 only when the user confirmed this and only one arrangement fits; its coordinates for them are 0. | diff --git a/docs/user_guide.md b/docs/user_guide.md index 1a3ac6f8..e479118d 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -575,8 +575,8 @@ continues. | `inconsistent_dimensions`, `inconsistent_mapping` | Error | Images have their dimensions in a different order. Export them the same way or process separately. | | `missing_axes` | Error | Some images lack a dimension. Confirm they have one plane along it (step 2), or export them again. | | `ambiguous_missing_axes` | Error | It is unclear which dimension an image lacks. Export it again. | -| `axes_not_named` | Warning | The metadata of some images does not name their axes (a plain stack of pages, `IYX` or `QYX`), so which dimension is depth, time or channel comes only from the roles you confirm. The images are read with them, and the run manifest records them as confirmed by you. Check the roles with the preview. Before segmenting these images in other tools, consider saving named-axis copies (OME-TIFF), so every tool reads the same axes. | -| `axes_yx_swapped` | Error | The roles you confirmed give the Y and X the image header names the other way round, so the images are not read (never transposed). The message names the likely cause. Two fixes: correct the axes (if the header is right, choose its roles), or convert (save named-axis copies with your roles with `morphobricks-preprocessing convert`, then run preprocessing on the copies). | +| `axes_not_named` | Warning | The metadata of some images does not name their axes (a plain stack of pages, `IYX` or `QYX`), so which dimension is depth, time or channel comes only from the roles you confirm. The images are read with them, and the roles are recorded as confirmed by you. Check the roles with the preview. Before segmenting these images in other tools, consider saving named-axis copies (OME-TIFF), so every tool reads the same axes. | +| `axes_yx_swapped` | Error | The roles you confirmed give the Y and X the image header names the other way round, so the images are not read (never transposed). The message names the likely cause. Two fixes: correct the axes (if the header is right, choose its roles), or convert (save named-axis copies with your roles, then run preprocessing on the copies). | | `axes_contradict_header` | Error | The roles you confirmed contradict the axes the image header names (for example the header says channels, C, and you chose time, T). The images are not read under a contradicted declaration. The message names the likely cause, often the tool that saved the files: tifffile labels a 3-D stack saved with `imagej=True` or `ome=True` as `CYX`, and an ImageJ stack of slices reads as depth even for a time-lapse. The same two fixes as `axes_yx_swapped`. | | `declared_axes_mismatch` | Error (other bricks; deprecated, removed in v0.5.0) | The axes a brick was told for an image or mask contradict the axes in its metadata (for example a depth stack declared as channels). Declare the axes the metadata gives, or export the image again with the right axes. | | `missing_xy_axis`, `duplicate_axis_role` | Error | X/Y missing or a role used twice. Correct the axis roles. | diff --git a/src/morphobricks_preprocessing/axis_check.py b/src/morphobricks_preprocessing/axis_check.py index f89dd1ef..617f4fc1 100644 --- a/src/morphobricks_preprocessing/axis_check.py +++ b/src/morphobricks_preprocessing/axis_check.py @@ -177,8 +177,7 @@ def axis_check_issues( warning( code, f"{n} do not name their axes in their metadata ({details}). They are read " - "with the roles you confirmed, recorded in the run manifest as confirmed " - "by you.", + "with the roles you confirmed, and recorded as confirmed by you.", paths, "Check the roles with the preview. Before segmenting these images in " "other tools, consider saving named-axis copies (OME-TIFF), so every tool " @@ -244,8 +243,8 @@ def likely_axis_cause(meta: TiffMetadata, mapping: AxisMapping, code: str) -> st _TWO_FIXES = ( "Two fixes. Correct the axes: if the header is right, choose its roles. Or convert: if " - "your roles are right, save these images as named-axis copies (OME-TIFF) with them " - "(morphobricks-preprocessing convert), then run preprocessing on the copies." + "your roles are right, save these images as named-axis copies (OME-TIFF) with them, " + "then run preprocessing on the copies." ) diff --git a/src/morphobricks_preprocessing/mask_checks.py b/src/morphobricks_preprocessing/mask_checks.py index bb91310f..8cbc4992 100644 --- a/src/morphobricks_preprocessing/mask_checks.py +++ b/src/morphobricks_preprocessing/mask_checks.py @@ -42,7 +42,7 @@ def mask_size_issues( skip Images not to check: those with an axis error (:attr:`.axes.AxisReview.axis_error_files`), which are never read, so a mask size would only be a misleading second message. Their - masks are checked when the images are saved as OME-Zarr. + masks are checked when the images are saved as named-axis copies. Returns ------- diff --git a/src/morphobricks_preprocessing/notebook_ui.py b/src/morphobricks_preprocessing/notebook_ui.py index a14a96da..12d4ec75 100644 --- a/src/morphobricks_preprocessing/notebook_ui.py +++ b/src/morphobricks_preprocessing/notebook_ui.py @@ -1540,7 +1540,7 @@ def _toggle_confirmation(self) -> None: *( [ f"{plural(len(confirmed), 'image')} do not name their axes: their " - "roles are recorded in the run manifest as confirmed by you." + "roles are recorded as confirmed by you." ] if confirmed else [] diff --git a/src/morphobricks_preprocessing/report.py b/src/morphobricks_preprocessing/report.py index efe157a7..1eecfbf9 100644 --- a/src/morphobricks_preprocessing/report.py +++ b/src/morphobricks_preprocessing/report.py @@ -156,7 +156,7 @@ def axis_check_summary(review: AxisReview) -> str: n = plural(len(review.axis_error_files), "image") return ( f"Axes check: {n} cannot be read with these roles (see the errors below): correct " - "the axes, or save named-axis copies with 'convert'." + "the axes, or save named-axis copies with your roles." ) if review.confirmed_files: n = plural(len(review.confirmed_files), "image") diff --git a/tests/test_axis_check.py b/tests/test_axis_check.py index e349efda..b04f09b0 100644 --- a/tests/test_axis_check.py +++ b/tests/test_axis_check.py @@ -121,6 +121,8 @@ def test_plain_stacks_never_name_their_axes(tmp_path, writer, roles): assert review.confirmed_files == ("a.tif",) assert f"header {meta.axes}, confirmed {roles}: a.tif" in found.message assert "read with the roles you confirmed" in found.message + # The run manifest (version 1) has no axes_from field: the message must not claim it. + assert "manifest" not in found.message assert "named-axis copies (OME-TIFF)" in found.action @@ -210,6 +212,8 @@ def test_every_file_is_named_with_its_header_and_confirmed_axes(tmp_path): found.message ) assert "Correct the axes" in found.action and "Or convert" in found.action + # There is no convert command yet (design v0.4 §5, PP-11): the fix must not name one. + assert "morphobricks-preprocessing convert" not in found.action def test_the_likely_cause_of_tifffiles_cyx_default_is_named(tmp_path): From b284034f5b90c098c39140316008b07f33396b61 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 11:38:25 +0000 Subject: [PATCH 5/5] Say the old axis functions go before v0.4.0, not in v0.5.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check_declared_axes and declared_axes_transposed are removed in v0.4.0 (design v0.4 revision 4, §6 and PP-2), so the deprecation warning, the docstrings, AGENTS.md, the contract, the user guide and plan Q45 no longer promise them until v0.5.0. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WX6rdMV1r5jUPUR94kxV5a --- AGENTS.md | 4 ++-- docs/contract.md | 2 +- docs/plan.md | 6 +++--- docs/user_guide.md | 2 +- src/morphobricks_preprocessing/axes.py | 9 +++++---- tests/test_header_axes.py | 7 ++++--- 6 files changed, 16 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1144a093..fb963ab4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,8 +86,8 @@ src/morphobricks_preprocessing/ validation.py ValidationIssue (severity, code, message, paths, action), error()/warning() builders axes.py AxisMapping (role per dimension), prefill, check_mapping, review_axes (one mapping applied to every file, missing axes as singletons only when confirmed), - check_declared_axes and declared_axes_transposed (deprecated in v0.4.0, - removed in v0.5.0) + check_declared_axes and declared_axes_transposed (deprecated; removed + before the v0.4.0 release) axis_check.py the axis check (contract §2): header_axis_issue, axis_disagreements, likely_axis_cause, check_header_axes; AXIS_ERROR_CODES, AXIS_CHECK_CODES planes.py plane-by-plane reader (iter_planes, read_plane, read_sample_plane), table rows as diff --git a/docs/contract.md b/docs/contract.md index 8ba7c28d..f62cf031 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -408,7 +408,7 @@ instead of keeping copies. | Plain-TIFF resolution (D-c) | `read_physical_sizes` | A TIFF's resolution tags give a pixel size only with OME or ImageJ metadata and a unit; a plain TIFF (scanner or image-editor dpi) is uncalibrated, never converted from inches or centimetres. | | Labels (D-b) | `check_labels(pixels, name)`, `LARGEST_LABEL` (`masks`) | A mask holds whole-number labels from 0 (background) to `LARGEST_LABEL` = 2,147,483,647: MorphoBricks labels are int32 in every brick. A negative label, a label above the limit, or a pixel type that is not integer or boolean is refused with a `ValueError` naming the mask and the value. `read_mask`, `piece_counts`, `pair_counts` and `scan_masks` call it. Since D86 the limit is int32; before, this brick refused only labels above 2^63 − 1 (int64), so a `uint32` or wider mask with a label of 3,000,000,000 is now refused. | | Header axes (D-d) | `header_axes(meta)`, `named_axes(meta)`, `series_axes_from_metadata(series)`, `PLACEHOLDER_AXES`, `OPEN_AXIS` (`metadata`) | The axes an image's own metadata names, as `tifffile` reads them with length-1 axes left out (`tifffile` keeps Y and X even of length 1, so an image one pixel high is `YX`): OME, ImageJ and `tifffile` "shaped" metadata (`{"axes": "CYX"}`, decision D-d), a plain 2D `YX` (or `YXS` RGB) TIFF, and an OME-Zarr image's OME-NGFF axes. None when an axis is a placeholder (`I`: a plain stack of pages; `Q`: shaped without axes, or an OME-NGFF axis of another type), or the image does not open: the file does not say which dimension is depth, time or channel. `named_axes` gives the rule position by position, with `?` (`OPEN_AXIS`) for a dimension the metadata does not name: a placeholder, and, in a plain stack of pages without axis metadata (`IYX`, `QYX`, `QQYX`; `TiffMetadata.axes_from_metadata` False, from `series_axes_from_metadata`; True, the strict reading, for a header built by hand), the `Y` and `X` too, which only say where each page's rows and columns are stored (a (Y, X, C) array written without metadata reads as `QYX`). `header_axes` gives axes exactly when nothing is open, and `check_declared_axes` then accepts those axes, them with `Y` and `X` swapped, and (without a C axis) `S` declared C; nothing else. | -| Declared axes | `check_declared_axes(meta, declared)`, `declared_axes_transposed(meta, declared)` (`axes`) | **Deprecated in v0.4.0, removed in v0.5.0** (they warn with a `DeprecationWarning`): bricks take no axis declarations and read the files of a preprocessing run whose axes were checked (§2); nothing is read transposed. Until then: axes a brick or user declares for an image or mask (`image_axes="CYX"`) may name what the header leaves open, never contradict it: one letter per header axis (any case), roles X, Y, Z, C, T each at most once, with `Y` and `X` declared when the header has them (a plain stack too); an axis the metadata names (`named_axes`) keeps its letter, except that the named `Y` and `X` may be declared swapped (header `CYX` declared `CXY`, `ZYX` declared `ZXY`): the image is read transposed, and `declared_axes_transposed` returns True so the caller can record it (owner decision of 31 October 2026; it raises `ValueError` for a declaration that does not fit); any other disagreement is refused; a dimension it leaves open may be declared any role, so a plain `QYX` stack may be declared `YXC` as well as `CYX` (since v0.3.4; before, its positional `Y` and `X` were binding and a placeholder could not be declared X or Y), while an ImageJ `ZYX` declared `CYX`, an OME `CYX` declared `YXC` or a shaped `CYX` declared `YXC` are refused; the RGB samples `S` may be declared C when the header has no C; any other header axis (tile `R`, ...) cannot be declared. Otherwise the error `declared_axes_mismatch` naming the image and both axes (`unreadable_tiff` or `unreadable_ome_zarr` when the header does not open). | +| Declared axes | `check_declared_axes(meta, declared)`, `declared_axes_transposed(meta, declared)` (`axes`) | **Deprecated; removed before the v0.4.0 release** (they warn with a `DeprecationWarning`): bricks take no axis declarations and read the files of a preprocessing run whose axes were checked (§2); nothing is read transposed. Until then: axes a brick or user declares for an image or mask (`image_axes="CYX"`) may name what the header leaves open, never contradict it: one letter per header axis (any case), roles X, Y, Z, C, T each at most once, with `Y` and `X` declared when the header has them (a plain stack too); an axis the metadata names (`named_axes`) keeps its letter, except that the named `Y` and `X` may be declared swapped (header `CYX` declared `CXY`, `ZYX` declared `ZXY`): the image is read transposed, and `declared_axes_transposed` returns True so the caller can record it (owner decision of 31 October 2026; it raises `ValueError` for a declaration that does not fit); any other disagreement is refused; a dimension it leaves open may be declared any role, so a plain `QYX` stack may be declared `YXC` as well as `CYX` (since v0.3.4; before, its positional `Y` and `X` were binding and a placeholder could not be declared X or Y), while an ImageJ `ZYX` declared `CYX`, an OME `CYX` declared `YXC` or a shaped `CYX` declared `YXC` are refused; the RGB samples `S` may be declared C when the header has no C; any other header axis (tile `R`, ...) cannot be declared. Otherwise the error `declared_axes_mismatch` naming the image and both axes (`unreadable_tiff` or `unreadable_ome_zarr` when the header does not open). | | Mask files | `read_mask_file(path, z=, t=)` (`masks`) | Reads one plane of a TIFF mask or of an OME-Zarr label image (`.zarr/labels/`) given by its path, with `read_mask`'s rules (§15): its axes are its own metadata's, Y and X plus at most a Z and a T axis named so, length-1 axes other than Y and X left out (a mask one pixel high or wide is read as a 2D array, plan §12 question 44); any other or unnamed dimension (a plain stack, `IYX`; a `C` axis) is refused, the message giving the header's axes and shape and the dimension that is not named, never taken for depth or time; `z` and `t` are required for a mask with that axis; integer or boolean pixels; labels checked with `check_labels`. A `ValueError` names the file and the problem. | | Identity (D-e) | `FOV_KEY`, `check_identity_text(value, field)`, `check_plane_index(value, axis)`, `check_identity_case(keys)`, `MISSING_TEXT`, `IdentityError` (`discovery`) | `FOV_KEY` = (`condition`, `biological_repeat`, `technical_repeat`, `position_id`), the position key of §1. `check_identity_text` accepts a label or `position_id` that is text without leading or trailing whitespace and not `""`, `NA`, `nan`, `none`, `null`, `n/a` or `` in any case (`MISSING_TEXT`); `check_plane_index` accepts None or an integer of at least 0, a Python `int` or a NumPy integer (returned as a plain `int`), and refuses everything else (bool, float, `"1"`) without converting it. Both raise `IdentityError` (a `ValueError`) naming the field and the value. Discovery applies the text rule to every folder and image name (`invalid_name`) and refuses names of one level that differ only in case anywhere in the experiment (`case_conflict`, §1). `position_id` keeps its own scope: unique within its technical repeat, ignoring case (`duplicate_position_id`). `check_identity_case` applies the same case rule to identities a brick reads from a table, one key per row in `FOV_KEY` order (or the three labels): a label of one level spelled two ways anywhere in the run, or two `position_id` values of one technical repeat equal ignoring case, raise `IdentityError` naming both spellings; the same key may repeat (several planes of one position). | | Sources of a run | `check_sources(manifest, experiment_root, relative_paths, image_axes=, identities=)`, `source_fingerprint`, `file_sha256` (`manifest`) | The checks a brick makes before using a run's table for some images (formerly CellPatcher `check_images` and MorphoFeature `histogram_fit`). Each image must be in the manifest (`images_not_in_manifest`), still exist (`images_missing`), and be unchanged by `source_fingerprint`, its size and modification time, for an OME-Zarr folder its files' total size and newest time, so a rewritten chunk counts (`images_changed`). Its recorded `position_id` must be the one its name gives; with `identities` (`FOV_KEY` order), its labels and `position_id` must be the caller's (`identity_differs`; `position_id_not_recorded` for a manifest written before positions were recorded). With `image_axes`, the caller's axes must give each axis the role recorded for the file (`FileEntry.roles`, or the run's mapping resolved for the header in older manifests); an axis without a role (length 1) fits any letter (`axis_mapping_mismatch`). All errors, naming their images; the table's own hash is checked with `file_sha256` against `csv_sha256`. | diff --git a/docs/plan.md b/docs/plan.md index 76be60be..5c926de8 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -1056,6 +1056,6 @@ Raised during review; to be decided before the milestones that need them: roles (they were compared with the header's letters; the role-based comparison still matters for plain stacks, whose positional Y and X need not be the roles), and not at all for an image with an axis error. `check_declared_axes` and - `declared_axes_transposed` are deprecated, to be removed in v0.5.0. Breaking for - v0.4.0: experiments run with roles that swap or contradict the header no longer run - as they are.* + `declared_axes_transposed` are deprecated, to be removed before the v0.4.0 release + (design v0.4 §6, PP-2). Breaking for v0.4.0: experiments run with roles that swap or + contradict the header no longer run as they are.* diff --git a/docs/user_guide.md b/docs/user_guide.md index e479118d..6696c0af 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -578,7 +578,7 @@ continues. | `axes_not_named` | Warning | The metadata of some images does not name their axes (a plain stack of pages, `IYX` or `QYX`), so which dimension is depth, time or channel comes only from the roles you confirm. The images are read with them, and the roles are recorded as confirmed by you. Check the roles with the preview. Before segmenting these images in other tools, consider saving named-axis copies (OME-TIFF), so every tool reads the same axes. | | `axes_yx_swapped` | Error | The roles you confirmed give the Y and X the image header names the other way round, so the images are not read (never transposed). The message names the likely cause. Two fixes: correct the axes (if the header is right, choose its roles), or convert (save named-axis copies with your roles, then run preprocessing on the copies). | | `axes_contradict_header` | Error | The roles you confirmed contradict the axes the image header names (for example the header says channels, C, and you chose time, T). The images are not read under a contradicted declaration. The message names the likely cause, often the tool that saved the files: tifffile labels a 3-D stack saved with `imagej=True` or `ome=True` as `CYX`, and an ImageJ stack of slices reads as depth even for a time-lapse. The same two fixes as `axes_yx_swapped`. | -| `declared_axes_mismatch` | Error (other bricks; deprecated, removed in v0.5.0) | The axes a brick was told for an image or mask contradict the axes in its metadata (for example a depth stack declared as channels). Declare the axes the metadata gives, or export the image again with the right axes. | +| `declared_axes_mismatch` | Error (other bricks; deprecated, removed in v0.4.0) | The axes a brick was told for an image or mask contradict the axes in its metadata (for example a depth stack declared as channels). Declare the axes the metadata gives, or export the image again with the right axes. | | `missing_xy_axis`, `duplicate_axis_role` | Error | X/Y missing or a role used twice. Correct the axis roles. | | `unassigned_axis` | Error | A dimension with several planes has no role. Choose Z, C or T. | | `unsupported_axis` | Error | A position or tile dimension. Export each position separately. | diff --git a/src/morphobricks_preprocessing/axes.py b/src/morphobricks_preprocessing/axes.py index e4e53425..47638a7b 100644 --- a/src/morphobricks_preprocessing/axes.py +++ b/src/morphobricks_preprocessing/axes.py @@ -291,8 +291,8 @@ def check_declared_axes(meta: TiffMetadata, declared: str) -> ValidationIssue | .. deprecated:: 0.4.0 Bricks no longer take axis declarations: they read the files of a preprocessing - run whose axes were checked (:func:`review_axes`, contract §2). This function will - be removed in v0.5.0. + run whose axes were checked (:func:`review_axes`, contract §2). This function is + removed before the v0.4.0 release. One rule for every brick that is told an image's (or mask's) axes, for example ``image_axes="CYX"`` (MorphoBricks D86): a declaration can name what the header leaves @@ -362,7 +362,8 @@ def declared_axes_transposed(meta: TiffMetadata, declared: str) -> bool: .. deprecated:: 0.4.0 Nothing is read transposed any more: a confirmed Y/X swap is the error - ``axes_yx_swapped`` (:func:`review_axes`). This function will be removed in v0.5.0. + ``axes_yx_swapped`` (:func:`review_axes`). This function is removed before the + v0.4.0 release. :func:`check_declared_axes` accepts a declaration that differs from the axes the metadata names only by swapping Y and X (header ``CYX`` declared ``CXY``): the image is @@ -425,7 +426,7 @@ def _declared_axes_problem(header: str, named: str, declared: str) -> str | None def _deprecated(name: str) -> None: warnings.warn( - f"morphobricks_preprocessing.{name} is deprecated and will be removed in v0.5.0: " + f"morphobricks_preprocessing.{name} is deprecated and will be removed in v0.4.0: " "bricks read the files of a preprocessing run whose axes were checked, and take no " "axis declarations.", DeprecationWarning, diff --git a/tests/test_header_axes.py b/tests/test_header_axes.py index 37b8be96..66720573 100644 --- a/tests/test_header_axes.py +++ b/tests/test_header_axes.py @@ -21,8 +21,9 @@ ) from tiff_tree import write_data -# check_declared_axes and declared_axes_transposed are deprecated in v0.4.0 (removed in -# v0.5.0); their rules stay tested until then, and the warning has its own test below. +# check_declared_axes and declared_axes_transposed are deprecated (removed before the v0.4.0 +# release, design v0.4 §6); their rules stay tested until then, and the warning has its own +# test below. pytestmark = pytest.mark.filterwarnings( "ignore:morphobricks_preprocessing.(check_declared_axes|declared_axes_transposed) is " "deprecated:DeprecationWarning" @@ -32,7 +33,7 @@ @pytest.mark.filterwarnings("error") def test_declared_axes_checks_are_deprecated(tmp_path): meta = inspect_image(write_data(tmp_path / "a.tif", np.zeros((2, 4, 5), np.uint8), "CYX")) - with pytest.warns(DeprecationWarning, match="check_declared_axes is deprecated.*v0.5.0"): + with pytest.warns(DeprecationWarning, match="check_declared_axes is deprecated.*v0.4.0"): assert check_declared_axes(meta, "CYX") is None with pytest.warns(DeprecationWarning, match="declared_axes_transposed is deprecated"): assert declared_axes_transposed(meta, "CXY")