From 8e1a3ed33be3344f6107846592596c5a5c4dd93b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 13:50:13 +0000 Subject: [PATCH 1/5] Set version 0.4.0rc5 and the notebook pin Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- README.md | 2 +- docs/user_guide.md | 2 +- notebooks/MorphoBricks_Preprocessing.ipynb | 6 +++--- notebooks/notebook_latest_versions.yaml | 2 +- pyproject.toml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 07175b0b..3e52f8dc 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ LabConstrictor's needs ipywidgets 8). Locally Step 0 prints the Python version, platform and working folder. No shell or line magics are used; pip runs with the kernel's Python. 3. When the package is missing (always on a fresh Colab runtime), Step 0 installs the release - named by `RELEASE_TAG` (currently `v0.4.0rc4`), never the moving `main` branch, and prints + named by `RELEASE_TAG` (currently `v0.4.0rc5`), never the moving `main` branch, and prints which source and version it uses. A checkout around the notebook, or an installed package, is used first. 4. `current_version` and `notebook_name` in Step 0 are the names LabConstrictor's version check diff --git a/docs/user_guide.md b/docs/user_guide.md index c46145a3..8b142e68 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -21,7 +21,7 @@ run **Step 0**. It detects local Jupyter or Google Colab and checks these option 2. **Installed package:** otherwise, use the package already available to this kernel. 3. **GitHub installation:** if the package is missing, install it with the notebook dependencies from the project's GitHub repository, at the release the notebook names - (`RELEASE_TAG` in Step 0, for example `v0.4.0rc4`), never the changing `main` branch. + (`RELEASE_TAG` in Step 0, for example `v0.4.0rc5`), never the changing `main` branch. Missing notebook dependencies are installed into the same Python environment as the kernel. A local checkout is installed in editable mode when it needs dependencies; diff --git a/notebooks/MorphoBricks_Preprocessing.ipynb b/notebooks/MorphoBricks_Preprocessing.ipynb index 0e18d40f..6a7047ad 100644 --- a/notebooks/MorphoBricks_Preprocessing.ipynb +++ b/notebooks/MorphoBricks_Preprocessing.ipynb @@ -90,7 +90,7 @@ "appears when everything is ready.\n", "\n", "> **Colab users:** Step 0 connects your Google Drive (allow access in the window that opens)\n", - "> and installs MorphoBricks preprocessing **v0.4.0rc4** from GitHub, which takes a minute or\n", + "> and installs MorphoBricks preprocessing **v0.4.0rc5** from GitHub, which takes a minute or\n", "> two. Keep your experiment folder and your results on Google Drive: in this notebook your Drive\n", "> is the folder **/content/drive/MyDrive**. Everything else on the runtime is wiped when it\n", "> disconnects; then run Step 0 again.\n", @@ -130,12 +130,12 @@ "\n", "# The notebook's version and name, as LabConstrictor's version check reads them\n", "# (notebooks/notebook_latest_versions.yaml lists the latest version).\n", - "current_version = \"0.4.0rc4\"\n", + "current_version = \"0.4.0rc5\"\n", "notebook_name = \"MorphoBricks_Preprocessing\"\n", "# The release installed from GitHub when the package is missing (Google Colab, or a\n", "# downloaded notebook). Set to the new tag, with current_version, before each release is\n", "# tagged, so a tagged notebook installs its own release (docs/plan.md, release checklist).\n", - "RELEASE_TAG = \"v0.4.0rc4\"\n", + "RELEASE_TAG = \"v0.4.0rc5\"\n", "REPOSITORY = \"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/CellMigrationLab/morphobricks-preprocessing.git\"\n", "# The package's extras to install; on Colab also zarr, to read OME-Zarr images.\n", "EXTRAS = \"notebook\"\n", diff --git a/notebooks/notebook_latest_versions.yaml b/notebooks/notebook_latest_versions.yaml index dcfd199f..90337645 100644 --- a/notebooks/notebook_latest_versions.yaml +++ b/notebooks/notebook_latest_versions.yaml @@ -1,3 +1,3 @@ # Latest version of each notebook, read by LabConstrictor's version check by notebook_name. # Equal to the notebook's current_version and its RELEASE_TAG without the "v" (README, "Notebook"). -"MorphoBricks_Preprocessing": "0.4.0rc4" +"MorphoBricks_Preprocessing": "0.4.0rc5" diff --git a/pyproject.toml b/pyproject.toml index 8d05bbb8..9fa8d755 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "morphobricks-preprocessing" -version = "0.4.0rc4" +version = "0.4.0rc5" description = "MorphoBricks preprocessing brick: per-plane intensity histograms and normalization methods for microscopy TIFFs." readme = "README.md" license = "MIT" From 6fc6e0b2be86bbb68911f0ad9d55205b69ea87a8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 13:54:55 +0000 Subject: [PATCH 2/5] Give a child tied between parents no parent: ambiguous_parent (plan Q74) pair_counts took the first parent in label order when two or more parents shared a child's largest overlap exactly, so inside and parents_with_N_children depended on the label numbering. A tie now gives no parent; when the tied share would make the child inside it is counted as ambiguous_parent. The report is version 3; masks, the notebook card and the relationship table show it. The overlap helpers move to the pure module mask_pairs (mask_scan stays under 600 lines). Tests: the tie, three tied parents, a tie below the largest overlap, random masks under permuted labels, best_parents against a reference. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- AGENTS.md | 2 + docs/contract.md | 18 +- docs/migration-0.3-to-0.4.md | 5 + docs/user_guide.md | 10 +- pyproject.toml | 1 + src/morphobricks_preprocessing/mask_pairs.py | 108 +++++++++++ src/morphobricks_preprocessing/mask_scan.py | 78 ++------ src/morphobricks_preprocessing/mask_text.py | 25 +++ .../notebook_mask_step.py | 9 + tests/test_mask_matching.py | 2 +- tests/test_mask_scan.py | 2 +- tests/test_mask_ties.py | 182 ++++++++++++++++++ tests/test_masks.py | 2 +- tests/test_notebook_mask_step.py | 19 ++ 14 files changed, 394 insertions(+), 69 deletions(-) create mode 100644 src/morphobricks_preprocessing/mask_pairs.py create mode 100644 tests/test_mask_ties.py diff --git a/AGENTS.md b/AGENTS.md index d4a9bba7..938d9fea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -180,6 +180,8 @@ src/morphobricks_preprocessing/ scan_masks -> MaskReport (kinds, relations, direct-parent hierarchy, evidence), write_mask_report (mask_relationships.json); DEFAULT_THRESHOLDS, mask_thresholds (checked changes) + mask_pairs.py pair_counts' pure helpers: best_parents (a tie between parents gives no + parent, ambiguous_parent; Q74), one_to_one_matches and match_counts copy_axes.py convert (PP-11, contract §17): the axes, shape, pixel sizes (by role; a relabelled axis gets an unknown scale; equal X and Y kept through a Y/X swap, question 59) and OME metadata of a named-axis copy diff --git a/docs/contract.md b/docs/contract.md index 63f1910b..b7cc620e 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -448,7 +448,7 @@ masks/ masks folder, outside the experiment folder | Warnings | `mask_missing` (images without a mask of a type that other images have: their objects of that type cannot be measured), `mask_without_image` (files matching no image of the experiment), `mask_of_excluded_image` (masks of excluded images, not used), `mask_outside_type_folder` (files directly in the masks folder that are not TIFF masks). | | Run | `run_experiment(..., masks_folder=..., mask_axes=...)` stops on a mask error and records the masks in the manifest's `masks` key (§9). Without masks, the run and its outputs are unchanged. | -### 15.1 The mask step (`mask_relationships.json`, report version 2) +### 15.1 The mask step (`mask_relationships.json`, report version 3) The optional mask step reads every image's masks, one image at a time, and describes each mask type and every ordered pair of types over the whole experiment. Pieces are counted with @@ -473,7 +473,12 @@ plane it is in, and `images` still counts images. For a child type and a parent type, the counts are summed over the images (and planes) with both masks. -Each child object's parent is the parent object sharing most of its pixels. +Each child object's parent is the parent object sharing most of its pixels. When two or more +parents share exactly that many (a child lying half in each of two cells), the child has +**no parent**, so that no count depends on how the objects are numbered (owner, 3 November +2026; before, the parent with the lower label was taken): it is never `inside`, and when +the tied share would have made it inside it is counted as `ambiguous_parent`. Each child is +counted once in `inside`, `ambiguous_parent`, `split_between_parents` or `outside_parents`. | Relation of child to parent | Rule (default thresholds; the values used are written in the report) | |---|---| @@ -485,9 +490,10 @@ Each child object's parent is the parent object sharing most of its pixels. | Evidence (per pair) | Meaning | |---|---| | `children`, `parents` | Objects of each type. | -| `inside`, `inside_parent_larger` | Children with at least half of their pixels in one parent; of those, children whose parent is larger. | -| `split_between_parents`, `outside_parents` | Children touching parents but with less than half of their pixels in any one; children touching no parent. | -| `parents_with_0_children`, `parents_with_1_child`, `parents_with_2_children`, `parents_with_3_or_more_children` | How many children each parent holds (two nuclei in one cell counts as 2). | +| `inside`, `inside_parent_larger` | Children with at least half of their pixels in one parent, which is their parent (no tie); of those, children whose parent is larger. | +| `ambiguous_parent` | Children with at least half of their pixels in a parent, for which two or more parents share that many pixels exactly: they have no parent and are not `inside` (report version 3). `masks`, the notebook's **Describe the masks** and the relationship table name them. | +| `split_between_parents`, `outside_parents` | Children touching parents but with less than half of their pixels in any one (tied or not); children touching no parent. | +| `parents_with_0_children`, `parents_with_1_child`, `parents_with_2_children`, `parents_with_3_or_more_children` | How many `inside` children each parent holds (two nuclei in one cell counts as 2); a child with no parent counts for none. | | `child_pixels`, `child_pixels_with_parent_label` | Child pixels; those whose label equals the parent label under them. | | `children_matched_one_to_one`, `parents_matched_one_to_one` | Objects matched one to one (the maximum matching above, at an intersection over union of at least 0.5); the two counts are equal. The number of matches does not depend on which of several equally large matchings is found, so it is deterministic. | | `children_not_matched`, `parents_not_matched` | Objects of each type left without a match. | @@ -512,7 +518,7 @@ The report is UTF-8 JSON (two-space indent, LF), written atomically to the outpu | Key | Meaning | |---|---| -| `report_version` | `2` (since issue #67: the one-to-one matches are a matching, and the four counts of unmatched and several-partner objects were added). | +| `report_version` | `3` (since v0.4.0rc5: a tie between parents gives no parent, and `ambiguous_parent` was added; plan §12 question 74). `2` since issue #67: the one-to-one matches are a matching, and the four counts of unmatched and several-partner objects were added. | | `thresholds` | The thresholds used, all seven. | | `images` | Images scanned (every image with at least one mask). | | `masks` | Per mask type: `kind`, `images`, `objects` (labels over all images) and `median_pieces_per_label`. | diff --git a/docs/migration-0.3-to-0.4.md b/docs/migration-0.3-to-0.4.md index 1d631f4a..5163917d 100644 --- a/docs/migration-0.3-to-0.4.md +++ b/docs/migration-0.3-to-0.4.md @@ -144,6 +144,11 @@ later bricks on the new run. - The notebook has a **Masks folder** field and a **Describe the masks** button (the `masks` step). - `same_objects` in `mask_relationships.json` is a one-to-one matching (report version 2). + - A child lying exactly as much in two or more parents has **no parent** (it went to + the parent with the lower label before, so the counts depended on the numbering): + it is not `inside` and is counted as `ambiguous_parent`, so `inside` and + `parents_with_N_children` may be lower than with v0.4.0rc4 (report version 3, + v0.4.0rc5). - The masks folder may hold only masks: an experiment, named-axis copies or the output folder inside it is refused (`experiment_inside_masks`, `copies_inside_masks`, `output_inside_masks`, before anything is written), and so is a `convert` folder diff --git a/docs/user_guide.md b/docs/user_guide.md index 8b142e68..8ad1b606 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -513,8 +513,9 @@ results or comparing another grouping. time (a progress bar names the image); the card then shows each mask type (kind, images, objects, median pieces per label), the relationships found (which type is inside or grouped by which, how many objects are inside, how many objects of the - other type hold 0, 1, 2 or 3 or more, and how many are outside or split), masks of - the same objects, and the hierarchy of direct parents. These are suggestions to + other type hold 0, 1, 2 or 3 or more, and how many are outside or split), the + objects given no parent because they lie equally in two or more objects of the other + type, masks of the same objects, and the hierarchy of direct parents. These are suggestions to check, not decisions. The description is written to **mask_relationships.json** in the output folder (its path is shown). The output folder's lock is held from before the first mask is read until the file is written: when another run holds it, **The output folder is locked** is shown with **Remove the lock of a @@ -690,7 +691,10 @@ Hierarchy (direct parents only): Nucleoli inside nuclei inside cells have the nuclei as their direct parent; two kinds of mask that segment the same objects are reported as such: at least 90% of the objects of both are matched one to one, each object matched at most once (two halves of one cell are -one match, not two). These are suggestions from the +one match, not two). An object that lies exactly as much in two or more objects of the +other kind (a nucleus half in each of two cells) is given **no parent**, whichever has the +lower number: it is not counted as inside, and the command lists it under "Objects given +no parent" (`ambiguous_parent` in the report, since v0.4.0rc5). These are suggestions from the pixels, not decisions: you name each kind of mask and check the relationships in the bricks that use them. The report is written to `mask_relationships.json` in the output folder (§6). The command takes `--output`, `--exclude` and `--log` like the others, and exits with diff --git a/pyproject.toml b/pyproject.toml index 9fa8d755..1752965b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -194,6 +194,7 @@ baseline = "tools/code_rules_baseline.json" "morphobricks_preprocessing.unrecorded" = "io" "morphobricks_preprocessing.testing" = "orchestration" "morphobricks_preprocessing.mask_scan" = "io" +"morphobricks_preprocessing.mask_pairs" = "pure" "morphobricks_preprocessing.masks" = "io" "morphobricks_preprocessing.masks.checks" = "io" "morphobricks_preprocessing.masks.files" = "io" diff --git a/src/morphobricks_preprocessing/mask_pairs.py b/src/morphobricks_preprocessing/mask_pairs.py new file mode 100644 index 00000000..4001cba0 --- /dev/null +++ b/src/morphobricks_preprocessing/mask_pairs.py @@ -0,0 +1,108 @@ +"""How the objects of two masks of one plane overlap: best parents and one-to-one matches. + +Pure helpers of :func:`.mask_scan.pair_counts` (``docs/contract.md`` §15.1), moved from +:mod:`.mask_scan`: each child's parent with the largest overlap, a tie giving none +(:func:`best_parents`, plan §12 question 74), and the maximum matching of ``same_objects`` +(:func:`one_to_one_matches`, issue #67). Every count they give is independent of the +label numbering. No files are read or written. +""" + +from __future__ import annotations + +import numpy as np +from scipy.sparse import csr_matrix +from scipy.sparse.csgraph import maximum_bipartite_matching + +__all__ = ["best_parents", "match_counts", "one_to_one_matches"] + + +def best_parents( + kid: np.ndarray, host: np.ndarray, shared: np.ndarray +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Return each child's parent sharing most of its pixels, and whether that is a tie. + + A child whose largest overlap is shared exactly by two or more parents has no parent: + it is tied, and the parent returned for it is the first of them and must not be used + (owner, 3 November 2026, plan §12 question 74). Before, the parent with the lower + label index was taken, so ``inside`` and ``parents_with_N_children`` depended on the + label numbering. Every other child has one parent that does not depend on it. + + Parameters + ---------- + kid, host, shared + Each overlapping (child, parent) pair as indices, sorted by child, and its number of + shared pixels; each pair at most once. + + Returns + ------- + tuple of numpy.ndarray + Per child with an overlap, in order: the child, its parent, the pixels they share, + and True when two or more parents share that many. + """ + new_child = np.r_[True, kid[1:] != kid[:-1]] + starts = np.flatnonzero(new_child) + child_of = np.cumsum(new_child) - 1 + most = np.maximum.reduceat(shared, starts) + top = np.flatnonzero(shared == most[child_of]) # every pair with its child's largest + ties = np.bincount(child_of[top], minlength=starts.size) > 1 + first = top[np.r_[True, child_of[top][1:] != child_of[top][:-1]]] + return kid[starts], host[first], most, ties + + +def one_to_one_matches(kid: np.ndarray, host: np.ndarray, children: int, parents: int) -> int: + """Return the number of one-to-one matches: a maximum matching of the eligible pairs. + + Each child and each parent is matched at most once, and the number of matches is the + largest possible (Hopcroft-Karp, :func:`scipy.sparse.csgraph.maximum_bipartite_matching`). + That number does not depend on which of several equally good matchings is found, so + the counts are deterministic. Above an intersection over union of 0.5 an object can + overlap only one partner that much, and every eligible pair is matched; at 0.5 or below, + two children covering one parent are one match, not two (issue #67). + + Parameters + ---------- + kid, host + The child and parent of each eligible pair (intersection over union at least the + threshold), as indices below ``children`` and ``parents``; each pair at most once. + children, parents + Numbers of child and parent objects. + + Returns + ------- + int + """ + if kid.size == 0: + return 0 + graph = csr_matrix((np.ones(kid.size, dtype=np.int8), (kid, host)), shape=(children, parents)) + partner = maximum_bipartite_matching(graph, perm_type="column") + return int(np.count_nonzero(partner >= 0)) + + +def match_counts(kid: np.ndarray, host: np.ndarray, children: int, parents: int) -> dict: + """Return the matched, unmatched and several-partner counts of the eligible pairs. + + Parameters + ---------- + kid, host + The child and parent of each eligible pair, as in :func:`one_to_one_matches`. + children, parents + Numbers of child and parent objects. + + Returns + ------- + dict + The six one-to-one counts of the evidence (contract §15.1), by name. + """ + matched = one_to_one_matches(kid, host, children, parents) + return { + "children_matched_one_to_one": matched, + "parents_matched_one_to_one": matched, + "children_not_matched": children - matched, + "parents_not_matched": parents - matched, + "children_over_iou_with_several_parents": int( + np.count_nonzero(np.bincount(kid, minlength=children) > 1) + ), + "parents_over_iou_with_several_children": int( + np.count_nonzero(np.bincount(host, minlength=parents) > 1) + ), + } diff --git a/src/morphobricks_preprocessing/mask_scan.py b/src/morphobricks_preprocessing/mask_scan.py index cc7dc1c7..b21424a8 100644 --- a/src/morphobricks_preprocessing/mask_scan.py +++ b/src/morphobricks_preprocessing/mask_scan.py @@ -22,7 +22,10 @@ - ``none`` otherwise. Several children per parent are counted (parents with 0, 1, 2 and 3 or more children), as - are children outside every parent and children split between parents. + are children outside every parent and children split between parents. A child whose + largest overlap is shared exactly by two or more parents has no parent, whatever the + label numbers: when that share would make it inside, it is counted as + ``ambiguous_parent``, not ``inside`` (owner, 3 November 2026, plan §12 question 74). - **Hierarchy:** the direct parents of each type: a parent is left out when it is the parent of another parent (nucleoli inside nuclei inside cells: the nucleoli's direct parent is the nuclei). A type may have several direct parents (cells inside an organoid and inside a @@ -53,11 +56,11 @@ import numpy as np from scipy import ndimage -from scipy.sparse import csr_matrix -from scipy.sparse.csgraph import maximum_bipartite_matching from ._io import write_text_atomic from .manifest import package_versions +from .mask_pairs import best_parents, match_counts +from .mask_pairs import one_to_one_matches as one_to_one_matches # moved there from .masks import MaskInventory, check_labels, read_mask logger = logging.getLogger(__name__) @@ -65,8 +68,9 @@ MASK_REPORT_FILENAME = "mask_relationships.json" """Name of the report written to the output folder.""" -MASK_REPORT_VERSION = "2" -"""Version of the report's format: 2 since the one-to-one matches are a matching (issue #67).""" +MASK_REPORT_VERSION = "3" +"""Version of the report's format: 2 since the one-to-one matches are a matching (issue #67), +3 since a tie between parents gives no parent and is counted as ``ambiguous_parent``.""" DEFAULT_THRESHOLDS = MappingProxyType( { @@ -88,6 +92,7 @@ "children", "parents", "inside", + "ambiguous_parent", "inside_parent_larger", "split_between_parents", "outside_parents", @@ -289,6 +294,11 @@ def pair_counts( Both are 2D label images of one position; label 0 is background. ``thresholds`` changes the default ``inside_pixel_share`` and ``same_objects_iou`` (:func:`mask_thresholds`). + Each child's parent is the parent sharing most of its pixels; a child for which two or + more parents share that many has none (:func:`.mask_pairs.best_parents`), so no count + depends on the label numbers. Each child is counted once in ``inside``, + ``ambiguous_parent`` (tied, with a share that would make it inside), + ``split_between_parents`` or ``outside_parents``. Raises ------ @@ -326,16 +336,16 @@ def pair_counts( return counts union = child_sizes[kid] + parent_sizes[host] - shared eligible = shared / union >= limits["same_objects_iou"] - counts.update(_match_counts(kid[eligible], host[eligible], child_labels.size, slots)) - best = np.lexsort((-shared, kid)) - lead = np.r_[True, kid[best][1:] != kid[best][:-1]] - best_kid, best_host, best_shared = kid[best][lead], host[best][lead], shared[best][lead] - inside = best_shared / child_sizes[best_kid] >= limits["inside_pixel_share"] + counts.update(match_counts(kid[eligible], host[eligible], child_labels.size, slots)) + best_kid, best_host, best_shared, tied = best_parents(kid, host, shared) + reaches = best_shared / child_sizes[best_kid] >= limits["inside_pixel_share"] + inside, ambiguous = reaches & ~tied, reaches & tied counts["inside"] = int(inside.sum()) + counts["ambiguous_parent"] = int(ambiguous.sum()) counts["inside_parent_larger"] = int( np.count_nonzero(parent_sizes[best_host[inside]] > child_sizes[best_kid[inside]]) ) - counts["split_between_parents"] = int(np.count_nonzero(~inside)) + counts["split_between_parents"] = int(np.count_nonzero(~reaches)) counts["outside_parents"] = child_labels.size - best_kid.size per_parent = np.bincount(best_host[inside], minlength=slots) counts["parents_with_0_children"] = int(np.count_nonzero(per_parent == 0)) @@ -345,52 +355,6 @@ def pair_counts( return counts -def one_to_one_matches(kid: np.ndarray, host: np.ndarray, children: int, parents: int) -> int: - """Return the number of one-to-one matches: a maximum matching of the eligible pairs. - - Each child and each parent is matched at most once, and the number of matches is the - largest possible (Hopcroft-Karp, :func:`scipy.sparse.csgraph.maximum_bipartite_matching`). - That number does not depend on which of several equally good matchings is found, so - the counts are deterministic. Above an intersection over union of 0.5 an object can - overlap only one partner that much, and every eligible pair is matched; at 0.5 or below, - two children covering one parent are one match, not two (issue #67). - - Parameters - ---------- - kid, host - The child and parent of each eligible pair (intersection over union at least the - threshold), as indices below ``children`` and ``parents``; each pair at most once. - children, parents - Numbers of child and parent objects. - - Returns - ------- - int - """ - if kid.size == 0: - return 0 - graph = csr_matrix((np.ones(kid.size, dtype=np.int8), (kid, host)), shape=(children, parents)) - partner = maximum_bipartite_matching(graph, perm_type="column") - return int(np.count_nonzero(partner >= 0)) - - -def _match_counts(kid: np.ndarray, host: np.ndarray, children: int, parents: int) -> dict: - """Return the matched, unmatched and several-partner counts of the eligible pairs.""" - matched = one_to_one_matches(kid, host, children, parents) - return { - "children_matched_one_to_one": matched, - "parents_matched_one_to_one": matched, - "children_not_matched": children - matched, - "parents_not_matched": parents - matched, - "children_over_iou_with_several_parents": int( - np.count_nonzero(np.bincount(kid, minlength=children) > 1) - ), - "parents_over_iou_with_several_children": int( - np.count_nonzero(np.bincount(host, minlength=parents) > 1) - ), - } - - def scan_masks( inventory: MaskInventory, experiment_root: str | os.PathLike, diff --git a/src/morphobricks_preprocessing/mask_text.py b/src/morphobricks_preprocessing/mask_text.py index 91d0476c..e849d081 100644 --- a/src/morphobricks_preprocessing/mask_text.py +++ b/src/morphobricks_preprocessing/mask_text.py @@ -60,6 +60,10 @@ def mask_report_summary(report: MaskReport) -> str: if report.same_objects: lines += ["", "Masks that look like two segmentations of the same objects:"] lines += [f" {first} and {second}" for first, second in report.same_objects] + ambiguous = mask_ambiguous_lines(report) + if ambiguous: + lines += ["", "Objects given no parent (equally in two or more objects of the other mask):"] + lines += [f" {line}" for line in ambiguous] lines += ["", "Hierarchy (direct parents only):"] lines += _hierarchy_lines(report.hierarchy) lines += ["", f"Thresholds: {mask_thresholds_used(report)}"] @@ -108,6 +112,7 @@ def mask_relationship_rows(report: MaskReport) -> list[list[str]]: "Other mask's objects with 0/1/2/3+", "Outside the other mask", "Split between two of its objects", + "Equally in two or more of its objects (no parent)", ] ] for pair in report.relationships: @@ -130,11 +135,27 @@ def mask_relationship_rows(report: MaskReport) -> list[list[str]]: "/".join(f"{n:,}" for n in counts), f"{e['outside_parents']:,}", f"{e['split_between_parents']:,}", + f"{e['ambiguous_parent']:,}", ] ) return rows +def mask_ambiguous_lines(report: MaskReport) -> list[str]: + """Return one line per pair of types with objects given no parent because of a tie. + + Such a child lies equally in two or more parent objects, with a share that would make + it inside (``ambiguous_parent``, contract §15.1): it is never given one of them, whatever + the relation of the pair, so every pair is listed, ``none`` included. + """ + return [ + f"{pair.child} in {pair.parent}: {plural(pair.evidence['ambiguous_parent'], 'object')}" + f" of {pair.evidence['children']:,} equally in two or more {pair.parent}" + for pair in report.pairs + if pair.evidence.get("ambiguous_parent") + ] + + def mask_hierarchy_lines(report: MaskReport) -> list[str]: """Return the hierarchy of the mask types (direct parents only) as an indented tree.""" return _hierarchy_lines(report.hierarchy) @@ -156,6 +177,10 @@ def _relationship_line(pair: PairRelation) -> str: extra.append(f"{e['outside_parents']} outside every {pair.parent}") if e["split_between_parents"]: extra.append(f"{e['split_between_parents']} split between {pair.parent}") + if e["ambiguous_parent"]: + extra.append( + f"{e['ambiguous_parent']} equally in two or more {pair.parent} (no parent given)" + ) return text + (f"; {', '.join(extra)}" if extra else "") diff --git a/src/morphobricks_preprocessing/notebook_mask_step.py b/src/morphobricks_preprocessing/notebook_mask_step.py index 4b96082d..3d15565e 100644 --- a/src/morphobricks_preprocessing/notebook_mask_step.py +++ b/src/morphobricks_preprocessing/notebook_mask_step.py @@ -27,6 +27,7 @@ from .manifest import MANIFEST_FILENAME from .mask_scan import MASK_REPORT_FILENAME from .mask_text import ( + mask_ambiguous_lines, mask_hierarchy_lines, mask_relationship_rows, mask_thresholds_used, @@ -250,6 +251,14 @@ def _show_mask_description(self, step: MaskStep, result: MaskStepResult) -> None _heading("Two segmentations of the same objects (to check)"), f"
    {same}
", ] + ambiguous = "".join( + f"
  • {html.escape(line)}
    • " for line in mask_ambiguous_lines(report) + ) + if ambiguous: + parts += [ + _heading("Objects given no parent (a tie, to check)"), + f"
      {ambiguous}
    ", + ] parts += [_heading("Hierarchy (direct parents only)"), f"
    {hierarchy}
    "] if step.issues: parts.append(issues_html(step.issues)) diff --git a/tests/test_mask_matching.py b/tests/test_mask_matching.py index f67f49ad..c80f7a2d 100644 --- a/tests/test_mask_matching.py +++ b/tests/test_mask_matching.py @@ -118,5 +118,5 @@ def test_a_split_segmentation_is_not_the_same_objects(tmp_path): assert (e["children_not_matched"], e["parents_not_matched"]) == (8, 0) assert e["parents_over_iou_with_several_children"] == 8 data = write_mask_report(report, tmp_path / "out.json").read_text(encoding="utf-8") - assert '"report_version": "2"' in data + assert '"report_version": "3"' in data assert '"children_over_iou_with_several_parents": 0' in data diff --git a/tests/test_mask_scan.py b/tests/test_mask_scan.py index 4c1fc8d4..c28c262f 100644 --- a/tests/test_mask_scan.py +++ b/tests/test_mask_scan.py @@ -226,7 +226,7 @@ def test_report_file(tmp_path): raw = path.read_bytes() assert b"\r\n" not in raw data = json.loads(raw) - assert data["report_version"] == "2" + assert data["report_version"] == "3" assert data["thresholds"]["inside_object_share"] == 0.9 assert data["hierarchy"]["Nucleoli"] == ["Nuclei", "NucleiAlt"] assert data["same_objects"] == [["Nuclei", "NucleiAlt"]] diff --git a/tests/test_mask_ties.py b/tests/test_mask_ties.py new file mode 100644 index 00000000..2506489c --- /dev/null +++ b/tests/test_mask_ties.py @@ -0,0 +1,182 @@ +"""A child that lies equally in two or more parents has no parent (owner, 3 November 2026). + +Before, ``pair_counts`` gave such a child the parent with the lower label index, so +``inside`` and ``parents_with_N_children`` depended on how the parents were numbered. An +exact tie now gives no parent; when the tied share would make the child inside, it is +counted as ``ambiguous_parent`` (plan §12 question 74, contract §15.1), and the report, +``masks`` text and notebook show it. +""" + +import numpy as np +import pytest + +from morphobricks_preprocessing import pair_counts +from morphobricks_preprocessing.mask_pairs import best_parents +from morphobricks_preprocessing.mask_scan import MASK_REPORT_VERSION +from morphobricks_preprocessing.mask_text import ( + mask_ambiguous_lines, + mask_relationship_rows, + mask_report_summary, +) +from test_mask_scan import boxes, by_pair, scan + +PER_PARENT = ( + "parents_with_0_children", + "parents_with_1_child", + "parents_with_2_children", + "parents_with_3_or_more_children", +) + + +def relabel(children, parents, rng): + """Give the labels other numbers, in another order: one map for both masks. + + One map keeps a child label equal to the parent label under it where it was + (``child_pixels_with_parent_label``, the evidence of ``grouped_by``). + """ + labels = np.unique(np.r_[children[children != 0], parents[parents != 0]]) + new = rng.permutation(np.arange(1, 4 * labels.size + 1))[: labels.size] * 3 + out = [] + for mask in (children, parents): + relabelled = np.zeros(mask.shape, dtype=np.int32) + for old, value in zip(labels.tolist(), new.tolist(), strict=True): + relabelled[mask == old] = value + out.append(relabelled) + return out + + +def halves(): + """Parents 1 and 2 side by side; child 5 exactly half in each, child 6 inside parent 2.""" + parents = boxes({1: (0, 10, 0, 10), 2: (0, 10, 10, 20)}) + children = boxes({5: (2, 4, 8, 12), 6: (6, 8, 14, 16)}) + return children, parents + + +def test_a_child_exactly_half_in_two_parents_has_no_parent(): + children, parents = halves() + counts = pair_counts(children, parents) + assert counts["children"] == 2 + assert counts["inside"] == 1 and counts["inside_parent_larger"] == 1 + assert counts["ambiguous_parent"] == 1 + assert counts["split_between_parents"] == 0 and counts["outside_parents"] == 0 + assert tuple(counts[k] for k in PER_PARENT) == (1, 1, 0, 0) + + +def test_the_tie_does_not_depend_on_which_parent_has_the_lower_label(): + # Before: (0, 0, 1, 0) with the parents swapped (child 5 went to the parent labelled 1, + # which then held both children), (0, 2, 0, 0) as written. + children, parents = halves() + swapped = parents.copy() + swapped[parents == 1], swapped[parents == 2] = 2, 1 + assert pair_counts(children, swapped) == pair_counts(children, parents) + + +def test_three_tied_parents_at_a_lower_inside_share(): + parents = boxes({1: (0, 9, 0, 3), 2: (0, 9, 3, 6), 3: (0, 9, 6, 9)}) + child = boxes({7: (0, 3, 0, 9)}) # 9 pixels, 3 in each parent + default = pair_counts(child, parents) + assert (default["inside"], default["ambiguous_parent"]) == (0, 0) + assert default["split_between_parents"] == 1 # a third is below one half: as before + lower = pair_counts(child, parents, thresholds={"inside_pixel_share": 0.3}) + assert (lower["inside"], lower["ambiguous_parent"], lower["split_between_parents"]) == ( + 0, + 1, + 0, + ) + assert lower["parents_with_0_children"] == 3 + + +def test_a_tie_below_the_largest_overlap_changes_nothing(): + parents = boxes({1: (0, 10, 0, 6), 2: (0, 10, 6, 8), 3: (0, 10, 8, 10)}) + child = boxes({4: (0, 1, 0, 10)}) # 6, 2 and 2 pixels: parent 1 holds it + counts = pair_counts(child, parents) + assert (counts["inside"], counts["ambiguous_parent"]) == (1, 0) + assert tuple(counts[k] for k in PER_PARENT) == (2, 1, 0, 0) + + +def random_masks(rng): + """Small blocky masks, so that exact ties are common.""" + shape = tuple(rng.integers(4, 12, 2)) + parents = np.kron(rng.integers(0, 5, (shape[0] // 2, shape[1] // 2)), np.ones((2, 2))) + children = rng.integers(0, 6, parents.shape) + return children.astype(np.int32), parents.astype(np.int32) + + +@pytest.mark.parametrize("share", [0.5, 0.25]) +def test_every_count_is_the_same_under_any_label_numbering(share): + rng = np.random.default_rng(74) + limits = {"inside_pixel_share": share} + ties = 0 + for _ in range(300): + children, parents = random_masks(rng) + counts = pair_counts(children, parents, thresholds=limits) + ties += counts["ambiguous_parent"] + for _ in range(3): + again = pair_counts(*relabel(children, parents, rng), thresholds=limits) + assert again == counts + assert counts["children"] == sum( + counts[k] + for k in ("inside", "ambiguous_parent", "split_between_parents", "outside_parents") + ) + assert sum(counts[k] for k in PER_PARENT) == counts["parents"] + assert ties > 20 # the data hold ties, so the test can fail + + +def reference_best(kid, host, shared): + """Each child's largest overlap and how many parents have it, by plain loops.""" + found = {} + for k, h, s in zip(kid.tolist(), host.tolist(), shared.tolist(), strict=True): + found.setdefault(k, []).append((s, h)) + result = {} + for k, pairs in found.items(): + most = max(s for s, _ in pairs) + hosts = sorted(h for s, h in pairs if s == most) + result[k] = (most, hosts) + return result + + +def test_best_parents_matches_a_reference(): + rng = np.random.default_rng(3) + for _ in range(200): + children, parents = rng.integers(1, 6), rng.integers(1, 6) + pairs = np.unique(rng.integers(0, children * parents, rng.integers(1, 15))) + kid, host = pairs // parents, pairs % parents + shared = rng.integers(1, 4, kid.size) + best_kid, best_host, most, tied = best_parents(kid, host, shared) + reference = reference_best(kid, host, shared) + assert best_kid.tolist() == sorted(reference) + found = zip(best_kid.tolist(), best_host.tolist(), most.tolist(), tied, strict=True) + for k, h, s, t in found: + assert s == reference[k][0] and bool(t) == (len(reference[k][1]) > 1) + assert h in reference[k][1] + + +def test_the_report_and_its_texts_show_the_tie(tmp_path): + children, parents = halves() + report = scan(tmp_path, {"Cells": parents, "Nuclei": children}) + pair = by_pair(report)[("Nuclei", "Cells")] + assert pair.evidence["ambiguous_parent"] == 2 # one per image, two images + assert pair.relation == "none" # 2 of 4 inside is below 90% + data = report.to_dict() + assert MASK_REPORT_VERSION == data["report_version"] == "3" + (written,) = [p for p in data["pairs"] if (p["child"], p["parent"]) == ("Nuclei", "Cells")] + assert written["evidence"]["ambiguous_parent"] == 2 + assert mask_ambiguous_lines(report) == [ + "Nuclei in Cells: 2 objects of 4 equally in two or more Cells" + ] + text = mask_report_summary(report) + assert "Objects given no parent (equally in two or more objects" in text + assert "Nuclei in Cells: 2 objects of 4" in text + + +def test_the_relationship_table_has_the_tied_children(tmp_path): + parents = boxes({1: (0, 30, 0, 30), 2: (0, 30, 30, 60)}) + inside = {n: (2 * n, 2 * n + 1, 2, 4) for n in range(1, 13)} + children = boxes({**inside, 20: (28, 29, 28, 32)}) # 12 inside, 1 tied: 92% inside + report = scan(tmp_path, {"Cells": parents, "Nuclei": children}) + assert by_pair(report)[("Nuclei", "Cells")].relation == "inside" + header, *rows = mask_relationship_rows(report) + assert header[-1] == "Equally in two or more of its objects (no parent)" + (row,) = [r for r in rows if r[0] == "Nuclei"] + assert row[3] == "24 of 26" and row[-1] == "2" + assert "2 equally in two or more Cells (no parent given)" in mask_report_summary(report) diff --git a/tests/test_masks.py b/tests/test_masks.py index ebe545a9..6712861d 100644 --- a/tests/test_masks.py +++ b/tests/test_masks.py @@ -320,7 +320,7 @@ def test_masks_command_writes_the_report(experiment, masks, tmp_path, capsys): assert "[3/3] drug/rep1/a.tif" in printed assert "Hierarchy (direct parents only):" in printed report = json.loads((out / "mask_relationships.json").read_text(encoding="utf-8")) - assert report["report_version"] == "2" + assert report["report_version"] == "3" assert sorted(report["masks"]) == ["Cells", "Nuclei"] assert report["same_objects"] == [["Cells", "Nuclei"]] diff --git a/tests/test_notebook_mask_step.py b/tests/test_notebook_mask_step.py index 218ea4cf..83fb673e 100644 --- a/tests/test_notebook_mask_step.py +++ b/tests/test_notebook_mask_step.py @@ -130,6 +130,25 @@ def test_describe_the_masks_writes_the_report_and_shows_the_cards(tmp_path, caps } +def test_a_child_equally_in_two_parents_is_shown_with_no_parent(tmp_path): + # Plan §12 question 74: a tie gives no parent, and the card names the objects. + root = experiment(tmp_path) + masks = nested_masks(tmp_path) + cells = np.zeros((16, 16), np.uint16) + cells[0:8, 0:8], cells[0:8, 8:16] = 1, 2 + nuclei = np.zeros((16, 16), np.uint16) + nuclei[3:5, 7:9] = 1 # two pixels in each cell + for rel in IMAGES: + write_data(masks / "cells" / rel, cells) + write_data(masks / "nuclei" / rel, nuclei) + nb, _ = ran(root, masks) + assert nb.run().written + nb.mask_step_button.click() + shown = nb.mask_step_result.value + assert "Objects given no parent (a tie, to check)" in shown + assert "
  • nuclei in cells: 4 objects of 4 equally in two or more cells
    • " in shown + + def test_a_locked_output_folder_shows_the_lock_card_and_writes_nothing(tmp_path): root = experiment(tmp_path) nb, _ = ran(root, nested_masks(tmp_path)) From b16159f8c5c4352367e9a4297848431d7309d72c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 13:56:09 +0000 Subject: [PATCH 3/5] Name an ImageJ unit without resolution tags, and a TIFF without pages, as invalid An ImageJ TIFF with a unit but neither XResolution nor YResolution read as uncalibrated with no reason; it is now invalid_physical_calibration, "a unit without a resolution" (owner, 3 November 2026). A TIFF file without pages, which tifffile opens with only a log message, gives the reason "no pages". A file with neither unit nor resolution is still uncalibrated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- docs/contract.md | 2 +- docs/user_guide.md | 2 +- .../physical_sizes.py | 8 ++- .../tiff_size_values.py | 15 ++++-- tests/test_tiff_size_read_errors.py | 50 +++++++++++++++++++ 5 files changed, 69 insertions(+), 8 deletions(-) diff --git a/docs/contract.md b/docs/contract.md index b7cc620e..e82284ad 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -342,7 +342,7 @@ row (follow-up P3b, normalization plan N43). | `different_pixel_sizes` | Warning | The physical pixel size (X by Y) differs between images, or the z-step differs between images with more than one depth slice. Sizes are compared in micrometres by the shared rule of decision D-a (§16): two sizes are the same when they differ by at most 0.1 % of the larger, on every axis. The warning is given exactly when, on some axis, the smallest and the largest size differ, that is when some two images differ. The images are grouped by single linkage (`pixel_size_groups`) and those outside the largest group are listed. When every image is linked to the others by near-equal sizes (1.0, 1.0009, 1.0018 µm), the sizes drift, no image stands out, and the message says that none is listed for that size. Until D86 sizes were rounded to four significant digits, so 1.0004 and 1.0006 µm differed (1 and 1.001); they are now the same. | | `missing_pixel_size` | Warning | The pixel size (or the z-step of images with more than one depth slice) is known for some images and not for others. Nothing is reported when no image has it. | | `unsupported_pixel_size_unit` | Warning | An image's metadata give a pixel size or z-step (OME `PhysicalSize{X,Y,Z}`, ImageJ resolution or `spacing`, OME-Zarr level-0 scale) in a length unit that is not one of decision D-a2 (§16), for example OME `in` or ImageJ `inch`. The size is not converted and the axis is treated as uncalibrated (as before); the warning names each image, its axes and the unit as written, so the missing calibration is never silent. An OME-Zarr scale of X, Y or Z without a unit is named the same way ("X and Y without a unit"; review of #136): it is recorded as unknown (audit I3), never taken as micrometres, except on the axes listed as `unknown_scale` (§10). A unit without a size, and a plain TIFF's resolution tags (decision D-c), are not reported. The run is not refused. | -| `invalid_physical_calibration` | Warning | An image's metadata give a calibration that cannot be used (issue #94): OME-Zarr `coordinateTransformations` that are not a list or of an unsupported type, more than one scale, a scale stored in a separate file or not one number per axis, or a scale value of X, Y, Z or T that is not a positive finite number; or, from any TIFF or OME-Zarr metadata, a size that is not a positive finite length in micrometres once converted (issue #93: `1e308 m`, `1e-320 pm`); or OME-XML in a TIFF that cannot be parsed (audit m6: before, any error reading the sizes read as merely uncalibrated); or TIFF metadata whose bytes cannot be read (`OSError`, `struct.error`, `TiffFileError` while a tag is read), or an ImageJ `XResolution` or `YResolution` tag that is missing or unreadable while the other is given (tifffile drops a tag it cannot read with only a log message), with a unit (round-2 audit A-m6); `read_physical_sizes` also gives a file it cannot open as a TIFF no sizes with the reason in `invalid` (in the folder check such a file is `unreadable_tiff`), and raises any other exception as the defect it is; or a size of OME-TIFF (`PhysicalSizeX/Y/Z`, `TimeIncrement`) or ImageJ metadata (`spacing`, `finterval` and the resolution tags, with a unit) that is itself not a positive finite number (`inf`, `NaN`, zero, negative or not a number; review of #133, before silently unknown); or an ImageJ `XResolution` or `YResolution` tag, with a unit, that is not two whole numbers (a tag of another type; review of #136, before silently unknown). The message names each image and what is wrong, with the axis and value. Its sizes are unknown on those axes (never a substituted value) and recorded as uncalibrated; the warning is in the `check` and `run` issues and the report (`discovery_report.json`). An image is not named once every size the malformed calibration leaves unknown (X and Y, and Z or T with more than one plane) is stated in `calibration.csv`; a time step cannot be stated there, so a lost time step of a time series is always named. A calibration that is not given (no transformations, or no scale) is not reported. The run is not refused. | +| `invalid_physical_calibration` | Warning | An image's metadata give a calibration that cannot be used (issue #94): OME-Zarr `coordinateTransformations` that are not a list or of an unsupported type, more than one scale, a scale stored in a separate file or not one number per axis, or a scale value of X, Y, Z or T that is not a positive finite number; or, from any TIFF or OME-Zarr metadata, a size that is not a positive finite length in micrometres once converted (issue #93: `1e308 m`, `1e-320 pm`); or OME-XML in a TIFF that cannot be parsed (audit m6: before, any error reading the sizes read as merely uncalibrated); or TIFF metadata whose bytes cannot be read (`OSError`, `struct.error`, `TiffFileError` while a tag is read), or an ImageJ `XResolution` or `YResolution` tag that is missing or unreadable while the other is given (tifffile drops a tag it cannot read with only a log message), with a unit (round-2 audit A-m6); or an ImageJ unit with neither resolution tag ("a unit without a resolution"; owner, 3 November 2026, v0.4.0rc5: before, uncalibrated with no reason); or a TIFF file without pages ("no pages", v0.4.0rc5: tifffile opens it with only a log message); `read_physical_sizes` also gives a file it cannot open as a TIFF no sizes with the reason in `invalid` (in the folder check such a file is `unreadable_tiff`), and raises any other exception as the defect it is; or a size of OME-TIFF (`PhysicalSizeX/Y/Z`, `TimeIncrement`) or ImageJ metadata (`spacing`, `finterval` and the resolution tags, with a unit) that is itself not a positive finite number (`inf`, `NaN`, zero, negative or not a number; review of #133, before silently unknown); or an ImageJ `XResolution` or `YResolution` tag, with a unit, that is not two whole numbers (a tag of another type; review of #136, before silently unknown). The message names each image and what is wrong, with the axis and value. Its sizes are unknown on those axes (never a substituted value) and recorded as uncalibrated; the warning is in the `check` and `run` issues and the report (`discovery_report.json`). An image is not named once every size the malformed calibration leaves unknown (X and Y, and Z or T with more than one plane) is stated in `calibration.csv`; a time step cannot be stated there, so a lost time step of a time series is always named. A calibration that is not given (no transformations, or no scale) is not reported. The run is not refused. | | `partial_pixel_size` | Warning | An image's recorded pixel size (header or `calibration.csv`) is known in micrometres along only one of X and Y; the other is unknown or stated as `pixels` (audit m5, plan §12 question 72). The message names each image with both sizes. Bricks that measure lengths refuse such an image (calibration decision 8, 2 November 2026): a partial calibration is never completed or approximated. The run is not refused. | | `non_square_pixels` | Warning | An image's recorded X and Y sizes are both known but not exactly equal in micrometres (calibration decision 1: no tolerance, so 0.5 and 0.5004 µm differ). The message names each image with both sizes. Bricks that measure lengths (perimeter, Feret diameter, axis lengths) refuse such an image; areas and intensities are exact. The run is not refused. | | `different_channel_counts` | Error | The number of channels differs between images: the `C` dimension, or the RGB samples `S` when there is no `C`; once the axis roles are confirmed, the length of the C axis (1 without one), which replaces the header's count. The images not having the most common number are listed, with each number's image count. Every image of an experiment must have the same channels: split the others into another experiment or exclude them. | diff --git a/docs/user_guide.md b/docs/user_guide.md index 8ad1b606..2c6636d1 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -1068,7 +1068,7 @@ continues. | `different_pixel_sizes` | Warning | The physical pixel size or z-step (from the metadata or `calibration.csv`) differs between images by more than 0.1 % (for example different objectives). Images outside the largest group of matching sizes are listed; when sizes only change by small steps from image to image, none is listed. Check the microscope settings if this is not intended. | | `missing_pixel_size` | Warning | Some images have a pixel size (from the metadata or `calibration.csv`) and others do not, so it cannot be checked; images stated as measured in pixels count as without one, and the message says how many. Export them again with metadata, or state their sizes with `calibrate`, if needed. | | `unsupported_pixel_size_unit` | Warning | An image gives its pixel size or z-step in a unit that is not converted (for example inches), or an OME-Zarr scale without a unit; only pm, Å, nm, µm, mm, cm and m are. The message names each image, axis and unit. Those sizes are not used: the image is treated as uncalibrated on those axes. Export it again with the pixel size in a supported unit, or state it with `calibrate`, if the physical size matters. Not reported for an axis whose size is stated. | -| `invalid_physical_calibration` | Warning | An image's metadata give a pixel size, z-step or time step that cannot be used (OME-XML in a TIFF that cannot be read; for OME-Zarr: `coordinateTransformations` that are malformed, of an unsupported type, stored in a separate file, or a scale that is zero, negative or not a number; for any image: a size too large or too small to be written in micrometres, such as `1e308 m`); the message names each image and what is wrong, with the value. The image is treated as uncalibrated on those axes; no size is made up. Correct the metadata, or state the sizes with the `calibrate` command: the warning goes once every lost pixel size and z-step is stated. A lost time step cannot be stated, so its warning stays until the metadata are corrected. | +| `invalid_physical_calibration` | Warning | An image's metadata give a pixel size, z-step or time step that cannot be used (OME-XML in a TIFF that cannot be read; an ImageJ TIFF with a unit but no resolution tags; a TIFF file without pages; for OME-Zarr: `coordinateTransformations` that are malformed, of an unsupported type, stored in a separate file, or a scale that is zero, negative or not a number; for any image: a size too large or too small to be written in micrometres, such as `1e308 m`); the message names each image and what is wrong, with the value. The image is treated as uncalibrated on those axes; no size is made up. Correct the metadata, or state the sizes with the `calibrate` command: the warning goes once every lost pixel size and z-step is stated. A lost time step cannot be stated, so its warning stays until the metadata are corrected. | | `partial_pixel_size` | Warning | An image has a pixel size along only one of X and Y (the other unknown or stated as pixels). Bricks that measure lengths will refuse it: nothing is completed or approximated. State both sizes with `calibrate`, or both as pixels. | | `non_square_pixels` | Warning | An image's X and Y pixel sizes are not exactly equal (for example 0.5 and 0.5004 µm). Bricks that measure lengths (perimeter, Feret diameter, axis lengths) will refuse it. If the pixels are square, state both sizes, equal, with `calibrate`; otherwise no action is needed if no lengths are measured. | | `calibration_axis_absent` | Warning | `calibration.csv` states a size for an axis some images do not have (for example a Z size for 2D images or projections); the message names them. No size is recorded for an axis an image does not have. Give the statement a narrower target, or no action if it is meant for the other images. | diff --git a/src/morphobricks_preprocessing/physical_sizes.py b/src/morphobricks_preprocessing/physical_sizes.py index 86385e57..95c36095 100644 --- a/src/morphobricks_preprocessing/physical_sizes.py +++ b/src/morphobricks_preprocessing/physical_sizes.py @@ -485,9 +485,12 @@ def _tiff_sizes(tif: tifffile.TiffFile) -> PhysicalSizes: OME-XML that cannot be parsed, or metadata whose bytes cannot be read (:data:`.tiff_size_values.METADATA_READ_ERRORS`), gives no sizes, with what is wrong in ``invalid`` (audits m6 and A-m6, the TIFF twin of issue #94): never read as merely - uncalibrated. + uncalibrated. So does a file without pages (rc5), which tifffile opens with a log + message only. """ try: + if not tif.pages: # tifffile opens it, but there is no image to have a size + return PhysicalSizes(invalid="the TIFF file has no pages (no image)") if tif.is_ome and tif.ome_metadata: try: return _ome_sizes(tif.ome_metadata) @@ -562,5 +565,6 @@ def _imagej_sizes(tif: tifffile.TiffFile) -> PhysicalSizes: ) # A malformed tag or spacing is named when its axis has a unit, so that it is a size. named = [problem for axis, (_, problem) in read.items() if units[axis] or written[axis]] - lost = _missing_resolution(page.tags) if units["X"] or units["Y"] else None + unit = next((_written(given[a]) for a in "XY" if units[a]), None) + lost = _missing_resolution(page.tags, unit) if unit else None return _with_invalid(found, [*named, lost, step[1]]) diff --git a/src/morphobricks_preprocessing/tiff_size_values.py b/src/morphobricks_preprocessing/tiff_size_values.py index e30153b1..5cb9493d 100644 --- a/src/morphobricks_preprocessing/tiff_size_values.py +++ b/src/morphobricks_preprocessing/tiff_size_values.py @@ -88,15 +88,22 @@ def _resolution(axis: str, tag: tifffile.TiffTag | None) -> tuple[float | None, return None, None if tag is None else f"{axis}: the {tag.name} tag {value!r} is malformed" -def _missing_resolution(tags: tifffile.TiffTags) -> str | None: - """Say which resolution tag is missing when only one of the pair is there (audit A-m6). +def _missing_resolution(tags: tifffile.TiffTags, unit: str | None = None) -> str | None: + """Say which resolution tag is missing when the unit is given (audit A-m6, rc5). ImageJ writes both. tifffile drops a tag it cannot read (a value beyond the end of the file, an unknown type) with only a log message, so a lone one means the other was lost: - never read as an axis that is merely uncalibrated. + never read as an axis that is merely uncalibrated. A unit with neither tag (owner, 3 + November 2026) is a calibration without its sizes, never read as no calibration + either; ``unit`` is the unit as written, for the message. """ present = {axis: tags.get(f"{axis}Resolution") is not None for axis in "XY"} - if present["X"] == present["Y"]: + if not any(present.values()): + return ( + f"X, Y: the unit {unit!r} is given but neither the XResolution nor the " + "YResolution tag is (a unit without a resolution)" + ) + if all(present.values()): return None lost, kept = ("Y", "X") if present["X"] else ("X", "Y") return f"{lost}: the {lost}Resolution tag is missing or unreadable, {kept}Resolution is given" diff --git a/tests/test_tiff_size_read_errors.py b/tests/test_tiff_size_read_errors.py index 36ade4e2..a160e6fe 100644 --- a/tests/test_tiff_size_read_errors.py +++ b/tests/test_tiff_size_read_errors.py @@ -191,3 +191,53 @@ def test_a_lone_resolution_tag_without_a_unit_is_not_a_calibration(tmp_path): assert tif.is_imagej and tif.pages.first.tags.get("XResolution") is None sizes = read_physical_sizes(path) assert (sizes.x, sizes.y, sizes.invalid) == (None, None, None) + + +def drop_resolution_tags(path): + """Renumber both resolution tags as private tags, so the file has neither.""" + raw = bytearray(path.read_bytes()) + for code, private in ((282, 65000), (283, 65001)): + offset = tag_entry(path, code) + raw[offset : offset + 2] = struct.pack(" Date: Tue, 3 Nov 2026 14:00:55 +0000 Subject: [PATCH 4/5] Record plan Q66, Q73 and Q74; changelog and migration notes for rc5 Q66 confirmed by the owner (3 November 2026); Q73's follow-up decided: neither a stated reshape nor a reorder in v0.4, kept in issue #144; Q74 records the tie-break and the consumers checked. The changelog gains a section for rc5 (PRs #139, #140, #142, #143) with migration notes: mounts refused, position_id required in v2, micrometre conversion within 1 ULP, --verify-content, and the tie-break. The migration guide gets the same. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- CHANGELOG.md | 66 +++++++++++++++++++++++++++++++++--- docs/migration-0.3-to-0.4.md | 31 ++++++++++++++--- docs/plan.md | 31 ++++++++++++++--- 3 files changed, 115 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7c1a064..f9fb046c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,15 +6,64 @@ The design is `docs/restart/preprocessing-design-v0.4.md` in MorphoFeature. The are in [docs/plan.md](docs/plan.md) §12. Upgrading from v0.3: see the [migration guide](docs/migration-0.3-to-0.4.md). -## 0.4.0 (unreleased; release candidates rc1 to rc4) +## 0.4.0rc5 (since v0.4.0rc4) + +What changed between v0.4.0rc4 and v0.4.0rc5: PRs #139 (cross-brick test of 3 November +2026), #140 (review of rc4), #142 (round-2 audit) and #143 (this release candidate). Each +item is described in full under 0.4.0 below. Decisions: [plan](docs/plan.md) §12 questions +66, 72, 73 and 74. + +### Migration notes (from rc4) + +- **Mount points are refused** (#140): a mount point below the experiment or masks folder + is the error `mounted_folder` in discovery, `check`, `run`, `convert` and `RunInputs`. + Copy the files into the experiment, or select the mounted folder itself. A file with + several hard links is the warning `hard_linked_file`. +- **`position_id` is required in a version 2 manifest** (#142, A-m5), equal to the image's + name without its extension; anything else is a `ManifestFormatError`. Manifests written + by rc1 to rc4 open unchanged (rc4's is tested); a hand-edited one may not. +- **Sizes in micrometres may change by up to 1 ULP** (#142, B-m7) when computed from a unit + other than µm (`9 nm` is now `0.009`, not `0.009000000000000001`). Manifests are + unchanged; exact comparisons of the same length in two units now agree. +- **`--verify-content`** (#140, I3), opt-in: rows are reused only for the same content + SHA-256 (`RunMode(verify_content=True)`, the notebook's **Verify each image's content**). + Turning it on or off recomputes every row; a default run's manifest has the rc4 keys. +- **A child tied between parents has no parent** (#143, plan Q74): a child lying exactly as + much in two or more parents is no longer given the one with the lower label; it is + counted as `ambiguous_parent`, not `inside`. `mask_relationships.json` is report version + 3; `inside` and `parents_with_N_children` may be lower than with rc4 when masks have such + ties. CellPatcher and MorphoFeature read neither count. + +### Changed and fixed + +- `pair_counts`: an exact tie between parents gives no parent; the new evidence + `ambiguous_parent` is shown by `masks`, the notebook's **Describe the masks** card and its + relationship table; report version 3 (#143, plan Q74). +- An ImageJ TIFF with a unit but neither resolution tag, and a TIFF without pages, are + `invalid_physical_calibration` with the reason; before, uncalibrated with no reason + (#143). +- A TIFF calibration that cannot be read is invalid, and defects are raised (#142, A-m6); + `position_id` required in v2 (#142, A-m5); one length in any unit is one double (#142, + B-m7). +- Mount points refused, hard links named (#140, B1/I1); the output folder flushed after the + commit (#140, I2); the identity rule stated and `--verify-content` added (#140, I3). +- `masks` accepts per-plane masks with a stated shared 2-D mask (F1); `run` needs no + `--axes` when every image names its axes (F9); a rerun that changes nothing keeps the + manifest byte-identical (F10); stale locks and resumed runs are worded (F11); `convert + --output` and the order of copies are explained (F8) (#139, plan Q73). +- Plan Q66 confirmed by the owner; Q73's follow-up (a stated page reshape or reordered + copies) is not in v0.4 (issue #144). +- Version 0.4.0rc5; the notebook installs `v0.4.0rc5`. + +## 0.4.0 (unreleased; release candidates rc1 to rc5) Preprocessing becomes the one step that discovers, validates, records and serves an experiment. Every later brick reads the run through `RunInputs`. Nothing is guessed: axes, calibration, masks and design come from the headers or from the user's recorded answers. Release candidates: **v0.4.0rc1** (after PP-6, #112), **v0.4.0rc2** (after PP-8d, #118), -**v0.4.0rc3** (#125, after PP-10d) and **v0.4.0rc4** (PP-13, for the round-2 data-flow audit -and the bricks' migration). +**v0.4.0rc3** (#125, after PP-10d), **v0.4.0rc4** (PP-13, for the round-2 data-flow audit +and the bricks' migration) and **v0.4.0rc5** (#143, after the round-2 audit fixes). ### Breaking @@ -111,7 +160,16 @@ and the bricks' migration). no calibration; any other exception is raised, and a value that is not a path is a `TypeError`. An ImageJ resolution tag that tifffile dropped because it could not read it (it only logs), next to the other one, is named instead of leaving that axis silently - uncalibrated. + uncalibrated. An ImageJ unit with neither resolution tag ("a unit without a resolution") + and a TIFF file without pages ("no pages") are `invalid_physical_calibration` too, no + longer uncalibrated with no reason (owner, 3 November 2026; #143). + +- **A child lying equally in two or more parents has no parent** (owner, 3 November 2026; + plan §12 question 74, #143): `pair_counts` gave it the parent with the lower label index, + so `inside` and `parents_with_N_children` depended on the label numbering. An exact equal + largest overlap now gives no parent; when the share would make the child inside it is + counted as `ambiguous_parent` (new evidence), shown by `masks`, the notebook and the + relationship table. `mask_relationships.json` is report version 3. - **A version 2 manifest names every image's position by its name** (round-2 audit, A-m5): the reader accepted a file entry without `position_id`, or with null, and `RunInputs` diff --git a/docs/migration-0.3-to-0.4.md b/docs/migration-0.3-to-0.4.md index 5163917d..e5917900 100644 --- a/docs/migration-0.3-to-0.4.md +++ b/docs/migration-0.3-to-0.4.md @@ -10,7 +10,7 @@ This guide lists every breaking change from v0.3.4. Each item gives **what chang (the scientific rule) and **what to do**. The first section is for people who run preprocessing (notebook, command line or Python). The second is for brick authors. The full rules are in the [contract](contract.md) and the [user guide](user_guide.md). The decisions -are in the [plan](plan.md), §12 questions 45 to 69. The complete list of changes is in the +are in the [plan](plan.md), §12 questions 45 to 74. The complete list of changes is in the [CHANGELOG](../CHANGELOG.md). **In short, for users:** install v0.4.0 and run preprocessing again on every experiment, @@ -100,7 +100,12 @@ later bricks on the new run. sizes** card or by hand. Each axis is recorded with its source (`header`, `stated` or `pixels`). A Z or T size is recorded only for an image that has that axis. A size that is not a positive finite length in micrometres is refused when stated, and unknown (with - `invalid_physical_calibration`) when it comes from a header. + `invalid_physical_calibration`) when it comes from a header. So is an ImageJ unit without + resolution tags, and a TIFF without pages (v0.4.0rc5). Since v0.4.0rc5 a length is + converted to micrometres from its shortest decimal, so one length written in any unit is + one double (`108.37 nm` is `0.10837 µm`): **a size in micrometres computed from another + unit may differ by 1 ULP** (the last bit) from v0.4.0rc4. Manifests are unchanged (each + size is recorded in its own unit), and the 0.1 % comparison is not affected. - **Why.** Pixel sizes belong to the experiment, not to each brick's configuration. A size nobody gave is uncalibrated, never a default. - **What to do.** Move the pixel sizes you gave to bricks (run files, FOV-table columns, @@ -217,6 +222,10 @@ later bricks on the new run. only when its real path stays inside that folder. Otherwise it is `link_outside_folder`. One file reached by two paths, hard links included, is `linked_twice`. A loop is `link_loop`. + - **Mount points below the experiment or masks folder are refused** (v0.4.0rc5): a folder + or file listed in `/proc/self/mountinfo`, or on another device than the folder + selected, is the error `mounted_folder`, in discovery, `check`, `run`, `convert` and + `RunInputs`. A file with more than one hard link is the warning `hard_linked_file`. - `--log` is checked before the file is opened. It is refused inside the experiment, the masks folder or the folder of copies, and as or inside a protected output file (the table, manifest, reports, design, `calibration.csv`, `mask_relationships.json`, lock, @@ -227,8 +236,8 @@ later bricks on the new run. - **Why.** A link that leaves the experiment brings in images that are not part of it, and one file seen twice is counted twice. A log must never overwrite a result. Scripts must be able to trust the exit status. -- **What to do.** Copy images into the experiment rather than linking to them from - elsewhere. Write logs next to the output folder. In scripts, check the exit status. +- **What to do.** Copy images into the experiment rather than linking to them or mounting + them from elsewhere. Write logs next to the output folder. In scripts, check the exit status. ### 11. Staging, resume and crash-consistent publication @@ -247,6 +256,13 @@ later bricks on the new run. `experiment_design.csv` (with a design), the reports, `mask_relationships.json`, `.publish` and `.staging`. It never touches `calibration.csv`, source images or masks. - New output files get your umask's permissions; existing files keep theirs. + - A source is identified by its fingerprint (size and modification time; for OME-Zarr the + inventory of its files), which misses a rewrite of the same size with the time set + back. The opt-in `--verify-content` (notebook: **Verify each image's content**, + `RunMode(verify_content=True)`, v0.4.0rc5) records each source's content SHA-256 and + reuses rows only for the same content; `RunInputs` checks it at `open`. Turning it on + or off recomputes every row; a default run's manifest is unchanged. + - The output folder is flushed to disk right after the manifest's rename (POSIX). - **Why.** A reader must see the old run or the new one, never a mix. A row is reused only when it is certainly the same row. - **What to do.** Nothing, except: read the outputs through `RunInputs` (bricks), or check @@ -299,6 +315,13 @@ brick refuses and gives the preprocessing command; there are no standalone excep for length features (audit m5). `read_plane`'s memory limit counts only reading (I2). - `histograms()`, `design()` (the strict `read_confirmed_design`, checked against the manifest's SHA-256) and `image_type()`. +- A version 2 manifest **requires** each file entry's `position_id`, equal to the image's + name without its extension (`position_id(relative_path)`); a missing, null or edited one + is a `ManifestFormatError` (v0.4.0rc5, audit A-m5). Every v0.4.0 writer wrote it, so runs + of rc1 to rc4 still open; a hand-edited manifest may not. +- `mask_relationships.json` is report version 3 (v0.4.0rc5): a child lying exactly as much + in two or more parents has no parent and is counted as `ambiguous_parent`, never + `inside`; `inside` and `parents_with_N_children` no longer depend on label numbers. - `RunInputs` is picklable for worker processes. `morphobricks_preprocessing.testing. make_clean_run(folder, ...)` writes a small clean run for tests. diff --git a/docs/plan.md b/docs/plan.md index 967940dd..2aaf5dd7 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -617,7 +617,8 @@ API, CLI). The notebook and CLI only call the API. - **Follow-ups recorded for the audit (coordinator, 2 November 2026; not changed here):** (a) `pair_counts` breaks an exact-half tie by the lower parent index, so `inside` and `parents_with_N_children` can depend on label numbering when a child lies exactly half in - each of two parents (the `same_objects` counts do not, question 67); (b) masks are not + each of two parents (the `same_objects` counts do not, question 67) (*done in v0.4.0rc5: + a tie gives no parent, `ambiguous_parent`, question 74*); (b) masks are not bound to a fingerprint during a run: they are fingerprinted when the manifest is written (PP-12 follow-up (a)); (c) a run killed while an atomic write's temporary file is open leaves a hidden `...tmp` in the output folder, for example after the @@ -644,6 +645,8 @@ API, CLI). The notebook and CLI only call the API. (the tests run it in a local kernel). - v0.4.0rc4 (PP-13): version and pin set; the owner tags it. The data-flow audit and MorphoFeature's migration run against it. + - v0.4.0rc5: version and pin set, the changelog's rc5 section (PRs #139, #140, #142 and + the rc5 PR: questions 66, 73 and 74, the round-2 audit fixes); the owner tags it. ## 11. Tests and acceptance criteria @@ -1695,9 +1698,9 @@ Raised during review; to be decided before the milestones that need them: (it was appended to). Contract §8; user guide §5. 66. What happens to a size that does not fit in micrometres, and what is the exit status of a command that reports an error (issue #93, review of #130)? - *Decided, pending the owner's confirmation (2 November 2026, design v0.4 §3: nothing - guessed, nothing silent; the coordinator's answers in the review of #133, recorded for - the owner to confirm: a header size that overflows or cannot be converted is a + *Decided (2 November 2026, design v0.4 §3: nothing guessed, nothing silent; the + coordinator's answers in the review of #133); **confirmed by the owner (3 November + 2026)**: a header size that overflows or cannot be converted is a warning, the image uncalibrated with the reason, as the owner decided for #94; and `design` exiting with 1 after a write that leaves a conflict is accepted):* (1) every conversion to micrometres goes through `to_micrometers`, whose result is always a positive finite number: a length that @@ -1858,4 +1861,22 @@ Raised during review; to be decided before the milestones that need them: the spec has no way to state one, so the user guide says to save the image again with its dimensions (for example as a Fiji hyperstack). Follow-up for the owner: whether `convert` should accept a stated reshape (for example `--pages CZ=2x3`) or reorder copies - into the common order. + into the common order. *Decided (owner, 3 November 2026):* neither in v0.4, neither a + stated reshape nor a reorder; F8a and F8c stay as above, and the post-matrix issue #144 + keeps both options. +74. When a child object lies equally in two or more parent objects, which is its parent + (`pair_counts`; PP-13 follow-up (a), §10)? *Decided (owner, 3 November 2026; contract + §15.1, implemented in v0.4.0rc5):* none. Before, the parent with the lower label index + was taken, so `inside` and `parents_with_N_children` depended on the label numbering + when a child lay exactly half in each of two parents. Every tie path is an exact equal + largest overlap of two or more parents (`mask_pairs.best_parents`); such a child is + never `inside` and counts for no parent, and when the tied share would make it inside + (at least `inside_pixel_share`) it is counted in the new evidence `ambiguous_parent`, + which `masks`, the notebook's **Describe the masks** and the relationship table show. A + tie below that share is `split_between_parents`, as before (no parent either way). Each + child is counted once in `inside`, `ambiguous_parent`, `split_between_parents` or + `outside_parents`. The `same_objects` counts were already independent of the numbering + (question 67). `mask_relationships.json` is report version 3. Consumers checked + read-only on 3 November 2026 (CellPatcher `79adc3a`, MorphoFeature `142c448`, their + main branches): neither reads `inside`, `parents_with_N_children`, + `mask_relationships.json` or `scan_masks`; the change reaches no brick's numbers (#143). From e5e6a740552b5385ad6c6533a1883bfd3c117bb0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 14:21:05 +0000 Subject: [PATCH 5/5] Say where a TIFF without pages is reported; test each mask relabelled on its own Review of #143. In check and run a TIFF without pages is the error unreadable_tiff (its header has no series), as before rc5; only read_physical_sizes gives it the "no pages" reason. The CHANGELOG, contract, migration guide and user guide said it was invalid_physical_calibration. They also said every tied child is ambiguous_parent; only a tie whose share reaches inside_pixel_share is, and a smaller one stays split, as the contract and plan Q74 state. Tests: the folder check of a TIFF without pages; every count but child_pixels_with_parent_label is unchanged when the child and parent masks are relabelled independently (three inside shares, ties present). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- CHANGELOG.md | 19 +++++++++++------- docs/contract.md | 2 +- docs/migration-0.3-to-0.4.md | 3 ++- docs/user_guide.md | 9 +++++---- tests/test_mask_ties.py | 30 +++++++++++++++++++++++++++++ tests/test_tiff_size_read_errors.py | 7 +++++++ 6 files changed, 57 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9fb046c..405ff15c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,8 +29,10 @@ item is described in full under 0.4.0 below. Decisions: [plan](docs/plan.md) §1 SHA-256 (`RunMode(verify_content=True)`, the notebook's **Verify each image's content**). Turning it on or off recomputes every row; a default run's manifest has the rc4 keys. - **A child tied between parents has no parent** (#143, plan Q74): a child lying exactly as - much in two or more parents is no longer given the one with the lower label; it is - counted as `ambiguous_parent`, not `inside`. `mask_relationships.json` is report version + much in two or more parents is no longer given the one with the lower label. When that + share reaches `inside_pixel_share` (at the default, exactly half in each of two parents) + it is counted as `ambiguous_parent`, not `inside`; a tie below it stays + `split_between_parents`, as before. `mask_relationships.json` is report version 3; `inside` and `parents_with_N_children` may be lower than with rc4 when masks have such ties. CellPatcher and MorphoFeature read neither count. @@ -39,9 +41,10 @@ item is described in full under 0.4.0 below. Decisions: [plan](docs/plan.md) §1 - `pair_counts`: an exact tie between parents gives no parent; the new evidence `ambiguous_parent` is shown by `masks`, the notebook's **Describe the masks** card and its relationship table; report version 3 (#143, plan Q74). -- An ImageJ TIFF with a unit but neither resolution tag, and a TIFF without pages, are - `invalid_physical_calibration` with the reason; before, uncalibrated with no reason - (#143). +- An ImageJ TIFF with a unit but neither resolution tag is `invalid_physical_calibration` + with the reason; before, uncalibrated with no reason. `read_physical_sizes` gives a TIFF + without pages no sizes with the reason in `invalid`; in `check` and `run` such a file is + the error `unreadable_tiff`, as before (#143). - A TIFF calibration that cannot be read is invalid, and defects are raised (#142, A-m6); `position_id` required in v2 (#142, A-m5); one length in any unit is one double (#142, B-m7). @@ -161,8 +164,10 @@ and the bricks' migration) and **v0.4.0rc5** (#143, after the round-2 audit fixe `TypeError`. An ImageJ resolution tag that tifffile dropped because it could not read it (it only logs), next to the other one, is named instead of leaving that axis silently uncalibrated. An ImageJ unit with neither resolution tag ("a unit without a resolution") - and a TIFF file without pages ("no pages") are `invalid_physical_calibration` too, no - longer uncalibrated with no reason (owner, 3 November 2026; #143). + is `invalid_physical_calibration` too, no longer uncalibrated with no reason (owner, 3 + November 2026; #143). `read_physical_sizes` gives a TIFF file without pages ("no pages") + no sizes with the reason in `invalid`; the folder check reports such a file as the error + `unreadable_tiff`, as before. - **A child lying equally in two or more parents has no parent** (owner, 3 November 2026; plan §12 question 74, #143): `pair_counts` gave it the parent with the lower label index, diff --git a/docs/contract.md b/docs/contract.md index e82284ad..19118530 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -342,7 +342,7 @@ row (follow-up P3b, normalization plan N43). | `different_pixel_sizes` | Warning | The physical pixel size (X by Y) differs between images, or the z-step differs between images with more than one depth slice. Sizes are compared in micrometres by the shared rule of decision D-a (§16): two sizes are the same when they differ by at most 0.1 % of the larger, on every axis. The warning is given exactly when, on some axis, the smallest and the largest size differ, that is when some two images differ. The images are grouped by single linkage (`pixel_size_groups`) and those outside the largest group are listed. When every image is linked to the others by near-equal sizes (1.0, 1.0009, 1.0018 µm), the sizes drift, no image stands out, and the message says that none is listed for that size. Until D86 sizes were rounded to four significant digits, so 1.0004 and 1.0006 µm differed (1 and 1.001); they are now the same. | | `missing_pixel_size` | Warning | The pixel size (or the z-step of images with more than one depth slice) is known for some images and not for others. Nothing is reported when no image has it. | | `unsupported_pixel_size_unit` | Warning | An image's metadata give a pixel size or z-step (OME `PhysicalSize{X,Y,Z}`, ImageJ resolution or `spacing`, OME-Zarr level-0 scale) in a length unit that is not one of decision D-a2 (§16), for example OME `in` or ImageJ `inch`. The size is not converted and the axis is treated as uncalibrated (as before); the warning names each image, its axes and the unit as written, so the missing calibration is never silent. An OME-Zarr scale of X, Y or Z without a unit is named the same way ("X and Y without a unit"; review of #136): it is recorded as unknown (audit I3), never taken as micrometres, except on the axes listed as `unknown_scale` (§10). A unit without a size, and a plain TIFF's resolution tags (decision D-c), are not reported. The run is not refused. | -| `invalid_physical_calibration` | Warning | An image's metadata give a calibration that cannot be used (issue #94): OME-Zarr `coordinateTransformations` that are not a list or of an unsupported type, more than one scale, a scale stored in a separate file or not one number per axis, or a scale value of X, Y, Z or T that is not a positive finite number; or, from any TIFF or OME-Zarr metadata, a size that is not a positive finite length in micrometres once converted (issue #93: `1e308 m`, `1e-320 pm`); or OME-XML in a TIFF that cannot be parsed (audit m6: before, any error reading the sizes read as merely uncalibrated); or TIFF metadata whose bytes cannot be read (`OSError`, `struct.error`, `TiffFileError` while a tag is read), or an ImageJ `XResolution` or `YResolution` tag that is missing or unreadable while the other is given (tifffile drops a tag it cannot read with only a log message), with a unit (round-2 audit A-m6); or an ImageJ unit with neither resolution tag ("a unit without a resolution"; owner, 3 November 2026, v0.4.0rc5: before, uncalibrated with no reason); or a TIFF file without pages ("no pages", v0.4.0rc5: tifffile opens it with only a log message); `read_physical_sizes` also gives a file it cannot open as a TIFF no sizes with the reason in `invalid` (in the folder check such a file is `unreadable_tiff`), and raises any other exception as the defect it is; or a size of OME-TIFF (`PhysicalSizeX/Y/Z`, `TimeIncrement`) or ImageJ metadata (`spacing`, `finterval` and the resolution tags, with a unit) that is itself not a positive finite number (`inf`, `NaN`, zero, negative or not a number; review of #133, before silently unknown); or an ImageJ `XResolution` or `YResolution` tag, with a unit, that is not two whole numbers (a tag of another type; review of #136, before silently unknown). The message names each image and what is wrong, with the axis and value. Its sizes are unknown on those axes (never a substituted value) and recorded as uncalibrated; the warning is in the `check` and `run` issues and the report (`discovery_report.json`). An image is not named once every size the malformed calibration leaves unknown (X and Y, and Z or T with more than one plane) is stated in `calibration.csv`; a time step cannot be stated there, so a lost time step of a time series is always named. A calibration that is not given (no transformations, or no scale) is not reported. The run is not refused. | +| `invalid_physical_calibration` | Warning | An image's metadata give a calibration that cannot be used (issue #94): OME-Zarr `coordinateTransformations` that are not a list or of an unsupported type, more than one scale, a scale stored in a separate file or not one number per axis, or a scale value of X, Y, Z or T that is not a positive finite number; or, from any TIFF or OME-Zarr metadata, a size that is not a positive finite length in micrometres once converted (issue #93: `1e308 m`, `1e-320 pm`); or OME-XML in a TIFF that cannot be parsed (audit m6: before, any error reading the sizes read as merely uncalibrated); or TIFF metadata whose bytes cannot be read (`OSError`, `struct.error`, `TiffFileError` while a tag is read), or an ImageJ `XResolution` or `YResolution` tag that is missing or unreadable while the other is given (tifffile drops a tag it cannot read with only a log message), with a unit (round-2 audit A-m6); or an ImageJ unit with neither resolution tag ("a unit without a resolution"; owner, 3 November 2026, v0.4.0rc5: before, uncalibrated with no reason); or, from `read_physical_sizes`, a TIFF file without pages ("no pages", v0.4.0rc5: tifffile opens it with only a log message; in the folder check such a file is `unreadable_tiff`, as before); `read_physical_sizes` also gives a file it cannot open as a TIFF no sizes with the reason in `invalid` (in the folder check such a file is `unreadable_tiff`), and raises any other exception as the defect it is; or a size of OME-TIFF (`PhysicalSizeX/Y/Z`, `TimeIncrement`) or ImageJ metadata (`spacing`, `finterval` and the resolution tags, with a unit) that is itself not a positive finite number (`inf`, `NaN`, zero, negative or not a number; review of #133, before silently unknown); or an ImageJ `XResolution` or `YResolution` tag, with a unit, that is not two whole numbers (a tag of another type; review of #136, before silently unknown). The message names each image and what is wrong, with the axis and value. Its sizes are unknown on those axes (never a substituted value) and recorded as uncalibrated; the warning is in the `check` and `run` issues and the report (`discovery_report.json`). An image is not named once every size the malformed calibration leaves unknown (X and Y, and Z or T with more than one plane) is stated in `calibration.csv`; a time step cannot be stated there, so a lost time step of a time series is always named. A calibration that is not given (no transformations, or no scale) is not reported. The run is not refused. | | `partial_pixel_size` | Warning | An image's recorded pixel size (header or `calibration.csv`) is known in micrometres along only one of X and Y; the other is unknown or stated as `pixels` (audit m5, plan §12 question 72). The message names each image with both sizes. Bricks that measure lengths refuse such an image (calibration decision 8, 2 November 2026): a partial calibration is never completed or approximated. The run is not refused. | | `non_square_pixels` | Warning | An image's recorded X and Y sizes are both known but not exactly equal in micrometres (calibration decision 1: no tolerance, so 0.5 and 0.5004 µm differ). The message names each image with both sizes. Bricks that measure lengths (perimeter, Feret diameter, axis lengths) refuse such an image; areas and intensities are exact. The run is not refused. | | `different_channel_counts` | Error | The number of channels differs between images: the `C` dimension, or the RGB samples `S` when there is no `C`; once the axis roles are confirmed, the length of the C axis (1 without one), which replaces the header's count. The images not having the most common number are listed, with each number's image count. Every image of an experiment must have the same channels: split the others into another experiment or exclude them. | diff --git a/docs/migration-0.3-to-0.4.md b/docs/migration-0.3-to-0.4.md index e5917900..c4f79b3f 100644 --- a/docs/migration-0.3-to-0.4.md +++ b/docs/migration-0.3-to-0.4.md @@ -101,7 +101,8 @@ later bricks on the new run. `pixels`). A Z or T size is recorded only for an image that has that axis. A size that is not a positive finite length in micrometres is refused when stated, and unknown (with `invalid_physical_calibration`) when it comes from a header. So is an ImageJ unit without - resolution tags, and a TIFF without pages (v0.4.0rc5). Since v0.4.0rc5 a length is + resolution tags (v0.4.0rc5); `read_physical_sizes` gives a TIFF without pages no sizes + with the reason in `invalid` (in `check` and `run` it is the error `unreadable_tiff`). Since v0.4.0rc5 a length is converted to micrometres from its shortest decimal, so one length written in any unit is one double (`108.37 nm` is `0.10837 µm`): **a size in micrometres computed from another unit may differ by 1 ULP** (the last bit) from v0.4.0rc4. Manifests are unchanged (each diff --git a/docs/user_guide.md b/docs/user_guide.md index 2c6636d1..a27fad46 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -692,9 +692,10 @@ Nucleoli inside nuclei inside cells have the nuclei as their direct parent; two mask that segment the same objects are reported as such: at least 90% of the objects of both are matched one to one, each object matched at most once (two halves of one cell are one match, not two). An object that lies exactly as much in two or more objects of the -other kind (a nucleus half in each of two cells) is given **no parent**, whichever has the -lower number: it is not counted as inside, and the command lists it under "Objects given -no parent" (`ambiguous_parent` in the report, since v0.4.0rc5). These are suggestions from the +other kind is given **no parent**, whichever has the lower number. When that share would +make it inside (a nucleus exactly half in each of two cells), it is not counted as inside +and the command lists it under "Objects given no parent" (`ambiguous_parent` in the +report, since v0.4.0rc5); a smaller tied share is counted as split, as before. These are suggestions from the pixels, not decisions: you name each kind of mask and check the relationships in the bricks that use them. The report is written to `mask_relationships.json` in the output folder (§6). The command takes `--output`, `--exclude` and `--log` like the others, and exits with @@ -1068,7 +1069,7 @@ continues. | `different_pixel_sizes` | Warning | The physical pixel size or z-step (from the metadata or `calibration.csv`) differs between images by more than 0.1 % (for example different objectives). Images outside the largest group of matching sizes are listed; when sizes only change by small steps from image to image, none is listed. Check the microscope settings if this is not intended. | | `missing_pixel_size` | Warning | Some images have a pixel size (from the metadata or `calibration.csv`) and others do not, so it cannot be checked; images stated as measured in pixels count as without one, and the message says how many. Export them again with metadata, or state their sizes with `calibrate`, if needed. | | `unsupported_pixel_size_unit` | Warning | An image gives its pixel size or z-step in a unit that is not converted (for example inches), or an OME-Zarr scale without a unit; only pm, Å, nm, µm, mm, cm and m are. The message names each image, axis and unit. Those sizes are not used: the image is treated as uncalibrated on those axes. Export it again with the pixel size in a supported unit, or state it with `calibrate`, if the physical size matters. Not reported for an axis whose size is stated. | -| `invalid_physical_calibration` | Warning | An image's metadata give a pixel size, z-step or time step that cannot be used (OME-XML in a TIFF that cannot be read; an ImageJ TIFF with a unit but no resolution tags; a TIFF file without pages; for OME-Zarr: `coordinateTransformations` that are malformed, of an unsupported type, stored in a separate file, or a scale that is zero, negative or not a number; for any image: a size too large or too small to be written in micrometres, such as `1e308 m`); the message names each image and what is wrong, with the value. The image is treated as uncalibrated on those axes; no size is made up. Correct the metadata, or state the sizes with the `calibrate` command: the warning goes once every lost pixel size and z-step is stated. A lost time step cannot be stated, so its warning stays until the metadata are corrected. | +| `invalid_physical_calibration` | Warning | An image's metadata give a pixel size, z-step or time step that cannot be used (OME-XML in a TIFF that cannot be read; an ImageJ TIFF with a unit but no resolution tags; for OME-Zarr: `coordinateTransformations` that are malformed, of an unsupported type, stored in a separate file, or a scale that is zero, negative or not a number; for any image: a size too large or too small to be written in micrometres, such as `1e308 m`); the message names each image and what is wrong, with the value. The image is treated as uncalibrated on those axes; no size is made up. Correct the metadata, or state the sizes with the `calibrate` command: the warning goes once every lost pixel size and z-step is stated. A lost time step cannot be stated, so its warning stays until the metadata are corrected. | | `partial_pixel_size` | Warning | An image has a pixel size along only one of X and Y (the other unknown or stated as pixels). Bricks that measure lengths will refuse it: nothing is completed or approximated. State both sizes with `calibrate`, or both as pixels. | | `non_square_pixels` | Warning | An image's X and Y pixel sizes are not exactly equal (for example 0.5 and 0.5004 µm). Bricks that measure lengths (perimeter, Feret diameter, axis lengths) will refuse it. If the pixels are square, state both sizes, equal, with `calibrate`; otherwise no action is needed if no lengths are measured. | | `calibration_axis_absent` | Warning | `calibration.csv` states a size for an axis some images do not have (for example a Z size for 2D images or projections); the message names them. No size is recorded for an axis an image does not have. Give the statement a narrower target, or no action if it is meant for the other images. | diff --git a/tests/test_mask_ties.py b/tests/test_mask_ties.py index 2506489c..c3d01bfa 100644 --- a/tests/test_mask_ties.py +++ b/tests/test_mask_ties.py @@ -122,6 +122,36 @@ def test_every_count_is_the_same_under_any_label_numbering(share): assert ties > 20 # the data hold ties, so the test can fail +def relabel_each(mask, rng): + """Give one mask's labels other numbers, in another order, independently of the other.""" + labels = np.unique(mask[mask != 0]) + new = rng.permutation(np.arange(1, 5 * labels.size + 1))[: labels.size] * 7 + relabelled = np.zeros(mask.shape, dtype=np.int32) + for old, value in zip(labels.tolist(), new.tolist(), strict=True): + relabelled[mask == old] = value + return relabelled + + +@pytest.mark.parametrize("share", [0.5, 1 / 3, 0.25]) +def test_the_counts_are_the_same_when_each_mask_is_numbered_on_its_own(share): + # Only child_pixels_with_parent_label compares label numbers across the masks. + rng = np.random.default_rng(143) + limits = {"inside_pixel_share": share} + ties = 0 + for _ in range(200): + children, parents = random_masks(rng) + counts = pair_counts(children, parents, thresholds=limits) + del counts["child_pixels_with_parent_label"] + ties += counts["ambiguous_parent"] + for _ in range(3): + again = pair_counts( + relabel_each(children, rng), relabel_each(parents, rng), thresholds=limits + ) + del again["child_pixels_with_parent_label"] + assert again == counts + assert ties > 20 + + def reference_best(kid, host, shared): """Each child's largest overlap and how many parents have it, by plain loops.""" found = {} diff --git a/tests/test_tiff_size_read_errors.py b/tests/test_tiff_size_read_errors.py index a160e6fe..f5582d78 100644 --- a/tests/test_tiff_size_read_errors.py +++ b/tests/test_tiff_size_read_errors.py @@ -241,3 +241,10 @@ def test_a_tiff_without_pages_is_an_invalid_calibration(tmp_path): sizes = read_physical_sizes(path) assert (sizes.x, sizes.y) == (None, None) assert sizes.invalid == "the TIFF file has no pages (no image)" + + +def test_a_tiff_without_pages_is_unreadable_in_the_folder_check(tmp_path): + # check and run never reach its sizes: the header has no image (as before rc5). + (tmp_path / "a.tif").write_bytes(b"II*\x00" + struct.pack("