From fc8edd59b6a2deffa92fe25c5143c3b01eb11896 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 7 Nov 2026 23:20:07 +0000 Subject: [PATCH] Semantic RunInputs from a published description (Stage 1, PR 12) Fov gains repeat_group, condition_id, technical_repeat_id and channel_ids beside the unchanged key; RunInputs gains channel_index, references, mask_semantics and mask_check_status. Only the published morphobricks.toml is read and it must be the file the run recorded: a draft or an unconfirmed file is A_DRAFT_REFUSED, and reading a mask file the mask step has not checked is A_MASK_UNCHECKED. A run without a description is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Scma6W3LC11s3qjXHZuGDm --- CHANGELOG.md | 13 + docs/contract.md | 21 +- pyproject.toml | 5 + .../description_consume.py | 59 +++++ .../fov_identity.py | 57 +++++ src/morphobricks_preprocessing/run_inputs.py | 84 +++---- .../run_inputs_error.py | 32 +++ .../run_inputs_image.py | 45 ++++ .../run_inputs_semantic.py | 232 ++++++++++++++++++ tests/spec_gate.py | 1 + tests/test_run_inputs_semantic.py | 153 ++++++++++++ tests/test_spec_run_inputs_semantic.py | 116 +++++++++ 12 files changed, 767 insertions(+), 51 deletions(-) create mode 100644 src/morphobricks_preprocessing/description_consume.py create mode 100644 src/morphobricks_preprocessing/fov_identity.py create mode 100644 src/morphobricks_preprocessing/run_inputs_error.py create mode 100644 src/morphobricks_preprocessing/run_inputs_image.py create mode 100644 src/morphobricks_preprocessing/run_inputs_semantic.py create mode 100644 tests/test_run_inputs_semantic.py create mode 100644 tests/test_spec_run_inputs_semantic.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 32caa8da..f6fc62d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,19 @@ are in [docs/plan.md](docs/plan.md) §12. Upgrading from v0.3: see the ## Unreleased +- **Semantic `RunInputs` (Stage 1, PR 12; stacked on PR 11).** For a run with a published + description, each `Fov` carries `repeat_group`, `condition_id`, `technical_repeat_id` and + `channel_ids` beside the unchanged key (owner decision Q8: the key keeps the source folder + labels); `RunInputs.channel_index(fov, id)` finds a channel by canonical ID and refuses an + absent one; `references()` gives the condition reference graph, `mask_semantics()` and + `mask_check_status()` the masks' meaning and check status. `RunInputs.open` and + `description()` read only the published `morphobricks.toml`, check it is the file the run + recorded, and refuse a draft or an unconfirmed file with `A_DRAFT_REFUSED`; `read_mask` of a + file the mask step has not checked is `A_MASK_UNCHECKED`. A run without a description is + unchanged. New modules `fov_identity`, `description_consume`, `run_inputs_error`, + `run_inputs_semantic`; `LazyImage` moved to `run_inputs_image` (still exported from + `run_inputs`). Conformance P14, P25 and the `A_MASK_UNCHECKED` part of P28 in + `tests/test_spec_run_inputs_semantic.py`. - **A run with a published experiment description (Stage 1, PR 11).** When the output folder holds a published `morphobricks.toml`, `run` reads it with the strict reader (a refusal is reported with its Spec code, table path and a plain message), runs the physical checks diff --git a/docs/contract.md b/docs/contract.md index 504774cf..cd740d80 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -301,7 +301,26 @@ It gives: reads `experiment_design.csv` with the strict `read_confirmed_design` (§16) and the manifest's `experiment_design_sha256` (issue #83, plan §12 question 62; it used the lenient reader before): a design that reader refuses raises `RunInputsError`, with the refusal - as its `__cause__`. + as its `__cause__`; +- **semantics of a published description** (Stage 1 PR 12; only for a run whose manifest has + `experiment_description`, §9; a run without one is unchanged: the fields below are `None`, + `description()`, `references()` and `mask_semantics()` return `None`, and `read_mask` is as + before). `Fov.repeat_group`, `Fov.condition_id`, `Fov.technical_repeat_id` (the canonical + IDs, added beside the unchanged `Fov.key`, whose condition, repeat and technical repeat stay + the source folder labels: owner decision Q8, experiment-v1 §3.4) and `Fov.channel_ids` (the + canonical channel ID of each C index, as the unit declares them; a unit whose channel count + is not the image's, or folders that name no declared unit, is a `RunInputsError`: nothing is + guessed). `channel_index(fov, channel_id)` gives the C index, `ChannelNotFoundError` for a + channel the image lacks (never a default). `references()` is the condition reference graph + (condition ID to the ID it references, or `None`); `mask_semantics()` maps mask ID to + `MaskSemantics(source_type, representation, source_channels)`; `mask_check_status(mask_id, + fov)` is `checked` or `not yet checked`. `description()` reads the published + `morphobricks.toml` again and returns it only if it is the file whose SHA-256 the run + recorded (`RunInputsError` otherwise, also at `open`); a draft, a file not named + `morphobricks.toml` (`description_consume.read_published_description`) or a published file + with an unconfirmed repeat is refused with `DraftRefusedError` (`A_DRAFT_REFUSED`). `read_mask` + of a mask file the mask step has not checked (§15; `mask_representation.json`) is refused with + `MaskUncheckedError` (`A_MASK_UNCHECKED`); the error classes carry the Spec code as `code`. A `RunInputs` is picklable, so worker processes can use it. `morphobricks_preprocessing.testing.make_clean_run(folder, ...)` writes a tiny experiment of diff --git a/pyproject.toml b/pyproject.toml index 01255571..10934e96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,6 +184,11 @@ baseline = "tools/code_rules_baseline.json" "morphobricks_preprocessing.physical_check" = "io" "morphobricks_preprocessing.description" = "pure" "morphobricks_preprocessing.description_write" = "pure" +"morphobricks_preprocessing.fov_identity" = "pure" +"morphobricks_preprocessing.description_consume" = "io" +"morphobricks_preprocessing.run_inputs_error" = "pure" +"morphobricks_preprocessing.run_inputs_semantic" = "io" +"morphobricks_preprocessing.run_inputs_image" = "pure" "morphobricks_preprocessing.description_layouts" = "pure" "morphobricks_preprocessing.description_record" = "pure" "morphobricks_preprocessing.description_run_text" = "pure" diff --git a/src/morphobricks_preprocessing/description_consume.py b/src/morphobricks_preprocessing/description_consume.py new file mode 100644 index 00000000..b58ed946 --- /dev/null +++ b/src/morphobricks_preprocessing/description_consume.py @@ -0,0 +1,59 @@ +"""Reading the published description as a consumer (experiment-v1 sections 10 and 12). + +A consumer reads only the published ``morphobricks.toml``. A draft (``morphobricks.draft.json``, +a reconciliation proposal, anything not named ``morphobricks.toml``) and a file with an +unconfirmed repeat are refused with ``A_DRAFT_REFUSED``; nothing is read from them. +""" + +from __future__ import annotations + +import os +from functools import lru_cache +from pathlib import Path + +from .description import Description, DescriptionError, description_from_bytes +from .description_file import DESCRIPTION_FILENAME, read_file_bytes, sha256_hex +from .description_rules import Code +from .run_inputs_error import DraftRefusedError, RunInputsError + + +@lru_cache(maxsize=8) +def _parsed(data: bytes) -> Description: + """Return the model of valid bytes (cached by content: a read is then a read and a hash).""" + return description_from_bytes(data) + + +def read_published_description(path: str | os.PathLike) -> tuple[Description, str]: + """Read the published description at ``path``; return it and the SHA-256 of its bytes. + + Raises + ------ + DraftRefusedError + ``A_DRAFT_REFUSED``: the file is not named ``morphobricks.toml`` (a draft or proposal) + or holds a repeat that is not confirmed. + RunInputsError + When the file is missing, a symbolic link, or not a valid description. + """ + path = Path(path) + if path.name != DESCRIPTION_FILENAME: + raise DraftRefusedError( + f"{path.name} is not the published {DESCRIPTION_FILENAME}: a draft or proposal " + "is never read (A_DRAFT_REFUSED). Publish the description first." + ) + try: + data = read_file_bytes(path) + if data is None: + raise RunInputsError(f"{path} is gone: the run recorded a description.") + return _parsed(data), sha256_hex(data) + except DescriptionError as exc: + codes = sorted({issue.code.value for issue in exc.issues}) + if any(issue.code is Code.E_REPEAT_UNCONFIRMED for issue in exc.issues): + raise DraftRefusedError( + f"{path.name} has a repeat that is not confirmed, so it is a draft " + "(A_DRAFT_REFUSED). Confirm and publish it first." + ) from exc + raise RunInputsError( + f"{path.name} is not a valid experiment description ({', '.join(codes)})." + ) from exc + except OSError as exc: + raise RunInputsError(f"{path.name} cannot be read: {exc}") from exc diff --git a/src/morphobricks_preprocessing/fov_identity.py b/src/morphobricks_preprocessing/fov_identity.py new file mode 100644 index 00000000..e1ba4f74 --- /dev/null +++ b/src/morphobricks_preprocessing/fov_identity.py @@ -0,0 +1,57 @@ +"""A position's canonical identity from the description (experiment-v1 section 3.4). + +The FOV key keeps the source folder labels. The canonical IDs are added beside it, each a +function of the key: the repeat ID (``repeat_group``) of the member folder, the condition ID of +the condition folder, the technical repeat ID of the unit folder, and the unit's channel IDs +by C index. Nothing is looked up by the label alone: a label that names no declared member, or +two, is refused. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from .description import Description + + +@dataclass(frozen=True) +class FovIdentity: + """The canonical identity of one position.""" + + repeat_group: str + condition_id: str + technical_repeat_id: str + channel_ids: tuple[str, ...] + + +def fov_identity(description: Description, labels: tuple[str, str, str]) -> FovIdentity: + """Return the identity of the position whose source folder labels are ``labels``. + + Parameters + ---------- + description + The published description. + labels + ``(condition folder, biological repeat folder, technical repeat folder)``. + + Raises + ------ + LookupError + When the labels name no declared unit, or more than one. + """ + condition, repeat, technical = labels + found = [ + FovIdentity(repeat_id, condition_id, unit_id, unit.channels) + for repeat_id, entry in description.repeats.items() + for condition_id, member in entry.members.items() + if description.conditions[condition_id].source_folder == condition + and member.source_repeat == repeat + for unit_id, unit in member.technical_repeats.items() + if unit.source_folder == technical + ] + if len(found) != 1: + raise LookupError( + f"the folders {condition!r} / {repeat!r} / {technical!r} name " + f"{len(found)} units of the description, not one" + ) + return found[0] diff --git a/src/morphobricks_preprocessing/run_inputs.py b/src/morphobricks_preprocessing/run_inputs.py index cfc84d67..1b8833ba 100644 --- a/src/morphobricks_preprocessing/run_inputs.py +++ b/src/morphobricks_preprocessing/run_inputs.py @@ -41,6 +41,7 @@ from .axes import AxisMapping from .calibration import CALIBRATION_FILENAME from .conversion_check import conversion_problem +from .description import Description from .design import DESIGN_FILENAME, DesignCheckError, ExperimentDesign, read_confirmed_design from .fingerprint import Fingerprint from .folder_rules import masks_folder_issues @@ -59,6 +60,9 @@ from .planes import DEFAULT_MEMORY_LIMIT, read_plane from .recorded_sizes import RecordedSize from .run.generation import published_path, read_stable +from .run_inputs_error import RunInputsError as RunInputsError # re-exported (moved) +from .run_inputs_image import LazyImage as LazyImage # re-exported (moved) +from .run_inputs_semantic import SemanticInputs, check_description, with_identity from .run_inputs_text import ( _ZARR_MISSING, _changed_text, @@ -72,10 +76,6 @@ from .zarr_input import NgffError, ZarrUnavailableError -class RunInputsError(ValueError): - """The run cannot be used as it is: run preprocessing again (the message says why).""" - - class UnrecordedFilesError(RunInputsError): """The experiment holds images or masks the run neither recorded nor excluded. @@ -122,6 +122,12 @@ class Fov: Significant bits from the metadata, or None. z_count, t_count, c_count Depth slices, time points and channels. + repeat_group, condition_id, technical_repeat_id + The canonical repeat, condition and technical repeat IDs of the position, added + beside the unchanged ``key`` (experiment-v1 §3.4). None for a run without a + published description. + channel_ids + The canonical channel ID of each C index, in order; None without a description. """ key: tuple[str, str, str, str] @@ -136,10 +142,14 @@ class Fov: z_count: int t_count: int c_count: int + repeat_group: str | None = None + condition_id: str | None = None + technical_repeat_id: str | None = None + channel_ids: tuple[str, ...] | None = None @dataclass(frozen=True) -class RunInputs: +class RunInputs(SemanticInputs): """The inputs of a clean preprocessing run, read as the run recorded them. Use :meth:`open`; the attributes are the snapshot taken then. @@ -224,6 +234,7 @@ def _read( design = published_path(output, DESIGN_FILENAME, manifest.experiment_design_sha256) _check_sha(design, manifest.experiment_design_sha256, "design") _check_calibration(output / CALIBRATION_FILENAME, manifest.calibration_sha256) + check_description(output, manifest) # A_DRAFT_REFUSED, or changed since the run root = Path(experiment_root or manifest.experiment_root).resolve() _check_conversion(root, manifest) masks = masks_root or (manifest.masks or {}).get("folder") @@ -244,7 +255,10 @@ def fovs(self) -> tuple[Fov, ...]: rows: dict[str, list[PlaneRecord]] = {} for record in self.records: rows.setdefault(record.source_relative_path, []).append(record) - return tuple(_fov(e, rows.get(e.relative_path, []), self.manifest) for e in self.files) + described = self.description() + return tuple( + _fov(e, rows.get(e.relative_path, []), self.manifest, described) for e in self.files + ) @property def files(self) -> tuple[FileEntry, ...]: @@ -330,6 +344,7 @@ def read_mask( entry = self._checked(fov) _check_planes(entry.relative_path, self._fov(entry), z, t) recorded = self._mask_entry(entry.relative_path, mask_type) + self.require_checked(recorded, mask_type) # A_MASK_UNCHECKED (a described run) if not self._mask_unchanged(recorded): raise RunInputsError(_changed_text([recorded["path"]], "mask")) mask = MaskFile(recorded["image"], recorded["type"], recorded["path"], recorded["source"]) @@ -454,47 +469,6 @@ def _mask_unchanged(self, recorded: Mapping) -> bool: except FileNotFoundError: return False - def _mask_entry(self, image: str, mask_type: str) -> Mapping: - """Return the recorded mask of an image and type.""" - for recorded in (self.manifest.masks or {}).get("files", ()): - if recorded["image"] == image and recorded["type"] == mask_type: - return recorded - raise RunInputsError(f"The run recorded no {mask_type} mask for {image}.") - - -@dataclass(frozen=True) -class LazyImage: - """An image of a clean run as a ``(T, Z, C, Y, X)`` array whose planes are read on demand. - - Attributes - ---------- - inputs - The run. - relative_path - The image. - shape - ``(T, Z, C, Y, X)``. - dtype - The stored pixel type. - """ - - inputs: RunInputs - relative_path: str - shape: tuple[int, ...] - dtype: np.dtype - - def __getitem__(self, index: tuple[int, int]) -> np.ndarray: - """Read the ``(C, Y, X)`` plane at ``[t, z]``.""" - t, z = index - return self.inputs.read_plane(self.relative_path, z=z, t=t) - - def __array__(self, dtype: np.dtype | None = None, copy: bool | None = None) -> np.ndarray: - """Read every plane into one array.""" - data = np.stack( - [np.stack([self[t, z] for z in range(self.shape[1])]) for t in range(self.shape[0])] - ) - return data if dtype is None else data.astype(dtype) - def _check_planes(relative_path: str, fov: Fov, z: int | None, t: int | None) -> None: """Refuse a ``z`` or ``t`` outside the image (audit I5).""" @@ -566,14 +540,23 @@ def _mapping(entry: FileEntry) -> AxisMapping: return AxisMapping(entry.axes, tuple(entry.roles or ()), entry.singleton_roles) -def _fov(entry: FileEntry, rows: list[PlaneRecord], manifest: RunManifest) -> Fov: - """Describe one position from its manifest entry and its table rows.""" +def _fov( + entry: FileEntry, + rows: list[PlaneRecord], + manifest: RunManifest, + description: Description | None = None, +) -> Fov: + """Describe one position from its manifest entry and its table rows. + + With the published ``description`` the canonical identity is added beside the key; the + unit's channels must be the image's channels (nothing is guessed when they are not). + """ zs = {r.z for r in rows} or {0} ts = {r.t for r in rows} or {0} channels = rows[0].channel_count if rows else len(entry.channel_names) or 1 if entry.position_id is None: # never an empty name in the key (round-2 audit A-m5) raise RunInputsError(f"The run records no position_id for {entry.relative_path}.") - return Fov( + fov = Fov( key=(*entry.labels, entry.position_id), relative_path=entry.relative_path, plane_ids=tuple(r.plane_id for r in rows), @@ -587,3 +570,4 @@ def _fov(entry: FileEntry, rows: list[PlaneRecord], manifest: RunManifest) -> Fo t_count=max(ts) + 1, c_count=channels, ) + return fov if description is None else with_identity(fov, entry, description) diff --git a/src/morphobricks_preprocessing/run_inputs_error.py b/src/morphobricks_preprocessing/run_inputs_error.py new file mode 100644 index 00000000..3c26174d --- /dev/null +++ b/src/morphobricks_preprocessing/run_inputs_error.py @@ -0,0 +1,32 @@ +"""The errors of :class:`.run_inputs.RunInputs` (moved from ``run_inputs``, which re-exports them). + +``RunInputsError`` is the general refusal: run preprocessing again. The Spec's analysis-layer +refusals (experiment-v1 section 8) are subclasses with their code, so a caller can match them: +:class:`DraftRefusedError` (``A_DRAFT_REFUSED``) and :class:`MaskUncheckedError` +(``A_MASK_UNCHECKED``). +""" + +from __future__ import annotations + + +class RunInputsError(ValueError): + """The run cannot be used as it is: run preprocessing again (the message says why).""" + + code: str | None = None + """The Spec's code when the refusal has one (``A_*``), else None.""" + + +class DraftRefusedError(RunInputsError): + """A draft or an unconfirmed description was given: only the published file is read.""" + + code = "A_DRAFT_REFUSED" + + +class MaskUncheckedError(RunInputsError): + """A mask file whose representation the mask step has not checked yet was used.""" + + code = "A_MASK_UNCHECKED" + + +class ChannelNotFoundError(RunInputsError): + """A canonical channel ID is not one of the image's channels (never a default index).""" diff --git a/src/morphobricks_preprocessing/run_inputs_image.py b/src/morphobricks_preprocessing/run_inputs_image.py new file mode 100644 index 00000000..4265ce07 --- /dev/null +++ b/src/morphobricks_preprocessing/run_inputs_image.py @@ -0,0 +1,45 @@ +"""A run's image as a lazy ``(T, Z, C, Y, X)`` array (moved from ``run_inputs``, PR 12).""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import numpy as np + +if TYPE_CHECKING: + from .run_inputs import RunInputs + + +@dataclass(frozen=True) +class LazyImage: + """An image of a clean run as a ``(T, Z, C, Y, X)`` array whose planes are read on demand. + + Attributes + ---------- + inputs + The run. + relative_path + The image. + shape + ``(T, Z, C, Y, X)``. + dtype + The stored pixel type. + """ + + inputs: RunInputs + relative_path: str + shape: tuple[int, ...] + dtype: np.dtype + + def __getitem__(self, index: tuple[int, int]) -> np.ndarray: + """Read the ``(C, Y, X)`` plane at ``[t, z]``.""" + t, z = index + return self.inputs.read_plane(self.relative_path, z=z, t=t) + + def __array__(self, dtype: np.dtype | None = None, copy: bool | None = None) -> np.ndarray: + """Read every plane into one array.""" + data = np.stack( + [np.stack([self[t, z] for z in range(self.shape[1])]) for t in range(self.shape[0])] + ) + return data if dtype is None else data.astype(dtype) diff --git a/src/morphobricks_preprocessing/run_inputs_semantic.py b/src/morphobricks_preprocessing/run_inputs_semantic.py new file mode 100644 index 00000000..f4a2991c --- /dev/null +++ b/src/morphobricks_preprocessing/run_inputs_semantic.py @@ -0,0 +1,232 @@ +"""The description's meaning for :class:`.run_inputs.RunInputs` (Stage 1 design section 3). + +A run with a published description records its SHA-256 in the manifest +(``experiment_description``, :mod:`.description_record`). :class:`SemanticInputs` is mixed into +``RunInputs`` and gives the consumer the canonical identities and semantics, read from the +published file after checking it is the file the run read: + +- canonical channel IDs per position and ``channel_index(fov, id)``: by ID, never by C index + alone, and an absent channel is an error, never a default; +- ``repeat_group``, ``condition_id`` and ``technical_repeat_id`` beside the unchanged FOV key + (:mod:`.fov_identity`), the condition reference graph and the masks' semantics; +- refusals: ``A_DRAFT_REFUSED`` for a draft or an unconfirmed file, ``A_MASK_UNCHECKED`` for a + mask file whose representation the mask step has not checked. + +A run without a description has none of this: ``description()`` and the graphs are None and +every other method of ``RunInputs`` is as before. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, replace +from pathlib import Path +from typing import TYPE_CHECKING + +from .description import Description +from .description_consume import read_published_description +from .description_file import DESCRIPTION_FILENAME +from .fov_identity import FovIdentity, fov_identity +from .manifest import FileEntry, RunManifest +from .mask_representation import CHECKED, file_status, read_checked +from .run_inputs_error import ChannelNotFoundError, MaskUncheckedError, RunInputsError + +if TYPE_CHECKING: + from .run_inputs import Fov + + +@dataclass(frozen=True) +class MaskSemantics: + """What the user confirmed about a declared mask (experiment-v1 section 4.5).""" + + source_type: str + representation: str + source_channels: tuple[str, ...] | None = None + + +def check_description(output: Path, manifest: RunManifest) -> Description | None: + """Return the published description a run recorded, checked against its SHA-256. + + Raises + ------ + DraftRefusedError + ``A_DRAFT_REFUSED``: the published file holds an unconfirmed repeat. + RunInputsError + When the file is gone, not valid, or not the file the run read (changed since). + """ + recorded = manifest.experiment_description + if recorded is None: + return None + description, sha256 = read_published_description(output / DESCRIPTION_FILENAME) + if sha256 != recorded["sha256"]: + raise RunInputsError( + f"The experiment description {DESCRIPTION_FILENAME} is not the one the run read " + "(changed or republished since). Run preprocessing again." + ) + return description + + +def with_identity(fov: Fov, entry: FileEntry, description: Description) -> Fov: + """Return the :class:`.run_inputs.Fov` with its canonical identity added. + + Raises + ------ + RunInputsError + When the entry's folders name no unit of the description, or the unit declares another + number of channels than the image has. + """ + identity = _identity(entry.labels, description) + if len(identity.channel_ids) != fov.c_count: + raise RunInputsError( + f"{entry.relative_path} has {fov.c_count} channels but its unit declares " + f"{len(identity.channel_ids)} ({', '.join(identity.channel_ids)}). " + "Run preprocessing again." + ) + return replace( + fov, + repeat_group=identity.repeat_group, + condition_id=identity.condition_id, + technical_repeat_id=identity.technical_repeat_id, + channel_ids=identity.channel_ids, + ) + + +def _identity(labels: tuple[str, str, str], description: Description) -> FovIdentity: + """Return a position's canonical identity, refusing labels the description lacks.""" + try: + return fov_identity(description, labels) + except LookupError as exc: + raise RunInputsError( + f"The run's image folders do not match its description: {exc}. Run preprocessing again." + ) from exc + + +class SemanticInputs: + """Methods of ``RunInputs`` that need the published description.""" + + output_folder: Path + manifest: RunManifest + + def description(self) -> Description | None: + """Return the published description the run read, or None when it had none. + + The file is read again and must still be the one the run recorded; only the + published ``morphobricks.toml`` is ever read, never a draft. + + Raises + ------ + DraftRefusedError + ``A_DRAFT_REFUSED``: the published file now holds an unconfirmed repeat. + RunInputsError + When the file is gone, invalid, or changed since the run. + """ + return check_description(self.output_folder, self.manifest) + + def channel_index(self, fov: object, channel_id: str) -> int: + """Return the C index of a canonical channel in one position's image. + + Parameters + ---------- + fov + A :class:`.run_inputs.Fov` or the image's relative path. + channel_id + The canonical channel ID. + + Raises + ------ + ChannelNotFoundError + When the image does not have that channel (nothing is defaulted). + RunInputsError + When the run had no description, so its channels have no IDs. + """ + ids = getattr(fov, "channel_ids", None) + if ids is None: + ids = self._channel_ids(fov) + if channel_id not in ids: + raise ChannelNotFoundError( + f"The channel {channel_id!r} is not in this image; its channels are " + f"{', '.join(ids)}." + ) + return ids.index(channel_id) + + def _channel_ids(self, fov: object) -> tuple[str, ...]: + description = self.description() + if description is None: + raise RunInputsError("This run has no experiment description, so no channel IDs.") + relative = getattr(fov, "relative_path", fov) + entry = self.manifest.file(str(relative)) + if entry is None: + raise RunInputsError(f"{relative} is not an image of this run.") + return _identity(entry.labels, description).channel_ids + + def references(self) -> dict[str, str | None] | None: + """Return the condition reference graph: condition ID to the ID it references. + + The value is None for a condition that references none; the whole result is None + when the run had no description. + """ + description = self.description() + if description is None: + return None + return {key: value.reference for key, value in sorted(description.conditions.items())} + + def mask_semantics(self) -> dict[str, MaskSemantics] | None: + """Return the declared masks by ID with their source type, representation and channels. + + None when the run had no description. + """ + description = self.description() + if description is None: + return None + return { + key: MaskSemantics(m.source_type, m.representation, m.source_channels) + for key, m in sorted(description.masks.items()) + } + + def mask_check_status(self, mask_id: str, fov: object) -> str: + """Return ``checked`` or ``not yet checked`` for one mask of one image. + + It is ``checked`` only when the mask step recorded this file with the fingerprint the + run recorded, for the representation and source type the description declares now. + + Raises + ------ + RunInputsError + For a run without a description, an undeclared mask ID, or an image with no + recorded file of that mask. + """ + description = self.description() + mask = None if description is None else description.masks.get(mask_id) + if mask is None: + raise RunInputsError(f"{mask_id!r} is not a mask of this run's description.") + relative = str(getattr(fov, "relative_path", fov)) + recorded = self._mask_entry(relative, mask.source_type) + table = read_checked(self.output_folder) + return file_status(table, mask_id, mask.source_type, mask.representation, recorded) + + def _mask_entry(self, image: str, mask_type: str) -> Mapping: + """Return the recorded mask of an image and type.""" + for recorded in (self.manifest.masks or {}).get("files", ()): + if recorded["image"] == image and recorded["type"] == mask_type: + return recorded + raise RunInputsError(f"The run recorded no {mask_type} mask for {image}.") + + def require_checked(self, recorded: Mapping, mask_type: str) -> None: + """Refuse a mask file the mask step has not checked (``A_MASK_UNCHECKED``). + + Only a run with a description refuses, and only for a mask type it declares. + """ + description = self.description() + if description is None: + return + table = read_checked(self.output_folder) + for mask_id, mask in description.masks.items(): + if mask.source_type != mask_type: + continue + state = file_status(table, mask_id, mask_type, mask.representation, recorded) + if state != CHECKED: + raise MaskUncheckedError( + f"The mask {mask_id!r} ({recorded['path']}) is not yet checked against " + f"its declared representation ({mask.representation}). Run the masks " + "step on this output folder first (A_MASK_UNCHECKED)." + ) diff --git a/tests/spec_gate.py b/tests/spec_gate.py index 9efea8c9..045a5772 100644 --- a/tests/spec_gate.py +++ b/tests/spec_gate.py @@ -41,6 +41,7 @@ "test_spec_refval_crosscheck.py", "test_spec_rules_table.py", "test_spec_run_description.py", + "test_spec_run_inputs_semantic.py", "test_spec_transitions.py", "test_spec_vendored_missing.py", ) diff --git a/tests/test_run_inputs_semantic.py b/tests/test_run_inputs_semantic.py new file mode 100644 index 00000000..7e8bf965 --- /dev/null +++ b/tests/test_run_inputs_semantic.py @@ -0,0 +1,153 @@ +"""What ``RunInputs`` gives from a published description, and that it gives nothing without one. + +The Spec scenarios (P14, P25, P28) are in ``test_spec_run_inputs_semantic.py``. Here: canonical +channel IDs and ``channel_index`` (an absent channel is an error, never a default), canonical IDs +that differ from the folder labels, the condition reference graph, the masks' semantics, a unit +that does not match its image, and a run without a description: every field None, nothing changed. +""" + +from __future__ import annotations + +import hashlib +import json +import pickle +import warnings +from dataclasses import replace + +import pytest + +from description_tree import MAPPING, build_experiment, write_mask +from morphobricks_preprocessing import run_experiment +from morphobricks_preprocessing.description import description_from_bytes +from morphobricks_preprocessing.description_write import canonical_bytes +from morphobricks_preprocessing.run_inputs import RunInputs, RunInputsError +from morphobricks_preprocessing.run_inputs_error import ChannelNotFoundError +from morphobricks_preprocessing.run_inputs_semantic import MaskSemantics +from spec_pin import fixtures + +VALID = {item.name.split("_")[0]: item for item in fixtures("valid")} + + +def _open(tmp_path, data, *, publish=True): + description = description_from_bytes(data) + root, out = tmp_path / "exp", tmp_path / "out" + out.mkdir() + build_experiment(root, description) + if publish: + (out / "morphobricks.toml").write_bytes(data) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + assert run_experiment(root, MAPPING, output_folder=out).written + return RunInputs.open(out), description + + +def test_channel_ids_are_the_units_in_c_order_and_channel_index_finds_them(tmp_path): + inputs, _ = _open(tmp_path, VALID["v01"].data) + fov = inputs.fovs()[0] + assert fov.channel_ids == ("DNA", "ACTIN") and fov.c_count == 2 + assert inputs.channel_index(fov, "DNA") == 0 + assert inputs.channel_index(fov, "ACTIN") == 1 + assert inputs.channel_index(fov.relative_path, "ACTIN") == 1 # by path as well + + +def test_an_absent_channel_is_an_error_never_a_default(tmp_path): + inputs, _ = _open(tmp_path, VALID["v01"].data) + fov = inputs.fovs()[0] + with pytest.raises(ChannelNotFoundError, match="PHASE.*DNA, ACTIN"): + inputs.channel_index(fov, "PHASE") + with pytest.raises(ChannelNotFoundError): + inputs.channel_index(fov, "dna") # case is part of the ID + + +def test_canonical_ids_are_added_beside_the_unchanged_key(tmp_path): + inputs, _ = _open(tmp_path, VALID["v33"].data) # condition folder "control", ID "ctrl" + fovs = inputs.fovs() + control = [f for f in fovs if f.condition_id == "ctrl"] + assert control and all(f.key[0] == "control" for f in control) + assert {f.technical_repeat_id for f in fovs} == {"single"} + assert all(f.key[2] == "single" for f in fovs) + + +def test_the_condition_reference_graph_uses_canonical_ids(tmp_path): + inputs, _ = _open(tmp_path, VALID["v33"].data) + assert inputs.references() == {"DrugA": "ctrl", "ctrl": None} + + +def test_a_reference_chain_is_given_as_it_is_declared(tmp_path): + inputs, description = _open(tmp_path, VALID["v10"].data) + graph = inputs.references() + assert graph == {k: v.reference for k, v in sorted(description.conditions.items())} + assert graph["VEC_SiCol6a1"] == "VEC_SiNC" and graph["VEC_SiNC"] == "VEC" + + +def test_mask_semantics_are_the_declared_ones(tmp_path): + inputs, _ = _open(tmp_path, VALID["v08"].data) + assert inputs.mask_semantics() == { + "Nuclei": MaskSemantics("nuclei", "instances", ("DNA",)), + "TumorRegion": MaskSemantics("tumor_region", "binary", None), + } + + +def test_a_unit_that_does_not_match_its_image_is_refused(tmp_path): + description = description_from_bytes(VALID["v01"].data) + root, out = tmp_path / "exp", tmp_path / "out" + out.mkdir() + build_experiment(root, description) + (out / "morphobricks.toml").write_bytes(VALID["v01"].data) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + assert run_experiment(root, MAPPING, output_folder=out).written + repeat = description.repeats["rep1"] + member = repeat.members["CTRL"] + unit = member.technical_repeats["single"] + short = replace(unit, channels=unit.channels[:1]) + changed = replace(member, technical_repeats={**member.technical_repeats, "single": short}) + repeats = {**description.repeats, "rep1": replace(repeat, members={"CTRL": changed})} + data = canonical_bytes(replace(description, repeats=repeats)) + # Same file bytes recorded by the run: forge the manifest's SHA so only the units differ. + manifest = json.loads((out / "run_manifest.json").read_text()) + manifest["experiment_description"]["sha256"] = hashlib.sha256(data).hexdigest() + (out / "run_manifest.json").write_text(json.dumps(manifest)) + (out / "morphobricks.toml").write_bytes(data) + inputs = RunInputs.open(out) + with pytest.raises(RunInputsError, match="declares 1"): + inputs.fovs() + + +def test_a_run_without_a_description_is_unchanged(tmp_path): + inputs, _ = _open(tmp_path, VALID["v01"].data, publish=False) + fov = inputs.fovs()[0] + assert (fov.repeat_group, fov.condition_id, fov.technical_repeat_id) == (None,) * 3 + assert fov.channel_ids is None + assert inputs.description() is None + assert inputs.references() is None and inputs.mask_semantics() is None + with pytest.raises(RunInputsError, match="no experiment description"): + inputs.channel_index(fov, "DNA") + + +def test_a_run_without_a_description_reads_masks_as_before(tmp_path): + description = description_from_bytes(VALID["v08"].data) + root, out, masks = tmp_path / "exp", tmp_path / "out", tmp_path / "masks" + found = build_experiment(root, description) + for image in found: + write_mask(masks, "tumor_region", image, [0, 1, 2, 3]) # would contradict "binary" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + assert run_experiment(root, MAPPING, output_folder=out, masks_folder=masks).written + inputs = RunInputs.open(out) + assert inputs.read_mask(inputs.fovs()[0], "tumor_region").max() == 3 + + +def test_a_description_published_after_a_run_without_one_changes_nothing(tmp_path): + inputs, _ = _open(tmp_path, VALID["v01"].data, publish=False) + (tmp_path / "out" / "morphobricks.toml").write_bytes(VALID["v01"].data) + again = RunInputs.open(tmp_path / "out") + assert again.description() is None and again.fovs()[0].channel_ids is None + assert inputs.output_folder == again.output_folder + + +def test_run_inputs_with_a_description_is_picklable(tmp_path): + inputs, description = _open(tmp_path, VALID["v01"].data) + again = pickle.loads(pickle.dumps(inputs)) + assert again.description() == description + assert again.fovs() == inputs.fovs() diff --git a/tests/test_spec_run_inputs_semantic.py b/tests/test_spec_run_inputs_semantic.py new file mode 100644 index 00000000..e434d648 --- /dev/null +++ b/tests/test_spec_run_inputs_semantic.py @@ -0,0 +1,116 @@ +"""Semantic ``RunInputs``: conformance P14 and P25, and ``A_MASK_UNCHECKED`` of P28 (PR 12). + +P14: only the published, confirmed ``morphobricks.toml`` is read; a draft or a file with an +unconfirmed repeat is ``A_DRAFT_REFUSED``. P25: every position carries the key with the source +label unchanged and the added ``repeat_group``; in ``v30`` the two ``rep1`` folders are in +different repeats, so grouping by the source label would merge what the description separates. +P28: a mask file the mask step has not checked is refused when it is read. +""" + +from __future__ import annotations + +import json + +import pytest + +from description_tree import MAPPING, build_experiment, write_mask +from morphobricks_preprocessing import run_experiment +from morphobricks_preprocessing.description import description_from_bytes +from morphobricks_preprocessing.description_consume import read_published_description +from morphobricks_preprocessing.run.mask_step import describe_masks, prepare_mask_step +from morphobricks_preprocessing.run_inputs import RunInputs, RunInputsError +from morphobricks_preprocessing.run_inputs_error import DraftRefusedError, MaskUncheckedError +from spec_pin import fixtures + +VALID = {item.name.split("_")[0]: item for item in fixtures("valid")} + + +def _described_run(tmp_path, name, images=2): + description = description_from_bytes(VALID[name].data) + root, out = tmp_path / "exp", tmp_path / "out" + out.mkdir() + (out / "morphobricks.toml").write_bytes(VALID[name].data) + found = build_experiment(root, description, images) + return root, out, found + + +def _run(root, out, masks=None): + assert run_experiment(root, MAPPING, output_folder=out, masks_folder=masks).written + + +@pytest.mark.parametrize(("name", "groups"), [("v03", {"Day1", "Day2"}), ("v30", {"Day1", "Day2"})]) +def test_p25_each_fov_carries_the_source_label_and_the_added_repeat_group(tmp_path, name, groups): + root, out, _ = _described_run(tmp_path, name) + description = description_from_bytes(VALID[name].data) + _run(root, out) + fovs = RunInputs.open(out).fovs() + assert {fov.repeat_group for fov in fovs} == groups + for fov in fovs: + condition, repeat, technical, _ = fov.key + member = description.repeats[fov.repeat_group].members[fov.condition_id] + assert repeat == member.source_repeat # the source label stays in the key + assert condition == description.conditions[fov.condition_id].source_folder + assert technical == member.technical_repeats[fov.technical_repeat_id].source_folder + + +def test_p25_in_v30_grouping_by_the_source_label_would_merge_different_repeats(tmp_path): + root, out, _ = _described_run(tmp_path, "v30") + _run(root, out) + fovs = RunInputs.open(out).fovs() + rep1 = [fov for fov in fovs if fov.key[1] == "rep1"] + assert {fov.key[0] for fov in rep1} == {"CTRL", "DrugA"} + by_label = {fov.key[1] for fov in rep1} + by_group = {fov.repeat_group for fov in rep1} + assert len(by_label) == 1 and by_group == {"Day1", "Day2"} + assert {(f.key[0], f.repeat_group) for f in rep1} == {("CTRL", "Day1"), ("DrugA", "Day2")} + + +def test_p14_a_draft_is_never_read(tmp_path): + root, out, _ = _described_run(tmp_path, "v01") + description = description_from_bytes(VALID["v01"].data) + _run(root, out) + (out / "morphobricks.draft.json").write_text('{"repeats": "garbage"}', encoding="utf-8") + assert RunInputs.open(out).description() == description # the draft is ignored + with pytest.raises(DraftRefusedError) as raised: + read_published_description(out / "morphobricks.draft.json") + assert raised.value.code == "A_DRAFT_REFUSED" + + +def test_p14_a_published_file_with_an_unconfirmed_repeat_is_refused(tmp_path): + root, out, _ = _described_run(tmp_path, "v01") + _run(root, out) + text = VALID["v01"].data.decode().replace("confirmed = true", "confirmed = false") + (out / "morphobricks.toml").write_text(text, encoding="utf-8") + with pytest.raises(DraftRefusedError, match="A_DRAFT_REFUSED"): + RunInputs.open(out) + + +def test_p14_a_description_changed_since_the_run_is_refused(tmp_path): + root, out, _ = _described_run(tmp_path, "v01") + _run(root, out) + (out / "morphobricks.toml").write_bytes(b"# edited\n" + VALID["v01"].data) + with pytest.raises(RunInputsError, match="not the one the run read") as raised: + RunInputs.open(out) + assert not isinstance(raised.value, DraftRefusedError) + + +def test_p28_a_mask_file_not_yet_checked_is_refused_when_read_until_the_step_checked_it(tmp_path): + root, out, found = _described_run(tmp_path, "v08") + masks, image = tmp_path / "masks", found[0] + _run(root, out) # no masks yet + write_mask(masks, "nuclei", image, [0, 1, 2]) + write_mask(masks, "tumor_region", image, [0, 1]) + _run(root, out, masks) + inputs = RunInputs.open(out) + fov = inputs.fovs()[0] + assert inputs.mask_check_status("TumorRegion", fov) == "not yet checked" + with pytest.raises(MaskUncheckedError) as raised: + inputs.read_mask(fov, "tumor_region") + assert raised.value.code == "A_MASK_UNCHECKED" + assert describe_masks(prepare_mask_step(None, out)).issues == () + inputs = RunInputs.open(out) + assert inputs.mask_check_status("TumorRegion", fov) == "checked" + assert inputs.read_mask(fov, "tumor_region").shape == (8, 8) + assert inputs.read_mask(fov, "nuclei").max() == 2 + manifest = json.loads((out / "run_manifest.json").read_text(encoding="utf-8")) + assert manifest["experiment_description"]["masks"]["TumorRegion"]["status"] == "not yet checked"