diff --git a/AGENTS.md b/AGENTS.md index 4f3c2bf1..ce1ee2f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,6 +90,8 @@ src/morphobricks_preprocessing/ design.py experiment design (P3): ExperimentDesign/DesignRow, default pairing, reconcile with the folders on reruns, design_issues, read/write experiment_design.csv, load_design; read_confirmed_design/ConfirmedDesign (the strict reader other bricks use, §16) + design_derive.py derive_design/write_derived_design: experiment_design.csv from a published + description (Spec experiment-v1 11), written by design.write_experiment_design design_values.py the design file's header check (no column twice or unnamed, #83), booleans, counts, dates and DesignFormatError (pure; re-exported by design.py) description_rules.py the one table of the experiment-description rules (Stage 1, Spec diff --git a/CHANGELOG.md b/CHANGELOG.md index a56a3404..e6d5d043 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,22 @@ are in [docs/plan.md](docs/plan.md) §12. Upgrading from v0.3: see the ## Unreleased +- **Derived `experiment_design.csv` and the cross-version parity suite (Stage 1, PR 10).** New + module `design_derive`: `derive_design(description)` and `write_derived_design(description, + path)` derive the legacy design file from a published description (Spec section 11): source + folder labels as `condition` and `biological_repeat`, the repeat ID as `repeat_group`, + `control` true for a condition another one references, `technical_repeat_count` 0 for the + single unit, `confirmed` true and the repeat's `confirmed_on`. It writes with today's + `write_experiment_design`, so the bytes equal the Spec's golden files for `v13`, `v33` and + `v34` and the reference validator's derivation of every valid fixture; renaming a condition or + technical-repeat ID never changes the file. `tests/test_parity_v0_4_0.py` runs the same + confirm-run-rerun script under the `v0.4.0` tag and under this code (`tests/released_code.py`) + on six legacy homogeneous datasets (one repeat, two conditions by two repeats, technical + repeats, flat, OME names, RGB): `plane_histograms.csv` and `experiment_design.csv` are + byte-identical, manifest file entries keep every v0.4.0 key and value (only the optional + `raw_channel_names` is added), a rerun over a v0.4.0 output folder recomputes no row, and the + derived design of each dataset's description equals the bytes v0.4.0 wrote. Not used by any + run, notebook or CLI command yet. Stacked on the Spec re-pin. - **Re-pin the vendored Spec to 7dbae0c (MorphoBricks-Spec PR 9).** New in the pin: the byte rules and golden files of the derived `experiment_design.csv` (section 11; `fixtures/expected/` for `v13`, `v33`, `v34`, not yet derived by any code: Stage 1 PR 10), the byte form of one diff --git a/pyproject.toml b/pyproject.toml index 9e7ec139..32e10c77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -177,6 +177,7 @@ baseline = "tools/code_rules_baseline.json" "morphobricks_preprocessing.dataset" = "pure" "morphobricks_preprocessing.design" = "io" "morphobricks_preprocessing.design_values" = "pure" +"morphobricks_preprocessing.design_derive" = "io" "morphobricks_preprocessing.description_identity" = "pure" "morphobricks_preprocessing.draft_ids" = "pure" "morphobricks_preprocessing.draft" = "io" diff --git a/src/morphobricks_preprocessing/design_derive.py b/src/morphobricks_preprocessing/design_derive.py new file mode 100644 index 00000000..e5b9c36f --- /dev/null +++ b/src/morphobricks_preprocessing/design_derive.py @@ -0,0 +1,64 @@ +"""The ``experiment_design.csv`` derived from a published description (experiment-v1 section 11). + +``derive_design`` maps the description to one :class:`.design.DesignRow` per member, and +``write_derived_design`` writes it with today's writer (:func:`.design.write_experiment_design`), +so the delimiter, quoting, final LF, absence of a byte-order mark and the sort order are the +writer's, and the bytes equal the Spec's golden files. The file carries the source folder labels +exactly as before and no condition, technical-repeat or channel ID: renaming an ID never changes +it. Nothing is guessed, defaulted or repaired: every value is the description's. + +Columns: ``condition`` is the condition's ``source_folder``; ``biological_repeat`` the member's +``source_repeat``; ``repeat_group`` the repeat ID; ``control`` is true when another condition +(or none other: never itself, 4.7) names the condition as its ``reference``; ``image_count`` +the member's; ``technical_repeat_count`` is 0 for the single unit ``single`` and else the number +of units; ``confirmed`` is true (the reader refuses a published repeat that is not); +``confirmed_on`` the repeat's date. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +from morphobricks_normalization.records import SINGLE + +from .description import Description +from .design import DesignRow, ExperimentDesign, write_experiment_design + + +def derive_design(description: Description) -> ExperimentDesign: + """Return the design the description derives, one row per member (section 11). + + Raises + ------ + DesignFormatError + From :func:`.design.write_experiment_design` when written, if a repeat ID is not a name + the legacy readers accept (it is never repaired). + """ + referenced = {c.reference for c in description.conditions.values() if c.reference is not None} + rows = [] + for repeat_id, repeat in description.repeats.items(): + for condition_id, member in repeat.members.items(): + units = member.technical_repeats + single = len(units) == 1 and next(iter(units.values())).source_folder == SINGLE + rows.append( + DesignRow( + description.conditions[condition_id].source_folder, + member.source_repeat, + repeat_id, + control=condition_id in referenced, + image_count=member.image_count, + technical_repeat_count=0 if single else len(units), + confirmed=repeat.confirmed, + confirmed_on=repeat.confirmed_on, + ) + ) + return ExperimentDesign(tuple(rows)) + + +def write_derived_design(description: Description, path: str | os.PathLike) -> Path: + """Write the derived ``experiment_design.csv`` atomically and return its path.""" + return write_experiment_design(derive_design(description), path) + + +__all__ = ["derive_design", "write_derived_design"] diff --git a/tests/parity_datasets.py b/tests/parity_datasets.py new file mode 100644 index 00000000..6cc82fc8 --- /dev/null +++ b/tests/parity_datasets.py @@ -0,0 +1,158 @@ +"""Legacy homogeneous datasets for the cross-version parity suite, and their descriptions. + +Every dataset is a dict of image paths (relative to the experiment folder) to a seed, written +as random pixels so each plane's histogram is distinct. ``described`` gives the experiment +description a user would publish for it: IDs equal to the folder labels (the legacy case, Spec +v13), one control referenced by every other condition, the same layout everywhere. +""" + +from __future__ import annotations + +import numpy as np +import tifffile + +from morphobricks_preprocessing.description import ( + Channel, + Condition, + Description, + Member, + Repeat, + Unit, +) +from morphobricks_preprocessing.description_write import canonical_bytes + +DATE = "2026-10-30" + + +def _write(path, data, axes, **kwargs): + path.parent.mkdir(parents=True, exist_ok=True) + tifffile.imwrite(path, data, photometric=kwargs.pop("photometric", "minisblack"), **kwargs) + + +def _pixels(seed, shape): + return np.random.default_rng(seed).integers(0, 255, shape, dtype=np.uint8) + + +def write_plain(root, images): + """Write 2-D YX images.""" + for seed, rel in enumerate(images, start=1): + _write(root / rel, _pixels(seed, (8, 8)), "YX", metadata={"axes": "YX"}) + + +def write_ome(root, images): + """Write two-channel OME-TIFF images whose headers name the channels.""" + for seed, rel in enumerate(images, start=1): + meta = {"axes": "CYX", "Channel": {"Name": ["DAPI", "GFP"]}} + _write(root / rel, _pixels(seed, (2, 8, 8)), "CYX", ome=True, metadata=meta) + + +def write_rgb(root, images): + """Write RGB images (samples axis S).""" + for seed, rel in enumerate(images, start=1): + _write(root / rel, _pixels(seed, (8, 8, 3)), "YXS", photometric="rgb") + + +# name -> (images, writer, axes, image_type, channels per image, control condition or None) +DATASETS = { + "one_repeat": (["A/rep1/a.tif", "A/rep1/b.tif"], write_plain, "YX", None, 1, None), + "two_by_two": ( + [f"{c}/{r}/{n}.tif" for c in ("CTRL", "DrugA") for r in ("rep1", "rep2") for n in "ab"], + write_plain, + "YX", + None, + 1, + "CTRL", + ), + "technical_repeats": ( + [ + f"{c}/{r}/{t}/{n}.tif" + for c in ("CTRL", "DrugA") + for r in ("rep1", "rep2") + for t in ("acq1", "acq2") + for n in "ab" + ], + write_plain, + "YX", + None, + 1, + "CTRL", + ), + "flat": (["a.tif", "b.tif", "c.tif"], write_plain, "YX", None, 1, None), + "ome_names": ( + [f"{c}/rep{r}/a.ome.tif" for c in ("CTRL", "DrugA") for r in (1, 2)], + write_ome, + "CYX", + None, + 2, + "CTRL", + ), + "rgb": ( + [f"{c}/rep{r}/a.tif" for c in ("CTRL", "DrugA") for r in (1, 2)], + write_rgb, + "YXS", + "rgb", + 3, + "CTRL", + ), +} + + +def _split(rel): + """Return ``(condition, repeat, technical)`` folders of an image path, ``single`` if absent.""" + folders = rel.split("/")[:-1] + return tuple(folders + ["single"] * (3 - len(folders))) + + +def described(images, channels, control, image_type, date=DATE) -> Description: + """Return the description of a dataset: folder labels as IDs, one reference per condition.""" + layout: dict = {} + for rel in images: + condition, repeat, technical = _split(rel) + members = layout.setdefault(repeat, {}).setdefault(condition, {}) + members[technical] = members.get(technical, 0) + 1 + names = ("R", "G", "B") if image_type == "rgb" else ("DAPI", "GFP")[:channels] + names = names if channels > 1 else ("C0",) + conditions = sorted({c for members in layout.values() for c in members}) + repeats = { + repeat: Repeat( + True, + date, + { + condition: Member( + repeat, + sum(units.values()), + { + technical: Unit( + technical, + names, + image_type="rgb" if image_type == "rgb" else None, + ) + for technical in sorted(units) + }, + ) + for condition, units in sorted(members.items()) + }, + ) + for repeat, members in sorted(layout.items()) + } + return Description( + { + c: Condition(c, reference=control if control and c != control else None) + for c in conditions + }, + {n: Channel() for n in names}, + repeats, + { + "conditions": tuple(conditions), + "channels": tuple(sorted(names)), + "repeats": tuple(sorted(repeats)), + "technical_repeats": tuple( + sorted({t for m in layout.values() for u in m.values() for t in u}) + ), + }, + ) + + +def description_bytes(*args, **kwargs) -> bytes: + """Return the canonical bytes of :func:`described` (it must be one the reader accepts).""" + return canonical_bytes(described(*args, **kwargs)) diff --git a/tests/test_parity_v0_4_0.py b/tests/test_parity_v0_4_0.py new file mode 100644 index 00000000..1767edcf --- /dev/null +++ b/tests/test_parity_v0_4_0.py @@ -0,0 +1,134 @@ +"""Cross-version parity: legacy homogeneous datasets are identical under v0.4.0 and this code. + +Stage 1 design section 4. The same script runs under the extracted ``v0.4.0`` tag and under +this checkout's ``src`` (``released_code``; in CI a missing tag fails, it does not skip): +confirm a design, run, rerun. Required equal: ``plane_histograms.csv`` and +``experiment_design.csv`` bytes (so plane IDs, FOV keys and ``position_id`` are the same), the +identity of every manifest file entry, and, over a v0.4.0 output folder, no row recomputed. +The derived ``experiment_design.csv`` of the dataset's description equals v0.4.0's bytes. +""" + +from __future__ import annotations + +import shutil + +import pytest + +from morphobricks_preprocessing import read_confirmed_design +from morphobricks_preprocessing.description import description_from_bytes +from morphobricks_preprocessing.design_derive import write_derived_design +from parity_datasets import DATASETS, DATE, description_bytes +from released_code import PRELUDE, ROOT, extract, run_with + +RELEASE = "v0.4.0" +ADDED_ENTRY_KEYS = {"raw_channel_names"} +"""Manifest file-entry keys added since v0.4.0 (optional, never read by the row-reuse check).""" + +RUN = ( + PRELUDE + + """ +import dataclasses +root, out, axes, image_type, control = sys.argv[2:7] +image_type = None if image_type == "-" else image_type +controls = set() if control == "-" else {control} +mapping = package.prefill_mapping(axes) +from morphobricks_preprocessing.dataset import biological_repeats +found = package.default_design(biological_repeats(package.discover_experiment(root))) +design = package.ExperimentDesign(tuple( + dataclasses.replace(r, control=r.condition in controls, confirmed=True, confirmed_on="@DATE@") + for r in found.rows +)) +first = package.run_experiment( + root, mapping, output_folder=out, image_type=image_type, design=design +) +assert first.written, first.issues +again = package.run_experiment(root, output_folder=out) +manifest = json.loads((Path(out) / "run_manifest.json").read_text(encoding="utf-8")) +print(json.dumps({ + "written": first.written, + "first": {"new": list(first.new), "read": list(first.read)}, + "again": {"written": again.written, "read": list(again.read), + "unchanged": list(again.unchanged), "rebuild_reason": again.rebuild_reason}, + "files": [{k: v for k, v in e.items() if k != "content_sha256"} for e in manifest["files"]], +})) +""".replace("@DATE@", DATE) +) + +RERUN = ( + PRELUDE + + """ +root, out = sys.argv[2:4] +result = package.run_experiment(root, output_folder=out) +print(json.dumps({"written": result.written, "read": list(result.read), + "new": list(result.new), "changed": list(result.changed), + "unchanged": list(result.unchanged), "rebuild_reason": result.rebuild_reason, + "resumed": list(result.resumed)})) +""" +) + + +@pytest.fixture(scope="module") +def released(tmp_path_factory): + return extract(RELEASE, tmp_path_factory.mktemp("released")) + + +def _args(root, out, name): + _, _, axes, image_type, _, control = DATASETS[name] + return root, out, axes, image_type or "-", control or "-" + + +def _build(tmp_path, name): + images, writer = DATASETS[name][:2] + root = tmp_path / "exp" + writer(root, images) + return root + + +@pytest.fixture(params=sorted(DATASETS)) +def case(request, tmp_path, released): + name = request.param + root = _build(tmp_path, name) + old = run_with(released, RUN, *_args(root, tmp_path / "old", name)) + new = run_with(ROOT / "src", RUN, *_args(root, tmp_path / "new", name)) + return name, root, tmp_path, old, new + + +def test_the_exports_are_byte_identical(case): + _, _, tmp_path, _, _ = case + for name in ("plane_histograms.csv", "experiment_design.csv"): + old, new = tmp_path / "old" / name, tmp_path / "new" / name + assert old.read_bytes() == new.read_bytes() + assert (tmp_path / "old" / "plane_histograms.csv").read_text().count("\n") > 1 + + +def test_the_identities_and_the_run_are_the_same(case): + _, _, _, old, new = case + for before, after in zip(old["files"], new["files"], strict=True): + assert set(after) - set(before) <= ADDED_ENTRY_KEYS + assert {k: after[k] for k in before} == before # every v0.4.0 key keeps its value + assert old["files"] + assert old["first"] == new["first"] + assert old["again"]["written"] == new["again"]["written"] + assert old["again"]["read"] == new["again"]["read"] == [] # a rerun recomputes no row + assert old["again"]["unchanged"] == new["again"]["unchanged"] + + +def test_this_code_reuses_every_row_of_a_v0_4_0_output_folder(case): + name, root, tmp_path, old, _ = case + copy = tmp_path / "old-copy" + shutil.copytree(tmp_path / "old", copy) + before = (copy / "plane_histograms.csv").read_bytes() + found = run_with(ROOT / "src", RERUN, root, copy) + assert found["rebuild_reason"] is None and found["new"] == [] and found["changed"] == [] + assert found["read"] == [] and found["resumed"] == [] # zero rows recomputed + assert sorted(found["unchanged"]) == sorted(old["first"]["new"]) + assert (copy / "plane_histograms.csv").read_bytes() == before + + +def test_the_derived_design_equals_the_bytes_v0_4_0_wrote(case): + name, _, tmp_path, _, _ = case + images, _, _, image_type, channels, control = DATASETS[name] + data = description_bytes(images, channels, control, image_type) + path = write_derived_design(description_from_bytes(data), tmp_path / "derived.csv") + assert path.read_bytes() == (tmp_path / "old" / "experiment_design.csv").read_bytes() + assert read_confirmed_design(path).design.rows diff --git a/tests/test_spec_derived_design.py b/tests/test_spec_derived_design.py new file mode 100644 index 00000000..14e14900 --- /dev/null +++ b/tests/test_spec_derived_design.py @@ -0,0 +1,57 @@ +"""The derived ``experiment_design.csv`` equals the Spec's golden files byte for byte (section 11). + +Conformance P30 and P32 (derived part). Each golden is the derivation of a named valid fixture; +the oracle's own derivation (``refval.derive_csv``) must agree with both. +""" + +from __future__ import annotations + +import tomllib + +import pytest + +from morphobricks_preprocessing import read_confirmed_design +from morphobricks_preprocessing.description import description_from_bytes +from morphobricks_preprocessing.design_derive import derive_design, write_derived_design +from spec_oracle import refval +from spec_pin import fixtures, spec_root + +VALID = {item.name.split("_")[0]: item for item in fixtures("valid")} +GOLDEN = sorted((spec_root() / "fixtures" / "expected").glob("*.experiment_design.csv")) + + +def test_the_golden_files_are_the_three_the_catalogue_lists(): + assert [p.name.split("_")[0] for p in GOLDEN] == ["v13", "v33", "v34"] + + +@pytest.mark.parametrize("golden", GOLDEN, ids=lambda p: p.name.split("_")[0]) +def test_the_derived_file_equals_the_golden_file_byte_for_byte(golden, tmp_path): + item = VALID[golden.name.split("_")[0]] + path = write_derived_design(description_from_bytes(item.data), tmp_path / "design.csv") + assert path.read_bytes() == golden.read_bytes() + + +@pytest.mark.parametrize("item", VALID.values(), ids=lambda item: item.name.split("_")[0]) +def test_the_derived_file_equals_the_oracles_on_every_valid_fixture(item, tmp_path): + oracle = refval() + expected = oracle.derive_csv(oracle.model(tomllib.loads(item.data.decode("utf-8")))) + path = write_derived_design(description_from_bytes(item.data), tmp_path / "design.csv") + assert path.read_bytes() == expected + + +@pytest.mark.parametrize("item", VALID.values(), ids=lambda item: item.name.split("_")[0]) +def test_the_derived_file_is_read_back_by_the_strict_reader(item, tmp_path): + description = description_from_bytes(item.data) + path = write_derived_design(description, tmp_path / "design.csv") + design = read_confirmed_design(path).design + assert design == derive_design(description) + assert all(row.confirmed for row in design.rows) + + +def test_the_file_names_folders_never_ids(): + description = description_from_bytes(VALID["v33"].data) # folder control, ID ctrl + rows = derive_design(description).rows + golden = (next(p for p in GOLDEN if p.name.startswith("v33"))).read_text().splitlines()[1:] + assert [row.condition for row in rows] == [line.split(",")[0] for line in golden] + assert "ctrl" not in {row.condition for row in rows} + assert {row.condition for row in rows if row.control} == {"control"}