Derived experiment_design.csv and cross-version parity suite (Stage 1, PR 10)

AGENTS.md

@@ -90,6 +90,8 @@ src/morphobricks_preprocessing/
   design.py         experiment design (P3): ExperimentDesign/DesignRow, default pairing, reconcile with
                     the folders on reruns, design_issues, read/write experiment_design.csv, load_design;
                     read_confirmed_design/ConfirmedDesign (the strict reader other bricks use, §16)
+  design_derive.py  derive_design/write_derived_design: experiment_design.csv from a published
+                    description (Spec experiment-v1 11), written by design.write_experiment_design
   design_values.py  the design file's header check (no column twice or unnamed, #83), booleans,
                     counts, dates and DesignFormatError (pure; re-exported by design.py)
   description_rules.py  the one table of the experiment-description rules (Stage 1, Spec

CHANGELOG.md

@@ -8,6 +8,22 @@ are in [docs/plan.md](docs/plan.md) §12. Upgrading from v0.3: see the
 
 ## Unreleased
 
+- **Derived `experiment_design.csv` and the cross-version parity suite (Stage 1, PR 10).** New
+  module `design_derive`: `derive_design(description)` and `write_derived_design(description,
+  path)` derive the legacy design file from a published description (Spec section 11): source
+  folder labels as `condition` and `biological_repeat`, the repeat ID as `repeat_group`,
+  `control` true for a condition another one references, `technical_repeat_count` 0 for the
+  single unit, `confirmed` true and the repeat's `confirmed_on`. It writes with today's
+  `write_experiment_design`, so the bytes equal the Spec's golden files for `v13`, `v33` and
+  `v34` and the reference validator's derivation of every valid fixture; renaming a condition or
+  technical-repeat ID never changes the file. `tests/test_parity_v0_4_0.py` runs the same
+  confirm-run-rerun script under the `v0.4.0` tag and under this code (`tests/released_code.py`)
+  on six legacy homogeneous datasets (one repeat, two conditions by two repeats, technical
+  repeats, flat, OME names, RGB): `plane_histograms.csv` and `experiment_design.csv` are
+  byte-identical, manifest file entries keep every v0.4.0 key and value (only the optional
+  `raw_channel_names` is added), a rerun over a v0.4.0 output folder recomputes no row, and the
+  derived design of each dataset's description equals the bytes v0.4.0 wrote. Not used by any
+  run, notebook or CLI command yet. Stacked on the Spec re-pin.
 - **Re-pin the vendored Spec to 7dbae0c (MorphoBricks-Spec PR 9).** New in the pin: the byte
   rules and golden files of the derived `experiment_design.csv` (section 11; `fixtures/expected/`
   for `v13`, `v33`, `v34`, not yet derived by any code: Stage 1 PR 10), the byte form of one

pyproject.toml

@@ -177,6 +177,7 @@ baseline = "tools/code_rules_baseline.json"
 "morphobricks_preprocessing.dataset" = "pure"
 "morphobricks_preprocessing.design" = "io"
 "morphobricks_preprocessing.design_values" = "pure"
+"morphobricks_preprocessing.design_derive" = "io"
 "morphobricks_preprocessing.description_identity" = "pure"
 "morphobricks_preprocessing.draft_ids" = "pure"
 "morphobricks_preprocessing.draft" = "io"

src/morphobricks_preprocessing/design_derive.py

@@ -0,0 +1,64 @@
+"""The ``experiment_design.csv`` derived from a published description (experiment-v1 section 11).
+
+``derive_design`` maps the description to one :class:`.design.DesignRow` per member, and
+``write_derived_design`` writes it with today's writer (:func:`.design.write_experiment_design`),
+so the delimiter, quoting, final LF, absence of a byte-order mark and the sort order are the
+writer's, and the bytes equal the Spec's golden files. The file carries the source folder labels
+exactly as before and no condition, technical-repeat or channel ID: renaming an ID never changes
+it. Nothing is guessed, defaulted or repaired: every value is the description's.
+
+Columns: ``condition`` is the condition's ``source_folder``; ``biological_repeat`` the member's
+``source_repeat``; ``repeat_group`` the repeat ID; ``control`` is true when another condition
+(or none other: never itself, 4.7) names the condition as its ``reference``; ``image_count``
+the member's; ``technical_repeat_count`` is 0 for the single unit ``single`` and else the number
+of units; ``confirmed`` is true (the reader refuses a published repeat that is not);
+``confirmed_on`` the repeat's date.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from morphobricks_normalization.records import SINGLE
+
+from .description import Description
+from .design import DesignRow, ExperimentDesign, write_experiment_design
+
+
+def derive_design(description: Description) -> ExperimentDesign:
+    """Return the design the description derives, one row per member (section 11).
+
+    Raises
+    ------
+    DesignFormatError
+        From :func:`.design.write_experiment_design` when written, if a repeat ID is not a name
+        the legacy readers accept (it is never repaired).
+    """
+    referenced = {c.reference for c in description.conditions.values() if c.reference is not None}
+    rows = []
+    for repeat_id, repeat in description.repeats.items():
+        for condition_id, member in repeat.members.items():
+            units = member.technical_repeats
+            single = len(units) == 1 and next(iter(units.values())).source_folder == SINGLE
+            rows.append(
+                DesignRow(
+                    description.conditions[condition_id].source_folder,
+                    member.source_repeat,
+                    repeat_id,
+                    control=condition_id in referenced,
+                    image_count=member.image_count,
+                    technical_repeat_count=0 if single else len(units),
+                    confirmed=repeat.confirmed,
+                    confirmed_on=repeat.confirmed_on,
+                )
+            )
+    return ExperimentDesign(tuple(rows))
+
+
+def write_derived_design(description: Description, path: str | os.PathLike) -> Path:
+    """Write the derived ``experiment_design.csv`` atomically and return its path."""
+    return write_experiment_design(derive_design(description), path)
+
+
+__all__ = ["derive_design", "write_derived_design"]

tests/parity_datasets.py

@@ -0,0 +1,158 @@
+"""Legacy homogeneous datasets for the cross-version parity suite, and their descriptions.
+
+Every dataset is a dict of image paths (relative to the experiment folder) to a seed, written
+as random pixels so each plane's histogram is distinct. ``described`` gives the experiment
+description a user would publish for it: IDs equal to the folder labels (the legacy case, Spec
+v13), one control referenced by every other condition, the same layout everywhere.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import tifffile
+
+from morphobricks_preprocessing.description import (
+    Channel,
+    Condition,
+    Description,
+    Member,
+    Repeat,
+    Unit,
+)
+from morphobricks_preprocessing.description_write import canonical_bytes
+
+DATE = "2026-10-30"
+
+
+def _write(path, data, axes, **kwargs):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tifffile.imwrite(path, data, photometric=kwargs.pop("photometric", "minisblack"), **kwargs)
+
+
+def _pixels(seed, shape):
+    return np.random.default_rng(seed).integers(0, 255, shape, dtype=np.uint8)
+
+
+def write_plain(root, images):
+    """Write 2-D YX images."""
+    for seed, rel in enumerate(images, start=1):
+        _write(root / rel, _pixels(seed, (8, 8)), "YX", metadata={"axes": "YX"})
+
+
+def write_ome(root, images):
+    """Write two-channel OME-TIFF images whose headers name the channels."""
+    for seed, rel in enumerate(images, start=1):
+        meta = {"axes": "CYX", "Channel": {"Name": ["DAPI", "GFP"]}}
+        _write(root / rel, _pixels(seed, (2, 8, 8)), "CYX", ome=True, metadata=meta)
+
+
+def write_rgb(root, images):
+    """Write RGB images (samples axis S)."""
+    for seed, rel in enumerate(images, start=1):
+        _write(root / rel, _pixels(seed, (8, 8, 3)), "YXS", photometric="rgb")
+
+
+# name -> (images, writer, axes, image_type, channels per image, control condition or None)
+DATASETS = {
+    "one_repeat": (["A/rep1/a.tif", "A/rep1/b.tif"], write_plain, "YX", None, 1, None),
+    "two_by_two": (
+        [f"{c}/{r}/{n}.tif" for c in ("CTRL", "DrugA") for r in ("rep1", "rep2") for n in "ab"],
+        write_plain,
+        "YX",
+        None,
+        1,
+        "CTRL",
+    ),
+    "technical_repeats": (
+        [
+            f"{c}/{r}/{t}/{n}.tif"
+            for c in ("CTRL", "DrugA")
+            for r in ("rep1", "rep2")
+            for t in ("acq1", "acq2")
+            for n in "ab"
+        ],
+        write_plain,
+        "YX",
+        None,
+        1,
+        "CTRL",
+    ),
+    "flat": (["a.tif", "b.tif", "c.tif"], write_plain, "YX", None, 1, None),
+    "ome_names": (
+        [f"{c}/rep{r}/a.ome.tif" for c in ("CTRL", "DrugA") for r in (1, 2)],
+        write_ome,
+        "CYX",
+        None,
+        2,
+        "CTRL",
+    ),
+    "rgb": (
+        [f"{c}/rep{r}/a.tif" for c in ("CTRL", "DrugA") for r in (1, 2)],
+        write_rgb,
+        "YXS",
+        "rgb",
+        3,
+        "CTRL",
+    ),
+}
+
+
+def _split(rel):
+    """Return ``(condition, repeat, technical)`` folders of an image path, ``single`` if absent."""
+    folders = rel.split("/")[:-1]
+    return tuple(folders + ["single"] * (3 - len(folders)))
+
+
+def described(images, channels, control, image_type, date=DATE) -> Description:
+    """Return the description of a dataset: folder labels as IDs, one reference per condition."""
+    layout: dict = {}
+    for rel in images:
+        condition, repeat, technical = _split(rel)
+        members = layout.setdefault(repeat, {}).setdefault(condition, {})
+        members[technical] = members.get(technical, 0) + 1
+    names = ("R", "G", "B") if image_type == "rgb" else ("DAPI", "GFP")[:channels]
+    names = names if channels > 1 else ("C0",)
+    conditions = sorted({c for members in layout.values() for c in members})
+    repeats = {
+        repeat: Repeat(
+            True,
+            date,
+            {
+                condition: Member(
+                    repeat,
+                    sum(units.values()),
+                    {
+                        technical: Unit(
+                            technical,
+                            names,
+                            image_type="rgb" if image_type == "rgb" else None,
+                        )
+                        for technical in sorted(units)
+                    },
+                )
+                for condition, units in sorted(members.items())
+            },
+        )
+        for repeat, members in sorted(layout.items())
+    }
+    return Description(
+        {
+            c: Condition(c, reference=control if control and c != control else None)
+            for c in conditions
+        },
+        {n: Channel() for n in names},
+        repeats,
+        {
+            "conditions": tuple(conditions),
+            "channels": tuple(sorted(names)),
+            "repeats": tuple(sorted(repeats)),
+            "technical_repeats": tuple(
+                sorted({t for m in layout.values() for u in m.values() for t in u})
+            ),
+        },
+    )
+
+
+def description_bytes(*args, **kwargs) -> bytes:
+    """Return the canonical bytes of :func:`described` (it must be one the reader accepts)."""
+    return canonical_bytes(described(*args, **kwargs))

tests/test_parity_v0_4_0.py

@@ -0,0 +1,134 @@
+"""Cross-version parity: legacy homogeneous datasets are identical under v0.4.0 and this code.
+
+Stage 1 design section 4. The same script runs under the extracted ``v0.4.0`` tag and under
+this checkout's ``src`` (``released_code``; in CI a missing tag fails, it does not skip):
+confirm a design, run, rerun. Required equal: ``plane_histograms.csv`` and
+``experiment_design.csv`` bytes (so plane IDs, FOV keys and ``position_id`` are the same), the
+identity of every manifest file entry, and, over a v0.4.0 output folder, no row recomputed.
+The derived ``experiment_design.csv`` of the dataset's description equals v0.4.0's bytes.
+"""
+
+from __future__ import annotations
+
+import shutil
+
+import pytest
+
+from morphobricks_preprocessing import read_confirmed_design
+from morphobricks_preprocessing.description import description_from_bytes
+from morphobricks_preprocessing.design_derive import write_derived_design
+from parity_datasets import DATASETS, DATE, description_bytes
+from released_code import PRELUDE, ROOT, extract, run_with
+
+RELEASE = "v0.4.0"
+ADDED_ENTRY_KEYS = {"raw_channel_names"}
+"""Manifest file-entry keys added since v0.4.0 (optional, never read by the row-reuse check)."""
+
+RUN = (
+    PRELUDE
+    + """
+import dataclasses
+root, out, axes, image_type, control = sys.argv[2:7]
+image_type = None if image_type == "-" else image_type
+controls = set() if control == "-" else {control}
+mapping = package.prefill_mapping(axes)
+from morphobricks_preprocessing.dataset import biological_repeats
+found = package.default_design(biological_repeats(package.discover_experiment(root)))
+design = package.ExperimentDesign(tuple(
+    dataclasses.replace(r, control=r.condition in controls, confirmed=True, confirmed_on="@DATE@")
+    for r in found.rows
+))
+first = package.run_experiment(
+    root, mapping, output_folder=out, image_type=image_type, design=design
+)
+assert first.written, first.issues
+again = package.run_experiment(root, output_folder=out)
+manifest = json.loads((Path(out) / "run_manifest.json").read_text(encoding="utf-8"))
+print(json.dumps({
+    "written": first.written,
+    "first": {"new": list(first.new), "read": list(first.read)},
+    "again": {"written": again.written, "read": list(again.read),
+              "unchanged": list(again.unchanged), "rebuild_reason": again.rebuild_reason},
+    "files": [{k: v for k, v in e.items() if k != "content_sha256"} for e in manifest["files"]],
+}))
+""".replace("@DATE@", DATE)
+)
+
+RERUN = (
+    PRELUDE
+    + """
+root, out = sys.argv[2:4]
+result = package.run_experiment(root, output_folder=out)
+print(json.dumps({"written": result.written, "read": list(result.read),
+                  "new": list(result.new), "changed": list(result.changed),
+                  "unchanged": list(result.unchanged), "rebuild_reason": result.rebuild_reason,
+                  "resumed": list(result.resumed)}))
+"""
+)
+
+
+@pytest.fixture(scope="module")
+def released(tmp_path_factory):
+    return extract(RELEASE, tmp_path_factory.mktemp("released"))
+
+
+def _args(root, out, name):
+    _, _, axes, image_type, _, control = DATASETS[name]
+    return root, out, axes, image_type or "-", control or "-"
+
+
+def _build(tmp_path, name):
+    images, writer = DATASETS[name][:2]
+    root = tmp_path / "exp"
+    writer(root, images)
+    return root
+
+
+@pytest.fixture(params=sorted(DATASETS))
+def case(request, tmp_path, released):
+    name = request.param
+    root = _build(tmp_path, name)
+    old = run_with(released, RUN, *_args(root, tmp_path / "old", name))
+    new = run_with(ROOT / "src", RUN, *_args(root, tmp_path / "new", name))
+    return name, root, tmp_path, old, new
+
+
+def test_the_exports_are_byte_identical(case):
+    _, _, tmp_path, _, _ = case
+    for name in ("plane_histograms.csv", "experiment_design.csv"):
+        old, new = tmp_path / "old" / name, tmp_path / "new" / name
+        assert old.read_bytes() == new.read_bytes()
+    assert (tmp_path / "old" / "plane_histograms.csv").read_text().count("\n") > 1
+
+
+def test_the_identities_and_the_run_are_the_same(case):
+    _, _, _, old, new = case
+    for before, after in zip(old["files"], new["files"], strict=True):
+        assert set(after) - set(before) <= ADDED_ENTRY_KEYS
+        assert {k: after[k] for k in before} == before  # every v0.4.0 key keeps its value
+    assert old["files"]
+    assert old["first"] == new["first"]
+    assert old["again"]["written"] == new["again"]["written"]
+    assert old["again"]["read"] == new["again"]["read"] == []  # a rerun recomputes no row
+    assert old["again"]["unchanged"] == new["again"]["unchanged"]
+
+
+def test_this_code_reuses_every_row_of_a_v0_4_0_output_folder(case):
+    name, root, tmp_path, old, _ = case
+    copy = tmp_path / "old-copy"
+    shutil.copytree(tmp_path / "old", copy)
+    before = (copy / "plane_histograms.csv").read_bytes()
+    found = run_with(ROOT / "src", RERUN, root, copy)
+    assert found["rebuild_reason"] is None and found["new"] == [] and found["changed"] == []
+    assert found["read"] == [] and found["resumed"] == []  # zero rows recomputed
+    assert sorted(found["unchanged"]) == sorted(old["first"]["new"])
+    assert (copy / "plane_histograms.csv").read_bytes() == before
+
+
+def test_the_derived_design_equals_the_bytes_v0_4_0_wrote(case):
+    name, _, tmp_path, _, _ = case
+    images, _, _, image_type, channels, control = DATASETS[name]
+    data = description_bytes(images, channels, control, image_type)
+    path = write_derived_design(description_from_bytes(data), tmp_path / "derived.csv")
+    assert path.read_bytes() == (tmp_path / "old" / "experiment_design.csv").read_bytes()
+    assert read_confirmed_design(path).design.rows