diff --git a/AGENTS.md b/AGENTS.md index a2beab56..4deb5344 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -331,7 +331,7 @@ src/morphobricks_preprocessing/ notebooks/MorphoBricks_Preprocessing.ipynb thin notebook: every code cell calls notebook_ui only notebooks/notebook_latest_versions.yaml the notebook's latest version, for LabConstrictor's version check tests/ pytest with simulated TIFFs (tests/tiff_tree.py, tests/simulation.py); the - schema-version-2 fixture (tests/v2_fixture.py, tests/data) + histogram science goldens (tests/histogram_goldens.py, tests/data) experiments/binning_study/ the study behind the binning decisions (not part of the package) ``` @@ -534,9 +534,9 @@ functions the tests use. - **Rows** are sorted by path, then z, t. - **Earlier versions:** the reader refuses schema 1 and 2 (`CsvFormatError`); a run recomputes them completely (manifest `schema_version` differs). Each channel's - histogram equals the version-2 row's, whole and in blocks - (`tests/test_schema_v2_equivalence.py` with the committed fixture in `tests/data`, - written by `tests/data/make_v2_fixture.py` with the pre-P3b code). + histogram equals the reference row's, whole and in blocks (science goldens: + `tests/test_histogram_goldens.py` with the committed reference in `tests/data`, written by + `tests/data/make_histogram_goldens.py` with an earlier, independent implementation). - **Step 4** summarises each channel of a group on its own (issue #104): one `GroupSummary` per group and channel (`channel`, `xyc_plane_count`, `channel_histogram_count`), a `channel` column in the table and one plot panel per diff --git a/tests/data/v2_float32.csv b/tests/data/golden_float32.csv similarity index 100% rename from tests/data/v2_float32.csv rename to tests/data/golden_float32.csv diff --git a/tests/data/v2_pixels.npz b/tests/data/golden_pixels.npz similarity index 100% rename from tests/data/v2_pixels.npz rename to tests/data/golden_pixels.npz diff --git a/tests/data/v2_uint16.csv b/tests/data/golden_uint16.csv similarity index 100% rename from tests/data/v2_uint16.csv rename to tests/data/golden_uint16.csv diff --git a/tests/data/make_v2_fixture.py b/tests/data/make_histogram_goldens.py similarity index 77% rename from tests/data/make_v2_fixture.py rename to tests/data/make_histogram_goldens.py index 3ad8f8af..06cbab46 100644 --- a/tests/data/make_v2_fixture.py +++ b/tests/data/make_histogram_goldens.py @@ -1,4 +1,4 @@ -"""Write the schema-version-2 fixture of ``test_schema_v2_equivalence.py``. +"""Write the reference histograms (science goldens) of ``test_histogram_goldens.py``. Run once, from the repository root, with the code of version 2: this repository at commit ``55f1074`` (before follow-up P3b) and morphobricks-normalization v0.1.0, @@ -7,13 +7,13 @@ git worktree add ../v2 55f1074 git clone -b v0.1.0 https://github.com/CellMigrationLab/morphobricks-normalization ../n1 pip install ../n1 tifffile zarr pytest - PYTHONPATH=../v2/src python tests/data/make_v2_fixture.py + PYTHONPATH=../v2/src python tests/data/make_histogram_goldens.py It writes, next to this script: -- ``v2_pixels.npz``: the pixels of every image of the experiments in - ``tests/v2_fixture.py``; -- ``v2_.csv``: the ``plane_histograms.csv`` that version's +- ``golden_pixels.npz``: the pixels of every image of the experiments in + ``tests/histogram_goldens.py``; +- ``golden_.csv``: the ``plane_histograms.csv`` that version's ``run_experiment`` wrote for them (schema version 2, one row per plane and channel). @@ -35,8 +35,8 @@ from morphobricks_normalization.csv_io import SCHEMA_VERSION # noqa: E402 +from histogram_goldens import EXPERIMENTS, PIXELS, make_pixels, write_experiment # noqa: E402 from morphobricks_preprocessing import prefill_mapping, run_experiment # noqa: E402 -from v2_fixture import EXPERIMENTS, PIXELS, make_pixels, write_experiment # noqa: E402 def main() -> None: @@ -52,7 +52,7 @@ def main() -> None: result = run_experiment(root, prefill_mapping(axes), bin_count=bins) if not result.written: sys.exit(f"{dtype}: {[issue.message for issue in result.issues]}") - shutil.copyfile(result.csv_path, HERE / f"v2_{dtype}.csv") + shutil.copyfile(result.csv_path, HERE / f"golden_{dtype}.csv") print(f"{dtype}: {result.plane_count} rows") diff --git a/tests/v2_fixture.py b/tests/histogram_goldens.py similarity index 87% rename from tests/v2_fixture.py rename to tests/histogram_goldens.py index 30e675bc..2830134a 100644 --- a/tests/v2_fixture.py +++ b/tests/histogram_goldens.py @@ -1,8 +1,9 @@ -"""The experiments of the schema-version-2 fixture in ``tests/data`` (follow-up P3b). +"""The experiments of the reference histograms (science goldens) in ``tests/data``. -``tests/data/make_v2_fixture.py`` wrote their ``plane_histograms.csv`` with the -code of version 2 (one row per plane and channel); ``test_schema_v2_equivalence.py`` -writes the same images again and compares the current table with it. +``tests/data/make_histogram_goldens.py`` wrote their ``plane_histograms.csv`` with an earlier, +independent implementation (one row per plane and channel, before follow-up P3b); +``test_histogram_goldens.py`` writes the same images again and compares the current table with +those values. """ from __future__ import annotations @@ -14,7 +15,7 @@ from tiff_tree import write_data DATA = Path(__file__).parent / "data" -PIXELS = DATA / "v2_pixels.npz" +PIXELS = DATA / "golden_pixels.npz" # Per pixel type: the bin count, the experiment axis roles and its images as # (relative path, axes, writer options). TIFF images share one arrangement of axes; @@ -74,7 +75,7 @@ def make_pixels(seed: int = 0) -> dict[str, np.ndarray]: MorphoBricks v1 analyses 2-D planes (owner decision of 5 November 2026): a Z-stack is refused, so the tests write each uint16 image with its planes along T instead. The pixels and every histogram are the reference's; only the reference's ``z`` is now ``t``. -``make_v2_fixture.py`` writes them as the reference was made (``as_made=True``). +``make_histogram_goldens.py`` writes them as the reference was made (``as_made=True``). """ diff --git a/tests/parity_datasets.py b/tests/parity_datasets.py deleted file mode 100644 index 6cc82fc8..00000000 --- a/tests/parity_datasets.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Legacy homogeneous datasets for the cross-version parity suite, and their descriptions. - -Every dataset is a dict of image paths (relative to the experiment folder) to a seed, written -as random pixels so each plane's histogram is distinct. ``described`` gives the experiment -description a user would publish for it: IDs equal to the folder labels (the legacy case, Spec -v13), one control referenced by every other condition, the same layout everywhere. -""" - -from __future__ import annotations - -import numpy as np -import tifffile - -from morphobricks_preprocessing.description import ( - Channel, - Condition, - Description, - Member, - Repeat, - Unit, -) -from morphobricks_preprocessing.description_write import canonical_bytes - -DATE = "2026-10-30" - - -def _write(path, data, axes, **kwargs): - path.parent.mkdir(parents=True, exist_ok=True) - tifffile.imwrite(path, data, photometric=kwargs.pop("photometric", "minisblack"), **kwargs) - - -def _pixels(seed, shape): - return np.random.default_rng(seed).integers(0, 255, shape, dtype=np.uint8) - - -def write_plain(root, images): - """Write 2-D YX images.""" - for seed, rel in enumerate(images, start=1): - _write(root / rel, _pixels(seed, (8, 8)), "YX", metadata={"axes": "YX"}) - - -def write_ome(root, images): - """Write two-channel OME-TIFF images whose headers name the channels.""" - for seed, rel in enumerate(images, start=1): - meta = {"axes": "CYX", "Channel": {"Name": ["DAPI", "GFP"]}} - _write(root / rel, _pixels(seed, (2, 8, 8)), "CYX", ome=True, metadata=meta) - - -def write_rgb(root, images): - """Write RGB images (samples axis S).""" - for seed, rel in enumerate(images, start=1): - _write(root / rel, _pixels(seed, (8, 8, 3)), "YXS", photometric="rgb") - - -# name -> (images, writer, axes, image_type, channels per image, control condition or None) -DATASETS = { - "one_repeat": (["A/rep1/a.tif", "A/rep1/b.tif"], write_plain, "YX", None, 1, None), - "two_by_two": ( - [f"{c}/{r}/{n}.tif" for c in ("CTRL", "DrugA") for r in ("rep1", "rep2") for n in "ab"], - write_plain, - "YX", - None, - 1, - "CTRL", - ), - "technical_repeats": ( - [ - f"{c}/{r}/{t}/{n}.tif" - for c in ("CTRL", "DrugA") - for r in ("rep1", "rep2") - for t in ("acq1", "acq2") - for n in "ab" - ], - write_plain, - "YX", - None, - 1, - "CTRL", - ), - "flat": (["a.tif", "b.tif", "c.tif"], write_plain, "YX", None, 1, None), - "ome_names": ( - [f"{c}/rep{r}/a.ome.tif" for c in ("CTRL", "DrugA") for r in (1, 2)], - write_ome, - "CYX", - None, - 2, - "CTRL", - ), - "rgb": ( - [f"{c}/rep{r}/a.tif" for c in ("CTRL", "DrugA") for r in (1, 2)], - write_rgb, - "YXS", - "rgb", - 3, - "CTRL", - ), -} - - -def _split(rel): - """Return ``(condition, repeat, technical)`` folders of an image path, ``single`` if absent.""" - folders = rel.split("/")[:-1] - return tuple(folders + ["single"] * (3 - len(folders))) - - -def described(images, channels, control, image_type, date=DATE) -> Description: - """Return the description of a dataset: folder labels as IDs, one reference per condition.""" - layout: dict = {} - for rel in images: - condition, repeat, technical = _split(rel) - members = layout.setdefault(repeat, {}).setdefault(condition, {}) - members[technical] = members.get(technical, 0) + 1 - names = ("R", "G", "B") if image_type == "rgb" else ("DAPI", "GFP")[:channels] - names = names if channels > 1 else ("C0",) - conditions = sorted({c for members in layout.values() for c in members}) - repeats = { - repeat: Repeat( - True, - date, - { - condition: Member( - repeat, - sum(units.values()), - { - technical: Unit( - technical, - names, - image_type="rgb" if image_type == "rgb" else None, - ) - for technical in sorted(units) - }, - ) - for condition, units in sorted(members.items()) - }, - ) - for repeat, members in sorted(layout.items()) - } - return Description( - { - c: Condition(c, reference=control if control and c != control else None) - for c in conditions - }, - {n: Channel() for n in names}, - repeats, - { - "conditions": tuple(conditions), - "channels": tuple(sorted(names)), - "repeats": tuple(sorted(repeats)), - "technical_repeats": tuple( - sorted({t for m in layout.values() for u in m.values() for t in u}) - ), - }, - ) - - -def description_bytes(*args, **kwargs) -> bytes: - """Return the canonical bytes of :func:`described` (it must be one the reader accepts).""" - return canonical_bytes(described(*args, **kwargs)) diff --git a/tests/test_schema_v2_equivalence.py b/tests/test_histogram_goldens.py similarity index 63% rename from tests/test_schema_v2_equivalence.py rename to tests/test_histogram_goldens.py index 085b536e..c1920870 100644 --- a/tests/test_schema_v2_equivalence.py +++ b/tests/test_histogram_goldens.py @@ -1,13 +1,13 @@ -"""The table holds exactly what version 2 held, channel by channel (follow-ups P3b, P6). - -The fixture in ``tests/data`` was written by the code before P3b (commit -``55f1074`` with morphobricks-normalization v0.1.0, ``make_v2_fixture.py``): -simulated pixels and the version-2 ``plane_histograms.csv`` its ``run_experiment`` -wrote for them, one row per plane and channel. Here the same images go through -the current code, read whole and in blocks; every value of every channel must be -the same text as in the version-2 row of that channel. Version 4 (follow-up P6) adds -underflow and overflow columns, which the default minimum-to-maximum range leaves -at zero. The reference's uint16 Z-stacks are written as time-lapses of the same pixels +"""Science goldens: every histogram value equals an independently computed reference. + +The reference in ``tests/data`` was computed by an earlier, independent implementation (commit +``55f1074`` with morphobricks-normalization v0.1.0, ``make_histogram_goldens.py``): simulated +pixels and the ``plane_histograms.csv`` it wrote for them, one row per plane and channel. Here +the same images go through the current code, read whole and in blocks; every value of every +channel must be the same text as in the reference row of that channel (owner ruling 6 of +10 November 2026: kept as science goldens, not as a version-parity check). The current table +also has underflow and overflow columns (follow-up P6), which the default minimum-to-maximum +range leaves at zero. The reference's uint16 Z-stacks are written as time-lapses of the same pixels since Z-stacks are refused (owner decision of 5 November 2026): its ``z`` is read as ``t``. """ @@ -18,24 +18,22 @@ import numpy as np import pytest -from morphobricks_normalization.csv_io import CsvFormatError, read_plane_histograms +from morphobricks_normalization.csv_io import read_plane_histograms from morphobricks_normalization.histogram import PlaneHistogram from morphobricks_normalization.records import PlaneRecord +from histogram_goldens import DATA, EXPERIMENTS, PIXELS, TIME_FOR_DEPTH, write_experiment from morphobricks_preprocessing import ( - RunMode, plane_layout, prefill_mapping, preflight, - read_manifest, review_axes, run_experiment, ) from morphobricks_preprocessing.manifest import file_sha256 from morphobricks_preprocessing.report import group_summaries, group_table -from v2_fixture import DATA, EXPERIMENTS, PIXELS, TIME_FOR_DEPTH, write_experiment -# Version-2 column -> version-3 column of channel c. +# Reference column -> current column of channel c. CHANNEL_COLUMNS = { "pixel_min": "Intensity_MinIntensity_C{c}", "pixel_max": "Intensity_MaxIntensity_C{c}", @@ -43,7 +41,7 @@ "binning_min": "Histogram_BinningMin_C{c}", "binning_max": "Histogram_BinningMax_C{c}", } -# Columns both versions share, with the same meaning. +# Columns the reference and the current table share, with the same meaning. SHARED_COLUMNS = ( "position_id", "image_file", @@ -87,7 +85,7 @@ def smallest_blocks_limit(root, mapping, bins): @pytest.mark.parametrize("blocks", [False, True], ids=["whole", "in-blocks"]) @pytest.mark.parametrize("dtype", list(EXPERIMENTS)) -def test_histograms_equal_those_of_version_2(tmp_path, pixels, dtype, blocks): +def test_histograms_equal_the_reference_values(tmp_path, pixels, dtype, blocks): needs_zarr(dtype) root = tmp_path / "exp" bins, axes = write_experiment(root, dtype, pixels) @@ -102,8 +100,8 @@ def test_histograms_equal_those_of_version_2(tmp_path, pixels, dtype, blocks): old = {} as_time = EXPERIMENTS[dtype][1] in TIME_FOR_DEPTH # the reference's z is now t - for row in read_rows(DATA / f"v2_{dtype}.csv"): - assert row["schema_version"] == "2" + for row in read_rows(DATA / f"golden_{dtype}.csv"): + assert row["schema_version"] == "2" # the reference table's own format z, t = ("0", row["z"]) if as_time else (row["z"], row["t"]) assert not as_time or row["t"] == "0" plane_id = f"{row['source_relative_path']}::z={z},t={t}" @@ -118,20 +116,20 @@ def test_histograms_equal_those_of_version_2(tmp_path, pixels, dtype, blocks): before = old[(row["plane_id"], c)] for column in SHARED_COLUMNS: assert row[column] == before[column], (row["plane_id"], column) - for v2_column, v3_column in CHANNEL_COLUMNS.items(): - assert row[v3_column.format(c=c)] == before[v2_column], (row["plane_id"], c) + for old_column, column in CHANNEL_COLUMNS.items(): + assert row[column.format(c=c)] == before[old_column], (row["plane_id"], c) counts = [row[f"Histogram_Count_C{c}_{i:05d}"] for i in range(bins)] assert counts == [before[f"histogram_bin_{i:05d}"] for i in range(bins)] matched += 1 - # Every version-2 row is a channel of exactly one version-3 row. + # Every reference row is a channel of exactly one current row. assert matched == len(old) assert len(new) == len(old) // int(new[0]["channel_count"]) -def v2_records(dtype): - """The version-2 rows as one-channel records, as the code before P3b read them.""" +def reference_records(dtype): + """The reference rows as one-channel records.""" records = [] - for row in read_rows(DATA / f"v2_{dtype}.csv"): + for row in read_rows(DATA / f"golden_{dtype}.csv"): number = int if np.dtype(row["dtype"]).kind in "iu" else float bins = int(row["bin_count"]) @@ -144,7 +142,7 @@ def value(text, number=number): binning_min=value(row["binning_min"]), binning_max=value(row["binning_max"]), # Normalization's next release refuses a histogram with valid pixels and no - # extrema (normalization#65, issue #47): passed explicitly from the v2 columns. + # extrema (normalization#65, issue #47): passed explicitly from the reference columns. pixel_min=value(row["pixel_min"]), pixel_max=value(row["pixel_max"]), counts=np.array([int(row[f"histogram_bin_{i:05d}"]) for i in range(bins)]), @@ -158,31 +156,33 @@ def value(text, number=number): @pytest.mark.parametrize("level", ["condition", "biological_repeat", "image"]) @pytest.mark.parametrize("dtype", list(EXPERIMENTS)) -def test_step_4_statistics_are_those_of_version_2(tmp_path, pixels, dtype, level): +def test_step_4_statistics_equal_the_reference_values(tmp_path, pixels, dtype, level): needs_zarr(dtype) root = tmp_path / "exp" bins, axes = write_experiment(root, dtype, pixels) result = run_experiment(root, prefill_mapping(axes), bin_count=bins) records = read_plane_histograms(result.csv_path) - # Issue #104: each channel on its own, from the version-2 rows of that channel. + # Issue #104: each channel on its own, from the reference rows of that channel. now = group_summaries(records, level, (1.0, 99.8)) - before = group_summaries(v2_planes(dtype), level, (1.0, 99.8)) + before = group_summaries(reference_planes(dtype), level, (1.0, 99.8)) assert now == before assert group_table(now, level) == group_table(before, level) - # The all-channel aggregate, asked for, has the numbers step 4 showed before #104. + # The all-channel aggregate, asked for, equals the reference's pooled numbers. pooled = group_summaries(records, level, (1.0, 99.8), pool_channels=True) - legacy = group_summaries(v2_records(dtype), level, (1.0, 99.8), pool_channels=True) + legacy = group_summaries(reference_records(dtype), level, (1.0, 99.8), pool_channels=True) statistics = ("pixel_count", "minimum", "maximum", "mean", "std", "median", "percentiles") for p, old in zip(pooled, legacy, strict=True): assert p.labels == old.labels - assert p.channel_histogram_count == old.xyc_plane_count # a v2 row was one channel + assert p.channel_histogram_count == old.xyc_plane_count # a reference row is one channel assert [getattr(p, f) for f in statistics] == [getattr(old, f) for f in statistics] -def v2_planes(dtype): - """The version-2 rows gathered into XYC planes, channels in order (``c``).""" +def reference_planes(dtype): + """The reference rows gathered into XYC planes, channels in order (``c``).""" planes = {} - for record, row in zip(v2_records(dtype), read_rows(DATA / f"v2_{dtype}.csv"), strict=True): + for record, row in zip( + reference_records(dtype), read_rows(DATA / f"golden_{dtype}.csv"), strict=True + ): key = (record.source_relative_path, row["z"], row["t"]) planes.setdefault(key, {})[int(row["c"])] = record return [ @@ -192,41 +192,14 @@ def v2_planes(dtype): ] -def test_a_version_2_table_is_recomputed_on_the_next_run(tmp_path, pixels): - root = tmp_path / "exp" - bins, axes = write_experiment(root, "float32", pixels) - mapping = prefill_mapping(axes) - first = run_experiment( - root, mapping, bin_count=bins, mode=RunMode("restart", verify_content=False) - ) - table = first.csv_path.read_bytes() - # Leave the output as a run of version 2 did: its table, and a manifest naming it. - shutil.copyfile(DATA / "v2_float32.csv", first.csv_path) - manifest = json.loads(first.manifest_path.read_text(encoding="utf-8")) - manifest["schema_version"] = "2" - manifest["csv_sha256"] = file_sha256(first.csv_path) - first.manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - with pytest.raises(CsvFormatError, match="schema version 2"): - read_plane_histograms(first.csv_path) - - second = run_experiment(root, mapping, bin_count=bins) - assert second.written - assert second.rebuild_reason == "the earlier table uses a different format version" - assert second.read == tuple(sorted(rel for rel, _, _ in EXPERIMENTS["float32"][2])) - assert second.csv_path.read_bytes() == table - data = json.loads(second.manifest_path.read_text(encoding="utf-8")) - assert (data["manifest_version"], data["schema_version"]) == ("2", "4") - assert read_manifest(second.manifest_path).csv_sha256 == file_sha256(second.csv_path) - - -def test_a_version_2_table_under_a_current_manifest_is_recomputed(tmp_path, pixels): +def test_a_table_of_an_older_format_is_recomputed(tmp_path, pixels): # A table replaced by hand with an earlier one cannot be read, so it is recomputed. root = tmp_path / "exp" bins, axes = write_experiment(root, "float32", pixels) mapping = prefill_mapping(axes) first = run_experiment(root, mapping, bin_count=bins) table = first.csv_path.read_bytes() - shutil.copyfile(DATA / "v2_float32.csv", first.csv_path) + shutil.copyfile(DATA / "golden_float32.csv", first.csv_path) manifest = json.loads(first.manifest_path.read_text(encoding="utf-8")) manifest["csv_sha256"] = file_sha256(first.csv_path) first.manifest_path.write_text(json.dumps(manifest), encoding="utf-8") diff --git a/tests/test_notebook_ui.py b/tests/test_notebook_ui.py index 4e15fd9d..cf576a06 100644 --- a/tests/test_notebook_ui.py +++ b/tests/test_notebook_ui.py @@ -1362,7 +1362,7 @@ def test_statistics_by_level_and_percentiles(experiment): def test_statistics_explain_a_table_of_an_earlier_version(experiment, tmp_path): nb = checked(experiment) - nb.csv_file.value = str(Path(__file__).parent / "data" / "v2_float32.csv") + nb.csv_file.value = str(Path(__file__).parent / "data" / "golden_float32.csv") nb.statistics_button.click() assert "The table cannot be read" in nb.stats_status.value assert "schema version 2" in nb.stats_status.value diff --git a/tests/test_parity_v0_4_0.py b/tests/test_parity_v0_4_0.py deleted file mode 100644 index 1767edcf..00000000 --- a/tests/test_parity_v0_4_0.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Cross-version parity: legacy homogeneous datasets are identical under v0.4.0 and this code. - -Stage 1 design section 4. The same script runs under the extracted ``v0.4.0`` tag and under -this checkout's ``src`` (``released_code``; in CI a missing tag fails, it does not skip): -confirm a design, run, rerun. Required equal: ``plane_histograms.csv`` and -``experiment_design.csv`` bytes (so plane IDs, FOV keys and ``position_id`` are the same), the -identity of every manifest file entry, and, over a v0.4.0 output folder, no row recomputed. -The derived ``experiment_design.csv`` of the dataset's description equals v0.4.0's bytes. -""" - -from __future__ import annotations - -import shutil - -import pytest - -from morphobricks_preprocessing import read_confirmed_design -from morphobricks_preprocessing.description import description_from_bytes -from morphobricks_preprocessing.design_derive import write_derived_design -from parity_datasets import DATASETS, DATE, description_bytes -from released_code import PRELUDE, ROOT, extract, run_with - -RELEASE = "v0.4.0" -ADDED_ENTRY_KEYS = {"raw_channel_names"} -"""Manifest file-entry keys added since v0.4.0 (optional, never read by the row-reuse check).""" - -RUN = ( - PRELUDE - + """ -import dataclasses -root, out, axes, image_type, control = sys.argv[2:7] -image_type = None if image_type == "-" else image_type -controls = set() if control == "-" else {control} -mapping = package.prefill_mapping(axes) -from morphobricks_preprocessing.dataset import biological_repeats -found = package.default_design(biological_repeats(package.discover_experiment(root))) -design = package.ExperimentDesign(tuple( - dataclasses.replace(r, control=r.condition in controls, confirmed=True, confirmed_on="@DATE@") - for r in found.rows -)) -first = package.run_experiment( - root, mapping, output_folder=out, image_type=image_type, design=design -) -assert first.written, first.issues -again = package.run_experiment(root, output_folder=out) -manifest = json.loads((Path(out) / "run_manifest.json").read_text(encoding="utf-8")) -print(json.dumps({ - "written": first.written, - "first": {"new": list(first.new), "read": list(first.read)}, - "again": {"written": again.written, "read": list(again.read), - "unchanged": list(again.unchanged), "rebuild_reason": again.rebuild_reason}, - "files": [{k: v for k, v in e.items() if k != "content_sha256"} for e in manifest["files"]], -})) -""".replace("@DATE@", DATE) -) - -RERUN = ( - PRELUDE - + """ -root, out = sys.argv[2:4] -result = package.run_experiment(root, output_folder=out) -print(json.dumps({"written": result.written, "read": list(result.read), - "new": list(result.new), "changed": list(result.changed), - "unchanged": list(result.unchanged), "rebuild_reason": result.rebuild_reason, - "resumed": list(result.resumed)})) -""" -) - - -@pytest.fixture(scope="module") -def released(tmp_path_factory): - return extract(RELEASE, tmp_path_factory.mktemp("released")) - - -def _args(root, out, name): - _, _, axes, image_type, _, control = DATASETS[name] - return root, out, axes, image_type or "-", control or "-" - - -def _build(tmp_path, name): - images, writer = DATASETS[name][:2] - root = tmp_path / "exp" - writer(root, images) - return root - - -@pytest.fixture(params=sorted(DATASETS)) -def case(request, tmp_path, released): - name = request.param - root = _build(tmp_path, name) - old = run_with(released, RUN, *_args(root, tmp_path / "old", name)) - new = run_with(ROOT / "src", RUN, *_args(root, tmp_path / "new", name)) - return name, root, tmp_path, old, new - - -def test_the_exports_are_byte_identical(case): - _, _, tmp_path, _, _ = case - for name in ("plane_histograms.csv", "experiment_design.csv"): - old, new = tmp_path / "old" / name, tmp_path / "new" / name - assert old.read_bytes() == new.read_bytes() - assert (tmp_path / "old" / "plane_histograms.csv").read_text().count("\n") > 1 - - -def test_the_identities_and_the_run_are_the_same(case): - _, _, _, old, new = case - for before, after in zip(old["files"], new["files"], strict=True): - assert set(after) - set(before) <= ADDED_ENTRY_KEYS - assert {k: after[k] for k in before} == before # every v0.4.0 key keeps its value - assert old["files"] - assert old["first"] == new["first"] - assert old["again"]["written"] == new["again"]["written"] - assert old["again"]["read"] == new["again"]["read"] == [] # a rerun recomputes no row - assert old["again"]["unchanged"] == new["again"]["unchanged"] - - -def test_this_code_reuses_every_row_of_a_v0_4_0_output_folder(case): - name, root, tmp_path, old, _ = case - copy = tmp_path / "old-copy" - shutil.copytree(tmp_path / "old", copy) - before = (copy / "plane_histograms.csv").read_bytes() - found = run_with(ROOT / "src", RERUN, root, copy) - assert found["rebuild_reason"] is None and found["new"] == [] and found["changed"] == [] - assert found["read"] == [] and found["resumed"] == [] # zero rows recomputed - assert sorted(found["unchanged"]) == sorted(old["first"]["new"]) - assert (copy / "plane_histograms.csv").read_bytes() == before - - -def test_the_derived_design_equals_the_bytes_v0_4_0_wrote(case): - name, _, tmp_path, _, _ = case - images, _, _, image_type, channels, control = DATASETS[name] - data = description_bytes(images, channels, control, image_type) - path = write_derived_design(description_from_bytes(data), tmp_path / "derived.csv") - assert path.read_bytes() == (tmp_path / "old" / "experiment_design.csv").read_bytes() - assert read_confirmed_design(path).design.rows diff --git a/tests/test_removed_in_v040.py b/tests/test_removed_in_v040.py deleted file mode 100644 index 208af382..00000000 --- a/tests/test_removed_in_v040.py +++ /dev/null @@ -1,96 +0,0 @@ -"""What v0.4.0 removes (spec revision 4, ยง8 PP-2), and what still reads. - -Removed: ``convert_tiffs`` with ``zarr_export`` (OME-Zarr copies in the output folder), the -deprecated stand-ins of the code moved to the normalization brick, and -``check_declared_axes``/``declared_axes_transposed``, which read a Y/X swap transposed. -""" - -from __future__ import annotations - -import importlib -import json - -import numpy as np -import pytest - -import morphobricks_preprocessing as pp -from morphobricks_preprocessing import prefill_mapping, read_manifest, run_experiment -from morphobricks_preprocessing._io import free_space -from morphobricks_preprocessing.cli import main -from tiff_tree import write_data - -REMOVED_NAMES = [ - "check_declared_axes", - "declared_axes_transposed", - "ConversionSettings", - "ConversionEstimate", - "OmeZarrWriter", - "estimate_conversion", - "ome_zarr_relative_path", - # The deprecated stand-ins of morphobricks_normalization's names. - "PlaneRecord", - "compute_plane_histogram", - "PooledHistogram", - "read_plane_histograms", - "min_max", - "SINGLE", -] - - -@pytest.mark.parametrize("name", REMOVED_NAMES) -def test_removed_names_are_gone(name): - assert not hasattr(pp, name) - assert name not in pp.__all__ - - -@pytest.mark.parametrize( - "module", - [ - "zarr_export", - "_moved", - "csv_io", - "dtypes", - "histogram", - "normalization", - "pooling", - "records", - ], -) -def test_removed_modules_are_gone(module): - with pytest.raises(ModuleNotFoundError): - importlib.import_module(f"morphobricks_preprocessing.{module}") - - -def test_the_run_takes_no_conversion(tmp_path): - write_data(tmp_path / "exp/a.tif", np.zeros((2, 6, 8), np.uint8), "TYX") - with pytest.raises(TypeError, match="convert_tiffs"): - run_experiment(tmp_path / "exp", prefill_mapping("TYX"), convert_tiffs=True) - with pytest.raises(TypeError, match="conversion"): - run_experiment(tmp_path / "exp", prefill_mapping("TYX"), conversion=None) - - -def test_the_cli_takes_no_conversion(tmp_path): - with pytest.raises(SystemExit) as stopped: - main(["run", str(tmp_path), "--axes", "TYX", "--convert-tiff"]) - assert stopped.value.code == 2 - - -def test_a_v03_manifest_with_conversion_keys_is_still_read(tmp_path): - root = tmp_path / "exp" - write_data(root / "a.tif", np.zeros((2, 6, 8), np.uint8), "TYX") - result = run_experiment(root, prefill_mapping("TYX")) - data = json.loads(result.manifest_path.read_text(encoding="utf-8")) - data["settings"]["convert_tiffs"] = True - data["settings"]["conversion"] = {"chunk_size": 1024} - data["files"][0]["ome_zarr_path"] = "ome_zarr/a.ome.zarr" - result.manifest_path.write_text(json.dumps(data), encoding="utf-8") - manifest = read_manifest(result.manifest_path) - assert manifest.file("a.tif") is not None - assert "convert_tiffs" not in json.dumps(manifest.to_dict()) - # The rows of the earlier run are kept: they were the TIFF's own. - again = run_experiment(root, prefill_mapping("TYX")) - assert again.unchanged == ("a.tif",) and again.read == () - - -def test_free_space_is_kept_for_the_disk_check(tmp_path): - assert free_space(tmp_path / "not" / "yet") > 0