From aafe8da3dc38443127174d5eed946706b224f007 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 07:30:06 +0000 Subject: [PATCH] Pin the normalization brick to its release v0.3.2 morphobricks-normalization v0.3.2 (normalization#19) checks each plane_histograms.csv row's stored plane_id, image_file and bit_depth against the row's other columns when reading, so an inconsistent row is refused (CsvFormatError) instead of silently recomputed (issue #39, CSV item). pyproject, README and AGENTS now name v0.3.2. The package version is not changed (the release is separate). Regression test test_a_row_whose_plane_id_disagrees_with_its_columns_is_refused: DID NOT RAISE with v0.3.1, passes with v0.3.2; a rerun finding such a table recomputes it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VbytvBRB9CmKQ5AAkhEFcv --- AGENTS.md | 4 ++-- README.md | 2 +- pyproject.toml | 10 ++++++---- tests/test_run.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4b6e9d99..64bec578 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,7 +32,7 @@ brick**. - **Statistics code (from the normalization brick):** the per-plane histogram, the CSV reader and writer, pooling (`PooledHistogram`) and normalization parameters (`min_max`, `percentile_range`, `z_score`, `robust_z_score`) live in the required dependency - `morphobricks_normalization` (pinned to its release v0.2.0, plan §12 questions 27 and + `morphobricks_normalization` (pinned to its release v0.3.2; first pinned at v0.2.0, plan §12 questions 27 and 28). This brick runs them over the experiment. Later bricks such as patch extraction use them to normalise any group of planes without rereading pixels. The old names stay importable from here, deprecated, @@ -439,7 +439,7 @@ Technical details go to `logging` (shown in the notebook's "Technical details" p | Topic | Decision | |---|---| | Binning | Per-plane min–max, same bin count per experiment, pooled counts for groups (binning study) | -| Core dependencies | `numpy`, `tifffile`, `scipy` (the mask step's piece counts, Q32) and `morphobricks-normalization` (git dependency pinned to its release tag v0.3.1; `[tool.hatch.metadata] allow-direct-references`). `ipywidgets` and `matplotlib` in the `notebook` extra; `zarr` in the `zarr` extra; `pytest` and `ruff` in `dev` | +| Core dependencies | `numpy`, `tifffile`, `scipy` (the mask step's piece counts, Q32) and `morphobricks-normalization` (git dependency pinned to its release tag v0.3.2, whose CSV reader checks each row's stored `plane_id`, `image_file` and `bit_depth`; `[tool.hatch.metadata] allow-direct-references`). `ipywidgets` and `matplotlib` in the `notebook` extra; `zarr` in the `zarr` extra; `pytest` and `ruff` in `dev` | | OME-Zarr (Q12) | Input may be TIFF and/or OME-Zarr (folders ending in `.zarr`), mixed. OME-Zarr axes come from their metadata and are matched by role; level 0 of a single image only, several images per store is `multiple_series`. TIFF choice once per experiment (`convert_tiffs`): read directly (default) or convert to `/ome_zarr` (OME-NGFF 0.4) and compute rows from the copies; switching recomputes TIFF rows and removes unused copies. Reads OME-NGFF 0.4 and 0.5 (0.5 needs zarr 3). In the notebook: its own step-3 card "(Optional) Image conversion" after Settings (Q13): radio buttons (read directly = recommended), a disk-space warning (amber, red when the estimate exceeds free space, never blocking), and conversion settings (chunk size, compression + level, pyramid levels) shown only when converting. CLI `--convert-tiff` with `--chunk-size`, `--compression`, `--compression-level`, `--pyramid-levels`. No overlap setting: chunks never overlap | | System files (Q5) | Ignored silently; `single` check and sibling uniqueness are case-insensitive | | Too-deep folders (Q7) | Error | diff --git a/README.md b/README.md index 403e212f..413c9b17 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Pre-alpha. Implemented and tested: and a command line (`morphobricks-preprocessing check` / `run` / `masks`) From the normalization brick (`morphobricks_normalization`, a required dependency, pinned -to its release v0.3.1): per-plane histograms with the robust range and the range check +to its release v0.3.2): per-plane histograms with the robust range and the range check (`histogram`, `ranges`), the histogram table (`csv_io`, schema version 4), pooled group statistics (`pooling`) and normalization parameters (`normalization`). Importing these names from `morphobricks_preprocessing` still works for one version, with a diff --git a/pyproject.toml b/pyproject.toml index c5175fd3..ed7e5928 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,11 +25,13 @@ dependencies = [ # Counting the connected pieces of each mask label in the optional mask step. "scipy>=1.10", # Per-plane histograms, the CSV table, pooling and normalization (moved there in P1). - # Pinned to the brick's release v0.3.1, which writes plane_histograms.csv schema + # Pinned to the brick's release v0.3.2, which writes plane_histograms.csv schema # version 4 (underflow and overflow columns, the robust histogram range and the - # range check; follow-up P6). Its tag v0.3.0 holds the same code but reports version - # 0.2.0. Schema version 3 with the channels in columns came with v0.2.0 (P3b). - "morphobricks-normalization @ git+https://github.com/CellMigrationLab/morphobricks-normalization.git@v0.3.1", + # range check; follow-up P6, since v0.3.1) and, when reading it, checks each row's + # stored plane_id, image_file and bit_depth against its other columns (data-flow + # audit, issue #39, normalization#19). Its tag v0.3.0 holds v0.3.1's code but reports + # version 0.2.0. Schema version 3 with the channels in columns came with v0.2.0 (P3b). + "morphobricks-normalization @ git+https://github.com/CellMigrationLab/morphobricks-normalization.git@v0.3.2", ] [project.optional-dependencies] diff --git a/tests/test_run.py b/tests/test_run.py index 160c130d..e1b8d2c7 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -946,3 +946,33 @@ def test_run_refuses_flags_that_are_not_booleans_before_reading(experiment, tmp_ with pytest.raises(ValueError, match=f"{flag} must be True or False"): run_experiment(experiment, ZYX, output_folder=output, **{flag: value}) assert not output.exists() + + +def _tamper_plane_id(csv_path): + """Change the stored plane_id of the first row, leaving its other columns as they are.""" + lines = csv_path.read_text(encoding="utf-8").splitlines(keepends=True) + header = lines[0].rstrip("\r\n").split(",") + column = header.index("plane_id") + values = lines[1].rstrip("\r\n").split(",") + values[column] = values[column] + "_x" + lines[1] = ",".join(values) + "\n" + csv_path.write_text("".join(lines), encoding="utf-8") + + +def test_a_row_whose_plane_id_disagrees_with_its_columns_is_refused(experiment): + # normalization v0.3.2 (normalization#19, issue #39): the reader checks the stored + # plane_id, image_file and bit_depth against the row's other columns. + from morphobricks_normalization.csv_io import CsvFormatError + + result = run_experiment(experiment, ZYX) + _tamper_plane_id(result.csv_path) + with pytest.raises(CsvFormatError, match="plane_id"): + read_plane_histograms(result.csv_path) + # A rerun that finds such a table (its checksum updated, so it is read) recomputes it. + manifest_path = result.output_folder / "run_manifest.json" + data = json.loads(manifest_path.read_text(encoding="utf-8")) + data["csv_sha256"] = hashlib.sha256(result.csv_path.read_bytes()).hexdigest() + manifest_path.write_text(json.dumps(data), encoding="utf-8") + rerun = run_experiment(experiment, ZYX) + assert rerun.rebuild_reason == "the earlier histogram table could not be read" + assert rerun.written and read_plane_histograms(rerun.csv_path)