diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8503da..790011e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,19 @@ jobs: # baseline (Ruff's own findings of v0.3.3 are in it, so plain `ruff check .` is not run). - run: python tools/check.py --rules + # The vendored Spec files (tests/spec_fixtures) must be byte for byte the Spec at the pinned + # commit (MorphoBricks-Spec docs/vendoring.md); tests/test_spec_pin.py checks the copy offline. + spec-pin: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: { repository: CellMigrationLab/MorphoBricks-Spec, path: spec, fetch-depth: 0 } + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python spec/tools/vendor.py --into . --check + test: runs-on: ${{ matrix.os }} strategy: diff --git a/AGENTS.md b/AGENTS.md index 602900b..da806db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ Read it, and the documents below, before changing anything. | File | What it fixes | |---|---| -| `docs/plan.md` | Goals, methods, scopes, milestones, the owner's decisions N1–N60 (§12) and what is still to be measured (§13) | +| `docs/plan.md` | Goals, methods, scopes, milestones, the owner's decisions N1–N64 (§12) and what is still to be measured (§13) | | `docs/contract.md` | Rules other bricks rely on: methods and formulas, interface, scopes and repeat groups, image types, the statistics file, intensity checks, parameter columns, validation | | `CHANGELOG.md` | Release notes and migration notes | | `docs/archive/` | Historical record: the two done follow-ups for the calling bricks | @@ -38,14 +38,17 @@ src/morphobricks_normalization/ their exact bound for 64-bit values float64 cannot hold (#28) ranges.py HistogramSettings, RangeCheck, range_check_issues (Milestone 5) integrity.py invariants every PlaneHistogram and HistogramTail checks when made - (counts, limits, tail sums) and the pixel type of every block + (counts, limits, tail moments) and the pixel type of every block + tail_moments.py exact centred moments of a tail (count, mean, std; schema version 5) + channels.py canonical channel IDs (the Spec's Token rule): check_channel_id(s) records.py PlaneRecord (one plane_histograms.csv row: an XYC plane, one - histogram per channel; position_id required), plane_id, group_records - csv_io.py write/read plane_histograms.csv schema version 4, channels in - CellProfiler-style columns, underflow/overflow columns (atomic write, - labels always text) + histogram per channel; position_id and channels (IDs) required), + plane_id, group_records; ChannelMeasurementFailure/Error (HV3) + csv_io.py write/read plane_histograms.csv schema version 5 (ADR 0003): channels + in CellProfiler-style columns named by channel ID, presence column + `channels`, underflow/overflow columns (atomic write, labels always text) pooling.py PooledHistogram: pooled counts, never mean-of-means, tails by - their means for percentiles and by their sums for mean and std; + their means for percentiles and by their moments for mean and std; WeightedHistogram: histograms added with a weight each (Milestone 2); linear_percentile: NumPy's linear interpolation (Milestone 3) normalization.py NormalizationParameters (offset, scale) and the four linear methods @@ -53,8 +56,9 @@ src/morphobricks_normalization/ scopes.py group_key, group_name, scope_groups: groups at each scope, repeat groups rgb.py StarDist's percentile normalization: rgb_percentile, apply_rgb (CSBDeep's float32 arithmetic), check_rgb - fitting.py fit, Parameters (apply, select, to_columns, from_columns with - Normalization_Offset_C and Normalization_Scale_C) + fitting.py fit, by channel ID + parameters.py Parameters (apply, select, to_columns, from_columns with + Normalization_Offset_ and Normalization_Scale_) summaries.py ImageSummary, image_summaries, summary_columns: one row per image, Z and T pooled, from plane_histograms.csv rows (Milestone 4) checks.py IntensityCheckSettings, intensity_checks: the four intensity warnings; @@ -115,7 +119,7 @@ range check. ## 3. How we work 1. **Read first:** the plan, the contract and the modules you touch. -2. **Ask before deciding.** Decisions N1–N60 are final (those marked withdrawn in plan §12 no longer apply); do not re-open them. A choice the +2. **Ask before deciding.** Decisions N1–N64 are final (those marked withdrawn in plan §12 no longer apply); do not re-open them. A choice the plan and the contract leave open, and any change of product behaviour or file format, is the owner's: ask with 2–4 options, the recommended one first. 3. **Outputs stay bit-identical** unless a change says otherwise. `tests/test_fit_goldens.py` keeps the fitted diff --git a/CHANGELOG.md b/CHANGELOG.md index 759fb91..e95b467 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,67 @@ The decisions are in [docs/plan.md](docs/plan.md) §12. ## Unreleased +Stage 3 S3.2 (MorphoBricks-Spec roadmap; ADR 0003, plan N61–N64). Breaking: `plane_histograms.csv` +schema version 5 and channels named by canonical channel ID everywhere. Version not bumped yet. + +- **`plane_histograms.csv` schema version 5** (HV1, HV2): one row per plane; the presence column + `channels` (the plane's IDs in source order, `;`-separated) after the 15 plane columns; the + per-channel columns of every channel ID of the experiment, sorted, named + `__` (`Histogram_Count_DNA_00042`). An absent channel is blank and a value + there is refused. `write_plane_histograms(records, path, *, channel_ids, failures=())` needs the + experiment's channel IDs; new `read_plane_histogram_table` (header IDs and records), + `table_header`, `header_channels`. +- **Failed measurements** (HV3): `ChannelMeasurementFailure`, `ChannelMeasurementError`, + `check_measurements`; the writer writes nothing when given a failure, and removes a table an + earlier run left at the path (`ChannelMeasurementError.removed` names it), so no out-of-date + table looks current. +- **Version 4 is refused** (HV4) with a message naming version 5 and how to recompute; no converter. +- **Channels by ID**: `PlaneRecord(..., *, position_id, channels)` (required), `channel_histogram(id)`, + `has_channel`; `PooledHistogram(..., channel=id)`; `fit(..., channels=)` (rows: which IDs, default + all; pixels: the ID of each entry, required except RGB); `Parameters.channels` are sorted IDs + (required), `select(ids)`, `apply(..., channels=)`, columns `Normalization_Offset_` / + `Normalization_Scale_`; `ImageSummary.channels`, `summary_columns(channel_ids)`; + `range_check_issues` takes channel IDs; `Issue.channel` is an ID. RGB uses the IDs `R`, `G`, `B`. + An index where an ID is expected is refused. +- **Heterogeneous images** (N43 replaced): rows may have different channels, and each channel is + pooled by its ID. A fitted group in which only some planes have a channel is refused with the + new `ChannelScopeIncompleteError` (the Spec's `A_SCOPE_INCOMPLETE`, "present in k of n + planes"), unless the user selects those planes with the new setting + `NormalizationSettings(partial_channels="planes_with_channel")`, stored by `to_dict` when + chosen and reported per fit by the new warning `channel_partially_present`. Planes of one image + with different channel lists are refused. +- **Centred tail moments** (owner decision of 11 November 2026, plan N65; #91's question): the + underflow and overflow columns are `Histogram_UnderflowCount_`, + `Histogram_UnderflowMean_`, `Histogram_UnderflowStd_` (and `Overflow`), replacing the + raw `...Sum_` and `...SumOfSquares_`. `HistogramTail(count, mean, std)` replaces + `HistogramTail(count, sum, sum_of_squares)`. The new module `tail_moments.py` accumulates every + tail pixel exactly (significand times a power of two, or the integer itself), so the mean and + the standard deviation are each the float64 nearest the exact value: no cancellation for large + values with a small spread, no overflow for finite pixels (no scale column needed), and the + same bits whole or in blocks (they could differ in the last bits before). Pooling adds + `count · std²` and `count · (mean − pooled mean)²`, and a pooled mean whose tail products + overflow is redone with exact power-of-two scaling. This supersedes #91's refusal of an + infinite tail sum of squares and of opposite infinite tail sums below: those tails now give + their finite mean and standard deviation. Numbers without tails (the default min-max range) + do not change; with tails, means, standard deviations and percentiles move by at most + about `1.4e-14` relative in the `tools/dump_outputs.py` cases, and every tail mean of the + `--corpus` is unchanged. +- **Parameter columns are versioned**: `to_columns` adds `normalization_parameters_version` = + `2` (appended to `COLUMNS`), and `from_columns` refuses a row without it (the + `Normalization_Offset_C` columns of 0.4, which would otherwise read as channel `C0`) or of + another version: fit again. +- `Parameters` moved to the new module `parameters.py` (re-exported by `fitting` and the package). +- Tests: the Spec's test material is vendored at `cc9820f` (`tests/spec_fixtures`, pin test and CI + check); described-tree tests for the owner's extra-stain case and fixtures v04, v05, v06, v13, + v17, v22. + +Numbers: for planes whose channels are all present, histograms and fits are bit-identical to +schema version 4 (v0.4.0 and the #91/#92 fixes below) for the same pixels: a one-off comparison +of 3600 fitted channels on `v13` trees, the 400 golden fits, and `tools/dump_outputs.py +--corpus` unchanged. `tools/dump_outputs.py` changes only in the table digests (new format), the +added `normalization_parameters_version` column, and `Issue.channel` and messages naming channel +`C0` instead of index `0`. + Numerical-correctness fixes. Each says which outputs change; every other output is bit-identical (`tools/dump_outputs.py` and `--corpus` lines, and the 400 golden fits). @@ -16,7 +77,7 @@ bit-identical (`tools/dump_outputs.py` and `--corpus` lines, and the 400 golden integers and the float64 standing in for them, rounded up, so a histogram is exact only when every representative equals its value: `[2**53 + 1]` is no longer reported exact with error 0 (its representative is `2**53`; error 1), and `normalization_exact` and - `Histogram_Exact_C` are false for it. Only int64 and uint64 planes with such values + `Histogram_Exact_` are false for it. Only int64 and uint64 planes with such values change; bin counts and limits never do. **Who is affected, and by how much.** Only int64 and uint64 images with a pixel of @@ -29,7 +90,7 @@ bit-identical (`tools/dump_outputs.py` and `--corpus` lines, and the 400 golden cannot hold it grows from 0 to at most half a float64 step (for example 1 for `[2**53 + 1]`, 170 and 341 for the constant planes `3074457345618258602` and `6148914691236517205`), so `is_exact`, `normalization_exact` and - `Histogram_Exact_C` change from true to false; for wider bins it changes in its last + `Histogram_Exact_` change from true to false; for wider bins it changes in its last bits only; - pooled means, standard deviations and percentiles, and fitted offsets and scales, move by at most about `3e-14` relative in the `tools/dump_outputs.py` cases (a zscore offset diff --git a/README.md b/README.md index 8764df2..56c9d35 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ out of scope for now. Pre-alpha; the latest release is v0.4.0. The library offers: - supported pixel types, bin counts and per-plane histograms (`dtypes`, `histogram`), with a robust range and a range check; -- plane records, `plane_histograms.csv` reading and writing (schema version 4) and pooling of group statistics (`records`, `csv_io`, `pooling`); +- plane records, `plane_histograms.csv` reading and writing and pooling of group statistics (`records`, `csv_io`, `pooling`): schema version 4 in v0.4.0; on `main` after the Stage 3 pull request, schema version 5 with channels named by canonical channel ID (ADR 0003, see CHANGELOG "Unreleased"); - settings, scopes, repeat groups and `fit`, with `Parameters` (`apply`, `to_columns`, `from_columns`), for fluorescence and RGB images; - per-image summaries and intensity checks, returned as `Issue` warnings. @@ -50,7 +50,7 @@ records = read_plane_histograms("plane_histograms.csv") groups, group_warnings = scope_groups(records, settings, repeat_groups=None) for key, rows in groups.items(): parameters = fit(rows, settings, group=group_name(key)) - columns = parameters.to_columns() # normalization_* and Normalization_*_C columns + columns = parameters.to_columns() # normalization_* and Normalization_*_ columns ``` ## Installation diff --git a/docs/contract.md b/docs/contract.md index 7a8e213..b27d4d2 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -22,6 +22,7 @@ with modality profiles, purpose rules, colour statistics and references is withd | 1.3 and 6. Channels across columns (`plane_histograms.csv` schema version 3); per-channel columns of 7.1 and 8.1 | Milestone 2b (done; the columns of 7.1 in Milestone 4, done) | | 7. Intensity checks | Milestone 4 (done) | | 1.4 Robust histogram range and range check; `plane_histograms.csv` schema version 4 (6.2) | Milestone 5 (done) | +| 1.5 and 6. Channels by canonical channel ID (`plane_histograms.csv` schema version 5); per-channel columns of 7.1 and 8.1 by ID | Stage 3 S3.2 ([ADR 0003](https://github.com/CellMigrationLab/MorphoBricks-Spec/blob/main/docs/adr/0003-plane-histograms-v5.md); draft, unreleased) | | 9. Validation | With the milestone of each check | Notation: `x` is a pixel value in source units; `n` a pixel count; channels are numbered @@ -66,14 +67,16 @@ behaviour of the names above, with one owner-approved exception (§1.3). Decisions N41–N44 change the layout of the moved code's table: `plane_histograms.csv` holds one row per XYC plane with every channel in its own columns (schema version 3, §6.2), instead -of one row per plane and channel. Accordingly, in Milestone 2b: +of one row per plane and channel. (Stage 3 replaces the C-index channel token and N43 by +channel IDs and per-row presence, §1.5; this section records Milestone 2b.) Accordingly, in +Milestone 2b: - `PlaneRecord` describes one XYC plane and holds one histogram per channel; `plane_id` has no channel coordinate. - `write_plane_histograms` and `read_plane_histograms` write and read schema version 3 only; a table of version 1 or 2 is refused with `CsvFormatError` naming its version (the preprocessing brick recomputes it on its next run). Since Milestone 5 they write and - read schema version 4 only (§1.4, N56). + read schema version 4 only (§1.4, N56), and since Stage 3 schema version 5 only (§1.5). - `group_records`, pooling and fitting take a channel index wherever they used the row's `c`; pooled statistics, parameters and the histograms themselves are unchanged, bit for bit. @@ -181,11 +184,10 @@ filled in: beyond it. Each of the four is given: a missing `pixel_min` or `pixel_max` is refused, never taken from `binning_min` or `binning_max`, even where a min-max histogram would have them equal (#47). In the statistics file the message names the blank column. -- An empty tail has zero sums. A populated tail has a sum that is a number and a sum of - squares of 0 or more, with `count · sum_of_squares ≥ sum²`, a mean within its range and - a sum of squares possible for values in it (relative rounding slack 1e-9). A sum of - squares (or sum) that overflows float64 to infinity is accepted only when the tail's - range makes the overflow possible (values beyond about 1e154). +- An empty tail has count, mean and standard deviation 0. A populated tail has a finite + mean within its range (between its extremum and its binning limit) and a finite standard + deviation of 0 or more and at most half that range (relative slack 1e-9 plus one float64 + step). Both are always finite for finite pixels (schema version 5, below). - Pooling adds the counts as integers, so a group's pixel count is exact; a group of more than 2**63 − 1 finite pixels is refused. @@ -198,7 +200,7 @@ tail's mean and its count. | Statistic | Definition | Accuracy | |---|---|---| | min, max | Smallest `pixel_min`, largest `pixel_max` | Exact | -| mean, std | Count-weighted mean and population std (÷N) of representatives; tails count with their exact sums and sums of squares. The weighted sums are correctly rounded sums of the products (`math.fsum`), so they never depend on the CPU thread count or BLAS library (#67). When the squares overflow float64 although every value is finite (`[-1e200, 1e200]`), the std is computed with the values divided by a power of two, exactly, so a finite spread is never reported as infinite (#33) | Exact for exact histograms; otherwise very close (errors cancel) | +| mean, std | Count-weighted mean and population std (÷N) of representatives; tails count with their centred moments (count, mean, std, §1.4). The weighted sums are correctly rounded sums of the products (`math.fsum`), so they never depend on the CPU thread count or BLAS library (#67). When the squares overflow float64 although every value is finite (`[-1e200, 1e200]`), the std is computed with the values divided by a power of two, exactly, so a finite spread is never reported as infinite (#33) | Exact for exact histograms; otherwise very close (errors cancel) | | percentile q | Smallest representative with cumulative count ≥ ceil(q/100 · N) (numpy `inverted_cdf`) | Within the group's largest per-plane bound | | median | 50th percentile | As percentiles | | MAD | Median of \|representative − median\| | Within twice the bound | @@ -254,22 +256,32 @@ at `robust_quantiles = (q_low, q_high)` in percent: the value of rank `max(1, ceil(q/100 · N))` among the `N` sorted finite pixels (NumPy `method="inverted_cdf"`, the percentile rule of §1.2.3), so both are pixel values. The bins span `[lo, hi]` with the rules of §1.2.2. Pixels `x < lo` form the **underflow** and pixels -`x > hi` the **overflow**; each tail keeps its count, the sum of its values and the sum of -their squares, in float64 (`HistogramTail`). `Histogram_BinningMin/Max_C` hold `lo` and -`hi`; `Intensity_Min/MaxIntensity_C` stay the exact extrema. A limit equals its extremum +`x > hi` the **overflow**; each tail keeps its **centred moments** (`HistogramTail`, schema +version 5; owner decision of 11 November 2026, N65): its count, its mean and its population +standard deviation about its mean. Every pixel is taken at its exact value (a float as its +significand times a power of two, an integer as itself), the sum and the sum of squares are +accumulated as exact rationals over all blocks, and the mean and the standard deviation are +each rounded once to the nearest float64 (`tail_moments.py`). So nothing cancels (the raw +float64 sums of squares of version 4 lost the spread of large values with a small spread), +nothing overflows (the mean lies in the tail's range, the standard deviation is at most half +of it, for every finite pixel; the raw squares overflowed beyond about `1.3e154`), and a plane +gives the same bits whole or in blocks of any size or order. No scale column is needed. `Histogram_BinningMin/Max_` hold `lo` and +`hi`; `Intensity_Min/MaxIntensity_` stay the exact extrema. A limit equals its extremum exactly when its tail is empty. With the default quantiles, a plane of at most 10,000 finite pixels keeps its extrema as limits; a 512 × 512 plane has 26 pixels in each tail. **Statistics with tails** (§1.2.3): -- `Histogram_ValidPixelCount_C` = the bins' counts + underflow count + overflow count. +- `Histogram_ValidPixelCount_` = the bins' counts + underflow count + overflow count. - Percentiles, median and MAD: each non-empty tail stands as one value, its mean, with its count. -- Mean and standard deviation use the tails' sums: as exact as without tails. -- A tail whose sum of squares is not finite (its pixels' squares overflowed float64 when the - file was written, values beyond about `1.3e154`) has no known spread: the standard - deviation, and so `zscore`, is refused with a message naming the tail (#33). Mean, - percentiles, median and MAD do not use it. +- Mean: each tail adds `count · mean`, summed with the bins' products in one correctly + rounded sum. When those products overflow although every value is finite (tails near the + float64 limits), the sum is redone with the values divided by a power of two, exactly. +- Standard deviation: each tail adds `count · std²` (its own spread) and + `count · (mean − pooled mean)²`, centred terms that never cancel; overflow is handled by the + same power-of-two scaling as the bins (#33). Pooling is exact given the stored moments, up + to one rounding of each product. - Min-max parameters use the exact extrema (`pixel_min`, `pixel_max`). - `max_representative_error` is the larger of the bins' bound and, for each non-empty tail, the larger distance from the tail's mean to its two ends (`max(mean − pixel_min, @@ -312,7 +324,7 @@ histogram's passes (table above). A `RangeCheck` holds: `narrow_histogram_range` warning (§9.2) per channel with a narrow plane, listing the planes in `items`, with `channel` set and the fewest central bins, the pixels outside the robust limits and the robust central bins in the message. Only the preprocessing brick reports it, -in its step 3. A channel must be a whole number 0 or above (nothing is rounded or parsed), +in its step 3. A channel must be a canonical channel ID (an index is refused, nothing is parsed), and the narrow planes of one channel must share one set of settings (range, central percentiles, robust quantiles, `min_central_bins`); otherwise `IncompatibleRangeChecksError` (a `ValueError`) names the planes of each setting (issue #59). `fit` adds no warning and its zero-spread error is unchanged (N57). @@ -329,6 +341,35 @@ The names that change in Milestone 5: | `csv_io.py` | `SCHEMA_VERSION = "4"`; `CHANNEL_FEATURES` gains the six tail features (§6.2). | | `pooling.py` | `PooledHistogram.values`/`counts` include one pair per tail; `bin_values`, `bin_counts` and `tails` give the bins alone and the tails' sums; `WeightedHistogram` the same, weighted. | +### 1.5 Channels by canonical channel ID (Stage 3, ADR 0003) + +The owner's decisions of 10 November 2026 (MorphoBricks-Spec ADR 0003, HV1-HV4; plan N61-N64) +replace the C index as the channel token: a channel is named by its **canonical channel ID** +from the confirmed experiment description (experiment-v1 §3.1, §4.3), whose Token rule +`^[A-Za-z][A-Za-z0-9]*(?:_[A-Za-z0-9]+)*$` it follows. Images of one experiment may have +different channels (N43 is replaced by per-row presence); a channel is pooled, summarized and +checked only over the planes that have it, never by its position in the image. A fitted group +in which only some planes have a channel is refused (`ChannelScopeIncompleteError`, the Spec's +`A_SCOPE_INCOMPLETE`, experiment-v1 §8) unless the user selects those planes with +`NormalizationSettings(partial_channels="planes_with_channel")`, which is stored with the run's +settings (§8.3) and reported per fit (`channel_partially_present`, §9.2). An index given where +an ID is expected is refused (`ValueError`), never read as an ID. + +| Module | Stage 3 (schema version 5) | +|---|---| +| `channels.py` (new) | `CHANNEL_ID_PATTERN`, `check_channel_id`, `check_channel_ids`. | +| `records.py` | `PlaneRecord(..., *, position_id, channels)`: `channels` (required, keyword-only) is the plane's IDs in source order (the unit's list), one per histogram; `has_channel(id)`, `channel_histogram(id)`. `ChannelMeasurementFailure(plane_id, channel, reason)`, `ChannelMeasurementError` (with `removed`, the earlier table the writer removed, §6.2) and `check_measurements` (HV3); `check_universe`. `different_channel_counts` is removed; `same_channels_per_image` refuses an image whose planes list different channels. | +| `csv_io.py` | `SCHEMA_VERSION = "5"`; `CHANNELS_COLUMN`; `write_plane_histograms(records, path, *, channel_ids, failures=())`; `read_plane_histogram_table(path)` gives `PlaneHistogramTable(channel_ids, records)`, `read_plane_histograms` its records; `table_header(ids, bin_count)`, `header_channels(header)`; `channel_column(feature, id)`, `bin_column(id, index)`, `channel_columns(id, bin_count)`. | +| `pooling.py` | `PooledHistogram(histograms, channel=None)`: `channel` is an ID; a record without it is refused, not skipped. | +| `settings.py` | `NormalizationSettings.partial_channels`: `"refuse"` (default) or `"planes_with_channel"` (§8.3). | +| `fitting.py`, `parameters.py` (new) | `fit(..., channels=None)`: for rows, the IDs to fit (default all the rows have; a channel only some rows have raises `ChannelScopeIncompleteError` unless `partial_channels="planes_with_channel"`); for pixels, the ID of each entry along `channel_axis` (required except for RGB, `R`, `G`, `B`). `Parameters` (moved to `parameters.py`, re-exported by `fitting`) holds `channels` as sorted IDs (required); `select(ids)`; `apply(pixels, channel_axis=, channels=)`; columns `Normalization_Offset_`, `Normalization_Scale_` and `normalization_parameters_version` = `2` (§8.1). | +| `summaries.py`, `checks.py`, `ranges.py`, `issues.py` | `ImageSummary.channels` (sorted IDs); `summary_columns(channel_ids)`; checks compare a channel among the images that have it; `range_check_issues` takes `(plane_id, channel_id, check)`; `Issue.channel` is an ID; new warning `channel_partially_present` (§9.2). | +| `rgb.py` | RGB rows and arrays have the channels `R`, `G`, `B` in this order (experiment-v1 §4.4b); `check_rgb_channels`. | + +For planes whose channels are all present, histograms and fitted numbers are those of schema +version 4 for the same pixels, bit for bit; only names change (and the parameter columns gain +their version column, §8.1). + --- ## 2. Interface @@ -337,10 +378,10 @@ The names that change in Milestone 5: | Object | Holds | Stored as | |---|---|---| -| `NormalizationSettings` | Image type, method and its options, scope, pooling weight, per-time-point option (§8.3) | Run configuration of the calling brick, once per run | +| `NormalizationSettings` | Image type, method and its options, scope, pooling weight, per-time-point option, what to do with a channel only some planes of a group have (`partial_channels`) (§8.3) | Run configuration of the calling brick, once per run | | Source statistics | Pooled plane histograms of a group (§1.2), or pixel arrays from which the library computes the same statistics | Not stored by the library | | `RepeatGroups` | Mapping from `(condition, biological_repeat)` to the repeat group's label (§4.2), built by the calling brick from `experiment_design.csv` | Not stored by the library | -| `Parameters` | Offset and scale per channel of one group, with the image's channel indices they belong to (`channels`, N46) (§8) | Per-row columns (N21, N23) | +| `Parameters` | Offset and scale per channel of one group, with the canonical channel IDs they belong to (`channels`, N46 as amended by HV2) (§8) | Per-row columns (N21, N23) | | `Issue` | Severity, code, message, affected items, suggested action, and for the intensity checks the channel (`channel`, N52) | Reported by the calling brick | | `ImageSummary` | Intensity summary of one image, all Z and T planes pooled per channel (§7.1) | Not stored by the library (N50) | | `IntensityCheckSettings` | The limits of the intensity checks (§7.2) | Run configuration of the calling brick | @@ -351,8 +392,8 @@ The names that change in Milestone 5: issues = check_settings(settings, *, statistics_file: bool) key = group_key(record, settings, repeat_groups=None) # group of one plane_histograms row parameters = fit(statistics, settings) # one group -normalized = parameters.apply(pixels, channel_axis=...) -patch_parameters = parameters.select([2]) # one channel of the image (§8.1) +normalized = parameters.apply(pixels, channel_axis=0, channels=["DNA", "ACTIN"]) +patch_parameters = parameters.select(["DNA"]) # one channel of the image (§8.1) columns = parameters.to_columns() # dict[str, str] parameters = Parameters.from_columns(columns, settings) settings.to_dict(); NormalizationSettings.from_dict(...) @@ -360,18 +401,18 @@ summaries = image_summaries(records, percentile=(1.0, 99.8)) # §7.1 row = summaries[0].to_row() # dict of the §7.1 columns issues = intensity_checks(summaries, settings=None) # §7.2, IntensityCheckSettings histogram, check = histogram_with_range_check(blocks, dtype, settings=HistogramSettings()) -issues = range_check_issues([(plane_id, channel, check), ...]) # §1.4 +issues = range_check_issues([(plane_id, channel_id, check), ...]) # §1.4 ``` | Function | Rule | |---|---| | `check_settings` | Checks the method against the image type (§5) and the scope against the available statistics (§4.5) using the settings alone, so a calling brick can refuse a choice before reading any image. Raises `MethodNotAllowedError` (a `ValueError`) with a message of §9.1. | | `group_key` | The group of a row at the settings' scope (§4.1). `biological_repeat` needs `repeat_groups`; without it, repeats match by name (§4.2). | -| `fit` | One entry point for every method; dispatches on `settings.image_type` and `settings.method`. Statistics are pooled histograms per channel, or pixels; with `pooling="conditions"` the histograms carry their condition (§4.3). Never reads files and never walks folders. | -| `apply` | Returns a new float32 array; the input is never modified. `channel_axis` gives the channel axis of `pixels`; it may be left out for one channel. No clipping (N19). Non-finite pixels stay non-finite. Computed in float64 for fluorescence (§3.2) and with CSBDeep's float32 arithmetic for `rgb` (§3.3). | -| `select` | The parameters of some of the image's channels, e.g. of a single-channel patch; `channels` keeps their indices (N46). For `rgb` only all of channels 0, 1 and 2 can be selected (§5: the three channels stay together); a subset is refused. | -| `to_columns` / `from_columns` | The columns of §8.1 as text in the format of §8.2; `from_columns(to_columns(p)) == p` exactly. `to_columns` writes the pairs of `channels` only; `from_columns` reads every channel whose pair is filled and skips channels whose pair is blank (N46). `from_columns` refuses a `normalization_scope` that is not one of §4.1's scopes, and one that differs from `settings.scope` (issue #37): parameters are read with the settings of the run that wrote them, method and scope alike, so image-scope parameters are never read as biological-repeat or patch ones. | -| Validity | `Parameters` and `NormalizationParameters` refuse, when made, an offset that is not finite and a scale that is not finite and above 0, naming the channel or method; `Parameters.channels` and `select` take whole numbers 0 or above only (Python or NumPy integers; booleans, fractions, whole floats and text are refused, never truncated). Non-finite pixels stay non-finite in `apply`. | +| `fit` | One entry point for every method; dispatches on `settings.image_type` and `settings.method`. Statistics are pooled histograms per channel, or pixels; with `pooling="conditions"` the histograms carry their condition (§4.3). Channels are pooled by ID; a channel only some rows of the group have raises `ChannelScopeIncompleteError` unless `settings.partial_channels` is `"planes_with_channel"` (§1.5). Never reads files and never walks folders. | +| `apply` | Returns a new float32 array; the input is never modified. `channel_axis` gives the channel axis of `pixels` and `channels` the ID of each entry along it (each entry gets its own ID's parameters; for `rgb` it defaults to `R`, `G`, `B`); both may be left out for one channel. No clipping (N19). Non-finite pixels stay non-finite. Computed in float64 for fluorescence (§3.2) and with CSBDeep's float32 arithmetic for `rgb` (§3.3). | +| `select` | The parameters of some of the image's channels by ID, e.g. of a single-channel patch; `channels` keeps their IDs (N46). For `rgb` only all of `R`, `G` and `B` can be selected (§5: the three channels stay together); a subset is refused. | +| `to_columns` / `from_columns` | The columns of §8.1 as text in the format of §8.2; `from_columns(to_columns(p)) == p` exactly. `from_columns` first refuses a row whose `normalization_parameters_version` is missing or not `2` (§8.1). `to_columns` writes the pairs of `channels` only; `from_columns` reads every channel whose pair is filled and skips channels whose pair is blank (N46). `from_columns` refuses a `normalization_scope` that is not one of §4.1's scopes, and one that differs from `settings.scope` (issue #37): parameters are read with the settings of the run that wrote them, method and scope alike, so image-scope parameters are never read as biological-repeat or patch ones. | +| Validity | `Parameters` and `NormalizationParameters` refuse, when made, an offset that is not finite and a scale that is not finite and above 0, naming the channel or method; `Parameters.channels`, `select` and `apply` take canonical channel IDs only (text following the Token rule; an index, a boolean or other text is refused, never converted). Non-finite pixels stay non-finite in `apply`. | | `to_dict` / `from_dict` | JSON-compatible dictionaries (§8.3) that round-trip exactly. | | `image_summaries`, `intensity_checks` | §7. | @@ -542,27 +583,36 @@ Every MorphoBricks table with values per channel puts the channels in columns, n with `position_id`, `image_file`, `z`, `t`, `condition`, `biological_repeat`, `technical_repeat`, `source_relative_path`, `plane_id`, in this order. Labels are always read as text. -- **Per-channel columns** follow CellProfiler's naming (N42): - `__C`, with an optional `_` after the channel, e.g. - `Intensity_MinIntensity_C0` or `Histogram_Count_C1_00042`. The channel token is `C` and the - zero-based channel index, in the image's channel order; channel names are recorded in the - preprocessing brick's manifest, never in column names. Category and feature are written in - CamelCase and contain no underscore, so names split unambiguously on `_`. -- Columns are grouped by channel: every column of `C0`, then of `C1`, and so on. -- Every row of a table has the same channels: all images of an experiment have the same - number of channels (N43). +- **Per-channel columns** follow CellProfiler's naming (N42, channel token amended by ADR 0003 + HV2): `__`, with an optional `_` after the channel, e.g. + `Intensity_MinIntensity_DNA` or `Histogram_Count_PAX_488_00042`. The channel token is the + **canonical channel ID** from the experiment description, never the C index. Category and + feature are CamelCase without `_`, but an ID may contain `_`, so names are **not** split on + `_`: a reader takes the table's IDs from the parameter-less feature + `Histogram_ValidPixelCount_` and a bin index from the last `_NNNNN` of its column, and + checks the whole header against the layout it implies. +- Columns are grouped by channel, the channel IDs of the experiment in sorted order (Python + string order, code point by code point: `ACTIN` < `DNA` < `PAX_488` < `actin`; HV1 says + "sorted" and this is the order chosen, the same on every platform and locale). +- **Which IDs.** The experiment's channel IDs are the **declared** channels of the confirmed + description (the keys of its `[channels]` table, experiment-v1 §4.3), including a declared + channel no imaging unit lists (it is blank in every row, `v22`). They are never + `used_ids.channels` (§4.9), which also keeps retired IDs no image can have. The calling + brick passes this list; the library cannot tell the two apart and takes the list it is given. +- Rows may have different channels (N43 replaced by per-row presence, HV1): a channel a row + does not have is blank there, never zero. Common format: UTF-8, comma-separated, one header row, rows in a deterministic order, floats written at full precision (Python `repr`), blanks for missing values, written to a temporary file and replaced only once writing succeeded. -### 6.2 `plane_histograms.csv` (schema version 4) +### 6.2 `plane_histograms.csv` (schema version 5) One row per XYC plane: every channel of one image at one Z and T position (an RGB plane is one -row with channels C0, C1, C2), with each channel's histogram of §1.2.2. The histograms are -those of schema version 2 in the layout of version 3 (§1.3), with the underflow and overflow -columns of the robust range (§1.4, N56) added in version 4; with the default min-max range -they are zero. +row with channels R, G, B), with each channel's histogram of §1.2.2. The row count is the plane +count (N41). The histograms are those of schema version 4 (the underflow and overflow columns +of the robust range, §1.4, N56; zero with the default min-max range); version 5 names the +channels by canonical channel ID and adds the presence column (ADR 0003 HV1, HV2). **Plane columns** @@ -576,29 +626,46 @@ they are zero. | `plane_id` | text | `source_relative_path::z=…,t=…`, always with both coordinates (`control/img.tif::z=0,t=0` for a 2D image). Row key: unique in a table, and in the rows given to `scope_groups`, `fit` and `image_summaries`. | | `dtype` | text | NumPy dtype name, e.g. `uint16`; the same for every channel. | | `bit_depth` | integer | Storage bits (8, 16, 32, 64). | -| `channel_count` | integer | Number of channels; the same in every row. | +| `channel_count` | integer | Number of channels of the source image, the length of `channels`. | | `total_pixel_count` | integer | Pixels in one channel of the plane (height × width). | | `bin_count` | integer | Number of histogram bins per channel; the same in every row. | -| `schema_version` | text | `4`. Versions 1 and 2 had one row per plane and channel with a `c` column; version 3 had no underflow and overflow columns; a table of another version is recomputed on the next run. | +| `schema_version` | text | `5`. Versions 1 and 2 had one row per plane and channel with a `c` column; version 3 had no underflow and overflow columns; version 4 named the channels by C index; a table of another version is refused and recomputed on the next run, never converted (HV4). | +| `channels` | text | The presence column (HV1): the canonical channel IDs of the plane, in source order (entry *i* is source channel C*i*, the imaging unit's confirmed `channels` list), `;`-separated. Never blank. It is the last plane column, right after `schema_version` and before every per-channel column (the Spec fixes the column, not its place; this is the place). There is no `failed_channels` column. | -**Per-channel columns**, for each channel `i` from 0 to `channel_count − 1`, in this order: +**Per-channel columns**, for every channel ID of the experiment (given to the writer from the +description, so every parallel shard has the same header), in sorted order, in this order: | Column | Type | Meaning | |---|---|---| -| `Intensity_MinIntensity_C`, `Intensity_MaxIntensity_C` | number or blank | Exact finite extrema of the channel. Floats written at full precision. | -| `Histogram_ValidPixelCount_C` | integer | Finite pixels; equals the sum of the channel's count columns plus its underflow and overflow counts. | -| `Histogram_BinningMin_C`, `Histogram_BinningMax_C` | number or blank | Range the channel's bins cover: its extrema, or its robust limits (§1.4). | -| `Histogram_UnderflowCount_C`, `Histogram_UnderflowSum_C`, `Histogram_UnderflowSumOfSquares_C` | integer, number, number | Pixels below `Histogram_BinningMin_C`: their count, the sum of their values and of their squares (float64). Always written; `0`, `0.0`, `0.0` with the min-max range and for exact histograms. | -| `Histogram_OverflowCount_C`, `Histogram_OverflowSum_C`, `Histogram_OverflowSumOfSquares_C` | integer, number, number | The same for the pixels above `Histogram_BinningMax_C`. | -| `Histogram_Count_C_00000` … `Histogram_Count_C_` | integer | Pixel count per bin, the bin number written with five digits. | +| `Intensity_MinIntensity_`, `Intensity_MaxIntensity_` | number or blank | Exact finite extrema of the channel. Floats written at full precision. | +| `Histogram_ValidPixelCount_` | integer | Finite pixels; equals the sum of the channel's count columns plus its underflow and overflow counts. | +| `Histogram_BinningMin_`, `Histogram_BinningMax_` | number or blank | Range the channel's bins cover: its extrema, or its robust limits (§1.4). | +| `Histogram_UnderflowCount_`, `Histogram_UnderflowMean_`, `Histogram_UnderflowStd_` | integer, number, number | Pixels below `Histogram_BinningMin_`: their count, their mean and their population standard deviation, each the float64 nearest its exact value (centred moments, §1.4, N65). Always written; `0`, `0.0`, `0.0` with the min-max range and for exact histograms. | +| `Histogram_OverflowCount_`, `Histogram_OverflowMean_`, `Histogram_OverflowStd_` | integer, number, number | The same for the pixels above `Histogram_BinningMax_`. | +| `Histogram_Count__00000` … `Histogram_Count__` | integer | Pixel count per bin, the bin number written with five digits. | A 16-bit image with exact histograms has 65,536 count columns per channel, so the table is very wide; read it with `read_plane_histograms`, not a spreadsheet program. -The reader checks every channel: the counts add up to `Histogram_ValidPixelCount_C`, an +The reader checks every channel: the counts add up to `Histogram_ValidPixelCount_`, an empty tail has zero sums, the extrema enclose the binning limits, and a binning limit differs from its extremum exactly when its tail is not empty. +**Presence, absence and failure (HV1, HV3).** A channel the row lists in `channels` is +*present*: all its cells hold a measured histogram (a measured channel without finite pixels +is an empty histogram, `Histogram_ValidPixelCount_` 0 and blank limits, as before). A +channel of the header the row does not list is *absent*: every cell is blank; the reader +refuses any value there, so a zero or NaN histogram can never stand for an absent channel. +Absent is not failed: a failed measurement is not a state of the table. If any channel +measurement of the run failed, the writer is given the failures +(`ChannelMeasurementFailure`) and writes nothing, raising `ChannelMeasurementError` with every +failing plane and channel. A file already at the path (an earlier run's table) is **removed**, +and the error names it (`removed`): left in place, it would sit beside this run's other outputs +and read as this run's table. Nothing else is touched, and every other error leaves an existing +file as it was (the temporary-file write of §6.1). The reader also refuses a +`channels` cell that is blank, repeats an ID, names an ID the header lacks or disagrees with +`channel_count`, and planes of one image that list different channels. + The biological-repeat pairing, its confirmation and the controls are not part of this file; they are in the preprocessing brick's `experiment_design.csv` (follow-up P3). @@ -613,32 +680,32 @@ Computed from `plane_histograms.csv` rows alone (N38). `image_summaries(records, percentile=(1.0, 99.8))` returns one `ImageSummary` per image, sorted by condition, biological repeat, technical repeat and path, pooling all Z and T planes of the image with the group statistics of §1.2.3 (inverse-CDF percentiles, exact for 8- and 16-bit -histograms). Images with different numbers of channels are refused with `ValueError` (N43), and so is an image whose Z or T planes have different pixel +histograms). Each summary holds the channels of its image by ID (`ImageSummary.channels`, sorted); images may have different channels, and an image whose planes list different channels is refused with `ValueError`, as is an image whose Z or T planes have different pixel types (`IncompatibleSummariesError`, a `ValueError`). An `ImageSummary` whose per-channel tuples have different lengths is refused when made, and `intensity_checks` refuses summaries with different numbers of channels or different percentile definitions, with the same error. `ImageSummary.to_row()` gives the columns below as a dictionary, in this order (numbers as -floats, a blank as `None`, `Histogram_Exact_C` as a bool), and `summary_columns(n)` their -names for `n` channels, so the calling brick can show the rows as a table or write them; the +floats, a blank as `None`, `Histogram_Exact_` as a bool; a channel the image does not +have is not in the row and is blank in a table), and `summary_columns(channel_ids)` the names +for the experiment's channel IDs, so the calling brick can show the rows as a table or write them; the library writes no file (N50). The columns are the shared columns `source_relative_path`, -`condition`, `biological_repeat`, `technical_repeat`, `image_file`, and for each channel `i` +`condition`, `biological_repeat`, `technical_repeat`, `image_file`, and for each channel ID (§6.1): | Column | Meaning | |---|---| -| `Intensity_MedianIntensity_C` | Median | -| `Intensity_LowPercentile_C`, `Intensity_HighPercentile_C` | Default the 1st and 99.8th percentiles | -| `Intensity_SaturatedFraction_C` | Pixels equal to the type's maximum over valid pixels, for integer types whose histogram holds the maximum value in a bin of its own; blank otherwise (float images, or inexact top bins) | -| `Histogram_Exact_C` | Whether every pooled histogram of the channel was exact; blank (`None`) for a channel without finite pixels, which has no statistics to be exact | +| `Intensity_MedianIntensity_` | Median | +| `Intensity_LowPercentile_`, `Intensity_HighPercentile_` | Default the 1st and 99.8th percentiles | +| `Intensity_SaturatedFraction_` | Pixels equal to the type's maximum over valid pixels, for integer types whose histogram holds the maximum value in a bin of its own; blank otherwise (float images, or inexact top bins) | +| `Histogram_Exact_` | Whether every pooled histogram of the channel was exact; blank (`None`) for a channel without finite pixels, which has no statistics to be exact | `iter_image_summaries(records, percentile=(1.0, 99.8))` is the streaming form for large tables (issue #58): a generator that holds one image's planes at a time and yields the same `ImageSummary` that `image_summaries` gives for that image. It requires every Z and T row of an image to be contiguous in `records`; it never sorts or stages rows, and refuses (`ValueError`) an image whose rows are not together, a source file under two sets of labels, repeated -`plane_id`s and images with different numbers of channels. Summaries come in the order the -images first appear (the eager order for rows sorted by condition, repeat and path). Channel -counts are only compared as images are read, so an error can follow summaries already yielded: +`plane_id`s and an image whose planes list different channels. Summaries come in the order the +images first appear (the eager order for rows sorted by condition, repeat and path). Rows are only checked as images are read, so an error can follow summaries already yielded: a run that raised is incomplete and its output must be discarded. `image_summaries` is unchanged. The checks below call these values `median`, `p_low`, `p_high` and `saturated_fraction`, the @@ -651,7 +718,8 @@ separate from `intensity_checks`, whose four codes are unchanged. ### 7.2 Checks `intensity_checks(summaries, settings=None)` compares images only within one condition, so -differences between conditions are never reported. `settings` is an `IntensityCheckSettings` +differences between conditions are never reported. A channel is compared only among the +images that have it (by ID); an image without it is neither compared nor counted. `settings` is an `IntensityCheckSettings` with the limits below (N51), stored by the calling brick with `to_dict()` and read back with `from_dict()`; None uses the defaults. A robust deviation is `MAD × 1.482602218505602` of the values compared with, around their median (`numpy.median`). @@ -668,9 +736,9 @@ biological repeats, each is compared with the other, so a shift may be reported Each finding is one `Issue` (N52): one per image or repeat, channel and compared value (an image whose median and high percentile are both outliers gives two). Its message names the -image or repeat, the channel (`C`), the value and the group value it was compared with, or +image or repeat, the channel (its ID), the value and the group value it was compared with, or the limit; `items` holds the image's `source_relative_path`, or `/` for a repeat; `channel` holds the channel index, so the calling brick can mark the +repeat>` for a repeat; `channel` holds the channel ID, so the calling brick can mark the image in that channel's plots. The issues come in the order of the table above, each sorted by condition, biological repeat, image and channel. @@ -689,15 +757,24 @@ written (N23); the settings are written once in the calling brick's run configur | `normalization_scope` | text | `biological_repeat`, `image` or `patch` | | `normalization_group` | text | §4.1 group text | | `normalization_exact` | `true`/`false` | `true` exactly when every pooled histogram has `max_representative_error == 0` (§1.2.4), at every scope | -| `Normalization_Offset_C`, `Normalization_Scale_C` | number or blank | One pair per channel `i` of the image (§6.1, N41), grouped by channel. A row whose patch holds only some channels (CellPatcher's `split_channels`) fills the pairs of those channels and leaves the others blank. For `rgb`, the shared value in C0, C1 and C2. | +| `normalization_parameters_version` | integer | `2`: the version of these columns. Version 1, the `Normalization_Offset_C` columns of normalization 0.4 and earlier, had no such column; `from_columns` refuses a row without it or of another version and asks to fit again (no conversion, ADR 0002 AR10). Read as text or as the number a table reader gives (`2`, `2.0`). | +| `Normalization_Offset_`, `Normalization_Scale_` | number or blank | One pair per canonical channel ID (§6.1, N41; token by HV2), grouped by channel in sorted ID order. A row whose patch holds only some channels, or whose image lacks a channel of the experiment, fills the pairs of its channels and leaves the others blank. For `rgb`, the shared value in R, G and B. | + +**Channel order.** Every table with per-channel columns orders them by sorted channel ID (HV1's +rule for `plane_histograms.csv`, which HV2 extends to all per-channel tables), and +`Parameters.channels`, `offset` and `scale` follow the same order. For `rgb` that is `B`, `G`, +`R`, while a plane's `channels` cell, an RGB pixel array and `apply(..., channels=)` use the +source order `R`, `G`, `B` (experiment-v1 §4.4b). Every lookup is by ID, so the order never +selects a channel; one rule for every table was chosen over an RGB exception. ### 8.2 Formatting - Numbers at full precision (Python `repr` of the float), read back unchanged. -- `Normalization_Offset_C` and `Normalization_Scale_C` replace the `;`-separated - `normalization_offset` and `normalization_scale` columns (N41); readers of earlier tables - may still accept those (CellPatcher follow-up C1). `from_columns` reads only the new - columns, so CellPatcher converts earlier rows itself (N47). +- `from_columns` reads only the `Normalization_Offset_` and `Normalization_Scale_` + columns (N47, token by HV2), taking every `` that follows the Token rule, once + `normalization_parameters_version` is `2`. A column of the version-4 naming, + `Normalization_Offset_C0`, is indistinguishable from the ID `C0` (a legal Token); the version + column tells them apart, so an old row is refused, never read as channel `C0`. - Column order within a file is the calling brick's; readers select columns by name. ### 8.3 Run level @@ -714,8 +791,10 @@ The calling brick stores `NormalizationSettings.to_dict()` once per run: | `scope` | `image` | | `pooling` | `pixels` (`biological_repeat` only) | | `per_time_point` | `false` | +| `partial_channels` | `refuse`: a fit of a group in which only some planes have a channel is refused (`ChannelScopeIncompleteError`, the Spec's `A_SCOPE_INCOMPLETE`). `planes_with_channel`: the user selects, for each such channel, the planes that have it; written only when chosen, so the choice is recorded with the run. Fluorescence only (reset for `rgb`). | -Keys that do not apply to the method or scope are omitted. With `biological_repeat`, the +Keys that do not apply to the method or scope are omitted, and so is `partial_channels` at its +default. With `biological_repeat`, the calling brick also records which `experiment_design.csv` it used (name and SHA-256), and the statistics file's name, `schema_version` and SHA-256. Raw patches are normalized on load from the row's columns alone. @@ -736,6 +815,7 @@ Raised as exceptions before any pixel is normalized; the calling brick blocks th | A method other than `percentile` with `rgb` | `MethodNotAllowedError` | "RGB images are normalized with StarDist's percentile method only, which scales the three colour channels together. Use method 'percentile'." | | An unknown scope, including the removed `condition`, `technical_repeat` and `experiment` | `ValueError` | "Scope '{scope}' does not exist. Choose 'biological_repeat' (one biological repeat across all conditions), 'image' or 'patch'." | | `biological_repeat` without the statistics file | `MethodNotAllowedError` | "The biological-repeat scope needs the preprocessing brick's plane_histograms.csv, which was not given. Run the preprocessing brick on this experiment, or use the image or patch scope." | +| `rgb` rows or arrays whose channels are not `R`, `G`, `B` in this order | `ValueError` | Names the plane or array and the IDs found, and asks to declare the unit `rgb` or choose `fluorescence` | | `rgb` without exactly three channels, or with a pixel type other than an 8- or 16-bit integer (float, 32- or 64-bit integer) | `ValueError` | "RGB normalization needs images with exactly three 8- or 16-bit colour channels, but these images have {n} channels of type {dtype}. Check the axis mapping (the samples axis must be the channel axis), or choose image type 'fluorescence'." | | A `(condition, biological_repeat)` missing from `repeat_groups` | `ValueError` | Names the missing pairs and asks to update the experiment design in the preprocessing brick. | | Unsupported pixel type; bin count not allowed (§1.2) | `UnsupportedDtypeError`, `ValueError` | Existing messages | @@ -746,27 +826,30 @@ Raised as exceptions before any pixel is normalized; the calling brick blocks th | Statistics file row whose `plane_id`, `image_file` or `bit_depth` differs from what its `source_relative_path`, `z`, `t` and `dtype` give | `CsvFormatError` | Names the file, line, column, stored value and implied value, and asks to run the preprocessing brick again | | Statistics file with the columns but no rows | `CsvFormatError` | Names the file and asks to run the preprocessing brick again | | Two rows with the same `plane_id`: in the statistics file, or given to `write_plane_histograms`, `scope_groups`, `fit` or `image_summaries` | `CsvFormatError` (file), `ValueError` | Names the plane and both lines (file) or every repeated plane, and asks to remove the repeats or run the preprocessing brick again | -| A histogram or tail that breaks the invariants of §1.2.2 (negative or wrapping counts, valid above total, missing or disordered limits, impossible tail sums) | `ValueError`; `CsvFormatError` in the file | Names the rule and the values; in the file also the line and channel | +| A histogram or tail that breaks the invariants of §1.2.2 (negative or wrapping counts, valid above total, missing or disordered limits, impossible tail moments) | `ValueError`; `CsvFormatError` in the file | Names the rule and the values; in the file also the line and channel | | A `PlaneRecord` without `position_id` (None, blank or not text); in the file a row with `position_id` blank | `ValueError`; `CsvFormatError` in the file | Names the image, says every plane needs its position and asks to run the preprocessing brick again | | A plane coordinate `z` or `t` that is not a whole number 0 or above | `ValueError`; `CsvFormatError` in the file | Names the image, coordinate and value | | A block of another pixel type than the plane's | `ValueError` | Names the block and both pixel types | | `repeat_groups` with an empty or non-text label, or two biological repeats of one condition in one repeat group | `ValueError` | Names the pairs and the label, and asks to correct the experiment design in the preprocessing brick | | A group of more than 2**63 − 1 finite pixels | `ValueError` | Asks to pool fewer planes | -| Statistics file of schema version 1 or 2 (one row per channel) | `CsvFormatError` | "This plane_histograms.csv was written with schema version {version}, one row per channel. Run the preprocessing brick again on this experiment: it rewrites the table with the channels in columns (schema version 4)." | -| Statistics file of schema version 3 (N56) | `CsvFormatError` | "This plane_histograms.csv was written with schema version 3, without the underflow and overflow columns. Run the preprocessing brick again on this experiment: it rewrites the table with these columns (schema version 4)." | +| Statistics file of schema version 1, 2, 3 or 4 (HV4) | `CsvFormatError` | "This plane_histograms.csv was written with schema version {version}, {what}. This version reads schema version 5 only, with every channel named by its canonical channel ID from the experiment description; old tables are not converted. Run the preprocessing brick again on this experiment: it recomputes the table (schema version 5)." — `{what}` is "one row per channel" (1, 2), "its channels named by index and without the underflow and overflow columns" (3), "its channels named by their index in the image (C0, C1, ...)" (4) | +| A failed channel measurement given to `write_plane_histograms` (HV3) | `ChannelMeasurementError` | Names every failing plane and channel and its reason, says plane_histograms.csv was not written (and, when it removed an earlier run's table at the path, names it) and asks to fix the cause and run the preprocessing brick again | +| A value in a cell of a channel the row does not list; a `channels` cell blank, with a repeated or unknown ID, or not matching `channel_count`; planes of one image listing different channels | `CsvFormatError` (file), `ValueError` (records) | Names the file, line, channel and columns | +| A fitted group in which only some planes have a channel to fit, with `partial_channels="refuse"` (default; Spec `A_SCOPE_INCOMPLETE`) | `ChannelScopeIncompleteError` (a `ValueError`; `channels` maps each such ID to the planes that lack it) | "A_SCOPE_INCOMPLETE: in group '{group}', channel {ID} is present in {k} of {n} planes (absent from ...). ..." naming every such channel, and the three ways on: groups whose planes all have it, `channels=[...]` without it, or `partial_channels='planes_with_channel'` | +| A parameter row without `normalization_parameters_version`, or of another version (§8.1) | `ValueError` | Says the row was written by normalization 0.4 or earlier (or names its version), that old parameters are not converted, and asks to fit again | +| A record or writer channel that is not a canonical channel ID (an index included), a record channel not in the writer's `channel_ids`, a requested channel no plane of the group has | `ValueError` | Names the value or channel and what to do; an absent channel is never fitted from zeros | | `HistogramSettings` with an unknown range, invalid quantiles or `min_central_bins` below 1 | `ValueError` | Names the value and the allowed ones | -| Rows or images with different numbers of channels (N43) | `CsvFormatError` (table), `ValueError` (pixels) | Names the images and their channel counts, and says that every image of an experiment must have the same channels | | `from_columns` with a missing or unreadable column, an unknown `normalization_scope`, or a channel without both its offset and scale | `ValueError` | Names the column | | `from_columns` with a `normalization_method` or `normalization_scope` other than the settings' | `ValueError` | Names the column, the row's value and the settings' value, and asks to read the parameters with the settings of the run that wrote them | | `NormalizationParameters.apply` with a result type other than float32 or float64, or clip limits that are not two finite numbers with `low < high` | `ValueError` | Names the type ("dtype must be float32 or float64; got 'uint8'") or the limits ("clip must be (low, high) with low < high; got low=1.0, high=0.0") | -| `Parameters` or `NormalizationParameters` with a non-finite offset, or a scale that is not finite and above 0; a channel index that is not a whole number 0 or above | `ValueError` | Names the channel (or method) and the value | +| `Parameters` or `NormalizationParameters` with a non-finite offset, or a scale that is not finite and above 0; a channel that is not a canonical channel ID | `ValueError` | Names the channel (or method) and the value | -Tables of schema versions 1 to 3 are recognised by the `schema_version` value of their rows alone and +Tables of schema versions 1 to 4 are recognised by the `schema_version` value of their rows alone and refused with the messages above; no reader or converter of their layouts exists (the owner's ruling -of 10 November 2026: no backward compatibility before the first pipeline release). A version 1 to 3 -header with no rows has no such value and is refused by its columns ("does not have the -plane-histogram columns" for versions 1 and 2, "unexpected per-channel column names" for version 3), -without the pointed message. +of 10 November 2026: no backward compatibility before the first pipeline release; ADR 0003 HV4 for +version 4). A version 1 to 4 header with no rows has no such value and is refused by its columns +("does not have the plane-histogram columns" for versions 1 and 2, "does not have the per-channel +columns of schema version 5" for versions 3 and 4), without the pointed message. ### 9.2 Warnings @@ -778,6 +861,7 @@ Returned as `Issue` objects; processing continues and the calling brick reports | `repeat_missing_conditions` | A repeat group without images from every condition (§4.2) | | `approximate_parameters` | Parameters from inexact histograms (`normalization_exact = false`) | | `planes_without_finite_pixels` | Planes without finite pixels excluded from a group that still has others | +| `channel_partially_present` | With `partial_channels="planes_with_channel"` only: a fitted channel that only some planes of the group have (ADR 0003); its parameters come from those planes, as selected, the others are listed in `items` and `channel` names it. Without that choice the fit is refused (§9.1) | | `patches_discarded` | Patches without spread or finite pixels, listed in one warning with a suggested scope | | The four codes of §7.2 | Intensity checks | | `channel_without_finite_pixels` | An image channel without finite pixels, which the intensity checks cannot check (§7.1, `unchecked_channels`) | diff --git a/docs/plan.md b/docs/plan.md index 808a7f4..a80c5b5 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -163,7 +163,7 @@ The preprocessing notebook shows the warnings and draws the plots (follow-up P5) ## 7. Statistics -`plane_histograms.csv`, schema version 4 (N41, N56): one row per XYC plane, that is per +`plane_histograms.csv`, schema version 5 (N41, N56, N61–N64; version 4 until Stage 3): one row per XYC plane, that is per image, Z and T, with every channel's exact extrema, pixel counts and binned counts side by side in per-channel columns named as in CellProfiler (`Histogram_Count_C0_00000`, …, N42). The histograms themselves are those of schema version 2; version 3 changed the layout and @@ -262,6 +262,11 @@ their formula is implemented here. `csv_io` (version 3 refused); tails in `pooling`. Tested on the issue's reproduction, against `np.quantile(method="inverted_cdf")` for every wide pixel type, whole and in blocks. The preprocessing brick adds the setting and the warning (follow-up P6).* +- **Stage 3 S3.2 — channels by canonical channel ID (ADR 0003):** `plane_histograms.csv` + schema version 5, `PlaneRecord.channels`, fits, parameters, summaries and checks by channel + ID (N61–N64). *Pull request #90 (10 November 2026; rebased on #91 and #92 and finished on + 11 November 2026, the owner having decided to start Stage 3 now), not merged; see CHANGELOG + "Unreleased".* - **Release `v0.3.1`**, pinned by the other bricks. *Released (28 October 2026): Milestone 5. The tag `v0.3.0` points at the same code but its package reports version 0.2.0, because `pyproject.toml` was not updated before tagging; `v0.3.1` corrects the version and nothing @@ -491,6 +496,57 @@ Added with the project owner on 28 October 2026, for Milestone 5 (issue #12): listing the affected planes in `items`, with `Issue.channel` set; the `RangeCheck` objects keep the per-plane numbers for tables or plots. +Added with the project owner on 10 November 2026, for Stage 3 (MorphoBricks-Spec ADR 0003, +decisions HV1-HV4, taken on the S3.1 benchmark; implemented in S3.2): + +55. **N61** (HV1) `plane_histograms.csv` schema version 5 is wide: one row per XYC plane, + per-channel columns by canonical channel ID, one presence column `channels` (the plane's IDs + in source order, `;`-separated) and no `failed_channels` column; the header lists every + channel ID of the experiment, taken from the description, sorted. An absent channel's cells + are blank, never a zero or NaN histogram, and a reader refuses a value there; absent is not + failed. +56. **N62** (HV2) N41 is kept; N42's channel token is the canonical channel ID instead of the C + index, for every per-channel table: this table, the N46/N47 parameter columns + (`Normalization_Offset_`), CellPatcher's patch columns and the summary columns. Names are + not split on `_`: the IDs come from `Histogram_ValidPixelCount_`, a bin index from the + last `_NNNNN`. N43 is replaced by per-row presence. +57. **N63** (HV3) A failed channel measurement fails the whole `plane_histograms.csv`: it is + not written and the run reports the plane and channel. The table has no failed state. +58. **N64** (HV4) Version 4 is a hard cut: the version 5 reader refuses it, naming version 5 + and how to recompute; there is no converter (ADR 0002 AR10). + +Added with the project owner on 11 November 2026, for Stage 3 (S3.2, pull request #90): + +59. **N65** Schema version 5 stores each tail's **centred moments**: count, mean and + population standard deviation (`Histogram_UnderflowMean_`, `Histogram_UnderflowStd_` + and the same for `Overflow`), each the float64 nearest its exact value, instead of the raw + sum and sum of squares, which cancel for large values with a small spread and overflow + beyond about `1.3e154` (#91's question). Pooling adds `count · std²` and + `count · (mean − pooled mean)²` (contract §1.4). +60. **N66** A channel that only some planes of a fitted group have is **refused by default** + (`ChannelScopeIncompleteError`, the Spec's `A_SCOPE_INCOMPLETE`); the user may select the + planes that have it with `partial_channels="planes_with_channel"`, stored with the settings + and reported per fit (the notebook asks explicitly) (contract §1.5, §8.3). + +Implementation choices of S3.2 where the Spec leaves the detail open (open questions 1-7 of +pull request #90), each taken on the side the Spec states or the safer one, and listed in that +pull request for the owner's review (not numbered decisions until the owner confirms them): + +- **Header IDs** are the description's declared channels (`[channels]`), never + `used_ids.channels` (retired IDs); the calling brick passes them (contract §6.1). +- **Sort order** is code point order; the `channels` column is the last plane column, before + every per-channel column (contract §6.1, §6.2). +- **Partial presence inside one fitted group**: confirmed by the owner, now N66. +- **Parameter columns have their own version**, `normalization_parameters_version` = `2`; a row + without it (the C-index columns of 0.4) is refused, never read as channel `C0` (contract + §8.1). +- **A failed measurement removes an earlier table** at the output path, so no out-of-date table + looks current (contract §6.2). +- **Summary columns** are named by ID already in S3.2, since records are by ID (HV2). +- **RGB parameter order** follows the one rule of every per-channel table, sorted IDs (`B`, `G`, + `R`); source order `R`, `G`, `B` is kept where order means position: a plane's `channels` + cell, pixel arrays and `apply` (contract §8.1). + ## 13. Still to measure - **Pooling weights** on real data: whether equal weighting per condition changes the diff --git a/pyproject.toml b/pyproject.toml index d5f89c8..a2af597 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,8 @@ addopts = "-ra" line-length = 100 target-version = "py311" src = ["src", "tests", "tools"] +# The vendored Spec files are byte-exact copies (tests/spec_fixtures/README.md). +extend-exclude = ["tests/spec_fixtures"] [tool.ruff.lint] select = [ @@ -97,6 +99,7 @@ baseline = "tools/code_rules_baseline.json" # others together. normalization is a pure library except the reader and writer of the file. "morphobricks_normalization" = "orchestration" "morphobricks_normalization._version" = "pure" +"morphobricks_normalization.channels" = "pure" "morphobricks_normalization.checks" = "pure" "morphobricks_normalization.csv_io" = "io" "morphobricks_normalization.dtypes" = "pure" @@ -107,6 +110,7 @@ baseline = "tools/code_rules_baseline.json" "morphobricks_normalization.issues" = "pure" "morphobricks_normalization.normalization" = "pure" "morphobricks_normalization.parameter_checks" = "pure" +"morphobricks_normalization.parameters" = "pure" "morphobricks_normalization.pooling" = "pure" "morphobricks_normalization.ranges" = "pure" "morphobricks_normalization.records" = "pure" @@ -116,4 +120,5 @@ baseline = "tools/code_rules_baseline.json" "morphobricks_normalization.summaries" = "pure" "morphobricks_normalization.bin_centres" = "pure" "morphobricks_normalization.bin_reachability" = "pure" +"morphobricks_normalization.tail_moments" = "pure" "morphobricks_normalization.sums" = "pure" diff --git a/src/morphobricks_normalization/__init__.py b/src/morphobricks_normalization/__init__.py index bf6a650..87c1d71 100644 --- a/src/morphobricks_normalization/__init__.py +++ b/src/morphobricks_normalization/__init__.py @@ -8,10 +8,18 @@ """ from ._version import __version__ +from .channels import check_channel_id, check_channel_ids from .checks import IntensityCheckSettings, intensity_checks, unchecked_channels -from .csv_io import OUTPUT_FILENAME, read_plane_histograms, write_plane_histograms +from .csv_io import ( + OUTPUT_FILENAME, + CsvFormatError, + PlaneHistogramTable, + read_plane_histogram_table, + read_plane_histograms, + write_plane_histograms, +) from .dtypes import UnsupportedDtypeError, default_bin_count -from .fitting import Parameters, fit +from .fitting import ChannelScopeIncompleteError, Parameters, fit from .histogram import ( HistogramTail, PlaneHistogram, @@ -40,7 +48,15 @@ RangeCheck, range_check_issues, ) -from .records import SINGLE, PlaneRecord, group_records, make_plane_id +from .records import ( + SINGLE, + ChannelMeasurementError, + ChannelMeasurementFailure, + PlaneRecord, + check_measurements, + group_records, + make_plane_id, +) from .rgb import apply_rgb, rgb_percentile from .scopes import group_key, group_name, scope_groups from .settings import MethodNotAllowedError, NormalizationSettings, check_settings @@ -55,6 +71,10 @@ __all__ = [ "OUTPUT_FILENAME", "SINGLE", + "ChannelMeasurementError", + "ChannelMeasurementFailure", + "ChannelScopeIncompleteError", + "CsvFormatError", "HistogramSettings", "HistogramTail", "ImageSummary", @@ -68,6 +88,7 @@ "NormalizationSettings", "Parameters", "PlaneHistogram", + "PlaneHistogramTable", "PlaneRecord", "PooledHistogram", "PooledStatistics", @@ -76,6 +97,9 @@ "WeightedHistogram", "__version__", "apply_rgb", + "check_channel_id", + "check_channel_ids", + "check_measurements", "check_settings", "compute_block_histogram", "compute_plane_histogram", @@ -93,6 +117,7 @@ "percentile_range", "pooled_statistics", "range_check_issues", + "read_plane_histogram_table", "read_plane_histograms", "rgb_percentile", "robust_z_score", diff --git a/src/morphobricks_normalization/channels.py b/src/morphobricks_normalization/channels.py new file mode 100644 index 0000000..b62fd6c --- /dev/null +++ b/src/morphobricks_normalization/channels.py @@ -0,0 +1,95 @@ +"""Canonical channel IDs: the channel token of every per-channel column (ADR 0003 HV2). + +A channel is named by its canonical channel ID from the confirmed experiment description +(MorphoBricks experiment-v1 §3.1, §4.3), never by its index in the image: the same index can +name different stains in two imaging units (Spec fixture ``v04``), and a unit can lack a +channel another unit has (``v05``, ``v17``). An ID follows the Spec's Token rule +(experiment-v1 §3.2), ``^[A-Za-z][A-Za-z0-9]*(?:_[A-Za-z0-9]+)*$``; it may contain ``_`` +(``PAX_488``), so column names are never split on ``_`` (:mod:`~morphobricks_normalization.csv_io`). + +IDs are compared exactly: nothing is trimmed, re-cased or normalized (experiment-v1 §3.3). +Which IDs exist and which channel of a file each one names is the calling brick's knowledge +(the description); this module only checks that a value is a well-formed ID. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable + +# experiment-v1 §3.2, the Token rule. It also keeps ``;`` (the separator of the ``channels`` +# cell) and ``,`` out of an ID. +CHANNEL_ID_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9]*(?:_[A-Za-z0-9]+)*") + + +def check_channel_id(channel: object, where: str = "") -> str: + """Return ``channel`` if it is a canonical channel ID, else refuse it. + + Parameters + ---------- + channel + The value to check. + where + Starts the message, e.g. ``"Plane a.tif::z=0,t=0"``; may be empty. + + Returns + ------- + str + ``channel``, unchanged. + + Raises + ------ + ValueError + For an index (an int), a value that is not text, or text that is not a Token. + """ + prefix = f"{where}: " if where else "" + if isinstance(channel, str) and CHANNEL_ID_PATTERN.fullmatch(channel): + return channel + if not isinstance(channel, str): + raise ValueError( + f"{prefix}a channel is named by its canonical channel ID from the experiment " + f"description (for example 'DNA'), never by its position in the image; got " + f"{channel!r}. Give the channel IDs of the imaging unit the image belongs to." + ) + raise ValueError( + f"{prefix}{channel!r} is not a canonical channel ID: an ID starts with a letter and " + "holds letters and digits, with single '_' between parts (for example 'DNA' or " + "'PAX_488'). Use the channel IDs of the confirmed experiment description." + ) + + +def check_channel_ids(channels: Iterable[object], where: str = "") -> tuple[str, ...]: + """Return channel IDs as a tuple, in the given order, refusing bad or repeated IDs. + + Parameters + ---------- + channels + The IDs to check; text, not a single string. + where + Starts the message; may be empty. + + Returns + ------- + tuple of str + The IDs, unchanged and in order. + + Raises + ------ + ValueError + For a single string instead of a list, an ID that :func:`check_channel_id` + refuses, or an ID given twice. + """ + prefix = f"{where}: " if where else "" + if isinstance(channels, str): + raise ValueError( + f"{prefix}expected a list of channel IDs, got the text {channels!r}; give " + f"[{channels!r}] for one channel." + ) + ids = tuple(check_channel_id(channel, where) for channel in channels) + repeated = sorted({channel for channel in ids if ids.count(channel) > 1}) + if repeated: + raise ValueError( + f"{prefix}channel(s) {', '.join(repeated)} are listed more than once; each " + "channel of an image has one ID, and each ID names one channel." + ) + return ids diff --git a/src/morphobricks_normalization/checks.py b/src/morphobricks_normalization/checks.py index 3f9d045..5c9a506 100644 --- a/src/morphobricks_normalization/checks.py +++ b/src/morphobricks_normalization/checks.py @@ -20,7 +20,12 @@ (identical values) skips the comparison instead of firing. A comparison with fewer than ``min_images`` values is skipped (N53). Each finding is one :class:`~morphobricks_normalization.issues.Issue` naming the image or repeat, -the channel (also in ``Issue.channel``), the value and the group value (N52). +the channel by its canonical ID (also in ``Issue.channel``), the value and the +group value (N52). + +Images may have different channels (ADR 0003): a channel is compared only among +the images that have it, by channel ID, never by position in the image, and an +image without the channel is neither compared nor counted. """ from __future__ import annotations @@ -43,7 +48,6 @@ Issue, ) from .normalization import NORMAL_CONSISTENCY -from .records import different_channel_counts from .summaries import ImageSummary, IncompatibleSummariesError _KEYS = ("library_version", "outlier_limit", "saturation_limit", "min_images") @@ -179,7 +183,6 @@ def intensity_checks( """ settings = IntensityCheckSettings() if settings is None else settings summaries = sorted(summaries, key=_order) - _check_comparable(summaries) _check_same_percentiles(summaries) return [ *_image_outliers(summaries, settings), @@ -214,14 +217,14 @@ def unchecked_channels(summaries: Iterable[ImageSummary]) -> list[Issue]: Issue( WARNING, CHANNEL_WITHOUT_FINITE_PIXELS, - f"Image {summary.source_relative_path}, channel C{channel}: it has no finite pixels " + f"Image {summary.source_relative_path}, channel {channel}: it has no finite pixels " "(every value is NaN or infinite), so its intensities were not checked.", items=(summary.source_relative_path,), action=_UNASSESSED_ACTION, channel=channel, ) for summary in sorted(summaries, key=_order) - for channel, median in enumerate(summary.median) + for channel, median in zip(summary.channels, summary.median, strict=True) if median is None ] @@ -233,14 +236,14 @@ def _image_outliers(summaries: list[ImageSummary], settings: IntensityCheckSetti found = [] for (condition, repeat), members in groups.items(): high = f"{_ordinal(members[0].percentile[1])} percentile" - for channel in range(members[0].channel_count): + for channel in _channel_ids(members): for order, (statistic, name) in enumerate( (("median", "median intensity"), ("p_high", high)) ): present = [ - (s, getattr(s, statistic)[channel]) + (s, value) for s in members - if getattr(s, statistic)[channel] is not None + if (value := _value(s, statistic, channel)) is not None ] if len(present) < settings.min_images: continue @@ -251,7 +254,7 @@ def _image_outliers(summaries: list[ImageSummary], settings: IntensityCheckSetti continue reference, deviation, distance = comparison message = ( - f"Image {summary.source_relative_path}, channel C{channel}: its {name} " + f"Image {summary.source_relative_path}, channel {channel}: its {name} " f"{_text(value)} is {distance:.3g} robust deviations from " f"{_text(reference)}, the median of the other {len(others)} images of " f"condition '{condition}', biological repeat '{repeat}' (robust " @@ -273,6 +276,10 @@ def _image_outliers(summaries: list[ImageSummary], settings: IntensityCheckSetti def _repeat_shifts(summaries: list[ImageSummary], settings: IntensityCheckSettings) -> list[Issue]: + """Return a ``repeat_intensity_shift`` warning per repeat and channel ID far from the rest. + + Per condition, each channel is compared among the images that have it. + """ conditions: dict[str, dict[str, list[ImageSummary]]] = {} for summary in summaries: repeats = conditions.setdefault(summary.condition, {}) @@ -282,9 +289,13 @@ def _repeat_shifts(summaries: list[ImageSummary], settings: IntensityCheckSettin if len(repeats) < 2: continue for repeat in repeats: - for channel in range(summaries[0].channel_count): + for channel in _channel_ids([s for members in repeats.values() for s in members]): medians = { - label: [s.median[channel] for s in members if s.median[channel] is not None] + label: [ + value + for s in members + if (value := _value(s, "median", channel)) is not None + ] for label, members in repeats.items() } own = medians[repeat] @@ -298,7 +309,7 @@ def _repeat_shifts(summaries: list[ImageSummary], settings: IntensityCheckSettin reference, deviation, distance = comparison message = ( f"Biological repeat '{repeat}' of condition '{condition}', channel " - f"C{channel}: the median of its {len(own)} image medians, {_text(value)}, is " + f"{channel}: the median of its {len(own)} image medians, {_text(value)}, is " f"{distance:.3g} robust deviations from {_text(reference)}, the median of " f"the {len(others)} image medians of the other biological repeats of " f"'{condition}' (robust deviation {_text(deviation)}; limit " @@ -322,12 +333,12 @@ def _saturated_images( ) -> list[Issue]: issues = [] for summary in summaries: - for channel, fraction in enumerate(summary.saturated_fraction): + for channel, fraction in zip(summary.channels, summary.saturated_fraction, strict=True): if fraction is None or not fraction > settings.saturation_limit: continue top = np.iinfo(summary.dtype).max message = ( - f"Image {summary.source_relative_path}, channel C{channel}: {100 * fraction:.3g} % " + f"Image {summary.source_relative_path}, channel {channel}: {100 * fraction:.3g} % " f"of its pixels are at the maximum of its pixel type, {top} ({summary.dtype}), " f"above the limit of {100 * settings.saturation_limit:g} %." ) @@ -345,14 +356,17 @@ def _saturated_images( def _no_spread(summaries: list[ImageSummary]) -> list[Issue]: + """Return a ``no_intensity_spread`` warning per image channel whose percentiles are equal.""" issues = [] for summary in summaries: low, high = (_ordinal(q) for q in summary.percentile) - for channel, (p_low, p_high) in enumerate(zip(summary.p_low, summary.p_high, strict=True)): + for channel, p_low, p_high in zip( + summary.channels, summary.p_low, summary.p_high, strict=True + ): if p_low is None or p_low != p_high: continue message = ( - f"Image {summary.source_relative_path}, channel C{channel}: its {low} and {high} " + f"Image {summary.source_relative_path}, channel {channel}: its {low} and {high} " f"percentiles are both {_text(p_low)}, so this channel has no intensity spread " "and cannot be normalized at the image scope with a percentile method." ) @@ -386,14 +400,16 @@ def _compare(value: float, others: list[float], limit: float) -> tuple[float, fl return (reference, deviation, distance) if distance > limit else None -def _check_comparable(summaries: list[ImageSummary]) -> None: - """Refuse summaries that do not describe the same channels (issue #56).""" - if len({s.channel_count for s in summaries}) > 1: - raise IncompatibleSummariesError( - different_channel_counts( - [(s.source_relative_path, s.channel_count) for s in summaries], "These images" - ) - ) +def _channel_ids(summaries: list[ImageSummary]) -> list[str]: + """Every channel ID of ``summaries``, sorted.""" + return sorted({channel for s in summaries for channel in s.channels}) + + +def _value(summary: ImageSummary, statistic: str, channel: str) -> float | None: + """``summary``'s value of ``statistic`` for ``channel``; None when absent or blank.""" + if channel not in summary.channels: + return None + return getattr(summary, statistic)[summary.channels.index(channel)] def _check_same_percentiles(summaries: list[ImageSummary]) -> None: diff --git a/src/morphobricks_normalization/csv_io.py b/src/morphobricks_normalization/csv_io.py index 36d92c8..9e1efb8 100644 --- a/src/morphobricks_normalization/csv_io.py +++ b/src/morphobricks_normalization/csv_io.py @@ -1,16 +1,29 @@ -"""Read and write the ``plane_histograms.csv`` table. +"""Read and write the ``plane_histograms.csv`` table, schema version 5 (ADR 0003). One row per XYC plane: every channel of one image at one Z and T position (contract §6.2, N41). The plane columns come first: the shared MorphoBricks columns (see ``docs/contract.md``), then ``dtype``, ``bit_depth``, -``channel_count``, ``total_pixel_count``, ``bin_count`` and ``schema_version``. -Then, for each channel ``i`` in turn, its columns named as in CellProfiler -(N42): ``Intensity_MinIntensity_C``, ``Intensity_MaxIntensity_C``, -``Histogram_ValidPixelCount_C``, ``Histogram_BinningMin_C``, -``Histogram_BinningMax_C``, the underflow and overflow columns of schema -version 4 (``Histogram_UnderflowCount_C``, ``Histogram_UnderflowSum_C``, -``Histogram_UnderflowSumOfSquares_C`` and the same three for ``Overflow``, -contract §1.4) and one column per bin (``Histogram_Count_C_00000`` ...). +``channel_count``, ``total_pixel_count``, ``bin_count`` and ``schema_version``, +then the presence column ``channels``: the canonical channel IDs of the plane, in +source order, ``;``-separated (HV1). Then, for every channel ID of the experiment +(taken from the description and given to the writer), sorted, its columns named +as in CellProfiler with the ID as the channel token (HV2): +``Intensity_MinIntensity_``, ``Intensity_MaxIntensity_``, +``Histogram_ValidPixelCount_``, ``Histogram_BinningMin_``, +``Histogram_BinningMax_``, the underflow and overflow columns +(``Histogram_UnderflowCount_``, ``Histogram_UnderflowMean_``, +``Histogram_UnderflowStd_`` and the same three for ``Overflow``, +contract §1.4) and one column per bin (``Histogram_Count__00000`` ...). + +A channel a plane does not list is *absent*: all its cells are blank, and the +reader refuses a value there (absent is not zero and not failed). A failed +measurement is not a state of the table: the writer refuses to write it at all +(HV3). A table of an earlier schema version is refused with the fix (HV4); it is +never converted. + +An ID may contain ``_`` (``PAX_488``), so column names are never split on +``_``: the reader takes the IDs from the parameter-less +``Histogram_ValidPixelCount_`` columns and requires the exact layout above. Labels are always read back as text, so folder names such as ``NA``, ``001`` or ``TRUE`` keep their exact spelling. Float values are written at full @@ -25,17 +38,28 @@ import tempfile from collections.abc import Iterable from pathlib import Path +from typing import NamedTuple import numpy as np +from .channels import check_channel_ids from .dtypes import bit_depth, check_supported, is_integer from .histogram import EMPTY_TAIL, HistogramTail, PlaneHistogram -from .records import PlaneRecord, different_channel_counts, duplicate_planes +from .records import ( + ChannelMeasurementError, + ChannelMeasurementFailure, + PlaneRecord, + check_measurements, + check_universe, + duplicate_planes, + same_channels_per_image, +) -# Version 4: the underflow and overflow columns of the robust histogram range -# (N56). Version 3 put one row per XYC plane, the channels in columns (N41); -# versions 1 and 2 had one row per plane and channel, with a ``c`` column. -SCHEMA_VERSION = "4" +# Version 5: channels named by canonical channel ID, with the presence column (ADR 0003). +# Version 4 added the underflow and overflow columns (N56); version 3 put one row per XYC +# plane with the channels in columns by C index (N41); versions 1 and 2 had one row per +# plane and channel, with a ``c`` column. +SCHEMA_VERSION = "5" OUTPUT_FILENAME = "plane_histograms.csv" METADATA_COLUMNS = ( @@ -55,6 +79,9 @@ "bin_count", "schema_version", ) +# The presence column (HV1), right after the plane columns. +CHANNELS_COLUMN = "channels" +CHANNEL_SEPARATOR = ";" # Per-channel columns before the bins, as _; the channel # token is appended by channel_column. CHANNEL_FEATURES = ( @@ -64,33 +91,51 @@ "Histogram_BinningMin", "Histogram_BinningMax", "Histogram_UnderflowCount", - "Histogram_UnderflowSum", - "Histogram_UnderflowSumOfSquares", + "Histogram_UnderflowMean", + "Histogram_UnderflowStd", "Histogram_OverflowCount", - "Histogram_OverflowSum", - "Histogram_OverflowSumOfSquares", + "Histogram_OverflowMean", + "Histogram_OverflowStd", ) BIN_COLUMN_PREFIX = "Histogram_Count" +# The parameter-less feature whose columns name the table's channel IDs (HV2). +_ID_FEATURE = "Histogram_ValidPixelCount" # Tables of these versions are refused by their ``schema_version`` value alone, with a message # that names the fix; no reader of their layouts is kept (they would have to be recomputed). -_EARLIER_VERSIONS = ("1", "2", "3") +_EARLIER_VERSIONS = ("1", "2", "3", "4") class CsvFormatError(ValueError): """Raised when a file does not follow the plane-histogram CSV format.""" -def channel_column(feature: str, channel: int) -> str: - """Column name of a per-channel feature, e.g. ``Intensity_MinIntensity_C0``.""" - return f"{feature}_C{channel}" +class PlaneHistogramTable(NamedTuple): + """A ``plane_histograms.csv`` read back: the header's channel IDs and the rows. + + Attributes + ---------- + channel_ids + Every channel ID of the header (the experiment's), sorted. + records + One :class:`~morphobricks_normalization.records.PlaneRecord` per row, in file + order; each holds the channels its row lists as present. + """ + channel_ids: tuple[str, ...] + records: list[PlaneRecord] -def bin_column(channel: int, index: int) -> str: - """Column name of bin ``index`` of ``channel``, e.g. ``Histogram_Count_C1_00042``.""" - return f"{BIN_COLUMN_PREFIX}_C{channel}_{index:05d}" +def channel_column(feature: str, channel: str) -> str: + """Column name of a per-channel feature, e.g. ``Intensity_MinIntensity_DNA``.""" + return f"{feature}_{channel}" -def channel_columns(channel: int, bin_count: int) -> list[str]: + +def bin_column(channel: str, index: int) -> str: + """Column name of bin ``index`` of ``channel``, e.g. ``Histogram_Count_DNA_00042``.""" + return f"{BIN_COLUMN_PREFIX}_{channel}_{index:05d}" + + +def channel_columns(channel: str, bin_count: int) -> list[str]: """Every column of ``channel`` in table order: the features, then the bins.""" return [ *(channel_column(feature, channel) for feature in CHANNEL_FEATURES), @@ -98,16 +143,79 @@ def channel_columns(channel: int, bin_count: int) -> list[str]: ] -def write_plane_histograms(records: Iterable[PlaneRecord], path: str | Path) -> Path: +def table_header(channel_ids: Iterable[str], bin_count: int) -> list[str]: + """Return the header of a table of these channel IDs (sorted here) and bin count. + + Every writer of a parallel shard gives the experiment's full list, so shards share + one header (ADR 0003, details). + """ + header = [*METADATA_COLUMNS, CHANNELS_COLUMN] + for channel in sorted(check_channel_ids(channel_ids, "The table's channel IDs")): + header.extend(channel_columns(channel, bin_count)) + return header + + +def header_channels(header: list[str]) -> tuple[tuple[str, ...], int] | None: + """Return the channel IDs (sorted) and bin count of a version-5 header, or None. + + The IDs are the tokens of the ``Histogram_ValidPixelCount_`` columns (HV2); the + header must then be exactly :func:`table_header` of those IDs and some bin count of + at least 1, so a bin index is the last ``_NNNNN`` of its column. + + Parameters + ---------- + header + The header row. + + Returns + ------- + tuple or None + ``(channel_ids, bin_count)``, or None when the header is not a version-5 header. + """ + n_meta = len(METADATA_COLUMNS) + 1 + if tuple(header[: n_meta - 1]) != METADATA_COLUMNS or header[n_meta - 1 : n_meta] != [ + CHANNELS_COLUMN + ]: + return None + prefix = f"{_ID_FEATURE}_" + found = [name[len(prefix) :] for name in header[n_meta:] if name.startswith(prefix)] + try: + ids = check_channel_ids(found) + except ValueError: + return None + columns = len(header) - n_meta + if not ids or columns % len(ids): + return None + n_bins = columns // len(ids) - len(CHANNEL_FEATURES) + if n_bins < 1 or header != table_header(ids, n_bins): + return None + return tuple(sorted(ids)), n_bins + + +def write_plane_histograms( + records: Iterable[PlaneRecord], + path: str | Path, + *, + channel_ids: Iterable[str], + failures: Iterable[ChannelMeasurementFailure] = (), +) -> Path: """Write records to a CSV file, replacing it only once writing succeeded. Parameters ---------- records - Plane records; all must have the same number of channels (N43) and use - the same bin count. + Plane records, in the order to write; all use the same bin count. Their + channels may differ (ADR 0003 HV1). path Destination file. + channel_ids + Every channel ID of the experiment, from the confirmed description; the header + lists them sorted. Give every shard of a run the same list, so shards share one + header. Each record's channels must be among them. + failures + The run's failed channel measurements (HV3). When there is any, nothing is + written, and a file already at ``path`` (an earlier run's table) is removed, so + that no out-of-date table is left that looks current (contract §6.2). Returns ------- @@ -116,38 +224,40 @@ def write_plane_histograms(records: Iterable[PlaneRecord], path: str | Path) -> Raises ------ + ChannelMeasurementError + When ``failures`` is not empty, naming each failing plane and channel and the + earlier table it removed (``removed``); checked before anything else. ValueError - If there are no records, two records have the same ``plane_id``, or their - channel counts or bin counts differ. + If there are no records, two records have the same ``plane_id``, their bin + counts differ, ``channel_ids`` is not a list of distinct channel IDs, a record + has a channel that is not in ``channel_ids``, or the planes of one image list + different channels. """ + _refuse_failures(failures, Path(path)) + universe = check_channel_ids(channel_ids, "The table's channel IDs") + if not universe: + raise ValueError("Give the experiment's channel IDs (channel_ids); the list is empty.") records = list(records) if not records: raise ValueError("There are no planes to write.") - channel_counts = {r.channel_count for r in records} - if len(channel_counts) != 1: - raise ValueError( - different_channel_counts( - [(r.source_relative_path, r.channel_count) for r in records], "These images" - ) - ) bin_counts = {r.histograms[0].bin_count for r in records} if len(bin_counts) != 1: raise ValueError( f"All planes in one table must use the same bin count; found {sorted(bin_counts)}." ) + check_universe(records, set(universe)) duplicate_planes(records, "The planes to write") + same_channels_per_image(records, "The planes to write") path = Path(path) - n_channels, n_bins = channel_counts.pop(), bin_counts.pop() - header = [*METADATA_COLUMNS] - for channel in range(n_channels): - header.extend(channel_columns(channel, n_bins)) + ids = tuple(sorted(universe)) + header = table_header(ids, bin_counts.pop()) handle, temporary = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) try: with os.fdopen(handle, "w", newline="", encoding="utf-8") as fh: writer = csv.writer(fh) writer.writerow(header) for record in records: - writer.writerow(_row(record)) + writer.writerow(_row(record, ids)) os.replace(temporary, path) except BaseException: Path(temporary).unlink(missing_ok=True) @@ -155,20 +265,51 @@ def write_plane_histograms(records: Iterable[PlaneRecord], path: str | Path) -> return path +def _refuse_failures(failures: Iterable[ChannelMeasurementFailure], path: Path) -> None: + """Raise for failed measurements, after removing an earlier table at ``path`` (HV3). + + An earlier run's table left in place would read as this run's, so it is removed and + the error names it (``ChannelMeasurementError.removed``). + """ + try: + check_measurements(failures) + except ChannelMeasurementError as error: + if not path.is_file(): + raise + path.unlink() + raise ChannelMeasurementError(error.failures, removed=str(path)) from None + + def read_plane_histograms(path: str | Path) -> list[PlaneRecord]: - """Read a plane-histogram CSV written by :func:`write_plane_histograms`. + """Read the rows of a plane-histogram CSV written by :func:`write_plane_histograms`. + + The records of :func:`read_plane_histogram_table`; use that function for the + header's channel IDs as well. + + Raises + ------ + CsvFormatError + As :func:`read_plane_histogram_table`. + """ + return read_plane_histogram_table(path).records + + +def read_plane_histogram_table(path: str | Path) -> PlaneHistogramTable: + """Read a plane-histogram CSV: its channel IDs and its rows. Raises ------ CsvFormatError If the header, schema version or a row does not match the format, including a row whose ``plane_id``, ``image_file`` or ``bit_depth`` differs from what its - path, coordinates and pixel type give, or whose statistics break the rules - of :mod:`~morphobricks_normalization.integrity`; if a row says the table was written - with an earlier schema version (1 and 2 had one row per channel, 3 no underflow and - overflow columns), which is refused by that value alone, without reading the layout; - if it has no rows, or two rows with the same ``plane_id``; or if its rows have - different numbers of channels. + path, coordinates and pixel type give, whose ``channels`` is blank, repeats an + ID, names an ID the header does not have or does not have ``channel_count`` + entries, which has a value in a cell of a channel it does not list (absent), or + whose statistics break the rules of :mod:`~morphobricks_normalization.integrity`; + if a row says the table was written with an earlier schema version (1 to 4), + which is refused by that value alone, without reading the layout, with the + message naming version 5 and how to recompute it (HV4); if it has no rows, two + rows with the same ``plane_id``, or planes of one image with different channels. """ with open(path, newline="", encoding="utf-8") as fh: reader = csv.reader(fh) @@ -182,55 +323,38 @@ def read_plane_histograms(path: str | Path) -> list[PlaneRecord]: if index < len(first) and first[index] in _EARLIER_VERSIONS: raise CsvFormatError(_earlier_version_message(path, first[index])) reader = itertools.chain([] if first is None else [first], reader) - n_meta = len(METADATA_COLUMNS) - if tuple(header[:n_meta]) != METADATA_COLUMNS: + if tuple(header[: len(METADATA_COLUMNS)]) != METADATA_COLUMNS: raise CsvFormatError(f"{path} does not have the plane-histogram columns.") - layout = _channel_layout(header[n_meta:]) + layout = header_channels(header) if layout is None: - raise CsvFormatError(f"{path} has unexpected per-channel column names.") - n_channels, n_bins = layout - records, other_counts, lines = [], [], {} + raise CsvFormatError( + f"{path} does not have the per-channel columns of schema version " + f"{SCHEMA_VERSION} (the '{CHANNELS_COLUMN}' column, then for every channel ID " + "in sorted order its features and bins). Run the preprocessing brick again on " + "this experiment to rewrite the table." + ) + ids, n_bins = layout + records, lines = [], {} for line, row in enumerate(reader, 2): - record = _record(row, n_channels, n_bins, path, line) - if not isinstance(record, PlaneRecord): - other_counts.append(record) - continue + record = _record(row, ids, n_bins, path, line) if record.plane_id in lines: raise CsvFormatError(_duplicate_message(path, record.plane_id, lines, line)) lines[record.plane_id] = line records.append(record) - if other_counts: - listed = [(r.source_relative_path, r.channel_count) for r in records] + other_counts - raise CsvFormatError(f"{path}: " + different_channel_counts(listed, "Its images")) if not records: raise CsvFormatError( f"{path} has the plane-histogram columns but no rows, so it holds no planes. The " "table is incomplete; run the preprocessing brick again on this experiment." ) - return records - - -def _channel_layout(columns: list[str]) -> tuple[int, int] | None: - # The bins of channel 0 give the bin count; the header must then hold the - # same columns for a whole number of channels, in order. None otherwise. - prefix = f"{BIN_COLUMN_PREFIX}_C0_" - n_bins = sum(name.startswith(prefix) for name in columns) - width = len(CHANNEL_FEATURES) + n_bins - n_channels = len(columns) // width - expected = [ - name - for c in range(n_channels) - for name in ( - *(channel_column(feature, c) for feature in CHANNEL_FEATURES), - *(bin_column(c, i) for i in range(n_bins)), - ) - ] - if n_bins == 0 or n_channels == 0 or columns != expected: - return None - return n_channels, n_bins + try: + same_channels_per_image(records, f"{path}: its rows") + except ValueError as error: + raise CsvFormatError(str(error)) from None + return PlaneHistogramTable(ids, records) -def _row(record: PlaneRecord) -> list[str]: +def _row(record: PlaneRecord, ids: tuple[str, ...]) -> list[str]: + """Return the cells of one row: the plane columns, then every channel of ``ids``.""" first = record.histograms[0] row = [ record.position_id, @@ -248,8 +372,14 @@ def _row(record: PlaneRecord) -> list[str]: str(first.total_pixel_count), str(first.bin_count), SCHEMA_VERSION, + CHANNEL_SEPARATOR.join(record.channels), ] - for hist in record.histograms: + width = len(CHANNEL_FEATURES) + first.bin_count + for channel in ids: + if not record.has_channel(channel): + row.extend([""] * width) # absent: blank, never a zero histogram (HV1) + continue + hist = record.channel_histogram(channel) row.extend( ( _text(hist.pixel_min), @@ -266,14 +396,14 @@ def _row(record: PlaneRecord) -> list[str]: def _record( - row: list[str], n_channels: int, n_bins: int, path: str | Path, line: int -) -> PlaneRecord | tuple[str, int]: - """Read one row; a row with another channel count is returned as ``(path, count)``.""" - n_meta = len(METADATA_COLUMNS) + row: list[str], ids: tuple[str, ...], n_bins: int, path: str | Path, line: int +) -> PlaneRecord: + """Read one row into a record of the channels it lists.""" + n_meta = len(METADATA_COLUMNS) + 1 width = len(CHANNEL_FEATURES) + n_bins - if len(row) != n_meta + n_channels * width: - raise CsvFormatError(f"{path}, line {line}: expected {n_meta + n_channels * width} values.") - meta = dict(zip(METADATA_COLUMNS, row[:n_meta], strict=True)) + if len(row) != n_meta + len(ids) * width: + raise CsvFormatError(f"{path}, line {line}: expected {n_meta + len(ids) * width} values.") + meta = dict(zip((*METADATA_COLUMNS, CHANNELS_COLUMN), row[:n_meta], strict=True)) if meta["schema_version"] != SCHEMA_VERSION: if meta["schema_version"] in _EARLIER_VERSIONS: raise CsvFormatError(_earlier_version_message(path, meta["schema_version"])) @@ -282,33 +412,35 @@ def _record( f"(this version reads {SCHEMA_VERSION!r})." ) try: - channel_count = int(meta["channel_count"]) - if channel_count != n_channels: - return meta["source_relative_path"], channel_count + present = _present_channels(meta, ids) dtype = check_supported(meta["dtype"]) if int(meta["bin_count"]) != n_bins: raise ValueError(f"its bin_count is {meta['bin_count']}, but it has {n_bins} bins.") - histograms = [] - for channel in range(n_channels): - values = row[n_meta + channel * width : n_meta + (channel + 1) * width] + by_channel = {} + for k, channel in enumerate(ids): + values = row[n_meta + k * width : n_meta + (k + 1) * width] + if channel not in present: + _check_absent(values, channel, n_bins) + continue try: total = meta["total_pixel_count"] - histograms.append(_histogram(values, dtype, n_bins, total, channel)) + by_channel[channel] = _histogram(values, dtype, n_bins, total, channel) except ValueError as error: raise CsvFormatError( - f"{path}, line {line}: the bin counts of channel {channel} are inconsistent: " - f"{error} The table was changed after it was written; run the preprocessing " - "brick again on this experiment to rewrite it." + f"{path}, line {line}: the bin counts of channel {channel} are " + f"inconsistent: {error} The table was changed after it was written; run " + "the preprocessing brick again on this experiment to rewrite it." ) from error record = PlaneRecord( source_relative_path=meta["source_relative_path"], condition=meta["condition"], biological_repeat=meta["biological_repeat"], technical_repeat=meta["technical_repeat"], - histograms=histograms, + histograms=[by_channel[channel] for channel in present], z=int(meta["z"]), t=int(meta["t"]), position_id=meta["position_id"], + channels=present, ) # These columns are written from the others, never read into the record, so a # row whose stored values disagree would otherwise pass with another identity. @@ -330,8 +462,49 @@ def _record( raise CsvFormatError(f"{path}, line {line}: {error}") from error +def _present_channels(meta: dict[str, str], ids: tuple[str, ...]) -> tuple[str, ...]: + """Return the row's ``channels``: present IDs in source order, checked against the header.""" + text = meta[CHANNELS_COLUMN] + if text == "": + raise ValueError( + f"its {CHANNELS_COLUMN} cell is blank: every plane has at least one channel, listed " + "by canonical channel ID." + ) + present = check_channel_ids(text.split(CHANNEL_SEPARATOR), f"its {CHANNELS_COLUMN} cell") + unknown = [channel for channel in present if channel not in ids] + if unknown: + raise ValueError( + f"its {CHANNELS_COLUMN} cell lists {', '.join(unknown)}, which the header does not " + f"have (it has {', '.join(ids)})." + ) + if meta["channel_count"] != str(len(present)): + raise ValueError( + f"its channel_count is {meta['channel_count']!r}, but its {CHANNELS_COLUMN} cell " + f"lists {len(present)} channel(s) ({text})." + ) + return present + + +def _check_absent(values: list[str], channel: str, n_bins: int) -> None: + """Refuse any value in the cells of a channel the row does not list (HV1).""" + filled = [ + name + for name, value in zip(channel_columns(channel, n_bins), values, strict=True) + if value != "" + ] + if filled: + shown = ", ".join(filled[:3]) + (f" and {len(filled) - 3} more" if len(filled) > 3 else "") + raise ValueError( + f"channel {channel} is absent from this plane (its {CHANNELS_COLUMN} cell does not " + f"list it), but {shown} {'holds a value' if len(filled) == 1 else 'hold values'}. " + "An absent channel has blank cells: it is not measured, and is never a zero or NaN " + "histogram. The table was changed after it was written; run the preprocessing " + "brick again on this experiment to rewrite it." + ) + + def _histogram( - values: list[str], dtype: np.dtype, n_bins: int, total: str, channel: int + values: list[str], dtype: np.dtype, n_bins: int, total: str, channel: str ) -> PlaneHistogram: """Read one channel's columns; the histogram checks its own invariants. @@ -343,7 +516,7 @@ def _histogram( blank = [ channel_column(feature, channel) for feature, value in zip(CHANNEL_FEATURES[:5], values[:5], strict=True) - if value == "" and feature != "Histogram_ValidPixelCount" + if value == "" and feature != _ID_FEATURE ] if blank and int(valid) > 0: raise ValueError( @@ -379,7 +552,7 @@ def _duplicate_message(path: str | Path, plane_id: str, lines: dict[str, int], l def _tail_text(tail: HistogramTail) -> tuple[str, str, str]: - return str(tail.count), repr(float(tail.sum)), repr(float(tail.sum_of_squares)) + return str(tail.count), repr(float(tail.mean)), repr(float(tail.std)) def _tail(texts: list[str]) -> HistogramTail: @@ -387,23 +560,28 @@ def _tail(texts: list[str]) -> HistogramTail: return EMPTY_TAIL if tail.is_empty else tail -def _earlier_version_message(path, version: str) -> str: - # The messages of contract §9.1, after the file's path. - if version == "3": - return ( - f"{path}: This plane_histograms.csv was written with schema version 3, without the " - "underflow and overflow columns. Run the preprocessing brick again on this " - "experiment: it rewrites the table with these columns (schema version " - f"{SCHEMA_VERSION})." - ) +def _earlier_version_message(path: str | Path, version: str) -> str: + """Return the refusal of a table of an earlier schema version (contract §9.1). + + A table is recomputed, never converted: the channel IDs come from the confirmed + description (HV4, ADR 0002 AR10). + """ + if version == "4": + what = "its channels named by their index in the image (C0, C1, ...)" + elif version == "3": + what = "its channels named by index and without the underflow and overflow columns" + else: + what = "one row per channel" return ( - f"{path}: This plane_histograms.csv was written with schema version {version}, one row " - "per channel. Run the preprocessing brick again on this experiment: it rewrites the " - f"table with the channels in columns (schema version {SCHEMA_VERSION})." + f"{path}: This plane_histograms.csv was written with schema version {version}, {what}. " + f"This version reads schema version {SCHEMA_VERSION} only, with every channel named by " + "its canonical channel ID from the experiment description; old tables are not " + "converted. Run the preprocessing brick again on this experiment: it recomputes the " + f"table (schema version {SCHEMA_VERSION})." ) -def _text(value) -> str: +def _text(value: object) -> str: if value is None: return "" if isinstance(value, float): @@ -411,5 +589,5 @@ def _text(value) -> str: return str(value) -def _value(text: str, number: type): +def _value(text: str, number: type) -> object: return None if text == "" else number(text) diff --git a/src/morphobricks_normalization/fitting.py b/src/morphobricks_normalization/fitting.py index 12fb3b9..6a18fe3 100644 --- a/src/morphobricks_normalization/fitting.py +++ b/src/morphobricks_normalization/fitting.py @@ -13,23 +13,33 @@ :mod:`~morphobricks_normalization.normalization`, applied to each channel separately. The RGB method is StarDist's percentile normalization of :mod:`~morphobricks_normalization.rgb`: the three channels are pooled together -and share one offset and scale. Parameters are stored as one -``Normalization_Offset_C`` and ``Normalization_Scale_C`` pair per channel -(contract §8.1, N41, N42); for RGB the shared value is written for C0, C1 and C2. +and share one offset and scale. + +Channels are addressed by canonical channel ID, never by their index in the image +(ADR 0003 HV2). Rows of one group may have different channels; a channel that only some +rows of the group have is refused (:class:`ChannelScopeIncompleteError`, the Spec's +``A_SCOPE_INCOMPLETE``) unless the settings record the user's choice to pool it over the +rows that have it (``partial_channels="planes_with_channel"``). Parameters are stored as one +``Normalization_Offset_`` and ``Normalization_Scale_`` pair per channel +(contract §8.1, N41, N42 as amended); for RGB the shared value is written for R, G +and B. """ from __future__ import annotations -import math -import re -from collections.abc import Iterable, Mapping -from dataclasses import dataclass, field, replace -from typing import Any +from collections.abc import Iterable import numpy as np +from .channels import check_channel_ids from .histogram import PlaneHistogram, compute_plane_histogram -from .issues import APPROXIMATE_PARAMETERS, PLANES_WITHOUT_FINITE_PIXELS, WARNING, Issue +from .issues import ( + APPROXIMATE_PARAMETERS, + CHANNEL_PARTIALLY_PRESENT, + PLANES_WITHOUT_FINITE_PIXELS, + WARNING, + Issue, +) from .normalization import ( NormalizationParameters, min_max, @@ -37,296 +47,42 @@ robust_z_score, z_score, ) -from .parameter_checks import channel_indices, check_offset_scale, check_scope +from .parameters import COLUMNS, Parameters, channel_axis_index, offset_column, scale_column from .pooling import NoValidPixelsError, PooledHistogram, WeightedHistogram -from .records import PlaneRecord, different_channel_counts, duplicate_planes +from .records import PlaneRecord, duplicate_planes from .rgb import ( - RGB_CHANNEL_COUNT, - apply_rgb, + RGB_CHANNELS, + check_rgb_channels, check_rgb_pixels, check_rgb_records, check_rgb_selection, rgb_percentile, ) -from .settings import SCOPES, NormalizationSettings, check_settings - -# Columns written once per row; the per-channel pairs follow (offset_column, -# scale_column). -COLUMNS = ( - "normalization_method", - "normalization_scope", - "normalization_group", - "normalization_exact", -) -OFFSET_COLUMN_PREFIX = "Normalization_Offset" -SCALE_COLUMN_PREFIX = "Normalization_Scale" -_CHANNEL_COLUMN = re.compile(r"Normalization_(?:Offset|Scale)_C(0|[1-9][0-9]*)") - - -def offset_column(channel: int) -> str: - """Column of channel ``channel``'s offset, e.g. ``Normalization_Offset_C0``.""" - return f"{OFFSET_COLUMN_PREFIX}_C{channel}" - - -def scale_column(channel: int) -> str: - """Column of channel ``channel``'s scale, e.g. ``Normalization_Scale_C0``.""" - return f"{SCALE_COLUMN_PREFIX}_C{channel}" - - -@dataclass(frozen=True) -class Parameters: - """Normalization of one group: ``(pixels - offset[k]) / scale[k]`` for its ``k``-th channel. - - Attributes - ---------- - method - Written method name, e.g. ``"percentile 1-99.8"`` (contract §3.1). - scope - ``"biological_repeat"``, ``"image"`` or ``"patch"``. - group - The ``normalization_group`` text (contract §4.1), or None when the - calling brick has not named the group. - exact - Whether every pooled histogram was exact. - offset, scale - One value per channel, in the order of ``channels``: finite offsets, - finite scales above zero. - channels - The image's channel indices the parameters belong to, in increasing - order; by default ``0, 1, ...``. A single-channel patch of channel 2 of - an image has ``channels == (2,)`` (contract §8.1). Whole numbers only: - booleans, fractions and text are refused, never truncated. - issues - Warnings about these parameters (``approximate_parameters``, - ``planes_without_finite_pixels``); not stored in the columns and not - compared. +from .settings import NormalizationSettings, check_settings + +# Re-exported: the parameter columns are part of this module's interface (contract §1.3). +__all__ = [ + "COLUMNS", + "ChannelScopeIncompleteError", + "Parameters", + "fit", + "offset_column", + "scale_column", +] +_MAX_LISTED = 5 + + +class ChannelScopeIncompleteError(ValueError): + """A channel to fit is absent from some planes of the group (Spec ``A_SCOPE_INCOMPLETE``). + + Raised by :func:`fit` unless the settings select the planes that have the channel + (``partial_channels="planes_with_channel"``). ``channels`` maps each such channel ID to + the ``plane_id`` of every plane of the group that lacks it. """ - method: str - scope: str - group: str | None - exact: bool - offset: tuple[float, ...] - scale: tuple[float, ...] - channels: tuple[int, ...] | None = None - issues: tuple[Issue, ...] = field(default=(), compare=False, repr=False) - - def __post_init__(self) -> None: - object.__setattr__(self, "offset", tuple(float(value) for value in self.offset)) - object.__setattr__(self, "scale", tuple(float(value) for value in self.scale)) - if not self.offset or len(self.offset) != len(self.scale): - raise ValueError( - f"Expected one offset and one scale per channel; got {len(self.offset)} " - f"offset(s) and {len(self.scale)} scale(s)." - ) - channels = range(len(self.offset)) if self.channels is None else self.channels - channels = channel_indices(channels) - increasing = all(b > a for a, b in zip((-1, *channels), channels, strict=False)) - if len(channels) != len(self.offset) or not increasing: - raise ValueError( - "Expected one channel index per offset, from 0 upwards in increasing order; " - f"got channels {channels} for {len(self.offset)} offset(s)." - ) - object.__setattr__(self, "channels", channels) - for channel, offset, scale in zip(channels, self.offset, self.scale, strict=True): - check_offset_scale(offset, scale, f"Channel {channel}") - - @property - def channel_count(self) -> int: - """Number of channels the parameters normalize.""" - return len(self.offset) - - @property - def image_type(self) -> str: - """``"rgb"`` for StarDist's RGB method (``rgb percentile ...``), else ``"fluorescence"``.""" - return "rgb" if self.method.startswith("rgb ") else "fluorescence" - - def select(self, channels: Iterable[int]) -> Parameters: - """Return the parameters of some of the channels only. - - For example ``parameters.select([2])`` gives the parameters of a - single-channel patch of channel 2, whose columns fill only - ``Normalization_Offset_C2`` and ``Normalization_Scale_C2`` (contract §8.1). - - Raises - ------ - ValueError - If a channel is not among :attr:`channels`, or is not a whole number; for RGB - parameters, if not all of R, G and B are selected (issue #50). - """ - wanted = sorted(set(channel_indices(channels))) - missing = [c for c in wanted if c not in self.channels] - if not wanted: - raise ValueError("Select at least one channel of these parameters.") - if missing: - raise ValueError( - f"These parameters have channel(s) {list(self.channels)}; channel(s) " - f"{missing} cannot be selected." - ) - if self.image_type == "rgb": - check_rgb_selection(wanted, self.channels) - index = [self.channels.index(c) for c in wanted] - return replace( - self, - offset=tuple(self.offset[i] for i in index), - scale=tuple(self.scale[i] for i in index), - channels=tuple(wanted), - ) - - def apply(self, pixels: np.ndarray, *, channel_axis: int | None = None) -> np.ndarray: - """Return the normalized pixels as a new float32 array. - - Computed in float64 as ``(pixels - offset) / scale`` and returned as - float32, without clipping; non-finite pixels stay non-finite and the - input is never modified. RGB parameters repeat CSBDeep's float32 - arithmetic instead, so the result is bit-identical to StarDist's - normalization (:func:`~morphobricks_normalization.rgb.apply_rgb`, N48). - - Parameters - ---------- - pixels - Pixel values of the group. - channel_axis - Axis of ``pixels`` that holds the channels, one entry per channel of - :attr:`channels` in that order; may be left out when the parameters - have one channel. - - Raises - ------ - ValueError - If the channel axis is missing or does not hold one entry per channel. - """ - values = np.asarray(pixels) - if self.image_type == "fluorescence": - values = values.astype(np.float64, copy=False) - if channel_axis is None: - if self.channel_count != 1: - raise ValueError( - f"These parameters have {self.channel_count} channels; give channel_axis, " - "the axis of the channels in the pixel array." - ) - offset, scale = self.offset[0], self.scale[0] - else: - axis = _axis_index(channel_axis, values.ndim) - if values.shape[axis] != self.channel_count: - raise ValueError( - f"The pixel array has {values.shape[axis]} channel(s) along axis " - f"{channel_axis}, but these parameters have {self.channel_count}." - ) - shape = [1] * values.ndim - shape[axis] = self.channel_count - offset = np.asarray(self.offset).reshape(shape) - scale = np.asarray(self.scale).reshape(shape) - if self.image_type == "rgb": - return apply_rgb(values, offset, scale) - return ((values - offset) / scale).astype(np.float32, copy=False) - - def to_columns(self) -> dict[str, str]: - """Return the columns of contract §8.1 as text, numbers in full. - - The four ``normalization_*`` columns, then ``Normalization_Offset_C`` - and ``Normalization_Scale_C`` for each channel ``i`` of - :attr:`channels`, grouped by channel. Columns of other channels are not - returned; a table holding rows of other channels leaves them blank. - """ - columns = { - "normalization_method": self.method, - "normalization_scope": self.scope, - "normalization_group": self.group or "", - "normalization_exact": "true" if self.exact else "false", - } - for channel, offset, scale in zip(self.channels, self.offset, self.scale, strict=True): - columns[offset_column(channel)] = repr(offset) - columns[scale_column(channel)] = repr(scale) - return columns - - @classmethod - def from_columns( - cls, columns: Mapping[str, Any], settings: NormalizationSettings - ) -> Parameters: - """Read parameters back from their columns; ``from_columns(p.to_columns()) == p``. - - Parameters - ---------- - columns - A row with the four ``normalization_*`` columns and the - ``Normalization_Offset_C`` and ``Normalization_Scale_C`` pair of - at least one channel; other columns are ignored. Values may be text - or numbers. A channel whose two columns are both blank or absent is - not part of the row's parameters (a single-channel patch fills only - its own channel). - settings - The settings of the run that wrote the row; its method must be the - row's ``normalization_method`` and its scope the row's - ``normalization_scope`` (one of the scopes of contract §4.1), so that - parameters fitted for one sampling population are never read as - another's (issue #37). - - Raises - ------ - ValueError - Naming the column that is missing or cannot be read, whose method - or scope differs from the settings, or whose scope is unknown; also - for a channel with only one of its two columns. - """ - missing = [name for name in COLUMNS if name not in columns] - if missing: - raise ValueError( - f"The normalization column(s) {', '.join(missing)} are missing. Read the " - "parameters from a table written with normalization." - ) - method = _text(columns["normalization_method"]) - if method != settings.method_name: - raise ValueError( - f"Column normalization_method is {method!r}, but the settings use " - f"{settings.method_name!r}. Read the parameters with the settings of the run " - "that wrote them." - ) - indices = sorted( - {int(match[1]) for name in columns if (match := _CHANNEL_COLUMN.fullmatch(str(name)))} - ) - if not indices: - raise ValueError( - f"The columns {offset_column(0)} and {scale_column(0)} are missing: the row has " - "no Normalization_Offset_C and Normalization_Scale_C columns. Read the " - "parameters from a table written with normalization." - ) - channels, offset, scale = [], [], [] - for channel in indices: - names = (offset_column(channel), scale_column(channel)) - texts = [_text(columns.get(name)) for name in names] - if texts == ["", ""]: - continue # a channel this row does not hold - if "" in texts: - blank, given = names if texts[0] == "" else names[::-1] - raise ValueError( - f"Column {blank} is blank or missing, but {given} holds a value; every " - "channel of a row needs both its offset and its scale." - ) - channels.append(channel) - offset.append(_number(columns, names[0])) - scale.append(_number(columns, names[1])) - if scale[-1] <= 0: - raise ValueError( - f"Column {names[1]} must hold a positive number; got {texts[1]!r}." - ) - if not channels: - raise ValueError( - "Every Normalization_Offset_C and Normalization_Scale_C column of this row " - "is blank, so it holds no parameters. Read the parameters from a row written " - "with normalization." - ) - if settings.image_type == "rgb": - _check_rgb_columns(channels, offset, scale) - return cls( - method=method, - scope=check_scope(_text(columns["normalization_scope"]), settings.scope, SCOPES), - group=_text(columns["normalization_group"]) or None, - exact=_flag(columns, "normalization_exact"), - offset=tuple(offset), - scale=tuple(scale), - channels=tuple(channels), - ) + def __init__(self, message: str, channels: dict[str, tuple[str, ...]]) -> None: + super().__init__(message) + self.channels = channels def fit( @@ -335,16 +91,19 @@ def fit( *, group: str | None = None, channel_axis: int | None = None, + channels: Iterable[str] | None = None, ) -> Parameters: - """Fit the settings' method to one group. + """Fit the settings' method to one group, channel by channel ID. Parameters ---------- statistics - The ``plane_histograms.csv`` rows of one group (each row an XYC plane; - each channel gets its own parameters, in channel order, or for RGB the - R, G and B values of every row are pooled together), or a pixel array - of one image or patch (image and patch scopes only). + The ``plane_histograms.csv`` rows of one group (each row an XYC plane), or a + pixel array of one image or patch (image and patch scopes only). Rows may + have different channels (ADR 0003): a channel is pooled by its canonical channel + ID, never by its position in the image. A channel only some rows have is refused + unless ``settings.partial_channels`` is ``"planes_with_channel"``; for RGB the R, + G and B values of every row are pooled together. settings The run's settings. group @@ -353,69 +112,59 @@ def fit( channel_axis For a pixel array, the axis that holds the channels; None for one channel. + channels + For rows: the channel IDs to fit; default every channel some row of the + group has (so a channel some rows lack is refused by default: leave it out by + listing the others, or opt in with ``partial_channels``). For a pixel array + (required): the channel ID of each entry along ``channel_axis``, in order, or + the one ID of an array without a channel axis. Returns ------- parameters : Parameters - The group's parameters, with ``issues`` listing ``approximate_parameters`` and - ``planes_without_finite_pixels`` warnings. + The group's parameters, channels in sorted ID order, with ``issues`` listing + ``approximate_parameters``, ``planes_without_finite_pixels`` and + ``channel_partially_present`` warnings. Raises ------ MethodNotAllowedError As :func:`~morphobricks_normalization.settings.check_settings`; also for the biological-repeat scope with pixels instead of rows. + ChannelScopeIncompleteError + If a channel to fit is absent from some rows of the group and + ``settings.partial_channels`` is ``"refuse"`` (default); names every such channel + and the planes that lack it ("present in k of n planes"). NoValidPixelsError If a channel of the group has no finite pixel. ValueError - If a fluorescence channel's scale is zero or not finite, the rows have - different numbers of channels (N43) or repeat a ``plane_id``, or RGB images - do not have exactly three channels of an integer type (contract §5). + If a fluorescence channel's scale is zero or not finite, the rows repeat a + ``plane_id``, a requested channel is in no row of the group, the channel IDs of + a pixel array are missing or do not match it, or RGB images do not have the + channels R, G and B of an integer type (contract §5). """ from_pixels = isinstance(statistics, np.ndarray) check_settings(settings, statistics_file=not from_pixels) rgb = settings.image_type == "rgb" - if rgb and from_pixels: # refuse before any histogram is built (#52) - check_rgb_pixels(statistics, channel_axis) if from_pixels: - # One histogram per channel of the array, pooled like the rows' channels. - items: list[PlaneRecord] | list[PlaneHistogram] = _pixel_histograms( - statistics, channel_axis - ) - channel_count = len(items) - excluded: list[str] = [] + items, wanted, issues = _pixel_items(statistics, channel_axis, channels, rgb) else: if channel_axis is not None: raise ValueError("channel_axis applies to pixel arrays only, not to histogram rows.") - items = _records(statistics, group) - channel_count = items[0].channel_count - if rgb: # refuse before the rows are pooled - check_rgb_records(items, group) - excluded = [ - f"{record.plane_id} (channel {channel})" - for record in items - for channel, histogram in enumerate(record.histograms) - if histogram.is_empty - ] - if rgb: - # Contract §3.3: R, G and B pooled together give one shared offset and scale. - channel_sets = [tuple(range(channel_count))] - else: - channel_sets = [(channel,) for channel in range(channel_count)] + items, wanted, issues = _record_items(statistics, channels, group, settings) + # Contract §3.3: R, G and B pooled together give one shared offset and scale. + channel_sets = [RGB_CHANNELS] if rgb else [(channel,) for channel in wanted] offset, scale, exact = [], [], True - for channels in channel_sets: - if rgb: - where = f"Group '{group}', channels R, G and B" if group else "Channels R, G and B" - else: - where = f"Group '{group}', channel {channels[0]}" if group else f"Channel {channels[0]}" + for pooled_ids in channel_sets: + where = _where(group, pooled_ids, rgb) try: - pooled = _pool(items, settings, channels) + pooled = _pool(items, settings, pooled_ids) except NoValidPixelsError as exc: raise NoValidPixelsError(f"{where}: {exc}") from exc if rgb: shared = rgb_percentile(pooled, *settings.percentile) - offset += [shared[0]] * len(channels) - scale += [shared[1]] * len(channels) + offset += [shared[0]] * len(wanted) + scale += [shared[1]] * len(wanted) else: try: parameters = _channel_parameters(pooled, settings) @@ -431,10 +180,58 @@ def fit( exact=exact, offset=tuple(offset), scale=tuple(scale), - issues=_fit_issues(group, exact, excluded), + channels=wanted, + issues=(*issues[0], *_fit_issues(group, exact, issues[1])), ) +def _pixel_items( + pixels: np.ndarray, channel_axis: int | None, channels: Iterable[str] | None, rgb: bool +) -> tuple[dict[str, PlaneHistogram], tuple[str, ...], tuple[list[Issue], list[str]]]: + """Return a pixel array's histograms by channel ID, the IDs to fit and no issues.""" + if rgb: # refuse before any histogram is built (#52) + check_rgb_pixels(pixels, channel_axis) + # An RGB unit's samples 0, 1, 2 are R, G and B by the description (§4.4b). + channels = RGB_CHANNELS if channels is None else channels + ids = _pixel_channel_ids(pixels, channel_axis, channels) + if rgb: + check_rgb_channels(ids, "The pixel array") + # One histogram per channel of the array, pooled like the rows' channels. + by_id = dict(zip(ids, _pixel_histograms(pixels, channel_axis), strict=True)) + return by_id, tuple(sorted(by_id)), ([], []) + + +def _record_items( + statistics: Iterable[PlaneRecord], + channels: Iterable[str] | None, + group: str | None, + settings: NormalizationSettings, +) -> tuple[list[PlaneRecord], tuple[str, ...], tuple[list[Issue], list[str]]]: + """Return the rows, the IDs to fit, partial-presence warnings and planes left out.""" + rgb = settings.image_type == "rgb" + items = _records(statistics, group) + if rgb: # refuse before the rows are pooled + check_rgb_records(items, group) + wanted = _wanted_channels(items, channels, group) + if rgb: + check_rgb_selection(list(wanted), tuple(sorted(RGB_CHANNELS))) + excluded = [ + f"{record.plane_id} (channel {channel})" + for record in items + for channel, histogram in zip(record.channels, record.histograms, strict=True) + if channel in wanted and histogram.is_empty + ] + lacking = {} if rgb else _lacking_planes(items, wanted) + if lacking and settings.partial_channels == "refuse": + raise _scope_incomplete(lacking, len(items), group) + return items, wanted, (_partial_presence(lacking, len(items), group), excluded) + + +def _where(group: str | None, channels: tuple[str, ...], rgb: bool) -> str: + what = "channels R, G and B" if rgb else f"channel {channels[0]}" + return f"Group '{group}', {what}" if group else what[0].upper() + what[1:] + + def _records(statistics: Iterable[PlaneRecord], group: str | None) -> list[PlaneRecord]: records = list(statistics) if not records: @@ -445,39 +242,142 @@ def _records(statistics: Iterable[PlaneRecord], group: str | None) -> list[Plane "or a pixel array." ) duplicate_planes(records, f"The rows of group '{group}'" if group else "The rows of this group") - if len({record.channel_count for record in records}) > 1: - subject = f"The images of group '{group}'" if group else "The images of this group" + return records + + +def _wanted_channels( + records: list[PlaneRecord], channels: Iterable[str] | None, group: str | None +) -> tuple[str, ...]: + """Return the channel IDs to fit, sorted: the requested ones, or every ID of the rows.""" + present = {channel for record in records for channel in record.channels} + if channels is None: + return tuple(sorted(present)) + wanted = check_channel_ids(channels, "fit") + if not wanted: + raise ValueError("Give at least one channel ID to fit, or leave channels out.") + absent = [channel for channel in wanted if channel not in present] + if absent: + name = f"group '{group}'" if group else "this group" raise ValueError( - different_channel_counts( - [(r.source_relative_path, r.channel_count) for r in records], subject - ) + f"No plane of {name} has channel(s) {', '.join(absent)} (its planes have " + f"{', '.join(sorted(present))}), so there is nothing to fit for them. An absent " + "channel is never fitted from zeros; fit it in a group whose images have it." ) - return records + return tuple(sorted(wanted)) + + +def _lacking_planes( + records: list[PlaneRecord], wanted: tuple[str, ...] +) -> dict[str, tuple[str, ...]]: + """Map each wanted channel some rows lack to the ``plane_id`` of those rows.""" + lacking = { + channel: tuple(record.plane_id for record in records if not record.has_channel(channel)) + for channel in wanted + } + return {channel: planes for channel, planes in lacking.items() if planes} + + +def _listed(planes: tuple[str, ...]) -> str: + """Return the first planes of a list, then how many more.""" + shown = ", ".join(planes[:_MAX_LISTED]) + more = f" and {len(planes) - _MAX_LISTED} more" if len(planes) > _MAX_LISTED else "" + return shown + more + + +def _scope_incomplete( + lacking: dict[str, tuple[str, ...]], total: int, group: str | None +) -> ChannelScopeIncompleteError: + """Return the refusal of channels some planes of the group lack (A_SCOPE_INCOMPLETE).""" + name = f"group '{group}'" if group else "this group" + parts = "; ".join( + f"channel {channel} is present in {total - len(planes)} of {total} planes (absent " + f"from {_listed(planes)})" + for channel, planes in lacking.items() + ) + return ChannelScopeIncompleteError( + f"A_SCOPE_INCOMPLETE: in {name}, {parts}. A channel is not fitted over part of a group " + "without your choice, and an absent channel is never filled in. Fit groups whose planes " + "all have the channel (for example one biological repeat that acquired it), leave the " + "channel out with channels=[...], or select the planes that have it with " + "NormalizationSettings(partial_channels='planes_with_channel'), which is stored with " + "the run's settings.", + lacking, + ) + + +def _partial_presence( + lacking: dict[str, tuple[str, ...]], total: int, group: str | None +) -> list[Issue]: + """One warning per channel fitted from the planes that have it (the user's subset).""" + name = f"group '{group}'" if group else "this group" + return [ + Issue( + WARNING, + CHANNEL_PARTIALLY_PRESENT, + f"Channel {channel} is absent from {len(planes)} of the {total} planes of {name} " + f"({_listed(planes)}); as selected (partial_channels='planes_with_channel'), its " + f"parameters come from the {total - len(planes)} planes that have it. Nothing is " + "filled in for the others.", + items=planes, + action="Check that these images were acquired without this channel. Their " + "channel has no parameters; normalize it only where it was acquired.", + channel=channel, + ) + for channel, planes in lacking.items() + ] + + +def _pixel_channel_ids( + pixels: np.ndarray, channel_axis: int | None, channels: Iterable[str] | None +) -> tuple[str, ...]: + """Return the channel IDs of a pixel array, one per entry along the channel axis.""" + if channels is None: + raise ValueError( + "Give channels, the canonical channel ID of each entry along channel_axis (or the " + "one ID of an array without a channel axis): parameters are named by channel ID, " + "never by position." + ) + ids = check_channel_ids(channels, "fit") + count = ( + 1 if channel_axis is None else pixels.shape[channel_axis_index(channel_axis, pixels.ndim)] + ) + if len(ids) != count: + where = "without a channel axis" if channel_axis is None else f"along axis {channel_axis}" + raise ValueError( + f"The pixel array has {count} channel(s) {where}, but {len(ids)} channel ID(s) were " + f"given ({', '.join(ids)})." + ) + return ids def _pixel_histograms(pixels: np.ndarray, channel_axis: int | None) -> list[PlaneHistogram]: if channel_axis is None: return [compute_plane_histogram(pixels)] - axis = _axis_index(channel_axis, pixels.ndim) + axis = channel_axis_index(channel_axis, pixels.ndim) return [ compute_plane_histogram(np.take(pixels, c, axis=axis)) for c in range(pixels.shape[axis]) ] def _channel_histograms( - items: list[PlaneRecord] | list[PlaneHistogram], channels: tuple[int, ...] + items: list[PlaneRecord] | dict[str, PlaneHistogram], channels: tuple[str, ...] ) -> list[PlaneHistogram]: - # The histograms of ``channels`` in every row, or of a pixel array's channels. - if isinstance(items[0], PlaneHistogram): + # The histograms of ``channels`` in every row that has them (only the same channel ID is + # pooled; rows without it are refused or, as selected, reported by _partial_presence), or + # of a pixel array. + if isinstance(items, dict): return [items[c] for c in channels] - return [record.channel_histogram(c) for record in items for c in channels] + return [ + record.channel_histogram(c) for record in items for c in channels if record.has_channel(c) + ] def _pool( - items: list[PlaneRecord] | list[PlaneHistogram], + items: list[PlaneRecord] | dict[str, PlaneHistogram], settings: NormalizationSettings, - channels: tuple[int, ...], + channels: tuple[str, ...], ) -> PooledHistogram | WeightedHistogram: + """Pool ``channels`` of the rows that have them, weighted per condition if set (§4.3).""" if settings.pooling != "conditions": return PooledHistogram(_channel_histograms(items, channels)) # Contract §4.3: each condition's counts are scaled to the same total. The @@ -485,7 +385,8 @@ def _pool( # counts get weight exactly 1 and the numbers equal those of "pixels". by_condition: dict[str, list[PlaneRecord]] = {} for record in items: - by_condition.setdefault(record.condition, []).append(record) + if all(record.has_channel(c) for c in channels): + by_condition.setdefault(record.condition, []).append(record) parts = [] for _, members in sorted(by_condition.items()): histograms = _channel_histograms(members, channels) @@ -539,60 +440,3 @@ def _fit_issues(group: str | None, exact: bool, excluded: list[str]) -> tuple[Is ) ) return tuple(issues) - - -def _check_rgb_columns(channels: list[int], offset: list[float], scale: list[float]) -> None: - # Contract §8.1: an RGB row holds the shared value in C0, C1 and C2. - expected = list(range(RGB_CHANNEL_COUNT)) - if channels != expected: - missing = [c for c in expected if c not in channels] - names = [name for c in missing for name in (offset_column(c), scale_column(c))] - extra = [c for c in channels if c not in expected] - problem = ( - f"the column(s) {', '.join(names)} are blank or missing" - if missing - else f"it also holds channel(s) {extra}" - ) - raise ValueError( - "An RGB row holds one offset and scale in each of channels C0, C1 and C2, but " - f"{problem}. Read the parameters from a row written with RGB normalization." - ) - if len(set(offset)) != 1 or len(set(scale)) != 1: - raise ValueError( - "An RGB row holds the same offset and scale in channels C0, C1 and C2, but " - f"Normalization_Offset_C0..C2 are {offset} and Normalization_Scale_C0..C2 are " - f"{scale}. Read the parameters from a row written with RGB normalization." - ) - - -def _axis_index(axis: int, ndim: int) -> int: - if not -ndim <= axis < ndim: - raise ValueError(f"channel_axis {axis} is outside the pixel array's {ndim} axes.") - return axis % ndim - - -def _text(value: Any) -> str: - if value is None or (isinstance(value, float) and math.isnan(value)): - return "" - return str(value) - - -def _number(columns: Mapping[str, Any], name: str) -> float: - text = _text(columns[name]) - try: - value = float(text) - except ValueError: - raise ValueError(f"Column {name} cannot be read as a number; got {text!r}.") from None - if not math.isfinite(value): - raise ValueError(f"Column {name} must hold a finite number; got {text!r}.") - return value - - -def _flag(columns: Mapping[str, Any], name: str) -> bool: - value = columns[name] - if isinstance(value, bool | np.bool_): - return bool(value) - text = _text(value).strip().lower() - if text not in ("true", "false"): - raise ValueError(f"Column {name} must be 'true' or 'false'; got {value!r}.") - return text == "true" diff --git a/src/morphobricks_normalization/histogram.py b/src/morphobricks_normalization/histogram.py index 36a3f9e..ec6a624 100644 --- a/src/morphobricks_normalization/histogram.py +++ b/src/morphobricks_normalization/histogram.py @@ -24,8 +24,9 @@ With ``HistogramSettings(histogram_range="robust")``, a histogram that would not be exact is binned between the plane's quantiles at ``robust_quantiles`` instead of its minimum and maximum, with the same rules. The pixels below - and above are kept as :class:`HistogramTail` counts and sums (underflow and - overflow). Exact histograms never change. + and above are kept as :class:`HistogramTail` counts, means and standard + deviations (underflow and overflow), exact to the last bit + (:mod:`~morphobricks_normalization.tail_moments`). Exact histograms never change. Every pixel in a bin lies within half a bin width of its bin's representative value, and every pixel of a tail between the tail's mean and the plane's @@ -55,7 +56,7 @@ from .integer_representatives import beyond_float_exact, integer_representatives, wide_integer_error from .integrity import check_histogram, check_tail, checked_blocks from .ranges import CENTRAL_PERCENTILES, HistogramSettings, RangeCheck -from .sums import weighted_sum +from .tail_moments import TailMoments, centred_moments, counted_sums # Above this many distinct values, integer planes are binned with a search over # bin starts instead of a full per-value count. @@ -78,18 +79,20 @@ class HistogramTail: ---------- count Number of pixels. - sum, sum_of_squares - Sum of their values and of their squared values, in float64; checked - by :func:`~morphobricks_normalization.integrity.check_tail`. + mean, std + Their mean, the value standing in for them, and their population standard + deviation about it: the float64 nearest the exact value of each (schema + version 5, contract §1.4), 0.0 for an empty tail; checked by + :func:`~morphobricks_normalization.integrity.check_tail`. """ count: int = 0 - sum: float = 0.0 - sum_of_squares: float = 0.0 + mean: float = 0.0 + std: float = 0.0 def __post_init__(self) -> None: - checked = check_tail(self.count, self.sum, self.sum_of_squares) - for name, value in zip(("count", "sum", "sum_of_squares"), checked, strict=True): + checked = check_tail(self.count, self.mean, self.std) + for name, value in zip(("count", "mean", "std"), checked, strict=True): object.__setattr__(self, name, value) @property @@ -97,11 +100,6 @@ def is_empty(self) -> bool: """True when no pixel lies in this tail.""" return self.count == 0 - @property - def mean(self) -> float | None: - """Mean value of the tail's pixels, the value standing in for them (None if empty).""" - return None if self.count == 0 else self.sum / self.count - EMPTY_TAIL = HistogramTail() @@ -583,7 +581,7 @@ def _bin_pass( a robust range: the pixels outside ``[lo, hi]`` then go to the tails. ``selection`` is fed every block on the way. """ - under, over = _TailSums(), _TailSums() + under, over = TailMoments(), TailMoments() if is_integer(dtype): counts = _integer_bins(blocks, dtype, n, lo, hi, extrema, selection, under, over) else: @@ -600,8 +598,8 @@ def _bin_pass( valid, pixel_min=extrema[0], pixel_max=extrema[1], - underflow=under.tail(), - overflow=over.tail(), + underflow=HistogramTail(*under.moments()), + overflow=HistogramTail(*over.moments()), ) @@ -613,8 +611,8 @@ def _integer_bins( hi: int, extrema: tuple[int, int] | None, selection: _RadixSelection | None, - under: _TailSums, - over: _TailSums, + under: TailMoments, + over: TailMoments, ) -> np.ndarray: width = hi + 1 - lo starts = _integer_bin_starts(width, n) @@ -653,8 +651,8 @@ def _float_bins( valid: int, extrema: tuple[float, float] | None, selection: _RadixSelection | None, - under: _TailSums, - over: _TailSums, + under: TailMoments, + over: TailMoments, ) -> np.ndarray: counts = np.zeros(n, dtype=np.int64) if lo == hi and extrema is None: @@ -679,28 +677,8 @@ def _float_bins( return counts -class _TailSums: - """Running count, sum and sum of squares of the pixels of one tail.""" - - def __init__(self) -> None: - self.count = 0 - self.sum = 0.0 - self.sum_of_squares = 0.0 - - def add(self, values: np.ndarray) -> None: - values = values.astype(np.float64) - self.count += int(values.size) - self.sum += float(np.sum(values)) - self.sum_of_squares += float(np.sum(values * values)) - - def tail(self) -> HistogramTail: - return ( - HistogramTail(self.count, self.sum, self.sum_of_squares) if self.count else EMPTY_TAIL - ) - - def _split_tails( - values: np.ndarray, lo: int | float, hi: int | float, under: _TailSums, over: _TailSums + values: np.ndarray, lo: int | float, hi: int | float, under: TailMoments, over: TailMoments ) -> np.ndarray: """Add the values outside ``[lo, hi]`` to the tails; return the values inside.""" below = values < lo @@ -714,14 +692,10 @@ def _split_tails( def _table_tail(per_value: np.ndarray, first_value: int) -> HistogramTail: - """Sum the tail of per-value counts whose first entry counts ``first_value``.""" + """Return the tail of per-value counts whose first entry counts ``first_value``.""" count = int(per_value.sum()) - if count == 0: - return EMPTY_TAIL - values = first_value + np.arange(per_value.size, dtype=np.float64) - return HistogramTail( - count, weighted_sum(per_value, values), weighted_sum(per_value, values * values) - ) + total, squares = counted_sums(per_value, first_value) + return HistogramTail(*centred_moments(count, total, squares)) if count else EMPTY_TAIL def _inverse_cdf_rank(q: float, total: int) -> int: diff --git a/src/morphobricks_normalization/integrity.py b/src/morphobricks_normalization/integrity.py index 66b72d4..5bc6760 100644 --- a/src/morphobricks_normalization/integrity.py +++ b/src/morphobricks_normalization/integrity.py @@ -7,18 +7,17 @@ that breaks a rule is refused with a ``ValueError``; nothing is clipped, rounded or filled in. -The sums of a tail are float64 accumulations of the tail's pixel values, so -the checks that compare them allow a relative rounding slack of -``MOMENT_TOLERANCE``. A float64 plane whose values exceed about ``1e154`` can -overflow its sum of squares to infinity (and its sum beyond ``1.8e308``); such -an overflow is accepted only where the tail's range makes it possible. +A tail holds its pixels' count, mean and population standard deviation, each the +float64 nearest its exact value (:mod:`~morphobricks_normalization.tail_moments`), so +both are finite for finite pixels: the mean lies between the tail's extremum and its +binning limit, and the standard deviation is at most half that range (with a rounding +slack of ``MOMENT_TOLERANCE`` relative and one float64 step). """ from __future__ import annotations import math import numbers -import sys from collections.abc import Iterable, Iterator from typing import TYPE_CHECKING @@ -28,8 +27,8 @@ from .histogram import HistogramTail, PlaneHistogram INT64_MAX = 2**63 - 1 -# Relative rounding slack of the float64 tail sums: summing N values in float64 -# loses at most about N * 2.2e-16 relative, far below this for any plane. +# Relative rounding slack of a tail's half range against its standard deviation, each +# rounded once from exact values (a few 2.2e-16 relative at most). MOMENT_TOLERANCE = 1e-9 # Below this float64 estimate, the int64 sum of non-negative counts cannot have # wrapped (its rounding is far smaller than the 2**62 left before 2**63). @@ -97,39 +96,29 @@ def exact_total(counts: np.ndarray) -> int: return sum(counts.tolist()) -def check_tail(count: object, total: object, squares: object) -> tuple[int, float, float]: - """Check one tail's count and sums on their own; return them as int and floats. +def check_tail(count: object, mean: object, std: object) -> tuple[int, float, float]: + """Check one tail's count, mean and standard deviation; return them as int and floats. Raises ------ ValueError - For a count that is not a whole number of pixels, sums of an empty tail - that are not zero, a NaN sum, a negative sum of squares, an infinite sum - with a finite sum of squares, or sums no set of values can have - (``count * sum_of_squares < sum**2``). + For a count that is not a whole number of pixels, an empty tail whose mean or + standard deviation is not zero, or a populated tail whose mean is not finite or + whose standard deviation is not a finite number 0 or above. """ count = pixel_count(count, "pixel count of an underflow or overflow") - total, squares = float(total), float(squares) + mean, std = float(mean), float(std) if count == 0: - if total != 0 or squares != 0: - raise ValueError("an empty underflow or overflow must have zero sums.") - return count, total, squares - if math.isnan(total) or not squares >= 0: + if mean != 0 or std != 0: + raise ValueError("an empty underflow or overflow must have mean and std 0.") + return count, mean, std + if not (math.isfinite(mean) and math.isfinite(std) and std >= 0): raise ValueError( - f"an underflow or overflow of {count} pixel(s) has sum {total!r} and sum of squares " - f"{squares!r}; the sum must be a number and the sum of squares 0 or more." + f"an underflow or overflow of {count} pixel(s) has mean {mean!r} and standard " + f"deviation {std!r}; the mean must be finite and the standard deviation a finite " + "number 0 or above." ) - if math.isinf(total) and math.isfinite(squares): - raise ValueError( - f"an underflow or overflow has an infinite sum {total!r} but a finite sum of " - f"squares {squares!r}." - ) - if math.isfinite(squares) and count * squares < total * total * (1 - MOMENT_TOLERANCE): - raise ValueError( - f"an underflow or overflow of {count} pixel(s) has sum {total!r} and sum of squares " - f"{squares!r}, which no {count} values can have (their variance would be negative)." - ) - return count, total, squares + return count, mean, std def check_histogram(histogram: PlaneHistogram) -> None: @@ -217,27 +206,13 @@ def _check_range( def _check_tail_range(name: str, tail: HistogramTail, low: float, high: float) -> None: - """Check that a tail's mean and sum of squares are possible for values in ``[low, high]``.""" - slack = MOMENT_TOLERANCE * max(abs(low), abs(high)) - largest = max(low * low, high * high) - smallest = 0.0 if low <= 0 <= high else min(low * low, high * high) - if math.isinf(tail.sum): - possible = ( - tail.count * max(abs(low), abs(high)) * (1 + MOMENT_TOLERANCE) >= sys.float_info.max - ) - else: - possible = low - slack <= tail.sum / tail.count <= high + slack - if math.isinf(tail.sum_of_squares): - possible = possible and tail.count * largest * (1 + MOMENT_TOLERANCE) >= sys.float_info.max - else: - possible = possible and ( - tail.count * smallest * (1 - MOMENT_TOLERANCE) - <= tail.sum_of_squares - <= tail.count * largest * (1 + MOMENT_TOLERANCE) - ) - if not possible: + """Check that a tail's mean and spread are possible for values in ``[low, high]``.""" + # Values in [low, high] have a mean in it and a standard deviation of at most half of it. + largest = 0.5 * high - 0.5 * low + slack = math.ulp(max(abs(low), abs(high))) + if not (low <= tail.mean <= high and tail.std <= largest * (1 + MOMENT_TOLERANCE) + slack): raise ValueError( - f"The {name} of {tail.count} pixel(s) between {low!r} and {high!r} has sum " - f"{tail.sum!r} and sum of squares {tail.sum_of_squares!r}, which values in that " - "range cannot have." + f"The {name} of {tail.count} pixel(s) between {low!r} and {high!r} has mean " + f"{tail.mean!r} and standard deviation {tail.std!r}, which values in that range " + "cannot have." ) diff --git a/src/morphobricks_normalization/issues.py b/src/morphobricks_normalization/issues.py index 3b7381c..3d2fed0 100644 --- a/src/morphobricks_normalization/issues.py +++ b/src/morphobricks_normalization/issues.py @@ -16,6 +16,8 @@ REPEAT_MISSING_CONDITIONS = "repeat_missing_conditions" APPROXIMATE_PARAMETERS = "approximate_parameters" PLANES_WITHOUT_FINITE_PIXELS = "planes_without_finite_pixels" +# A channel only some planes of a fitted group have (ADR 0003): pooled over those planes. +CHANNEL_PARTIALLY_PRESENT = "channel_partially_present" # Intensity checks of contract §7.2, Milestone 4. IMAGE_INTENSITY_OUTLIER = "image_intensity_outlier" @@ -46,9 +48,10 @@ class Issue: action What the user can do about it. channel - The zero-based channel the warning is about, for the intensity checks - (N52), so a calling brick can mark the image in that channel's plots; - None for warnings about whole images or groups. + The canonical channel ID the warning is about (ADR 0003 HV2), for the + intensity checks (N52), the range check and ``channel_partially_present``, + so a calling brick can mark the image in that channel's plots; None for + warnings about whole images or groups. """ severity: str @@ -56,4 +59,4 @@ class Issue: message: str items: tuple[str, ...] = () action: str = "" - channel: int | None = None + channel: str | None = None diff --git a/src/morphobricks_normalization/parameter_checks.py b/src/morphobricks_normalization/parameter_checks.py index 62af2d3..c666524 100644 --- a/src/morphobricks_normalization/parameter_checks.py +++ b/src/morphobricks_normalization/parameter_checks.py @@ -1,4 +1,6 @@ -"""Checks of normalization parameters: offsets, scales, channel indices, scope (contract §8). +"""Checks of normalization parameters: offsets, scales, scope (contract §8). + +Channel IDs are checked by :mod:`~morphobricks_normalization.channels`. :class:`~morphobricks_normalization.normalization.NormalizationParameters` and :class:`~morphobricks_normalization.fitting.Parameters` check their values when @@ -10,11 +12,8 @@ from __future__ import annotations import math -import numbers from collections.abc import Iterable -import numpy as np - def check_offset_scale(offset: float, scale: float, where: str) -> None: """Refuse an offset that is not finite, or a scale that is not finite and above zero. @@ -24,7 +23,7 @@ def check_offset_scale(offset: float, scale: float, where: str) -> None: offset, scale The values of one channel. where - Names the channel or method in the message, e.g. ``"Channel 2"``. + Names the channel or method in the message, e.g. ``"Channel DNA"``. Raises ------ @@ -43,33 +42,6 @@ def check_offset_scale(offset: float, scale: float, where: str) -> None: ) -def channel_indices(channels: Iterable[object]) -> tuple[int, ...]: - """Return channel indices as ints, refusing anything that is not a whole number 0 or above. - - Python and NumPy integers are accepted; booleans, fractions (even whole - floats such as ``1.0``), text and negative numbers are refused rather than - rounded, truncated or parsed. - - Raises - ------ - ValueError - Naming the value that is not a channel index. - """ - indices = [] - for channel in channels: - if ( - isinstance(channel, bool | np.bool_) - or not isinstance(channel, numbers.Integral) - or channel < 0 - ): - raise ValueError( - f"A channel index is a whole number counted from 0; got {channel!r}. Give the " - "channels as integers (0 for the first channel)." - ) - indices.append(int(channel)) - return tuple(indices) - - def check_scope(scope: str, expected: str, scopes: Iterable[str]) -> str: """Return a parameter row's scope if it is a scope and the one the settings use. diff --git a/src/morphobricks_normalization/parameters.py b/src/morphobricks_normalization/parameters.py new file mode 100644 index 0000000..7a41334 --- /dev/null +++ b/src/morphobricks_normalization/parameters.py @@ -0,0 +1,443 @@ +"""Normalization parameters and their columns (contract §2, §8; ADR 0003 HV2). + +:class:`Parameters` holds an offset and a scale per channel of one group, each channel named +by its canonical channel ID; :func:`~morphobricks_normalization.fitting.fit` makes them. They +are stored as one ``Normalization_Offset_`` and ``Normalization_Scale_`` pair per +channel (contract §8.1, N41, N46, N47 with the channel token amended by HV2); for RGB the +shared value is written for R, G and B. ``from_columns(to_columns(p)) == p`` exactly. + +The columns carry their own version, ``normalization_parameters_version`` (contract §8.1): +the version-4 columns ``Normalization_Offset_C0`` would otherwise read as channel ``C0`` (a +legal channel ID), so a row without the version, or of another version, is refused and never +converted (ADR 0002 AR10). +""" + +from __future__ import annotations + +import math +import re +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field, replace +from typing import Any + +import numpy as np + +from .channels import CHANNEL_ID_PATTERN, check_channel_ids +from .issues import Issue +from .parameter_checks import check_offset_scale, check_scope +from .rgb import RGB_CHANNELS, apply_rgb, check_rgb_channels, check_rgb_selection +from .settings import SCOPES, NormalizationSettings + +# Version of the parameter columns (contract §8.1). Version 1, the C-index columns of +# normalization 0.4 and earlier, had no version column. +PARAMETERS_VERSION = 2 +VERSION_COLUMN = "normalization_parameters_version" +# Columns written once per row; the per-channel pairs follow (offset_column, +# scale_column). +COLUMNS = ( + "normalization_method", + "normalization_scope", + "normalization_group", + "normalization_exact", + VERSION_COLUMN, +) +OFFSET_COLUMN_PREFIX = "Normalization_Offset" +SCALE_COLUMN_PREFIX = "Normalization_Scale" +# The channel token is a canonical channel ID (ADR 0003 HV2), which may contain "_". +_CHANNEL_COLUMN = re.compile(rf"Normalization_(?:Offset|Scale)_({CHANNEL_ID_PATTERN.pattern})") + + +def offset_column(channel: str) -> str: + """Column of channel ``channel``'s offset, e.g. ``Normalization_Offset_DNA``.""" + return f"{OFFSET_COLUMN_PREFIX}_{channel}" + + +def scale_column(channel: str) -> str: + """Column of channel ``channel``'s scale, e.g. ``Normalization_Scale_DNA``.""" + return f"{SCALE_COLUMN_PREFIX}_{channel}" + + +@dataclass(frozen=True) +class Parameters: + """Normalization of one group: ``(pixels - offset[k]) / scale[k]`` for channel ``channels[k]``. + + Attributes + ---------- + method + Written method name, e.g. ``"percentile 1-99.8"`` (contract §3.1). + scope + ``"biological_repeat"``, ``"image"`` or ``"patch"``. + group + The ``normalization_group`` text (contract §4.1), or None when the + calling brick has not named the group. + exact + Whether every pooled histogram was exact. + offset, scale + One value per channel, in the order of ``channels``: finite offsets, + finite scales above zero. + channels + The canonical channel IDs the parameters belong to, sorted and distinct + (ADR 0003 HV2); never C indices. A single-channel patch of channel ``DNA`` + has ``channels == ("DNA",)`` (contract §8.1). Required: nothing is guessed. + issues + Warnings about these parameters (``approximate_parameters``, + ``planes_without_finite_pixels``, ``channel_partially_present``); not stored + in the columns and not compared. + """ + + method: str + scope: str + group: str | None + exact: bool + offset: tuple[float, ...] + scale: tuple[float, ...] + channels: tuple[str, ...] + issues: tuple[Issue, ...] = field(default=(), compare=False, repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "offset", tuple(float(value) for value in self.offset)) + object.__setattr__(self, "scale", tuple(float(value) for value in self.scale)) + if not self.offset or len(self.offset) != len(self.scale): + raise ValueError( + f"Expected one offset and one scale per channel; got {len(self.offset)} " + f"offset(s) and {len(self.scale)} scale(s)." + ) + channels = check_channel_ids(self.channels, "Parameters") + if len(channels) != len(self.offset) or list(channels) != sorted(channels): + raise ValueError( + "Expected one channel ID per offset, in sorted order; got channels " + f"{list(channels)} for {len(self.offset)} offset(s)." + ) + object.__setattr__(self, "channels", channels) + for channel, offset, scale in zip(channels, self.offset, self.scale, strict=True): + check_offset_scale(offset, scale, f"Channel {channel}") + + @property + def channel_count(self) -> int: + """Number of channels the parameters normalize.""" + return len(self.offset) + + @property + def image_type(self) -> str: + """``"rgb"`` for StarDist's RGB method (``rgb percentile ...``), else ``"fluorescence"``.""" + return "rgb" if self.method.startswith("rgb ") else "fluorescence" + + def select(self, channels: Iterable[str]) -> Parameters: + """Return the parameters of some of the channels only, by channel ID. + + For example ``parameters.select(["DNA"])`` gives the parameters of a + single-channel patch of channel ``DNA``, whose columns fill only + ``Normalization_Offset_DNA`` and ``Normalization_Scale_DNA`` (contract §8.1). + + Raises + ------ + ValueError + If a channel is not among :attr:`channels`, or is not a channel ID (an + index is refused); for RGB parameters, if not all of R, G and B are + selected (issue #50). + """ + wanted = sorted(set(check_channel_ids(channels, "Select"))) + missing = [c for c in wanted if c not in self.channels] + if not wanted: + raise ValueError("Select at least one channel of these parameters.") + if missing: + raise ValueError( + f"These parameters have channel(s) {', '.join(self.channels)}; channel(s) " + f"{', '.join(missing)} cannot be selected." + ) + if self.image_type == "rgb": + check_rgb_selection(wanted, self.channels) + index = [self.channels.index(c) for c in wanted] + return replace( + self, + offset=tuple(self.offset[i] for i in index), + scale=tuple(self.scale[i] for i in index), + channels=tuple(wanted), + ) + + def apply( + self, + pixels: np.ndarray, + *, + channel_axis: int | None = None, + channels: Iterable[str] | None = None, + ) -> np.ndarray: + """Return the normalized pixels as a new float32 array. + + Computed in float64 as ``(pixels - offset) / scale`` and returned as + float32, without clipping; non-finite pixels stay non-finite and the + input is never modified. RGB parameters repeat CSBDeep's float32 + arithmetic instead, so the result is bit-identical to StarDist's + normalization (:func:`~morphobricks_normalization.rgb.apply_rgb`, N48). + + Parameters + ---------- + pixels + Pixel values of the group. + channel_axis + Axis of ``pixels`` that holds the channels; may be left out when the + array holds one channel. + channels + The canonical channel ID of each entry along ``channel_axis``, in that + order (or the one ID of an array without a channel axis): each entry is + normalized with the parameters of its ID, never by its position. Required + with ``channel_axis`` (for RGB parameters it defaults to ``R``, ``G``, ``B``, + the samples of an RGB unit); may be left out without it when the parameters + have one channel. Every ID must be among :attr:`channels`; the array may hold + fewer channels than the parameters. + + Raises + ------ + ValueError + If the channel axis or the channel IDs are missing, do not match the array, + or name a channel these parameters do not have. + """ + values = np.asarray(pixels) + if self.image_type == "fluorescence": + values = values.astype(np.float64, copy=False) + order = self._apply_order(values, channel_axis, channels) + if channel_axis is None: + offset, scale = self.offset[order[0]], self.scale[order[0]] + else: + axis = channel_axis_index(channel_axis, values.ndim) + shape = [1] * values.ndim + shape[axis] = len(order) + offset = np.asarray([self.offset[k] for k in order]).reshape(shape) + scale = np.asarray([self.scale[k] for k in order]).reshape(shape) + if self.image_type == "rgb": + return apply_rgb(values, offset, scale) + return ((values - offset) / scale).astype(np.float32, copy=False) + + def _apply_order( + self, values: np.ndarray, channel_axis: int | None, channels: Iterable[str] | None + ) -> list[int]: + """Positions in :attr:`channels` of the array's channels, checked against the array.""" + if channels is None and self.image_type == "rgb" and channel_axis is not None: + channels = RGB_CHANNELS # an RGB unit's samples (experiment-v1 §4.4b) + if channels is None: + if channel_axis is not None or self.channel_count != 1: + raise ValueError( + f"These parameters have channel(s) {', '.join(self.channels)}; give " + "channels, the channel ID of each entry along channel_axis in order, so " + "that every channel is normalized with its own parameters." + ) + return [0] + ids = check_channel_ids(channels, "apply") + count = ( + 1 + if channel_axis is None + else values.shape[channel_axis_index(channel_axis, values.ndim)] + ) + if len(ids) != count: + where = ( + "without a channel axis" if channel_axis is None else f"along axis {channel_axis}" + ) + raise ValueError( + f"The pixel array has {count} channel(s) {where}, but {len(ids)} channel ID(s) " + f"were given ({', '.join(ids)})." + ) + missing = [channel for channel in ids if channel not in self.channels] + if missing: + raise ValueError( + f"These parameters have channel(s) {', '.join(self.channels)}, not " + f"{', '.join(missing)}; normalize those channels with parameters fitted for them." + ) + if self.image_type == "rgb": + check_rgb_channels(ids, "The pixel array") + return [self.channels.index(channel) for channel in ids] + + def to_columns(self) -> dict[str, str]: + """Return the columns of contract §8.1 as text, numbers in full. + + The five ``normalization_*`` columns (the last the columns' version, ``2``), + then ``Normalization_Offset_`` + and ``Normalization_Scale_`` for each channel ID of :attr:`channels`, + grouped by channel in sorted order. Columns of other channels are not + returned; a table holding rows of other channels leaves them blank. + """ + columns = { + "normalization_method": self.method, + "normalization_scope": self.scope, + "normalization_group": self.group or "", + "normalization_exact": "true" if self.exact else "false", + VERSION_COLUMN: str(PARAMETERS_VERSION), + } + for channel, offset, scale in zip(self.channels, self.offset, self.scale, strict=True): + columns[offset_column(channel)] = repr(offset) + columns[scale_column(channel)] = repr(scale) + return columns + + @classmethod + def from_columns( + cls, columns: Mapping[str, Any], settings: NormalizationSettings + ) -> Parameters: + """Read parameters back from their columns; ``from_columns(p.to_columns()) == p``. + + Parameters + ---------- + columns + A row with the five ``normalization_*`` columns (version 2) and the + ``Normalization_Offset_`` and ``Normalization_Scale_`` pair of at + least one channel ID; other columns are ignored. Values may be text or + numbers. A channel whose two columns are both blank or absent is not part + of the row's parameters (a single-channel patch fills only its own + channel, and a channel absent from the row's image is blank). + settings + The settings of the run that wrote the row; its method must be the + row's ``normalization_method`` and its scope the row's + ``normalization_scope`` (one of the scopes of contract §4.1), so that + parameters fitted for one sampling population are never read as + another's (issue #37). + + Raises + ------ + ValueError + Naming the column that is missing or cannot be read, whose method + or scope differs from the settings, or whose scope is unknown; also + for a channel with only one of its two columns, and for a row of another + version of the columns (without ``normalization_parameters_version``: the + C-index columns of normalization 0.4 and earlier, never read as channel IDs). + """ + _check_version(columns) + missing = [name for name in COLUMNS if name not in columns] + if missing: + raise ValueError( + f"The normalization column(s) {', '.join(missing)} are missing. Read the " + "parameters from a table written with normalization." + ) + method = _text(columns["normalization_method"]) + if method != settings.method_name: + raise ValueError( + f"Column normalization_method is {method!r}, but the settings use " + f"{settings.method_name!r}. Read the parameters with the settings of the run " + "that wrote them." + ) + ids = sorted( + {match[1] for name in columns if (match := _CHANNEL_COLUMN.fullmatch(str(name)))} + ) + if not ids: + raise ValueError( + "The row has no Normalization_Offset_ and Normalization_Scale_ columns " + "(one pair per canonical channel ID). Read the parameters from a table written " + "with normalization." + ) + channels, offset, scale = [], [], [] + for channel in ids: + names = (offset_column(channel), scale_column(channel)) + texts = [_text(columns.get(name)) for name in names] + if texts == ["", ""]: + continue # a channel this row does not hold + if "" in texts: + blank, given = names if texts[0] == "" else names[::-1] + raise ValueError( + f"Column {blank} is blank or missing, but {given} holds a value; every " + "channel of a row needs both its offset and its scale." + ) + channels.append(channel) + offset.append(_number(columns, names[0])) + scale.append(_number(columns, names[1])) + if scale[-1] <= 0: + raise ValueError( + f"Column {names[1]} must hold a positive number; got {texts[1]!r}." + ) + if not channels: + raise ValueError( + "Every Normalization_Offset_ and Normalization_Scale_ column of this " + "row is blank, so it holds no parameters. Read the parameters from a row " + "written with normalization." + ) + if settings.image_type == "rgb": + _check_rgb_columns(channels, offset, scale) + return cls( + method=method, + scope=check_scope(_text(columns["normalization_scope"]), settings.scope, SCOPES), + group=_text(columns["normalization_group"]) or None, + exact=_flag(columns, "normalization_exact"), + offset=tuple(offset), + scale=tuple(scale), + channels=tuple(channels), + ) + + +def _check_version(columns: Mapping[str, Any]) -> None: + """Refuse a row whose parameter columns are not of :data:`PARAMETERS_VERSION`. + + The version is text as written, or the number a table reader gives (``2``, or ``2.0`` in + a column with blanks). + """ + value = columns.get(VERSION_COLUMN) + number = isinstance(value, int | float | np.integer | np.floating) and not isinstance( + value, bool | np.bool_ + ) + if (number and value == PARAMETERS_VERSION) or _text(value) == str(PARAMETERS_VERSION): + return + text = _text(value).strip() + found = ( + f"has {VERSION_COLUMN} {text!r}" + if text + else f"has no {VERSION_COLUMN} column, so it was written by normalization 0.4 or " + "earlier, with channels named by their index in the image (Normalization_Offset_C0)" + ) + raise ValueError( + f"This row of parameter columns {found}. This version reads parameter columns of " + f"version {PARAMETERS_VERSION} only, with channels named by canonical channel ID; old " + "parameters are not converted. Fit the parameters again with this version." + ) + + +def _check_rgb_columns(channels: list[str], offset: list[float], scale: list[float]) -> None: + # Contract §8.1: an RGB row holds the shared value in R, G and B. + expected = sorted(RGB_CHANNELS) + if channels != expected: + missing = [c for c in expected if c not in channels] + names = [name for c in missing for name in (offset_column(c), scale_column(c))] + extra = [c for c in channels if c not in expected] + problem = ( + f"the column(s) {', '.join(names)} are blank or missing" + if missing + else f"it also holds channel(s) {extra}" + ) + raise ValueError( + "An RGB row holds one offset and scale in each of channels R, G and B, but " + f"{problem}. Read the parameters from a row written with RGB normalization." + ) + if len(set(offset)) != 1 or len(set(scale)) != 1: + raise ValueError( + "An RGB row holds the same offset and scale in channels R, G and B, but " + f"Normalization_Offset_B, _G, _R are {offset} and Normalization_Scale_B, _G, _R are " + f"{scale}. Read the parameters from a row written with RGB normalization." + ) + + +def channel_axis_index(axis: int, ndim: int) -> int: + """Return ``axis`` as a non-negative axis of an array of ``ndim`` axes, or refuse it.""" + if not -ndim <= axis < ndim: + raise ValueError(f"channel_axis {axis} is outside the pixel array's {ndim} axes.") + return axis % ndim + + +def _text(value: Any) -> str: + if value is None or (isinstance(value, float) and math.isnan(value)): + return "" + return str(value) + + +def _number(columns: Mapping[str, Any], name: str) -> float: + text = _text(columns[name]) + try: + value = float(text) + except ValueError: + raise ValueError(f"Column {name} cannot be read as a number; got {text!r}.") from None + if not math.isfinite(value): + raise ValueError(f"Column {name} must hold a finite number; got {text!r}.") + return value + + +def _flag(columns: Mapping[str, Any], name: str) -> bool: + value = columns[name] + if isinstance(value, bool | np.bool_): + return bool(value) + text = _text(value).strip().lower() + if text not in ("true", "false"): + raise ValueError(f"Column {name} must be 'true' or 'false'; got {value!r}.") + return text == "true" diff --git a/src/morphobricks_normalization/pooling.py b/src/morphobricks_normalization/pooling.py index eebd8a2..c3a19c2 100644 --- a/src/morphobricks_normalization/pooling.py +++ b/src/morphobricks_normalization/pooling.py @@ -81,8 +81,10 @@ class PooledHistogram: per channel. Planes without finite pixels are skipped and counted in ``n_excluded_planes``. channel - For records, the channel to pool (zero-based); it may be left out when - every record has one channel. Plane histograms are used as given. + For records, the canonical ID of the channel to pool; it may be left out when + every record has one channel. Every record must have that channel: only the + same channel ID is pooled, and a plane without it is refused, never skipped. + Plane histograms are used as given. Raises ------ @@ -90,8 +92,9 @@ class PooledHistogram: If none of the planes has a finite pixel. ValueError If a record has more than one channel and ``channel`` is not given, or - has no channel ``channel``; or if the planes hold more than ``2**63 - 1`` - finite pixels together, more than the int64 counts can hold. + does not have channel ``channel`` (it is absent there); or if the planes + hold more than ``2**63 - 1`` finite pixels together, more than the int64 + counts can hold. Notes ----- @@ -100,11 +103,11 @@ class PooledHistogram: """ def __init__( - self, histograms: Iterable[PlaneHistogram | PlaneRecord], channel: int | None = None + self, histograms: Iterable[PlaneHistogram | PlaneRecord], channel: str | None = None ) -> None: values, weights = [], [] bin_values, bin_weights = [], [] - # (count, sum, sum of squares) of every non-empty tail. + # (count, mean, standard deviation) of every non-empty tail. self.tails: list[tuple[float, float, float]] = [] self.n_planes = 0 self.n_excluded_planes = 0 @@ -126,7 +129,7 @@ def __init__( if not tail.is_empty: values.append(np.array([tail.mean])) weights.append(np.array([tail.count], dtype=np.int64)) - self.tails.append((tail.count, tail.sum, tail.sum_of_squares)) + self.tails.append((tail.count, tail.mean, tail.std)) self.minimum = min(self.minimum, float(hist.pixel_min)) self.maximum = max(self.maximum, float(hist.pixel_max)) self.max_representative_error = max( @@ -148,21 +151,21 @@ def __init__( self.pixel_count = int(self.counts.sum()) self._cumulative = np.cumsum(self.counts) if self.tails: - # The bins alone, for the mean and standard deviation (the tails use their sums). + # The bins alone, for the mean and standard deviation (the tails use their moments). self.bin_values, self.bin_counts = _merge(bin_values, bin_weights) else: self.bin_values, self.bin_counts = self.values, self.counts @property def mean(self) -> float: - """Count-weighted mean; pixels outside the bins count with their exact values.""" + """Count-weighted mean; pixels outside the bins count by their tail's mean.""" if self.tails: return _mean_with_tails(self.bin_values, self.bin_counts, self.tails, self.pixel_count) - return weighted_sum(self.counts, self.values) / self.pixel_count + return _mean_with_tails(self.values, self.counts, [], self.pixel_count) @property def std(self) -> float: - """Count-weighted population standard deviation; tails count with their exact sums.""" + """Count-weighted population standard deviation; tails count with their moments.""" if self.tails: return _std_with_tails( self.bin_values, self.bin_counts, self.tails, self.pixel_count, self.mean @@ -271,11 +274,11 @@ def __init__(self, parts: Sequence[PooledHistogram], weights: Sequence[float]): self.weights = np.bincount(inverse, weights=scaled) self._cumulative = np.cumsum(self.weights) self.total_weight = float(self._cumulative[-1]) - # Tails scaled by their part's weight, for the mean and standard deviation. + # Tails with their count scaled by their part's weight (mean and spread unchanged). self.tails = [ - (count * float(w), total * float(w), squares * float(w)) + (count * float(w), mean, std) for part, w in zip(parts, weights, strict=True) - for count, total, squares in part.tails + for count, mean, std in part.tails ] if self.tails: merged, inverse = np.unique( @@ -291,16 +294,16 @@ def __init__(self, parts: Sequence[PooledHistogram], weights: Sequence[float]): @property def mean(self) -> float: - """Weighted mean; pixels outside the bins count with their exact values.""" + """Weighted mean; pixels outside the bins count by their tail's mean.""" if self.tails: return _mean_with_tails( self.bin_values, self.bin_weights, self.tails, self.total_weight ) - return weighted_sum(self.weights, self.values) / self.total_weight + return _mean_with_tails(self.values, self.weights, [], self.total_weight) @property def std(self) -> float: - """Weighted population standard deviation; tails count with their exact sums.""" + """Weighted population standard deviation; tails count with their moments.""" if self.tails: return _std_with_tails( self.bin_values, self.bin_weights, self.tails, self.total_weight, self.mean @@ -345,7 +348,7 @@ def pooled_statistics( histograms: Iterable[PlaneHistogram | PlaneRecord], percentiles: Sequence[float] = DEFAULT_PERCENTILES, *, - channel: int | None = None, + channel: str | None = None, ) -> PooledStatistics: """Pool a group of planes and return its summary statistics. @@ -357,29 +360,24 @@ def pooled_statistics( def _mean_with_tails( values: np.ndarray, weights: np.ndarray, tails: list[tuple[float, float, float]], total: float ) -> float: - sums = [s for _, s, _ in tails] - if not all(math.isfinite(s) for s in sums): - # math.fsum would raise a bare ValueError or OverflowError from the standard library. - raise ValueError(_non_finite_tail_message(sums)) - return (weighted_sum(weights, values) + math.fsum(sums)) / total - - -def _non_finite_tail_message(sums: list[float]) -> str: - """Refusal text for tail sums that are infinite or NaN, naming each such tail.""" - bad = ", ".join( - f"tail {number} of {len(sums)} has sum {s!r}" - for number, s in enumerate(sums, 1) - if not math.isfinite(s) - ) - opposing = any(s == math.inf for s in sums) and any(s == -math.inf for s in sums) - reason = ( - "infinite sums of opposite sign have no mean" if opposing else "a sum that is not finite" - ) - return ( - f"The mean cannot be pooled: {bad} ({reason}). The pixels outside the robust range " - "are too large for float64 sums. Use a method that does not need the mean, or " - "another range." - ) + """Return the weighted mean of the bins and of each tail's mean, weighted by its count. + + Without tails this is the plain weighted mean, bit for bit (a sum that overflows gives + ``inf`` or ``nan``, which the methods refuse by name). A tail's mean is finite, so when + the products of the tails overflow although every value is finite (pixels near the + float64 limit at both ends), the sum is redone with every value divided by a power of + two near the largest, which is exact, and the finite mean is found. + """ + if not tails: + return weighted_sum(weights, values) / total + weights = np.concatenate([np.asarray(weights, dtype=np.float64), [t[0] for t in tails]]) + values = np.concatenate([values, [t[1] for t in tails]]) + with np.errstate(over="ignore", invalid="ignore"): # an overflow is redone below + mean = weighted_sum(weights, values) / total + if math.isfinite(mean) or not np.isfinite(values).all(): + return mean + scale = _power_of_two_near(float(np.max(np.abs(values)))) + return weighted_sum(weights, values / scale) / total * scale def _std_with_tails( @@ -391,15 +389,13 @@ def _std_with_tails( ) -> float: """Return the population standard deviation of the bins (and tails) around ``mean`` (#33). - The squares are first summed as they are, which keeps every finite result bit for - bit. When that overflows although the mean and every value are finite (``[-1e200, - 1e200]`` has standard deviation ``1e200`` but squares of ``1e400``), the sum is redone - with every value divided by a power of two near the largest, which is exact, so the finite - spread is found. A tail whose stored sum of squares is not finite is refused by name. + Each tail adds its own spread, ``count * std**2``, and its mean's distance from ``mean`` + (centred moments: no cancellation). The squares are first summed as they are, which + keeps every finite result bit for bit. When that overflows although the mean and every + value are finite (``[-1e200, 1e200]`` has standard deviation ``1e200`` but squares of + ``1e400``), the sum is redone with every value divided by a power of two near the + largest, which is exact, so the finite spread is found. """ - squares = [s2 for _, _, s2 in tails] - if not all(math.isfinite(s2) for s2 in squares): - raise ValueError(_non_finite_squares_message(squares)) try: with np.errstate(over="ignore", invalid="ignore"): # an overflow is redone below std = _std_scaled(values, weights, tails, total, mean, 1.0) @@ -408,11 +404,13 @@ def _std_with_tails( if math.isfinite(std) or not (math.isfinite(mean) and np.isfinite(values).all()): return std largest = max(float(np.max(np.abs(values), initial=0.0)), abs(mean)) - largest = max([largest] + [abs(s / count) for count, s, _ in tails]) - # Values divided by 2**(exponent - 1) are below 2 in magnitude, and the scale is finite. - return _std_scaled( - values, weights, tails, total, mean, math.ldexp(1.0, math.frexp(largest)[1] - 1) - ) + largest = max([largest] + [max(abs(m), s) for _, m, s in tails]) + return _std_scaled(values, weights, tails, total, mean, _power_of_two_near(largest)) + + +def _power_of_two_near(largest: float) -> float: + """Return the power of two at or just below ``largest``; values divided by it are below 2.""" + return math.ldexp(1.0, math.frexp(largest)[1] - 1) def _std_scaled( @@ -427,39 +425,20 @@ def _std_scaled( mean = mean / scale deviations = values / scale - mean squares = weighted_sum(weights, deviations * deviations) - for count, s, s2 in tails: - # Each tail adds its spread around its own mean and its mean's distance - # from the overall mean; the spread never goes negative through rounding. - s, s2 = s / scale, s2 / scale / scale - tail_mean = s / count - squares += max(s2 - s * tail_mean, 0.0) + count * (tail_mean - mean) ** 2 + for count, tail_mean, tail_std in tails: + spread, offset = tail_std / scale, tail_mean / scale - mean + squares += count * (spread * spread) + count * (offset * offset) return float(np.sqrt(squares / total)) * scale -def _non_finite_squares_message(squares: list[float]) -> str: - """Refusal text for tails whose sum of squares overflowed float64 when it was computed.""" - bad = ", ".join( - f"tail {number} of {len(squares)} has sum of squares {s2!r}" - for number, s2 in enumerate(squares, 1) - if not math.isfinite(s2) - ) - return ( - f"The standard deviation cannot be pooled: {bad}. The squares of pixels outside the " - "robust range overflowed float64 when the statistics file was written (pixel values " - "beyond about 1.3e154), so their spread is not known, although it may be finite. Use " - "a method that does not need the standard deviation (percentile or robust z-score), " - "or the min-max range." - ) - - -def _channel_histogram(record: PlaneRecord, channel: int | None) -> PlaneHistogram: +def _channel_histogram(record: PlaneRecord, channel: str | None) -> PlaneHistogram: if channel is None: if record.channel_count != 1: raise ValueError( - f"Plane {record.plane_id} has {record.channel_count} channels; give the " - f"channel to pool, from 0 to {record.channel_count - 1}." + f"Plane {record.plane_id} has {record.channel_count} channels " + f"({', '.join(record.channels)}); give the ID of the channel to pool." ) - channel = 0 + return record.histograms[0] return record.channel_histogram(channel) diff --git a/src/morphobricks_normalization/ranges.py b/src/morphobricks_normalization/ranges.py index 003de30..d620975 100644 --- a/src/morphobricks_normalization/ranges.py +++ b/src/morphobricks_normalization/ranges.py @@ -29,8 +29,8 @@ import numpy as np from ._version import __version__ +from .channels import check_channel_id from .issues import NARROW_HISTOGRAM_RANGE, WARNING, Issue -from .parameter_checks import channel_indices HISTOGRAM_RANGES = ("minmax", "robust") DEFAULT_ROBUST_QUANTILES = (0.01, 99.99) @@ -204,7 +204,7 @@ def _settings_of(check: RangeCheck) -> tuple: ) -def _check_one_setting(channel: int, found: list[tuple[str, RangeCheck]]) -> None: +def _check_one_setting(channel: str, found: list[tuple[str, RangeCheck]]) -> None: """Refuse narrow checks of one channel that were made with different settings.""" by_settings: dict[tuple, list[str]] = {} for plane_id, check in found: @@ -216,40 +216,40 @@ def _check_one_setting(channel: int, found: list[tuple[str, RangeCheck]]) -> Non for sig, ids in sorted(by_settings.items(), key=lambda kv: repr(kv[0])) ) raise IncompatibleRangeChecksError( - f"The narrow range checks of channel C{channel} were made with different " + f"The narrow range checks of channel {channel} were made with different " f"settings ({parts}). One warning cannot describe them, and no setting is " "taken as representative. Check all planes with the same histogram settings, " "or pass the planes of each setting separately." ) -def range_check_issues(checks: Iterable[tuple[str, int, RangeCheck | None]]) -> list[Issue]: +def range_check_issues(checks: Iterable[tuple[str, str, RangeCheck | None]]) -> list[Issue]: """Warnings for the narrow central ranges among ``checks``, one per channel (N60). Parameters ---------- checks - ``(plane_id, channel, check)`` for every channel plane; a None check - (an exact histogram) is skipped. + ``(plane_id, channel, check)`` for every channel plane, ``channel`` its + canonical channel ID; a None check (an exact histogram) is skipped. Returns ------- list of Issue One ``narrow_histogram_range`` warning per channel with at least one - narrow plane, in channel order; ``items`` lists the planes and + narrow plane, in sorted channel-ID order; ``items`` lists the planes and ``channel`` holds the channel. Empty when no range is narrow. Raises ------ ValueError - For a channel that is not a whole number 0 or above (nothing is rounded - or parsed). + For a channel that is not a canonical channel ID (an index is refused; + nothing is parsed). IncompatibleRangeChecksError For narrow planes of one channel whose checks used different settings. """ - by_channel: dict[int, list[tuple[str, RangeCheck]]] = {} + by_channel: dict[str, list[tuple[str, RangeCheck]]] = {} for plane_id, channel, check in checks: - (channel,) = channel_indices([channel]) + check_channel_id(channel, f"Range check of plane {plane_id}") if check is not None and check.narrow: by_channel.setdefault(channel, []).append((plane_id, check)) issues = [] @@ -289,7 +289,7 @@ def range_check_issues(checks: Iterable[tuple[str, int, RangeCheck | None]]) -> Issue( WARNING, NARROW_HISTOGRAM_RANGE, - f"In {len(planes)} plane(s), the central intensity range of channel C{channel} " + f"In {len(planes)} plane(s), the central intensity range of channel {channel} " f"(its {low}-{high} percentile range) covers fewer than " f"{first.min_central_bins:,} of the histogram's bins (as few as {fewest:,}): " f"{shown}{more}. Percentile parameters from these planes are coarse, or cannot " diff --git a/src/morphobricks_normalization/records.py b/src/morphobricks_normalization/records.py index 8c08b2c..1beb23d 100644 --- a/src/morphobricks_normalization/records.py +++ b/src/morphobricks_normalization/records.py @@ -8,6 +8,7 @@ import numpy as np +from .channels import check_channel_id, check_channel_ids from .histogram import PlaneHistogram # Label used for hierarchy levels a branch of the folder tree does not have. @@ -18,7 +19,7 @@ # among siblings (bio_rep1 under two conditions are different samples). LEVELS = ("condition", "biological_repeat", "technical_repeat") -# Images named per channel count in one message before the rest are counted. +# Planes or measurements named in one message before the rest are counted. _MAX_LISTED = 5 @@ -27,8 +28,8 @@ class PlaneRecord: """An XYC plane's identity, experimental labels and per-channel histograms. One plane is every channel of one image at one Z and T position (contract - §1.3, N41), so a record holds one histogram per channel, in the image's - channel order. + §1.3, N41), so a record holds one histogram per channel. Each channel is named + by its canonical channel ID (ADR 0003 HV2), never by its index in the image. Attributes ---------- @@ -39,8 +40,9 @@ class PlaneRecord: Folder-derived labels; ``"single"`` when the level is absent. histograms One :class:`~morphobricks_normalization.histogram.PlaneHistogram` per - channel, channel 0 first. Every channel has the same pixel type, bin - count and size. A single histogram is accepted for a one-channel plane. + channel, in the order of ``channels``. Every channel has the same pixel + type, bin count and size. A single histogram is accepted for a one-channel + plane. z, t Zero-based plane coordinates. An axis the image does not have, or has with length 1, has coordinate 0; None is stored as 0. Booleans, @@ -49,6 +51,13 @@ class PlaneRecord: Required, keyword-only, non-empty text: the XY position within its technical repeat (MorphoBricks decision D02), from reliable metadata, or the file stem when one file is one position. A record without it is refused. + channels + Required, keyword-only: the canonical channel IDs of the plane's channels in + source order, entry ``i`` naming source channel ``C`` (the imaging unit's + confirmed ``channels`` list, experiment-v1 §4.4). These are the channels + *present* in the plane; a channel of the experiment that is not listed is + absent, which is not a measurement and never a zero histogram (ADR 0003 HV1). + Never guessed or filled in: a record without it is refused. """ source_relative_path: str @@ -59,6 +68,7 @@ class PlaneRecord: z: int = 0 t: int = 0 position_id: str = field(kw_only=True) + channels: tuple[str, ...] = field(kw_only=True) def __post_init__(self) -> None: # Every plane has both coordinates (contract §6.2), so a plane is @@ -70,27 +80,37 @@ def __post_init__(self) -> None: histograms = (histograms,) object.__setattr__(self, "histograms", tuple(histograms)) self._check_position_id() + object.__setattr__( + self, "channels", check_channel_ids(self.channels, f"Plane {self.plane_id}") + ) self._check_channels() @property def channel_count(self) -> int: - """Number of channels of the plane.""" + """Number of channels of the plane (of its source image).""" return len(self.histograms) - def channel_histogram(self, channel: int) -> PlaneHistogram: - """Histogram of channel ``channel`` (zero-based). + def has_channel(self, channel: str) -> bool: + """Whether the plane has (lists as present) the channel with ID ``channel``.""" + return channel in self.channels + + def channel_histogram(self, channel: str) -> PlaneHistogram: + """Histogram of the channel with canonical ID ``channel``. Raises ------ ValueError - If the plane has no such channel. + If ``channel`` is not a channel ID (an index is refused), or the plane does + not have that channel (it is absent from the plane's imaging unit). """ - if not 0 <= channel < self.channel_count: + check_channel_id(channel, f"Plane {self.plane_id}") + if channel not in self.channels: raise ValueError( - f"Plane {self.plane_id} has {self.channel_count} channel(s), numbered from 0, " - f"so it has no channel {channel}." + f"Plane {self.plane_id} has channel(s) {', '.join(self.channels)}; channel " + f"{channel} is absent from it (its imaging unit does not list it), so it has no " + "histogram. An absent channel is not measured and is never taken as zero." ) - return self.histograms[channel] + return self.histograms[self.channels.index(channel)] @property def image_file(self) -> str: @@ -126,6 +146,7 @@ def _check_position_id(self) -> None: ) def _check_channels(self) -> None: + """Refuse no histogram, one ID per histogram missing, or channels that differ.""" if not self.histograms: raise ValueError(f"Plane {self.plane_id} has no channels; give one histogram each.") if not all(isinstance(h, PlaneHistogram) for h in self.histograms): @@ -133,8 +154,15 @@ def _check_channels(self) -> None: f"Plane {self.plane_id}: the histograms must be PlaneHistogram objects, one " "per channel." ) + if len(self.channels) != len(self.histograms): + raise ValueError( + f"Plane {self.plane_id} has {len(self.histograms)} histogram(s) but " + f"{len(self.channels)} channel ID(s) ({', '.join(self.channels) or 'none'}). " + "Give one channel ID per histogram, in source order: the channels list of the " + "image's imaging unit in the experiment description." + ) first = self.histograms[0] - for channel, hist in enumerate(self.histograms[1:], 1): + for channel, hist in zip(self.channels[1:], self.histograms[1:], strict=True): for what, a, b in ( ("pixel type", first.dtype, hist.dtype), ("bin count", first.bin_count, hist.bin_count), @@ -143,10 +171,110 @@ def _check_channels(self) -> None: if a != b: raise ValueError( f"Plane {self.plane_id}: channel {channel} has {what} {b}, but channel " - f"0 has {a}. All channels of a plane have the same {what}." + f"{self.channels[0]} has {a}. All channels of a plane have the same " + f"{what}." ) +@dataclass(frozen=True) +class ChannelMeasurementFailure: + """A channel of a plane whose histogram could not be measured (ADR 0003 HV3). + + The calling brick records one when computing a channel's histogram fails (a read + error, an unsupported value); the plane-histogram table then cannot be written + (:func:`~morphobricks_normalization.csv_io.write_plane_histograms`). It is not an + absent channel (a channel the imaging unit does not list), and not an empty + histogram (a measured channel without finite pixels, which is a measurement). + + Attributes + ---------- + plane_id + The plane, as :func:`make_plane_id` gives it. + channel + The canonical channel ID of the channel that failed. + reason + What went wrong, in plain language, for the run report. + """ + + plane_id: str + channel: str + reason: str + + def __post_init__(self) -> None: + if not isinstance(self.plane_id, str) or not self.plane_id: + raise ValueError( + f"A failed measurement names its plane (plane_id); got {self.plane_id!r}." + ) + check_channel_id(self.channel, f"Failed measurement of plane {self.plane_id}") + if not isinstance(self.reason, str) or not self.reason.strip(): + raise ValueError( + f"The failed measurement of plane {self.plane_id}, channel {self.channel}, " + "needs a reason for the run report." + ) + + +class ChannelMeasurementError(ValueError): + """A channel measurement failed, so the plane-histogram table is not written (HV3). + + Attributes + ---------- + failures + Every failed measurement, in the order given. + removed + The table of an earlier run that the writer removed, so that no stale table is + left that looks current; None when there was none. + """ + + def __init__( + self, failures: Sequence[ChannelMeasurementFailure], removed: str | None = None + ) -> None: + self.failures = tuple(failures) + self.removed = removed + listed = "; ".join( + f"plane {f.plane_id}, channel {f.channel}: {f.reason}" + for f in self.failures[:_MAX_LISTED] + ) + more = ( + f"; and {len(self.failures) - _MAX_LISTED} more" + if len(self.failures) > _MAX_LISTED + else "" + ) + stale = ( + f" The table of an earlier run, {removed}, was removed, so that no out-of-date " + "table is left that looks current." + if removed + else "" + ) + super().__init__( + f"{len(self.failures)} channel measurement(s) failed ({listed}{more}), so " + "plane_histograms.csv was not written: the table holds a measured histogram for " + f"every channel a plane has, and never a partial table.{stale} Fix the cause (read " + "the failing image and channel), then run the preprocessing brick again." + ) + + +def check_measurements(failures: Iterable[ChannelMeasurementFailure]) -> None: + """Refuse a run with any failed channel measurement (ADR 0003 HV3). + + Parameters + ---------- + failures + The run's failed measurements; empty when every channel was measured. + + Raises + ------ + ChannelMeasurementError + Naming every failing plane and channel, when ``failures`` is not empty. + TypeError + For an entry that is not a :class:`ChannelMeasurementFailure`. + """ + failures = list(failures) + if not all(isinstance(f, ChannelMeasurementFailure) for f in failures): + raise TypeError("Give the failed measurements as ChannelMeasurementFailure objects.") + if failures: + raise ChannelMeasurementError(failures) + + def _coordinate(value: object, name: str, record: PlaneRecord) -> int: """Return a plane coordinate as an int: None is 0, other values must be whole and >= 0.""" if value is None: @@ -188,6 +316,66 @@ def duplicate_planes(records: Iterable[PlaneRecord], subject: str) -> None: ) +def check_universe(records: list[PlaneRecord], universe: set[str]) -> None: + """Refuse records with a channel the experiment's list does not have.""" + outside: dict[str, list[str]] = {} + for record in records: + for channel in record.channels: + if channel not in universe: + outside.setdefault(channel, []).append(record.plane_id) + if outside: + parts = "; ".join( + f"{channel} (in {', '.join(ids[:3])}{' ...' if len(ids) > 3 else ''})" + for channel, ids in sorted(outside.items()) + ) + raise ValueError( + f"Some planes have channels that are not in the experiment's channel IDs " + f"({', '.join(sorted(universe))}): {parts}. Give channel_ids every channel ID of the " + "experiment description." + ) + + +def same_channels_per_image(records: Iterable[PlaneRecord], subject: str) -> None: + """Refuse an image whose planes list different channels. + + One source file has one channel layout: every Z and T plane of it has the same + channels in the same source order. Planes that differ would pool one stain under + another's ID or drop a channel for some planes, so they are refused, never merged. + + Parameters + ---------- + records + The records of one table or group. + subject + Starts the message, e.g. ``"The planes to write"``. + + Raises + ------ + ValueError + Naming each such image and the channel lists of its planes. + """ + layouts: dict[str, dict[tuple[str, ...], list[str]]] = {} + for record in records: + planes = layouts.setdefault(record.source_relative_path, {}) + planes.setdefault(record.channels, []).append(record.plane_id) + mixed = {path: found for path, found in layouts.items() if len(found) > 1} + if not mixed: + return + parts = [] + for path, found in list(mixed.items())[:_MAX_LISTED]: + lists = "; ".join( + f"{';'.join(channels)} in {', '.join(ids[:2])}{' ...' if len(ids) > 2 else ''}" + for channels, ids in found.items() + ) + parts.append(f"{path} ({lists})") + more = f" and {len(mixed) - _MAX_LISTED} more" if len(mixed) > _MAX_LISTED else "" + raise ValueError( + f"{subject} hold images whose planes list different channels: {', '.join(parts)}{more}. " + "Every plane of one image has the channels of its imaging unit, in the same order; " + "run the preprocessing brick again on this experiment." + ) + + def make_plane_id(source_relative_path: str, *, z: int | None = 0, t: int | None = 0) -> str: """Build a plane identifier such as ``"cond/rep1/img.tif::z=3,t=0"``. @@ -209,25 +397,3 @@ def group_records( for record in records: groups.setdefault(record.group_key(level), []).append(record) return dict(sorted(groups.items())) - - -def different_channel_counts(images: list[tuple[str, int]], subject: str) -> str: - """Message for images with different numbers of channels (N43). - - ``images`` lists ``(source_relative_path, channel_count)``, repeats allowed; - ``subject`` starts the sentence, e.g. ``"These images"``. - """ - by_count: dict[int, dict[str, None]] = {} - for image, count in images: - by_count.setdefault(count, {})[image] = None # each image once, in order - parts = [] - for count, found in sorted(by_count.items()): - names = list(found) - shown = ", ".join(names[:_MAX_LISTED]) - more = f" and {len(names) - _MAX_LISTED} more" if len(names) > _MAX_LISTED else "" - parts.append(f"{count} channel(s): {shown}{more}") - return ( - f"{subject} have different numbers of channels ({'; '.join(parts)}). Every image of an " - "experiment must have the same channels; split or exclude the other images, then run " - "the preprocessing brick again." - ) diff --git a/src/morphobricks_normalization/rgb.py b/src/morphobricks_normalization/rgb.py index eb7625b..47c62e1 100644 --- a/src/morphobricks_normalization/rgb.py +++ b/src/morphobricks_normalization/rgb.py @@ -33,6 +33,8 @@ RGB_CHANNEL_COUNT = 3 # The pixel types of RGB images: 8- and 16-bit integers, every value exact in float32 (#23). RGB_DTYPES = frozenset(np.dtype(name) for name in ("uint8", "uint16", "int8", "int16")) +# The fixed channel IDs of an RGB imaging unit, in source order (experiment-v1 §4.4b). +RGB_CHANNELS = ("R", "G", "B") def rgb_percentile( @@ -111,6 +113,31 @@ def check_rgb(channel_count: int, dtypes: Iterable[np.dtype]) -> None: ) +def check_rgb_channels(channels: Iterable[str], where: str) -> None: + """Refuse RGB channels that are not ``R``, ``G`` and ``B`` in this order (§4.4b). + + Parameters + ---------- + channels + The channel IDs, in source order. + where + Starts the message, e.g. ``"Plane a.tif::z=0,t=0"``. + + Raises + ------ + ValueError + Naming the IDs found. + """ + channels = tuple(channels) + if channels != RGB_CHANNELS: + raise ValueError( + f"{where}: RGB normalization needs the channels R, G and B of an RGB imaging unit, " + f"in this order (experiment-v1 §4.4b), but it has {', '.join(map(str, channels))}. " + "Declare the unit as image type 'rgb' in the experiment description, or choose " + "image type 'fluorescence'." + ) + + def check_rgb_pixels(pixels: np.ndarray, channel_axis: int | None) -> None: """Refuse a pixel array that is not three 8- or 16-bit channels, before any work (#52, #23). @@ -144,6 +171,8 @@ def check_rgb_records(records: list[PlaneRecord], group: str | None) -> None: or converted; the message names the images of each pixel type. """ check_rgb(records[0].channel_count, [h.dtype for r in records for h in r.histograms]) + for record in records: + check_rgb_channels(record.channels, f"Plane {record.plane_id}") by_dtype: dict[str, list[str]] = {} for record in records: files = by_dtype.setdefault(record.histograms[0].dtype.name, []) @@ -160,7 +189,7 @@ def check_rgb_records(records: list[PlaneRecord], group: str | None) -> None: ) -def check_rgb_selection(wanted: list[int], channels: tuple[int, ...]) -> None: +def check_rgb_selection(wanted: list[str], channels: tuple[str, ...]) -> None: """Refuse a strict subset of R, G and B: RGB parameters keep the channels together (#50). A subset would serialize as a single channel that ``from_columns`` reads back as diff --git a/src/morphobricks_normalization/settings.py b/src/morphobricks_normalization/settings.py index c99e466..99eb86f 100644 --- a/src/morphobricks_normalization/settings.py +++ b/src/morphobricks_normalization/settings.py @@ -27,6 +27,10 @@ RGB_METHODS = ("percentile",) SCOPES = ("biological_repeat", "image", "patch") POOLING_WEIGHTS = ("pixels", "conditions") +# What a fit does with a channel that only some planes of its group have (ADR 0003; Spec +# experiment-v1 §8, A_SCOPE_INCOMPLETE): refuse (default), or pool the planes that have it, +# the user's recorded choice of that subset. +PARTIAL_CHANNELS = ("refuse", "planes_with_channel") DEFAULT_PERCENTILE = (1.0, 99.8) # Written names of the methods without options (contract §3.1); the same names @@ -42,6 +46,7 @@ "scope", "pooling", "per_time_point", + "partial_channels", ) @@ -80,6 +85,14 @@ class NormalizationSettings: (default, each pixel once) or ``"conditions"`` (each condition equally). per_time_point Whether each time point is a group of its own (default False: time points pooled). + partial_channels + What :func:`~morphobricks_normalization.fitting.fit` does with a channel that only + some planes of the group have: ``"refuse"`` (default; the Spec's + ``A_SCOPE_INCOMPLETE``, never a fit over part of a group without the user's choice) + or ``"planes_with_channel"`` (the user selects that subset: the channel is pooled + over the planes that have it, the others are named in a + ``channel_partially_present`` warning, and the choice is stored with the settings). + Fluorescence only: RGB rows always hold R, G and B. Raises ------ @@ -97,6 +110,7 @@ class NormalizationSettings: scope: str = "image" pooling: str = "pixels" per_time_point: bool = False + partial_channels: str = "refuse" def __post_init__(self) -> None: if self.image_type not in IMAGE_TYPES: @@ -122,6 +136,12 @@ def __post_init__(self) -> None: f"Pooling {self.pooling!r} does not exist. Choose 'pixels' (each pixel counts " "once) or 'conditions' (each condition weighs the same)." ) + if self.partial_channels not in PARTIAL_CHANNELS: + raise ValueError( + f"partial_channels {self.partial_channels!r} does not exist. Choose 'refuse' " + "(a channel some planes of a group lack is refused) or 'planes_with_channel' " + "(it is fitted from the planes that have it, and the others are reported)." + ) if not isinstance(self.per_time_point, bool): raise ValueError(f"per_time_point must be true or false; got {self.per_time_point!r}.") percentile = DEFAULT_PERCENTILE @@ -134,6 +154,8 @@ def __post_init__(self) -> None: object.__setattr__(self, "consistency", consistency) if self.scope != "biological_repeat": object.__setattr__(self, "pooling", "pixels") + if self.image_type == "rgb": # RGB rows always hold R, G and B: no partial channel + object.__setattr__(self, "partial_channels", "refuse") @property def method_name(self) -> str: @@ -162,6 +184,8 @@ def to_dict(self) -> dict[str, Any]: if self.scope == "biological_repeat": data["pooling"] = self.pooling data["per_time_point"] = self.per_time_point + if self.partial_channels != "refuse": # the user's recorded choice of a subset + data["partial_channels"] = self.partial_channels return data @classmethod diff --git a/src/morphobricks_normalization/summaries.py b/src/morphobricks_normalization/summaries.py index 78f1da1..af6ed0f 100644 --- a/src/morphobricks_normalization/summaries.py +++ b/src/morphobricks_normalization/summaries.py @@ -13,8 +13,11 @@ binning. Otherwise the fraction is blank (None). :meth:`ImageSummary.to_row` gives the columns of contract §7.1, channels in -CellProfiler-style columns (N41, N42), for the calling brick to show or write; -the library writes no file (N50). +CellProfiler-style columns named by canonical channel ID (N41, N42 as amended by +ADR 0003 HV2), for the calling brick to show or write; the library writes no file +(N50). Images of one experiment may have different channels; each summary holds +the channels of its image, and a channel the image does not have has no value +(blank in a table), never zero. """ from __future__ import annotations @@ -26,10 +29,11 @@ import numpy as np +from .channels import check_channel_ids from .csv_io import channel_column from .dtypes import is_integer from .pooling import NoValidPixelsError, PooledHistogram -from .records import PlaneRecord, different_channel_counts, duplicate_planes +from .records import PlaneRecord, duplicate_planes, same_channels_per_image from .settings import DEFAULT_PERCENTILE SHARED_COLUMNS = ( @@ -53,8 +57,9 @@ class IncompatibleSummariesError(ValueError): """Raised for image summaries that cannot be compared or pooled with each other. - Different numbers of channels, percentile definitions or pixel types make the - values measure different things; nothing is converted or guessed. + Different percentile definitions or pixel types make the values measure different + things, and per-channel values without one value per channel describe no channel; + nothing is converted or guessed. """ @@ -70,9 +75,11 @@ class ImageSummary: Pixel type of the image. percentile ``(low, high)`` in percent: the percentiles held by ``p_low`` and ``p_high``. + channels + The canonical channel IDs of the image, sorted; the per-channel values below + follow this order. median, p_low, p_high - One value per channel, channel 0 first; None for a channel without - finite pixels. + One value per channel; None for a channel without finite pixels. saturated_fraction Per channel, the pixels equal to the pixel type's maximum over the valid pixels; None for float images, for a maximum that shares its bin with @@ -89,6 +96,7 @@ class ImageSummary: technical_repeat: str dtype: np.dtype percentile: tuple[float, float] + channels: tuple[str, ...] median: tuple[float | None, ...] p_low: tuple[float | None, ...] p_high: tuple[float | None, ...] @@ -97,17 +105,24 @@ class ImageSummary: def __post_init__(self) -> None: """Refuse per-channel tuples of different lengths, or none (issue #56).""" + channels = check_channel_ids(self.channels, f"Image {self.source_relative_path}") + if list(channels) != sorted(channels): + raise IncompatibleSummariesError( + f"Image {self.source_relative_path}: its channels {list(channels)} are not in " + "sorted order. Build the summary with image_summaries()." + ) + object.__setattr__(self, "channels", channels) lengths = { name: len(getattr(self, name)) - for name in ("median", "p_low", "p_high", "saturated_fraction", "exact") + for name in ("channels", "median", "p_low", "p_high", "saturated_fraction", "exact") } if len(set(lengths.values())) > 1 or lengths["median"] == 0: found = ", ".join(f"{name}: {n}" for name, n in lengths.items()) raise IncompatibleSummariesError( f"Image {self.source_relative_path}: its per-channel values do not have one " f"entry per channel ({found}), so the channels of the image are not defined. " - "Give median, p_low, p_high, saturated_fraction and exact the same, non-zero " - "length, or build the summary with image_summaries()." + "Give channels, median, p_low, p_high, saturated_fraction and exact the same, " + "non-zero length, or build the summary with image_summaries()." ) @property @@ -123,11 +138,13 @@ def channel_count(self) -> int: def to_row(self) -> dict[str, Any]: """Return the columns of contract §7.1, in order. - The shared columns as text, then for each channel the median, low and - high percentiles and saturated fraction as floats (None for a blank) and - ``Histogram_Exact_C`` as a bool (None for a channel without finite - pixels). A list of rows can be passed to ``pandas.DataFrame`` or - ``csv.DictWriter`` (with :func:`summary_columns`). + The shared columns as text, then for each channel of the image, in sorted ID + order, the median, low and high percentiles and saturated fraction as floats + (None for a blank) and ``Histogram_Exact_`` as a bool (None for a channel + without finite pixels). Channels the image does not have are not in the row; a + table of the experiment (:func:`summary_columns`) leaves them blank, for + example ``csv.DictWriter(..., restval="")``. A list of rows can be passed to + ``pandas.DataFrame`` or ``csv.DictWriter``. """ row: dict[str, Any] = { "source_relative_path": self.source_relative_path, @@ -136,28 +153,29 @@ def to_row(self) -> dict[str, Any]: "technical_repeat": self.technical_repeat, "image_file": self.image_file, } - for channel in range(self.channel_count): + for k, channel in enumerate(self.channels): values = ( - self.median[channel], - self.p_low[channel], - self.p_high[channel], - self.saturated_fraction[channel], - self.exact[channel], + self.median[k], + self.p_low[k], + self.p_high[k], + self.saturated_fraction[k], + self.exact[k], ) for feature, value in zip(SUMMARY_FEATURES, values, strict=True): row[channel_column(feature, channel)] = value return row -def summary_columns(channel_count: int) -> list[str]: - """Column names of :meth:`ImageSummary.to_row` for images with ``channel_count`` channels.""" +def summary_columns(channel_ids: Iterable[str]) -> list[str]: + """Column names of a table of :meth:`ImageSummary.to_row` rows of these channels. + + ``channel_ids`` are the experiment's channel IDs (sorted here), as for the header + of ``plane_histograms.csv``. + """ + ids = sorted(check_channel_ids(channel_ids, "summary_columns")) return [ *SHARED_COLUMNS, - *( - channel_column(feature, channel) - for channel in range(channel_count) - for feature in SUMMARY_FEATURES - ), + *(channel_column(feature, channel) for channel in ids for feature in SUMMARY_FEATURES), ] @@ -183,20 +201,15 @@ def image_summaries( Raises ------ ValueError - For invalid percentiles, rows that repeat a ``plane_id``, or images with - different numbers of channels (N43). + For invalid percentiles, rows that repeat a ``plane_id``, or an image whose + planes list different channels. IncompatibleSummariesError For an image whose Z or T planes have different pixel types (issue #54). """ low, high = _check_percentile(percentile) records = list(records) duplicate_planes(records, "These rows") - if len({r.channel_count for r in records}) > 1: - raise ValueError( - different_channel_counts( - [(r.source_relative_path, r.channel_count) for r in records], "These images" - ) - ) + same_channels_per_image(records, "These rows") images: dict[tuple[str, ...], list[PlaneRecord]] = {} for record in records: images.setdefault(record.group_key("image"), []).append(record) @@ -242,13 +255,13 @@ def iter_image_summaries( ValueError At the call, for invalid percentiles. While iterating: for an image whose rows are not contiguous, a source file listed under two sets of labels, rows that repeat a - ``plane_id``, or images with different numbers of channels (N43). + ``plane_id``, or an image whose planes list different channels. IncompatibleSummariesError For an image whose Z or T planes have different pixel types (issue #54). Notes ----- - Whether all images have the same number of channels is only known as they are read, so an + Whether the rows are valid is only known as they are read, so an error can come after earlier summaries were emitted. Those summaries are valid on their own, but a run that ended with an error is incomplete: discard its output; do not publish it as the result for the experiment. :func:`image_summaries` validates every row before @@ -264,7 +277,6 @@ def _stream_summaries( """Group rows by adjacent image, check each group and yield its summary.""" seen_keys: set[tuple[str, ...]] = set() labels_of_path: dict[str, tuple[str, ...]] = {} - first_channels: tuple[str, int] | None = None for key, group in groupby(records, key=lambda r: r.group_key("image")): planes = list(group) path = key[-1] @@ -283,18 +295,7 @@ def _stream_summaries( "the preprocessing brick again on this experiment." ) duplicate_planes(planes, "These rows") - counts = {r.channel_count for r in planes} - if first_channels is None: - first_channels = (path, planes[0].channel_count) - counts.add(first_channels[1]) - if len(counts) > 1: - raise ValueError( - different_channel_counts( - [(first_channels[0], first_channels[1])] - + [(r.source_relative_path, r.channel_count) for r in planes], - "These images", - ) - ) + same_channels_per_image(planes, "These rows") yield _summary(planes, percentile) del planes @@ -315,11 +316,12 @@ def _check_one_dtype(planes: list[PlaneRecord]) -> None: def _summary(planes: list[PlaneRecord], percentile: tuple[float, float]) -> ImageSummary: - """Pool the planes of one image, channel by channel.""" + """Pool the planes of one image, channel by channel ID.""" _check_one_dtype(planes) first = planes[0] + channels = tuple(sorted(first.channels)) median, p_low, p_high, saturated, exact = [], [], [], [], [] - for channel in range(first.channel_count): + for channel in channels: histograms = [plane.channel_histogram(channel) for plane in planes] try: pooled = PooledHistogram(histograms) @@ -341,6 +343,7 @@ def _summary(planes: list[PlaneRecord], percentile: tuple[float, float]) -> Imag technical_repeat=first.technical_repeat, dtype=first.histograms[0].dtype, percentile=percentile, + channels=channels, median=tuple(median), p_low=tuple(p_low), p_high=tuple(p_high), diff --git a/src/morphobricks_normalization/tail_moments.py b/src/morphobricks_normalization/tail_moments.py new file mode 100644 index 0000000..e477df4 --- /dev/null +++ b/src/morphobricks_normalization/tail_moments.py @@ -0,0 +1,121 @@ +"""Exact centred moments of the pixels of a histogram tail (contract §1.4; schema version 5). + +The pixels outside a robust range are stored by their count, mean and standard deviation, +not by their sum and sum of squares as schema version 4 did. The raw sum of squares lost +the spread of a tail of large values with a small spread by cancellation +(``sum_of_squares - sum * mean``), and it overflowed float64 for values beyond about +``1e154`` although the spread was finite (#91, #33). + +Here every pixel value is turned into an exact integer multiple of a power of two (its +float64 significand and exponent, or the integer itself), so the sum and the sum of squares +are kept as exact rationals over any number of blocks, in any order. The mean is then the +float64 nearest the exact mean, and the standard deviation the float64 nearest the exact +population standard deviation about the exact mean. Both are finite for finite pixels (the +standard deviation is at most half the tail's range), and a plane gives the same bits read +whole or in blocks. +""" + +from __future__ import annotations + +import math +from fractions import Fraction + +import numpy as np + +_SIGNIFICAND_BITS = 53 +# Extra bits of the integer square root before the one rounding to float64. +_SQRT_BITS = 2 * _SIGNIFICAND_BITS + 16 + + +class TailMoments: + """Running exact count, sum and sum of squares of one tail's pixels, block by block.""" + + def __init__(self) -> None: + self.count = 0 + self.total = Fraction(0) + self.squares = Fraction(0) + + def add(self, values: np.ndarray) -> None: + """Add the pixels ``values`` (finite; integers or floats) to the tail.""" + if values.size == 0: + return + total, squares = exact_sums(values) + self.count += int(values.size) + self.total += total + self.squares += squares + + def moments(self) -> tuple[int, float, float]: + """Return ``(count, mean, std)`` of the pixels added so far (zeros when none).""" + return centred_moments(self.count, self.total, self.squares) + + +def exact_sums(values: np.ndarray) -> tuple[Fraction, Fraction]: + """Return the exact sum and sum of squares of finite integer or float values.""" + if values.dtype.kind in "iu": + whole = values.tolist() # Python integers: exact for every 64-bit value + return Fraction(sum(whole)), Fraction(sum(v * v for v in whole)) + significands, exponents = np.frexp(values.astype(np.float64, copy=False)) + # Each value is whole * 2**shift exactly: frexp's significand has at most 53 bits. + whole = (significands * 2.0**_SIGNIFICAND_BITS).astype(np.int64) + shifts = exponents.astype(np.int64) - _SIGNIFICAND_BITS + order = np.argsort(shifts, kind="stable") + unique, starts = np.unique(shifts[order], return_index=True) + total, squares = 0, 0 + lowest = int(unique[0]) + for shift, group in zip(unique.tolist(), np.split(whole[order], starts[1:]), strict=True): + group = group.tolist() + offset = shift - lowest # every group on the lowest power of two, as integers + total += sum(group) << offset + squares += sum(m * m for m in group) << (2 * offset) + return _times_power_of_two(total, lowest), _times_power_of_two(squares, 2 * lowest) + + +def counted_sums(counts: np.ndarray, first_value: int) -> tuple[Fraction, Fraction]: + """Return the exact sum and sum of squares of per-value ``counts`` from ``first_value``.""" + index = np.flatnonzero(counts) + pairs = zip(counts[index].tolist(), (index + first_value).tolist(), strict=True) + total, squares = 0, 0 + for count, value in pairs: + total += count * value + squares += count * value * value + return Fraction(total), Fraction(squares) + + +def centred_moments(count: int, total: Fraction, squares: Fraction) -> tuple[int, float, float]: + """Return ``(count, mean, std)`` of values with exact ``total`` and ``squares``. + + The mean and the population standard deviation are each rounded once, to the nearest + float64, from their exact values; zeros for no values. + """ + if count == 0: + return 0, 0.0, 0.0 + mean = total / count + variance = squares / count - mean * mean # exact, so never negative + return count, float(mean), sqrt_nearest(variance) + + +def sqrt_nearest(value: Fraction) -> float: + """Return the float64 nearest the square root of a non-negative rational.""" + if value < 0: + raise ValueError(f"A variance cannot be negative; got {value}.") + if value == 0: + return 0.0 + numerator, denominator = value.numerator, value.denominator + # 2**scale * sqrt(value) has about _SQRT_BITS bits, far more than float64 keeps. + scale = _SQRT_BITS - (numerator.bit_length() - denominator.bit_length()) // 2 + if scale >= 0: + scaled, remainder = divmod(numerator << (2 * scale), denominator) + else: + scaled, remainder = divmod(numerator, denominator << (-2 * scale)) + root = math.isqrt(scaled) + # A sticky last bit for the part below the integer root, so that rounding to float64 + # once more cannot land on a false tie. + inexact = remainder != 0 or root * root != scaled + return float(_times_power_of_two(2 * root + int(inexact), -(scale + 1))) + + +def _times_power_of_two(whole: int, exponent: int) -> Fraction: + """Return ``whole * 2**exponent`` as an exact rational.""" + if exponent >= 0: + return Fraction(whole << exponent) + return Fraction(whole, 1 << -exponent) diff --git a/tests/described.py b/tests/described.py new file mode 100644 index 0000000..0685c3c --- /dev/null +++ b/tests/described.py @@ -0,0 +1,136 @@ +"""Image trees built from experiment descriptions, for the Stage 3 tests (ADR 0003). + +A description is read from the vendored MorphoBricks-Spec fixtures (``tests/spec_fixtures``, +pinned in ``PIN.toml``) with ``tomllib``, or written here as the same TOML text (the owner's +case is not a Spec fixture). Only what the statistics need is taken from it: every imaging +unit (repeat, condition, technical repeat) with its folder labels and its ordered ``channels`` +list (experiment-v1 §4.4), and the experiment's channel IDs: the keys of its ``[channels]`` +table, the declared IDs (open question 1 of PR #90: never ``used_ids.channels``, which also +keeps retired IDs no image can have). + +Each image is simulated as the calling brick holds it, a ``(C, Y, X)`` array in the unit's +source channel order; its histograms are computed per C index, exactly as the preprocessing +brick does, and the record is labelled with the unit's list. The pixels of a channel depend on +the image, the time point and the channel *ID* only, never on its index, so two layouts of the +same stains (``v04``) hold the same pixels per ID. +""" + +import tomllib +import zlib +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from morphobricks_normalization import SINGLE, PlaneRecord, compute_plane_histogram + +FIXTURES = Path(__file__).parent / "spec_fixtures" / "fixtures" / "valid" +# Brightness per stain, so pooling one stain under another's ID changes every number. +STAIN_LEVEL = {"DNA": 1.0, "ACTIN": 2.5, "TUBULIN": 0.6, "PAXILLIN": 1.7, "PHASE": 4.0} + + +@dataclass(frozen=True) +class Unit: + """One imaging unit of a description: where its images are and which channels they have.""" + + repeat: str + condition: str + source_repeat: str + technical_repeat: str + channels: tuple[str, ...] + + def folder(self) -> str: + parts = [self.condition, self.source_repeat] + if self.technical_repeat != SINGLE: + parts.append(self.technical_repeat) + return "/".join(parts) + + +@dataclass(frozen=True) +class Description: + """The units and channel IDs of a description.""" + + units: tuple[Unit, ...] + channel_ids: tuple[str, ...] + + def repeat_groups(self) -> dict[tuple[str, str], str]: + """``(condition, source repeat)`` to the canonical repeat ID (contract §4.2).""" + return {(u.condition, u.source_repeat): u.repeat for u in self.units} + + +def parse(text: str) -> Description: + data = tomllib.loads(text) + units = [] + for repeat, entry in sorted(data["repeats"].items()): + for condition, member in sorted(entry["members"].items()): + folder = data["conditions"][condition]["source_folder"] + for _, tech in sorted(member["technical_repeats"].items()): + units.append( + Unit(repeat, folder, member["source_repeat"], tech["source_folder"], + tuple(tech["channels"])) + ) # fmt: skip + return Description(tuple(units), tuple(data["channels"])) # declared, not used_ids + + +def fixture(name: str) -> Description: + """The description of a vendored Spec fixture, e.g. ``"v04_swapped_channel_order"``.""" + return parse((FIXTURES / f"{name}.toml").read_text(encoding="utf-8")) + + +def renamed(description: Description, old: str, new: str) -> Description: + """The same description with channel ID ``old`` renamed ``new`` (no fixture has a ``_``).""" + + def swap(ids): + return tuple(new if c == old else c for c in ids) + + units = tuple( + Unit(u.repeat, u.condition, u.source_repeat, u.technical_repeat, swap(u.channels)) + for u in description.units + ) + return Description(units, swap(description.channel_ids)) + + +def channel_pixels(path: str, t: int, channel: str, repeat: str, shape=(12, 12)) -> np.ndarray: + """The pixels of one channel plane: a function of the image, time point and channel ID.""" + rng = np.random.default_rng(zlib.crc32(f"{path}|{t}|{channel}".encode())) + level = STAIN_LEVEL.get(channel, 1.0 + zlib.crc32(channel.encode()) % 7 / 3) + level *= 1.0 + 0.25 * (zlib.crc32(repeat.encode()) % 4) + return np.clip(rng.normal(40 * level, 6 * level, shape), 0, 255).astype(np.uint8) + + +def images(description: Description, n_images: int = 2, n_times: int = 2): + """Yield ``(unit, path, t, pixels)``, ``pixels`` a ``(C, Y, X)`` array in source order.""" + for unit in description.units: + for i in range(n_images): + path = f"{unit.folder()}/img_{i}.tif" + for t in range(n_times): + planes = [channel_pixels(path, t, c, unit.repeat) for c in unit.channels] + yield unit, path, t, np.stack(planes) + + +def record(unit: Unit, path: str, t: int, pixels: np.ndarray, channels=None) -> PlaneRecord: + """The record the preprocessing brick writes: histograms by C index, the unit's IDs.""" + return PlaneRecord( + path, + unit.condition, + unit.source_repeat, + unit.technical_repeat, + [compute_plane_histogram(plane) for plane in pixels], + t=t, + position_id=path.rsplit("/", 1)[-1].removesuffix(".tif"), + channels=unit.channels if channels is None else channels, + ) + + +def records(description: Description, **kwargs) -> list[PlaneRecord]: + return [record(*item) for item in images(description, **kwargs)] + + +def pixels_of(description: Description, channel: str, where=lambda unit, path, t: True, **kwargs): + """Every pixel of ``channel`` (by ID) in the images ``where`` selects, as float64.""" + found = [ + pixels[unit.channels.index(channel)].ravel() + for unit, path, t, pixels in images(description, **kwargs) + if channel in unit.channels and where(unit, path, t) + ] + return np.concatenate(found).astype(np.float64) diff --git a/tests/simulation.py b/tests/simulation.py index 9d03332..95601d7 100644 --- a/tests/simulation.py +++ b/tests/simulation.py @@ -14,6 +14,20 @@ from morphobricks_normalization import SINGLE, PlaneRecord, compute_plane_histogram +# Channel IDs of the simulated images, by source index. Sorted they keep the index order, so a +# fit's channels (sorted IDs) line up with the simulated channel indices. +RGB = ("R", "G", "B") + + +def channel_id(c): + """The canonical channel ID of simulated source channel ``c``.""" + return f"ch{c}" + + +def channel_ids(n): + """The channel IDs of an image with ``n`` simulated channels, in source order.""" + return tuple(channel_id(c) for c in range(n)) + def simulate_plane(rng, dtype, shape=(64, 64), brightness=1.0, n_cells=6): yy, xx = np.mgrid[0 : shape[0], 0 : shape[1]] @@ -58,7 +72,11 @@ def make_records(planes, bin_count=None): """One single-channel record per plane of :func:`simulate_experiment`.""" return [ PlaneRecord( - path, *labels, histograms=compute_plane_histogram(pixels, bin_count), position_id="p" + path, + *labels, + histograms=compute_plane_histogram(pixels, bin_count), + position_id="p", + channels=channel_ids(1), ) for labels, path, pixels in planes ] @@ -127,7 +145,12 @@ def make_design_records(planes, bin_count=None): channels[c] = compute_plane_histogram(pixels, bin_count) return [ PlaneRecord( - path, *labels, histograms=[channels[c] for c in sorted(channels)], t=t, position_id="p" + path, + *labels, + histograms=[channels[c] for c in sorted(channels)], + t=t, + position_id="p", + channels=channel_ids(len(channels)), ) for (path, t), (labels, channels) in by_plane.items() ] @@ -197,6 +220,7 @@ def make_rgb_records(images, bin_count=None): histograms=[compute_plane_histogram(pixels[..., c], bin_count) for c in range(3)], t=t, position_id="p", + channels=RGB, ) for labels, path, t, pixels in images ] diff --git a/tests/spec_fixtures/.gitattributes b/tests/spec_fixtures/.gitattributes new file mode 100644 index 0000000..011ff88 --- /dev/null +++ b/tests/spec_fixtures/.gitattributes @@ -0,0 +1,3 @@ +# The vendored Spec files are byte-exact copies. Git must not touch line endings +# (the pin is a hash of the bytes; a CRLF checkout would break it). +* -text diff --git a/tests/spec_fixtures/PIN.toml b/tests/spec_fixtures/PIN.toml new file mode 100644 index 0000000..dcc4a91 --- /dev/null +++ b/tests/spec_fixtures/PIN.toml @@ -0,0 +1,14 @@ +# The MorphoBricks-Spec commit these files are copied from (conformance plan section 1). +# Updating the pin is a reviewed change: see README.md in this folder. +repository = "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/CellMigrationLab/MorphoBricks-Spec" +commit = "cc9820f5b84f01bbbb5c8e3f54e70ee13cb5e9eb" + +# `git rev-parse :fixtures`, the git tree-object ID of the fixtures/ folder. The +# vendored folder must hash to it, so one changed, added or missing byte fails a test. +fixtures_tree = "40fe48c6d735266ca7608fab0c3a564affc68171" + +# `git rev-parse :`, the git blob ID of each other vendored file. +[files] +"docs/experiment-v1.md" = "d21388ad9e48608ce42adb0f7b0a4c600cb2c9a7" +"tools/audit/refval.py" = "0ea83094cfa8c1bcb6ea596e17b3f0c948ad5dfb" +"tools/audit/refrecords.py" = "ef06a1ea59a93c695dbba21d2ee792667ad8e8eb" diff --git a/tests/spec_fixtures/README.md b/tests/spec_fixtures/README.md new file mode 100644 index 0000000..37d1118 --- /dev/null +++ b/tests/spec_fixtures/README.md @@ -0,0 +1,19 @@ +# Vendored MorphoBricks-Spec files + +**Do not edit anything in this folder.** It is a byte-exact, pinned copy of +[MorphoBricks-Spec](https://github.com/CellMigrationLab/MorphoBricks-Spec) made by the Spec's shared sync tool. Regenerate it with +`vendor.py`, never by hand: + + python /tools/vendor.py --into --commit + +`PIN.toml` names the Spec commit and the git hash of each pinned path. The pin test hashes +this folder against it offline, and a changed, added or missing byte fails it. The copy is a +test fixture, not a runtime dependency. A fixture that a test fails is a finding for the Spec +repository, not something to change here. How to re-pin: `docs/vendoring.md` in the Spec. + +| Here | In the Spec | +|---|---| +| `fixtures/` | `fixtures/` | +| `docs/experiment-v1.md` | `docs/experiment-v1.md` | +| `tools/audit/refval.py` | `tools/audit/refval.py` | +| `tools/audit/refrecords.py` | `tools/audit/refrecords.py` | diff --git a/tests/spec_fixtures/docs/experiment-v1.md b/tests/spec_fixtures/docs/experiment-v1.md new file mode 100644 index 0000000..d21388a --- /dev/null +++ b/tests/spec_fixtures/docs/experiment-v1.md @@ -0,0 +1,885 @@ +# MorphoBricks experiment description, schema version 1 (normative) + +- **Status:** Normative specification, Stage 0 (6 November 2026). No reader implements it yet. + The fixed-SHA audit of current consumers (roadmap S0.1) is done + ([PR #5](https://github.com/CellMigrationLab/MorphoBricks-Spec/pull/5), audit + [`s01-consumer-audit-2026-11-06.md`](audits/s01-consumer-audit-2026-11-06.md)); the owner answered + its questions AQ1-AQ13 on 6 November 2026 ([ADR 0001](adr/0001-experiment-description-v1.md), + third round) and this text applies them. A fourth decision of the same day (ADR 0001, fourth + round: folder labels are never restricted) gives conditions and technical repeats a canonical ID + separate from their source folder label (§4.6a). A statement about a consumer is as audited as that + audit; it is not a conformance claim for any brick. +- **Decisions:** [ADR 0001](adr/0001-experiment-description-v1.md). +- **Fixtures:** [fixtures/README.md](../fixtures/README.md). **Conformance:** + [conformance-plan.md](conformance-plan.md). + +The key words MUST, MUST NOT, SHOULD and MAY are normative. "Reader" means any program that +parses the file (in v1, preprocessing is the only reference reader); "writer" means any program +that produces it. + +## 1. Scope + +The file `morphobricks.toml` describes **what one preprocessing experiment is**: conditions, +canonical channels, biological-repeat pairing, technical-repeat acquisition layouts, mask +semantics and an optional primary reference per condition. It MUST NOT contain file inventory, +fingerprints, calibration, exclusions, per-FOV statements, axes, brick names or routing, feature +requests, normalization, statistics, histogram or model settings. Those stay in +`calibration.csv`, the run manifest, analysis requests and MorphoCoverage configuration. + +v1 analyses 2-D planes only. A Z stack is projected by a recorded preprocessing step +(preprocessing #176) before it is described; the file has no Z, axes or shared-plane field. +Time-lapses (T > 1) are allowed: the file has no T field, and the homogeneity of an imaging unit +(§8, `R_UNIT_NOT_HOMOGENEOUS`) compares what each axis means, never the number of time points. + +## 2. File, encoding and parsing + +1. The file name is `morphobricks.toml`, in the preprocessing output folder. It is written by the + preprocessing notebook/CLI under the output folder's lock, atomically, and never by `run`. + Readers MUST refuse a symlinked file (`R_FILE_SYMLINK`). +2. The bytes MUST be UTF-8 without a byte-order mark (`E_FILE_ENCODING`) and MUST be valid TOML + 1.0 (`E_TOML_SYNTAX`). A key declared twice, in any spelling TOML treats as the same key + (bare vs quoted, table vs dotted vs inline), MUST be refused as `E_DUPLICATE_KEY`. +3. Readers MUST NOT convert types. A value whose TOML type differs from §4 is `E_TYPE`. In + particular a TOML integer, float, boolean, date, time or datetime where text is required, a + string where an integer or boolean is required, and a boolean where an integer is required + (implementations whose booleans are integers MUST check the exact type). +4. Every table has a closed set of fields. A field not listed in §4 is `E_UNKNOWN_FIELD`, except + the names in §4.7, which are `E_FORBIDDEN_FIELD`. There is no extension namespace. +5. Comments are allowed and carry no meaning. Writers do not preserve them (§9). +6. The SHA-256 of the exact bytes read is the file's provenance identity (run manifest). It is not + a cache key (§9.3). +7. A reader has no source of rules other than the normative text and fixtures of the Spec commit it + pins. It MUST NOT carry a fallback copy or older variant of the rules, and MUST NOT obtain any + enumeration (in v1 only `representation`, §4.5) or vocabulary from outside the reader at run + time. An enumeration it cannot resolve is a refusal, never free text. + +## 3. Identities + +### 3.1 Kinds of identity + +| Identity | Where | Canonical or source | Rule set | +|---|---|---|---| +| Condition ID | key of `conditions` | Canonical (user-confirmed; proposed = the source folder label when allowed, §4.6a) | Label | +| Condition source folder | `conditions..source_folder` | Source (condition folder label, or `single`; not an identity key) | Label | +| Canonical repeat ID | key of `repeats` | Canonical (user-confirmed; proposed = the source folder label when allowed, §4.6a) | Label | +| Source repeat | `source_repeat` | Source (biological-repeat folder label) | Label | +| Technical repeat ID | key of `technical_repeats` | Canonical (user-confirmed; proposed = the source folder label when allowed, §4.6a; or `single`) | Label | +| Technical-repeat source folder | `technical_repeats..source_folder` | Source (technical-repeat folder label, or `single`; not an identity key) | Label | +| Image type | `image_type` | Declared by the user per imaging unit (§4.4b) | Enumeration | +| Channel ID | key of `channels` | Canonical | Token | +| Mask ID | key of `masks` | Canonical (object-set name) | Token + object set | +| Mask source type | `source_type` | Source (discovered mask type) | Label | +| Display name | `display_name` | Display only, never an identity | Non-empty text (§4.8) | + +### 3.2 Rule sets + +- **Label** (preprocessing decision D-e, `check_identity_text`, preprocessing v0.4.0 `bc7b872`): text; + no leading or trailing whitespace (nothing is trimmed: refused as written); not `""`, `NA`, `nan`, `none`, `null`, `n/a` or `` in any + case (`E_ID_TEXT`). +- **Token** (MorphoFeature `naming.py`, MorphoFeature v0.1.0 `182b373`): the Label rules, plus the + regular expression `^[A-Za-z][A-Za-z0-9]*(?:_[A-Za-z0-9]+)*$` (`E_ID_SYNTAX`; this also excludes + the feature-column separator `__`). The two checks are independent rules: an identity that breaks + both is reported with both codes (§6, one code per broken rule). +- **Reserved list** (one list, shared by channel IDs and mask IDs; AQ4). A channel ID or a mask ID + whose case-folded form (`str.casefold()`) is in the list is `E_ID_RESERVED`; `ring`, `Ring` and + `RING` are all refused. An entry of `used_ids.channels` is refused the same way, since a listed + ID reserves that ID (§4.9). The list is closed: a new entry needs a new `schema_version`. + + | Names | Source | + |---|---| + | `whole`, `cell`, `nucleus`, `cytoplasm`, `ring`, `expanded_nucleus` | MorphoRelate compartment names; MorphoFeature `naming.py:44-59` (`182b373`) | + | `condition`, `biological_repeat`, `repeat`, `technical_repeat`, `position_id`, `image_file`, `image_path`, `mask_file`, `patch_file`, `patch_id`, `label_id`, `z`, `z0`, `z1`, `c`, `t` | MorphoRelate reserved column names, `registry/_expand.py:41-59` (MorphoRelate v0.1.1 `160ad23`; name rules `:68-115`) | + + The reserved list is not the forbidden-name list of §4.7, which is compared exactly and applies + to field names and ID keys. A name in both (`z`) is reported by the earlier phase only. Neither + list ever applies to a folder label: a `source_folder` or `source_repeat` value is never + restricted by them (§4.6a). +- **Object set** (MorphoFeature `is_reserved_object_set`): in addition to the reserved list, a mask + ID whose case-folded `_`-separated parts include `in` is `E_ID_RESERVED`. +- **RGB tokens:** `R`, `G` and `B` are the fixed channel IDs of RGB units (§4.4b); they are not on + the reserved list and are valid channel IDs, with the extra rules of §4.4b. +- **`single`:** an identity of any kind whose case-folded form is `single` (a source folder + included) MUST be exactly `single`. A technical repeat `single` MUST be the only technical + repeat of its member; a member with `source_repeat = "single"` MUST be the only member of its + condition in the file. A technical repeat ID is `single` if and only if its `source_folder` is + `single` (`single` stands for "no technical-repeat folder level"; `E_SINGLE_MISUSE`; + preprocessing contract §1). Conditions follow the same rule: a condition ID is `single` if and + only if its `source_folder` is `single` (`single` stands for "no condition folder level", an + experiment folder that is itself the leaf), and a condition whose `source_folder` is `single` + MUST be the only condition in the file (`E_SINGLE_MISUSE`). + +### 3.3 Normalization, uniqueness and case + +- **NFC.** Identities (the keys and identity fields of §3.1, and every entry of `used_ids`) are + Unicode NFC. A writer MUST emit NFC. A reader MUST refuse an identity that is not NFC + (`E_ID_NOT_NFC`, reported for every such identity, including `reference` values, members' keys, + `source_folder` values, unit `channels` entries and `source_channels` entries). The check of a reference is NFC only and + belongs to phase 4, so a non-NFC channel reference is `E_ID_NOT_NFC` and never + `E_UNKNOWN_CHANNEL` (AQ13). A reader never transforms an identity: no normalization, trimming, case folding, parsing + or repair of any kind, and so it never resolves a reference through a transformed spelling. + Display names are not identities and are not checked (§4.8). A folder label read from disk that + is not NFC is refused by preprocessing (`R_FOLDER_NOT_NFC`), never normalized and never renamed; + two folders equal after NFC are therefore also refused, because one of them is not NFC. The + stored `source_folder` is the label exactly as read, so it is NFC by the same rule. +- Identities are compared exactly, code point by code point. +- **No repair.** No reader or writer ever repairs, re-cases, pads, trims or renumbers an + identity. A malformed, colliding or non-NFC ID is refused with its code and left as written. + There is no ID allocator: IDs have no width, prefix or number rule (`R999`, `R1000` and `cl001` + are valid and stay as they are). +- Within each of these scopes, two identities that differ after NFC but whose `str.casefold()` + forms are equal are `E_ID_CASE_COLLISION`: condition IDs; condition source folders; repeat IDs; + source repeats over the whole file; technical repeat IDs over the whole file; technical-repeat + source folders over the whole file; channel IDs; mask IDs; mask source types. + (Preprocessing `check_identity_case`: a label is spelled one way in the whole run.) Case folding + is never applied when resolving a reference (`ctrl` does not resolve to `CTRL`). +- Identities are never parsed. `rep1`, `2026-01-01`, `012` and `acquisition_2` are opaque text; + no number, date, order or meaning is derived from them. +- **Immutability and no reuse.** Once a repeat, condition, technical repeat or channel is + confirmed, its ID never changes (§4.6) and is never reused after removal (§4.9). Only + `display_name` may change, at any time. The `source_folder` of an existing ID is not freely + editable: a change is a reassignment (§4.6). + +### 3.4 Keys downstream: the source label stays, the canonical repeat ID is added (AQ5) + +- The existing FOV key (`condition`, `biological_repeat`, `technical_repeat`, `position_id`) is + **unchanged**. `biological_repeat` in it stays the **source repeat**, the source folder label. + FOV, patch and object identities, fitted-scaling keys, caches and exports keep their current + spelling; a consumer MUST NOT replace the source label by the canonical repeat ID in any of + them. +- The canonical repeat ID is **added** next to the key, as the field `repeat_group` (the column + name of the derived `experiment_design.csv`, §11). Grouping and pairing MUST use `repeat_group`: pooling of + normalization and scaling, pairing of conditions and families in MorphoCoverage, and any + statement about biological N. A consumer MUST NOT group or pair by `biological_repeat` when a + description exists, and MUST NOT fall back to folder-name grouping. +- `repeat_group` is a function of the FOV key (a member belongs to exactly one repeat, + `E_SOURCE_REPEAT_REUSED`), so adding it creates no new identity: two FOVs with equal keys have + equal `repeat_group`. Products that pool over a repeat key their caches on the `repeats` section + digest (§9.3); products of one FOV keep keying on the FOV key. +- Consequence: `rep1` of `CTRL` and `rep1` of `DrugA` may be different repeats (`v30`) and the same + source label may be in two repeats under different conditions; the key stays unique because it + includes the condition. +- **Conditions and technical repeats follow the same pattern (fourth round).** In the existing + FOV key, `condition` and `technical_repeat` stay the **source folder labels** + (`conditions..source_folder`, `technical_repeats..source_folder`); a consumer MUST NOT + replace them by the canonical IDs in FOV, patch or object identities, fitted-scaling keys, + caches or exports. The canonical IDs are **added** next to the key, beside `repeat_group`, under + the names `condition_id` and `technical_repeat_id`, which Stage 1 fixes for every consumer. + Each is a function of the FOV key (one folder, one ID, §4.6a), so adding them creates no new + identity. Everything that names a condition by identity (`reference`, families, pairing, + contrasts, `used_ids`) uses the canonical condition ID, and a consumer MUST NOT group, pair or + reference by the source folder label when a description exists. + +## 4. Tables and fields + +Types are TOML types. "Text" is a TOML string. "Date" is a TOML **string** `YYYY-MM-DD` naming a +real calendar day (`E_DATE_FORMAT`); a TOML date literal is `E_TYPE`. "Non-empty text" refuses +`""` and whitespace-only strings (`E_TEXT_EMPTY`). "Req" = required; a missing required field is +`E_MISSING_FIELD`. An empty table or array where at least one entry is required is `E_EMPTY`. + +### 4.1 Top level + +| Key | Type | Req | Rule | +|---|---|---|---| +| `schema` | text | yes | Exactly `morphobricks-experiment` (`E_SCHEMA_ID`) | +| `schema_version` | integer | yes | Exactly `1` (missing: `E_SCHEMA_VERSION_MISSING`; other integer: `E_SCHEMA_VERSION_UNSUPPORTED`) | +| `conditions` | table of condition | yes | ≥ 1 entry | +| `channels` | table of channel | yes | ≥ 1 entry | +| `repeats` | table of repeat | yes | ≥ 1 entry | +| `masks` | table of mask | no | May be absent or empty | +| `used_ids` | table of ID lists | yes | Append-only registry of every ID ever confirmed, per kind (§4.9) | + +### 4.2 `conditions.` + +| Key | Type | Req | Rule | +|---|---|---|---| +| `source_folder` | text | yes | Exact condition folder label as read from disk, or `single` when the experiment has no condition folder level (Label rules, NFC); a source value, **not an identity key**; not restricted by the reserved or forbidden names (§4.6a) | +| `display_name` | non-empty text | no | Display only (§4.8); MAY be the folder label | +| `reference` | text | no | One condition ID: the primary reference (§5) | +| `target` | non-empty text | no | Descriptive plain string, e.g. `SRC` (§4.2a) | +| `treatment_key` | non-empty text | no | Descriptive plain string, e.g. the same perturbation under different backgrounds (§4.2a) | + +Every condition MUST be the condition of at least one member (`E_CONDITION_WITHOUT_MEMBERS`). + +**§4.2a Descriptive only.** `target` and `treatment_key` are descriptive free text, written and +shown as typed. They are not identities: no Label or Token rule, no case-collision scope, no +uniqueness (two conditions may carry the same value, or values differing only in case). A consumer +MUST NOT use either for grouping, pairing, joins, contrasts, family definitions or any cache key +(§9.3 omits them from every digest). A future use as a grouping key needs a new `schema_version` +that makes it an identity. + +**Prefill only (AQ3).** A consumer MAY offer the values as a *suggestion* to prefill an input of +its own, for example MorphoCoverage's reagent grouping. The suggestion is shown to the user, who +confirms or edits it **inside that consumer**; the confirmed grouping is the consumer's own input, +recorded in its own configuration, and is the only thing the analysis reads. An unconfirmed +prefill is never used, and a later edit of `target` or `treatment_key` in the file changes no +confirmed grouping, no digest and no cache key. + +### 4.3 `channels.` + +| Key | Type | Req | Rule | +|---|---|---|---| +| `display_name` | non-empty text | no | Display only (§4.8), e.g. `DAPI` | +| `header_names` | array of non-empty text | no (default `[]`) | Header channel names the user confirmed as evidence for this channel; stored as typed (not identities: no Label, Token or NFC check; outer whitespace is part of the name); no duplicates, where two entries equal after NFC are duplicates (`E_DUPLICATE_VALUE`); order not semantic. Not allowed for `R`, `G`, `B` (§4.4b) | + +A channel that appears in no technical repeat's `channels` is **valid**; the reader reports the warning `W_UNUSED_CHANNEL`. A declared but unused channel is never an error, and never an implicit zero or default. + +Header evidence (enforced by preprocessing, §8): a non-empty header channel name read at source +index *i* of an image in a unit MUST match a member of `header_names` of the channel at position +*i* of that unit's list, else `R_HEADER_CONTRADICTION`. **Matching rule (AQ7):** the name read from +the header and the `header_names` entry are each converted to NFC, and the results are compared +code point by code point. Nothing else is applied: **no trimming** (outer whitespace is part of +the name, so a header ` DAPI` matches only an entry ` DAPI`), no case folding, no similarity. The +NFC form is a comparison key only; it is never stored, and the header name is recorded as read. +Any other spelling needs a user-confirmed alias (a new `header_names` entry). Preprocessing +therefore MUST stop trimming header channel names (it trims today, `channel_names_trimmed`, +preprocessing `zarr_input.py:68`): this is a Stage 1 change and the raw name is what is recorded +and compared. This holds even when the header +name equals the channel ID or display name: aliases are only ever user-confirmed, never inferred from +name similarity, case or pixels. An image without header names needs no evidence. The same header +name MAY be evidence for different channels (fluorophores are reused across acquisitions). + +### 4.4 `repeats.` + +| Key | Type | Req | Rule | +|---|---|---|---| +| `display_name` | non-empty text | no | Display only (§4.8) | +| `confirmed` | boolean | yes | MUST be `true` in a published file (`E_REPEAT_UNCONFIRMED`): the N35 answer "prepared and acquired under the same circumstances" | +| `confirmed_on` | date | yes | Date of the user's confirmation (§4.4a) | +| `members` | table of member | yes | ≥ 1 entry | + +`repeats..members.` (one member = one source biological-repeat folder of one +condition; at most one per condition per repeat, guaranteed by TOML key uniqueness): + +| Key | Type | Req | Rule | +|---|---|---|---| +| `source_repeat` | text | yes | Exact source folder label (Label rules) | +| `image_count` | integer ≥ 1 | yes | Images to analyse at confirmation (N35 evidence; `E_VALUE_RANGE`) | +| `technical_repeats` | table of technical repeat | yes | ≥ 1 entry | + +`repeats..members..technical_repeats.` (one imaging unit): + +| Key | Type | Req | Rule | +|---|---|---|---| +| `source_folder` | text | yes | Exact technical-repeat folder label as read from disk, or `single` when the member has no such folder level (Label rules, NFC); a source value, **not an identity key**; not restricted by the reserved or forbidden names (§4.6a) | +| `display_name` | non-empty text | no | Display only (§4.8); MAY be the folder label | +| `channels` | array of channel IDs | yes | ≥ 1 entry; ordered: entry *i* is source channel C*i*; every entry declared (`E_UNKNOWN_CHANNEL`); no repeats (`E_DUPLICATE_CHANNEL_IN_UNIT`) | +| `image_type` | text | no | Only the value `rgb` (`E_ENUM`: exact, lower case); absent = an ordinary channel image. User-declared, never inferred (§4.4b) | + +**§4.4b RGB imaging units (AQ6; supersedes the D8 refusal).** RGB (interleaved colour) is an +accepted, **declared** image type, per imaging unit: + +- A unit with `image_type = "rgb"` MUST list exactly `["R", "G", "B"]`, in this order, as its + `channels` (source samples 0, 1, 2): any other count, order, case or IDs is + `E_RGB_UNIT_CHANNELS`. +- `R`, `G`, `B` are fixed tokens. A unit without `image_type = "rgb"` MUST NOT list a channel whose + case-folded ID is `r`, `g` or `b` (`E_RGB_TOKEN_MISUSE`). +- They have **no aliases**: a channel whose case-folded ID is `r`, `g` or `b` MUST NOT have a + non-empty `header_names` (`E_RGB_ALIAS`). `display_name` is allowed (display only). +- The image type is declared by the user and is **never inferred** from pixels, dtype, channel + count or file name. Preprocessing checks it against the image **header metadata** only (the + colour interpretation and samples per pixel that the format states, for example TIFF + photometric interpretation or OME `Interleaved`), in both directions: a unit declared `rgb` whose + header says it is not, or a unit not declared `rgb` whose header says it is, is + `R_IMAGE_TYPE_CONTRADICTS_HEADER`. A format whose header states no colour interpretation cannot + contradict the declaration, which then stands on user confirmation and is recorded in the + manifest as not header-verified. +- The spec gives RGB **no scientific meaning**: no grey conversion, luminance, stain separation + or joint-channel rule. Each brick owns the meaning and MUST declare whether it supports RGB; + undeclared means unsupported. An analysis request that includes an RGB unit's channels for a + brick without RGB support is refused at analysis (`A_RGB_NOT_SUPPORTED`, §8), never converted or + split into grey channels silently. +- **Machine-readable declaration (`rgb_support`).** A brick that publishes a provider manifest + declares RGB support there, as the optional top-level key `rgb_support`, whose value is a + boolean. `true` declares support (the brick owns what RGB means for it); `false` and an absent key + both mean unsupported, so a manifest written before this key existed is read as unsupported and + needs no edit. A value that is not a boolean is a manifest error: it is never read as `true`. A + brick without a provider manifest states the same declaration in its documented capabilities. + The key declares support only; it gives RGB no meaning (the bullet above), and it does not + change a description: `image_type = "rgb"` is still user-declared and never inferred. +- The refusal applies to a **described** run (one opened from a published description). There is + no other run: a run without a published description is refused with `A_DESCRIPTION_REQUIRED` + before any capability is checked (ADR 0002 AR10, which supersedes owner decision RC1's clause + that such runs keep working with a warning). +- A file MAY hold RGB and non-RGB units; their lists differ, so that layout is heterogeneous + (`R_HETEROGENEOUS_LAYOUT_NOT_ENABLED` in Stage 1-2). A unit's `image_type` is part of the + `repeats` section digest (§9.3). + +**§4.4a `confirmed_on` is a user fact.** It is the calendar date of the user's confirmation +action. A writer MUST NOT default, infer or refresh it (not from the write time, the read time, +file times, another repeat or a template); a reader supplies no default (absent is +`E_MISSING_FIELD`). A reconciliation that does not reopen the N35 confirmation (§10) MUST keep +`confirmed_on` of every unchanged repeat byte-identical; a candidate that changes it without a +reopening and a new user confirmation is refused (`R_CONFIRMED_ON_CHANGED`). + +Repeat rules: + +- The member key MUST be a declared condition ID (`E_UNKNOWN_CONDITION`). +- A source folder `(condition, source_repeat)` MUST belong to exactly one repeat + (`E_SOURCE_REPEAT_REUSED`). +- If a canonical repeat ID R equals, ignoring case, the `source_repeat` S of a member, that member + MUST belong to R (`E_REPEAT_ID_SHADOWS_SOURCE`). This forbids swapped labels such as repeat + `rep1` holding folder `rep2` while repeat `rep2` holds folder `rep1`. +- One folder, one ID (§4.6a): a condition `source_folder` belongs to exactly one condition ID and + a technical-repeat `source_folder` to exactly one technical repeat ID over the whole file + (`E_SOURCE_FOLDER_REUSED`); one technical repeat ID carries the same `source_folder` in every + unit that declares it (`E_SOURCE_FOLDER_CONFLICT`). +- A repeat MAY lack some conditions (warning `W_REPEAT_MISSING_CONDITIONS`). +- Technical repeats inside one biological repeat MAY have different channel counts and orders. + They remain one biological repeat; they never create additional biological replicates. + +### 4.5 `masks.` + +| Key | Type | Req | Rule | +|---|---|---|---| +| `source_type` | text | yes | Exact discovered mask type (Label rules); one mask per source type (`E_MASK_SOURCE_TYPE_REUSED`) | +| `representation` | text | yes | `instances` or `binary` (`E_ENUM`); user-confirmed, never inferred from dtype, values or names. A mask declared `binary` with more than two distinct values is refused by preprocessing's mask step (`R_MASK_CONTRADICTS_REPRESENTATION`, §4.5a); the user must declare `instances` | +| `source_channels` | array of channel IDs | no | Provenance only (what the mask was derived from); not a restriction on measured channels; each declared (`E_UNKNOWN_CHANNEL`); no duplicates (`E_DUPLICATE_VALUE`); order not semantic | +| `display_name` | non-empty text | no | Display only (§4.8) | + +Mask files, pairing, completeness and fingerprints stay in the run manifest. A declared mask with +files for none, some or all FOVs is valid (progressive segmentation, 0/1000 → 1000/1000). + +**§4.5a When the representation is checked (AQ10).** `R_MASK_CONTRADICTS_REPRESENTATION` needs +mask pixels. It is checked **in the preprocessing mask step** (the step that reads mask pixels), +for every mask file as it is added: for a progressive mask, **for each new batch**, on the new +files. Publishing `morphobricks.toml` MUST NOT read mask pixels; it records, per mask and file +set in the run manifest, the status `checked` (with the checked files and their fingerprints) or +`not yet checked`. A run that uses a mask file whose status is `not yet checked` MUST refuse +(`A_MASK_UNCHECKED`, §8); it never checks implicitly and never assumes. A new batch of files is +`not yet checked` until the mask step has read it. Pixels cannot prove `instances` (an integer +mask holding only 0 and 1 is accepted as declared). + +**§4.5b Grouped (`parent_labels`) masks are out of scope in v1 (AQ8).** MorphoRelate's third +representation `parent_labels` (preprocessing kind `grouped`) has no v1 value: +`representation = "parent_labels"` is `E_ENUM`. A grouped mask cannot be declared `instances` +(wrong objects) or `binary` (more than two values: `R_MASK_CONTRADICTS_REPRESENTATION`), so it +is not described, and not analysed through a description, in v1. A design for it follows with a +later `schema_version` (v1.1); nothing here predicts its form. + +**§4.5c A binary mask is never an object set by default (AQ9).** A request that uses a `binary` +mask as an object set is `A_MASK_NOT_INSTANCES`. The only way around it is an **explicit, +per-request, recorded conversion** to instances chosen in the request, for example CellPatcher +`binary_mask_policy="connected_components"`: it is recorded in the run manifest with the mask ID, +the method and its parameters, applies to that request only, and is never automatic, never a +default and never stored in `morphobricks.toml`, where `representation` stays `binary` +(`binary_mask_policy` and `mask_conversion` are `E_FORBIDDEN_FIELD`, §4.7). The numerical method +stays in the brick that owns it. + +### 4.6 Immutable IDs + +- Once confirmed, the ID of a repeat, condition, technical repeat or channel MUST NOT change. + There is no rename and no retired-ID table. A rename in the file would be a different ID for the + same data. +- **What a changed entity is (AQ1).** An entity is what the published file says an ID holds: for a + repeat, its member folders `(condition, source_repeat)`; for a condition, its `source_folder` + and its source repeat folders over all repeats; for a technical repeat, its `source_folder` and + the members `(condition, source_repeat)` that declare it; for a channel, its positions `(condition, source_repeat, technical + repeat, index)` and its `header_names`. An entity is *changed* only when something it held in + the published file is **removed or reassigned**. **Adding** does not change it: a new member + folder joining an existing repeat, a new repeat folder of an existing condition, a new unit + of an existing technical repeat ID, a new unit or position of a channel, a new alias, is not + `R_CONFIRMED_ID_CHANGED`. (It may reopen the N35 + confirmation, §10; reopening is not an ID change.) +- Preprocessing MUST refuse, as `R_CONFIRMED_ID_CHANGED`, a candidate file (compared with the + published file it was drafted from, §10) in which (a) a repeat with the same member folders + `(condition, source_repeat)` carries another ID, (b) a channel's ID is replaced by another ID at + every unit position it held with the same header evidence, (b2) a condition or technical repeat + with the same `source_folder` carries another ID, or (c) an ID that is declared in both + files names a changed entity (some member folder, position or alias it held is removed or + reassigned, or its `source_folder` is another folder). Removing or reassigning is accepted only with an explicit, recorded user + confirmation (the same record as for a mistyped ID, below); the confirmation of a removal that + §10 requires is that record, and every removal is also written as a `removed` record (below). An ID that was removed and comes back is `R_USED_ID_REUSED` + (§4.9). A condition or technical repeat ID is not its folder label (§4.6a): a folder that is + renamed or replaced on disk under an existing ID is a removed folder plus a new pending one + (§7), and giving the existing ID the new `source_folder` is a reassignment under (c). +- A mistyped ID is fixed only by an explicit, recorded re-confirmation: a user action that + writes a new file and records old ID, new ID, date and both file SHA-256. The record is made + at publication, which happens outside a run: preprocessing appends it, under the output + folder's publication lock (§10), to the append-only file `morphobricks.records.jsonl` next to + `morphobricks.toml`, and copies the records unchanged into the manifest of the next run. The + records file is never rewritten or truncated, and nothing about a re-confirmation is stored in + `morphobricks.toml`. Products keyed on that ID are invalidated. +- **Encoding of the records file (owner decisions RF1, RF4).** UTF-8 without a byte-order mark, + one JSON object per line, each line ending in LF (a CR before the LF is not allowed), keys in + code point order at every level (the objects inside `technical_repeats` included), non-ASCII + characters unescaped, the separators `, ` between members and `: ` after a key, and no other + whitespace outside strings. A string uses only the escapes `\"`, `\\`, `\b`, `\f`, `\n`, `\r`, + `\t` and, for the other control characters U+0000 to U+001F, `\u00xx` with lower-case hex; no + other character is escaped. A file of zero bytes holds no record and is accepted. The key + `type` tells the record types + apart: `"reconfirmation"`, `"image_count_changed"` and `"removed"`. Each type has its own fixed + key set; a line of one type carries no key of another. The first type is the only one written + before this addition, and its lines are unchanged. Keys of `reconfirmation`: `type`, + `id_kind` (`conditions`, `channels`, `repeats` or `technical_repeats`), `old_id`, `new_id` + (`null` when the same ID lost or was reassigned something), `confirmed_on` (`YYYY-MM-DD`, the + date of the user's action), `old_sha256` (of the published file replaced, or `absent`) and + `new_sha256` (of the file written). A SHA-256 is 64 lower-case hex digits. The records are + appended after the file is replaced. +- **Refusal of a records file (owner decisions RF2, RF3, RF4, RF5; `R_RECORDS_FILE_INVALID`).** A + reader refuses the whole file, before anything is published, when any of these holds (the reason + is given in parentheses, one file may have several): the bytes are not UTF-8 or start with a + byte-order mark (`E_RECORDS_ENCODING`); the last line lacks its LF (`E_RECORDS_NO_FINAL_LF`); a + line is not a JSON object, including an empty line and an object with a key twice + (`E_RECORDS_NOT_JSON_OBJECT`); `type` is missing or none of the three (`E_RECORDS_UNKNOWN_TYPE`); + the keys are not exactly the key set of the type (`E_RECORDS_KEY_SET`); keys are not in code point + order at every level (`E_RECORDS_KEY_ORDER`); a non-ASCII character is written as an escape + (`E_RECORDS_NON_ASCII_ESCAPED`); a value is invalid: an enumeration (`id_kind`, `entity`) outside + its values, a date that is not a real `YYYY-MM-DD`, a SHA-256 that is not 64 lower-case hex + digits (`old_sha256` may also be `absent`), a key that must be `null` (or must not be) in a + `removed` line, a count that is not an integer of at least 1, `old_image_count` equal to + `new_image_count`, `technical_repeats` empty or not sorted by `id` (`E_RECORDS_VALUE`); or the + line differs from the canonical bytes above in anything else, whitespace, a CR, an optional + escape (`E_RECORDS_FORM`). The reasons are named by the reference validator + (`tools/audit/refrecords.py`); the code is the Spec's. The record values that name IDs and + folder labels are text; the Spec states no further rule for them here. +- **Record of a removal (`removed`, owner decision RR2).** Written for **every** removal of an + imaging unit, technical repeat, repeat or condition that the user confirms (§10), including a + removal whose ID is held nowhere else afterwards; it is written in addition to a + `reconfirmation` where §4.6 requires one. Keys: `type`, `entity` (`unit`, `technical_repeat`, + `repeat` or `condition`), `condition_id`, `condition_source_folder`, `repeat_id`, + `source_repeat`, `technical_repeat_id`, `technical_repeat_source_folder`, and `confirmed_on` + (`YYYY-MM-DD`, the date of the user's confirmation; a `removed` line is itself that + confirmation). A key that the entity does not name is `null`: a `unit` (one technical repeat + entry of one member of one repeat) names all six; a `technical_repeat` names its two keys; a + `repeat` its `repeat_id`; a `condition` its two condition keys. The source folder labels are the + exact labels of the removed published file. A removed member folder is written as one `unit` + record per imaging unit it held, and an entity that held units is followed by the `unit` + records of those units, so every removed folder label appears in a record. Removing an entity + never removes its ID from `used_ids` (§4.9). +- **Record of an image-count change (`image_count_changed`, owner decisions RR1 and RR3).** + Written for **every** confirmed member whose `image_count` changes at a publication, including + a publication that also changes something else (a removal, a new member, an ID correction, + §10); the other changes do not suppress it and it suppresses none of their records. It needs no + confirmation and reopens nothing: a count change alone is not a reopening, and + `R_CONFIRMED_ON_CHANGED` (§4.4a) is unchanged. Keys: `type`, `condition_id`, + `condition_source_folder`, `repeat_id`, `source_repeat`, `technical_repeats` (the member's + imaging units as a non-empty array of objects `{"id", "source_folder"}`, sorted by `id` in code + point order), `old_image_count`, `new_image_count` (integers, each at least 1, and different: + the record states a change) and `recorded_on` + (`YYYY-MM-DD`, the date the change was recorded at publication; it is not a confirmation and is + never copied into `confirmed_on`). +- **Order and content of the `removed` lines of one publication (RR2, clarified with RR3).** Each + vanished ID is one entity line; the `unit` lines of the units those entities held are written + once, after all entity lines of the publication, each unit once. An entity takes the + `confirmed_on` date of the first removal answer that touches it. When a removal is combined + with an ID correction in the same publication, the lines carry the IDs as published (before the + correction). +- **Order of the lines of one publication (owner decision RF6).** Preprocessing writes them as one + append: the `reconfirmation` lines (by `id_kind`, `old_id`, `new_id`, `confirmed_on`), then the + `removed` lines (entity lines, then `unit` lines, above), then the `image_count_changed` lines + by `condition_id`, then `repeat_id`. The file has no publication boundary, so a reader of the + file alone cannot check this order or that each unit is written once; the writer tests of + scenarios P34 and P35 check it, and a file that breaks it is not refused for that. +- **A case-only correction needs a new ID (AQ2).** `ACTIN` cannot be corrected to `Actin`: the + retired spelling stays in `used_ids` and the new one collides with it + (`E_ID_CASE_COLLISION`, §4.9). The correction is a new, non-colliding ID (for example + `ActinStain`) with the recorded re-confirmation above, and `display_name` (here `Actin`) carries + the appearance. The old spelling stays listed and is never reused. +- The friendly name of any entity is `display_name` (§4.8), not the ID. +- **Publication reports a set (AQ2).** When several `R_*` refusals apply to one candidate, + preprocessing reports **every** applicable code, not the first (`t02` is + `R_CONFIRMED_ID_CHANGED` and `R_USED_ID_DROPPED`). + +### 4.6a Folder labels and canonical IDs (fourth round) + +Conditions and technical repeats are identified exactly like biological repeats: the key is a +**canonical ID** (immutable, user-confirmed, the only thing identity uses), and the folder it was +discovered under is recorded separately in the required field `source_folder`. + +- **`source_folder` is a source value, not an identity.** It holds the folder label exactly as + read from disk (Label rules, NFC; never trimmed, normalized or renamed). It is never a table + key, a `reference` value, a member key or a `used_ids` entry, and no reader resolves an ID + through it. +- **Folder labels are never restricted by the reserved list (§3.2) or the forbidden names + (§4.7).** Any folder label that satisfies the Label rules and NFC may be a `source_folder`, + including `control`, `files` or `z`. The restrictions stay on the **IDs** (keys), where the + forbidden names would collide with fields. +- **One folder, one ID.** Over the whole file a condition `source_folder` names exactly one + condition ID, and a technical-repeat `source_folder` exactly one technical repeat ID + (`E_SOURCE_FOLDER_REUSED` when two IDs of one kind share a folder; `E_SOURCE_FOLDER_CONFLICT` + when one technical repeat ID is declared with two different folders). A condition has one + folder; a technical repeat ID has the same folder in every unit that declares it. Folders that + differ only in case are `E_ID_CASE_COLLISION` (§3.3), as before. +- **Proposed ID.** When preprocessing meets a new folder it proposes the folder label as the ID + if that is allowed: it passes the Label rules, is not an exact forbidden name (§4.7), and does + not collide (exactly or by case folding) with a declared or `used_ids` ID of its kind. If it + is not allowed (for example the folder `control`), preprocessing proposes a different ID that + is allowed, and the user confirms it or types another; the user MAY also set `display_name` to + the folder label. A proposal is never adopted unconfirmed, and no ID is repaired or derived by + a reader (§3.3). +- **Repeat IDs follow the same rule.** For a new biological-repeat folder the proposed repeat ID is + the folder label if it is allowed in the same sense (Label rules, not an exact forbidden name, + no exact or case-folded collision with a declared or `used_ids` repeat ID). If it is not + allowed (for example the folder `z`), preprocessing makes **no proposal**: it never repairs or + derives an ID from the label (§3.3), and the user names the repeat ID. The reader validates what + the user typed like any ID; `source_repeat` keeps the folder label. +- **Existing ID, other folder.** Giving an existing condition or technical repeat ID another + `source_folder` is a reassignment (§4.6): `R_CONFIRMED_ID_CHANGED` without the recorded user + confirmation. A new folder next to the existing ones is a new pending member or unit (§10). +- **`single`.** The technical repeat `single` has `source_folder = "single"`: the label denotes + "no technical-repeat folder level", not a folder (§3.2). The same holds for a condition: the + condition `single` has `source_folder = "single"`, which denotes "no condition folder level" (an + experiment folder that is itself the leaf, with the member `source_repeat = "single"` and the + technical repeat `single`), and it is the only condition of the file (§3.2). +- A folder label that breaks the Label rules (for example `NA`) is still refused at discovery as + today; this decision adds no new code for it. + +### 4.7 Forbidden fields + +These names are refused as `E_FORBIDDEN_FIELD` in **any** table, with a message naming where the +information belongs, and they apply both to **field names** and to **ID keys** (the key of a +condition, channel, repeat, member, technical repeat or mask; AQ13). The comparison is exact and +case-sensitive. A forbidden name is reported once, at its name; its value is not examined. A +source folder label equal to one of these names (a condition folder `control`, a technical-repeat +folder `files`) cannot be used as the **ID** (the key) but **is** describable: it is stored as the +`source_folder` value, which is never restricted (§4.6a), and the user confirms another ID +(`ctrl`, `acq1`) that preprocessing proposes. The folder is never renamed on disk. A +`source_repeat` value is not a key and was never affected. + +| Field | Belongs to | +|---|---| +| `axes`, `shared`, `plane_scope`, `z`, `projection` | Image/manifest records; v1 is 2-D only (preprocessing #173, #176) | +| `pixel_size`, `calibration` | `calibration.csv` | +| `fovs`, `files`, `paths`, `overrides`, `exclude`, `fingerprints` | Run manifest / preprocessing inputs; no per-FOV statements | +| `bricks`, `routes`, `features`, `normalization`, `statistics`, `contrasts` | Analysis requests / brick and MorphoCoverage configuration | +| `binary_mask_policy`, `mask_conversion` | The analysis request that records a per-request binary-to-instances conversion (§4.5c) | +| `control`, `roles`, `experiment` | Derived from `reference` / analysis requests / MorphoCoverage FDR families | +| `source_index`, `channel_count` | Implied by the order and length of `channels` | +| `retired_repeats`, `replaced_by` | None: IDs are immutable (§4.6); the only registry is `used_ids` (§4.9), IDs only and never a mapping | +| `labchronicle`, `provenance`, `source_database` | Run manifest provenance (preprocessing, §9.3): lab-tool links are never part of this file | + +### 4.8 Display names + +`display_name` (conditions, repeats, technical repeats, channels, masks) is for display only (reports, notebooks, +plots). It MUST NOT be used for identity, joins, pairing, references or cache keys, and changing +or adding it invalidates nothing (§9.3). A consumer MUST NOT accept a display name where an ID is +expected, even when the two strings are equal and even when the display name is unique: a +`reference`, member key or unit channel entry that names a display name is refused with the code +of an unknown ID (`E_REFERENCE_UNKNOWN`, `E_UNKNOWN_CONDITION`, `E_UNKNOWN_CHANNEL`). + +Display names are kept exactly as typed: never normalized, trimmed or case-changed on read or +write. They are unique within one kind (conditions, repeats, technical repeats, channels, masks; +the same string may appear in two kinds, and a display name may equal a `source_folder`): two +display names of one kind whose keys are equal are `E_DISPLAY_NAME_COLLISION`; for technical +repeats the names of one technical repeat ID in several units never collide with each other, only +with those of another ID, and need not agree; where the key of a name *x* is `NFC(casefold(NFC(x)))` with +`str.casefold()` (so `Control` and `control`, `Straße` and `STRASSE`, decomposed and composed +`Café` collide). The key is used for the comparison only and is never stored or used for lookup. + +### 4.9 Used IDs + +`used_ids` is an append-only registry of IDs, per kind. It holds IDs only: never a mapping, a +date or a reason. + +| Key | Type | Req | Rule | +|---|---|---|---| +| `conditions` | array of text | yes | Every condition ID ever confirmed | +| `channels` | array of text | yes | Every channel ID ever confirmed | +| `repeats` | array of text | yes | Every canonical repeat ID ever confirmed | +| `technical_repeats` | array of text | yes | Every canonical technical repeat ID ever confirmed (`single` included) | + +- Entries follow the identity rules of their kind (Label for conditions, repeats and technical + repeats, Token for channels; NFC, §3.3), have no duplicates (`E_DUPLICATE_VALUE`), and share the case-collision + scope of the kind's declared IDs (`E_ID_CASE_COLLISION`): a retired `ctrl` cannot come back as + `CTRL`. At least one entry per kind (`E_EMPTY`). +- **Reserved names in `used_ids.channels`** (owner decision UR1, 7 November 2026). A channel entry + follows the reserved list of §3.2 like a declared channel ID: an entry whose case-folded form is + in the list is `E_ID_RESERVED`. A listed ID reserves that ID, so a name no channel may bear + cannot be reserved by being listed. The registry has no list for mask IDs (it holds the four + kinds above), so the list applies to channels only; the other three kinds are unchanged. +- Source folder labels are not IDs and are never listed. Every declared ID of a kind MUST be + listed (`E_USED_ID_MISSING`). A listed ID that is not + declared is a **retired** ID: its entity was removed. +- Publication MUST refuse a candidate that (a) omits an ID listed in the published file it was + drafted from (`R_USED_ID_DROPPED`: removing an entity keeps its ID listed), or (b) declares as + an entity an ID that is listed but not declared there (`R_USED_ID_REUSED`). Rule (b) applies + even when the same source folder returns: it needs a new ID. The registry carries no mapping, + so it cannot say which entity an ID once named. A case-only correction therefore needs a new, + non-colliding ID (§4.6). +- Scope of the guarantee: the published file's lineage inside one output folder (§10 compares each + publication with the file it replaces). A new output folder starts a new lineage and the + registry cannot see across it; the run manifest history is the only cross-folder record + (ADR 0001, decision Q1). +- `used_ids` is not part of any section digest (§9.3). + +## 5. Reference graph + +- `reference` names the primary matched control of a condition. It is experiment metadata, not + a contrast table; further contrasts belong to MorphoCoverage. +- It MUST name a declared condition, exactly (`E_REFERENCE_UNKNOWN`; `ctrl` does not resolve to + `CTRL`), MUST NOT name its own condition (`E_REFERENCE_SELF`), and the graph of all `reference` + edges MUST be acyclic (`E_REFERENCE_CYCLE`). A condition has at most one `reference` (an array + is `E_TYPE`). +- Chains are allowed (`VEC_SiCol6a1 → VEC_SiNC → VEC`). A condition is a *control* iff some + condition references it. +- If a condition and its reference share no repeat, MorphoCoverage cannot pair them: warning + `W_REFERENCE_UNPAIRED`. + +## 6. Validation phases and error catalogue + +A reader MUST run the phases in order and stop after the first phase that reports an error. It +MUST report every error of that phase, each with the code, the table path (e.g. +`repeats.Day1.members.CTRL.technical_repeats.single.channels[2]`) and the offending value. +Warnings are reported only for a file without errors and never block publication. Conformance +compares the **set of codes** of the reported phase ([fixtures](../fixtures/README.md)). +**One code per broken rule (AQ13):** each rule has its own code and a reader reports every rule +that an identity or value breaks, so ` DNA` as a channel ID is `E_ID_TEXT` (outer whitespace) and +`E_ID_SYNTAX` (the regular expression), and a decomposed key is `E_ID_NOT_NFC` in addition to any +other rule it breaks. A rule is never reported twice under two codes, and a reference is checked +for NFC only (phase 4) before it is resolved (phase 5). + +| Phase | Code | Meaning | +|---|---|---| +| 1 Decode | `E_FILE_ENCODING` | Not UTF-8, or starts with a byte-order mark | +| 1 | `E_TOML_SYNTAX` | Not TOML 1.0 (includes unquoted text such as `NO`, and `012`) | +| 1 | `E_DUPLICATE_KEY` | A key or table declared twice | +| 2 Header | `E_SCHEMA_ID` | `schema` missing or not `morphobricks-experiment` | +| 2 | `E_SCHEMA_VERSION_MISSING` | `schema_version` missing | +| 2 | `E_SCHEMA_VERSION_UNSUPPORTED` | `schema_version` an integer other than 1 | +| 2 | `E_TYPE` | `schema` or `schema_version` of the wrong TOML type | +| 3 Structure | `E_UNKNOWN_FIELD` | Field not defined for its table | +| 3 | `E_FORBIDDEN_FIELD` | Field name or ID key from §4.7 | +| 3 | `E_MISSING_FIELD` | Required field absent | +| 3 | `E_TYPE` | Wrong TOML type (no conversion) | +| 3 | `E_EMPTY` | Required table/array is empty | +| 3 | `E_TEXT_EMPTY` | Non-empty text is empty or whitespace only | +| 3 | `E_ENUM` | Value outside its enumeration | +| 3 | `E_DATE_FORMAT` | Not a real `YYYY-MM-DD` date string | +| 3 | `E_VALUE_RANGE` | Integer outside its range | +| 3 | `E_REPEAT_UNCONFIRMED` | `confirmed = false` in a published file | +| 4 Identity | `E_ID_TEXT` | Label rule violated | +| 4 | `E_ID_SYNTAX` | Token regular expression violated | +| 4 | `E_ID_RESERVED` | Name on the reserved list of §3.2 (channel and mask IDs, and entries of `used_ids.channels`, §4.9), or a mask ID with an `in` part | +| 4 | `E_ID_NOT_NFC` | An identity (key, reference, source repeat, `source_folder`, unit `channels` or `source_channels` entry, `used_ids` entry) is not Unicode NFC; never normalized | +| 4 | `E_ID_CASE_COLLISION` | Two identities of one scope equal ignoring case (`used_ids` entries share the scope of their kind; source folders have their own scopes, §3.3) | +| 4 | `E_SINGLE_MISUSE` | `single` in another case or not alone, or a technical repeat ID or condition ID `single` whose `source_folder` is not `single` (or the reverse) | +| 5 Semantics | `E_UNKNOWN_CONDITION` | Member key is not a declared condition ID | +| 5 | `E_UNKNOWN_CHANNEL` | Unit or `source_channels` names an undeclared channel ID | +| 5 | `E_DUPLICATE_CHANNEL_IN_UNIT` | A channel twice in one unit | +| 5 | `E_DUPLICATE_VALUE` | Duplicate in `header_names` (equal after NFC), `source_channels` or a `used_ids` list | +| 5 | `E_RGB_UNIT_CHANNELS` | A unit with `image_type = "rgb"` whose `channels` is not exactly `["R", "G", "B"]` (§4.4b) | +| 5 | `E_RGB_TOKEN_MISUSE` | A unit without `image_type = "rgb"` lists a channel whose case-folded ID is `r`, `g` or `b` (§4.4b) | +| 5 | `E_RGB_ALIAS` | A channel whose case-folded ID is `r`, `g` or `b` has non-empty `header_names` (§4.4b) | +| 5 | `E_DISPLAY_NAME_COLLISION` | Two display names of one kind equal after NFC and case folding (§4.8) | +| 5 | `E_USED_ID_MISSING` | A declared condition, channel, repeat or technical repeat ID is not in `used_ids` (§4.9) | +| 5 | `E_SOURCE_FOLDER_REUSED` | One `source_folder` on two condition IDs, or on two technical repeat IDs (§4.6a) | +| 5 | `E_SOURCE_FOLDER_CONFLICT` | One technical repeat ID declared with two different `source_folder` values (§4.6a) | +| 5 | `E_CONDITION_WITHOUT_MEMBERS` | Declared condition in no repeat | +| 5 | `E_SOURCE_REPEAT_REUSED` | One source folder in two repeats | +| 5 | `E_REPEAT_ID_SHADOWS_SOURCE` | Repeat ID equal to the source folder name of another repeat's member | +| 5 | `E_MASK_SOURCE_TYPE_REUSED` | Two masks with one `source_type` | +| 5 | `E_REFERENCE_UNKNOWN` | `reference` names no declared condition ID (a display name is not an ID) | +| 5 | `E_REFERENCE_SELF` | `reference` names its own condition | +| 5 | `E_REFERENCE_CYCLE` | The reference graph has a cycle | +| warn | `W_UNUSED_CHANNEL` | A declared channel is in no unit | +| warn | `W_REPEAT_MISSING_CONDITIONS` | A repeat lacks some declared conditions | +| warn | `W_REFERENCE_UNPAIRED` | A condition and its reference share no repeat | + +Phase-2 `E_TYPE` concerns only `schema`/`schema_version`. An unsupported or missing version stops +validation: a reader MUST NOT try to interpret fields of a version it does not support. + +## 7. Versioning and migration + +- `schema_version` is an integer. Any change to fields, types, rules or meanings requires a new + version. Readers MUST refuse any version they do not implement (`E_SCHEMA_VERSION_UNSUPPORTED`) + with a message naming the supported versions; they MUST NOT read a newer file "best effort". + The supported set is a constant of the reader (in v1: `{1}`). It is checked without network + access, against nothing fetched. A missing version is `E_SCHEMA_VERSION_MISSING` and is never + mapped to a default version. +- A migration between versions MUST be an explicit, recorded operation that writes a new file and + never changes canonical identities silently. +- **Repeat, condition, technical repeat and channel IDs are immutable** (§4.6) and never reused + (§4.9). There is no rename migration; a friendlier name is `display_name`. A mistyped ID is corrected only by a + recorded re-confirmation, which retires the old ID in `used_ids`. No reader or writer repairs, + re-cases or renumbers an ID (§3.3). +- **Source folder renamed or moved:** a removed member plus a new, unpaired, pending member. It is + never auto-paired with the removed one; the user may assign it to the existing canonical repeat + (the canonical ID does not change). For a **condition** or **technical repeat** folder the same + holds (the folder is removed, the new folder is pending with a proposed ID, §4.6a); giving the + new folder to the existing ID changes that ID's `source_folder`, which is a reassignment and + follows the removed/reassigned rules of §4.6 (recorded user confirmation, else + `R_CONFIRMED_ID_CHANGED`). The ID never changes. +- **No migration from a legacy `experiment_design.csv`** (ADR 0002 AR10 (9), owner decision of + 10 November 2026). Preprocessing MUST NOT read an `experiment_design.csv` found in an output + folder to propose or prefill a description or a draft: a description is created only from + discovery of the experiment folder and the user's answers (§10). The derived + `experiment_design.csv` of §11 is an output only and is never read back. + +## 8. Validation boundaries + +| Layer | Owns | Codes | +|---|---|---| +| Spec (this document) | Syntax, types, identities, cross-references, graphs; no files read except this one | `E_*`, `W_*` | +| Preprocessing (reference reader) | Physical agreement of the file with the experiment folder; reconciliation; publication | `R_*` | +| Analysis layer (MorphoFeature, MorphoRelate, fiberfeatures, MorphoEmbed, MorphoCoverage, CellPatcher, normalization) | Applicability of a request to the described units | `A_*` | + +Required preprocessing refusals (`R_*`): + +| Code | Meaning | +|---|---| +| `R_FILE_SYMLINK` | `morphobricks.toml` is a symlink | +| `R_CONFIRMED_ID_CHANGED` | A confirmed ID was changed, or a declared ID names a changed entity (something it held was removed or reassigned, a `source_folder` included; adding is not a change), without a recorded re-confirmation (§4.6) | +| `R_USED_ID_REUSED` | A candidate declares an ID that `used_ids` lists as retired (§4.9) | +| `R_USED_ID_DROPPED` | A candidate omits an ID listed in `used_ids` of the file it replaces (§4.9) | +| `R_CONFIRMED_ON_CHANGED` | `confirmed_on` of a repeat whose confirmation was not reopened differs from the published file (§4.4a) | +| `R_RECORDS_FILE_INVALID` | `morphobricks.records.jsonl` breaks the encoding, key set or value rules of §4.6; reasons `E_RECORDS_ENCODING`, `E_RECORDS_NO_FINAL_LF`, `E_RECORDS_NOT_JSON_OBJECT`, `E_RECORDS_UNKNOWN_TYPE`, `E_RECORDS_KEY_SET`, `E_RECORDS_KEY_ORDER`, `E_RECORDS_NON_ASCII_ESCAPED`, `E_RECORDS_VALUE`, `E_RECORDS_FORM` | +| `R_PUBLISHED_FILE_CHANGED` | The SHA-256 of the published file at publication differs from the draft's recorded base (§10) | +| `R_DRAFT_BASE_MISSING` | A draft has no recorded base SHA-256 (§10) | +| `R_FOLDER_NOT_NFC` | A folder label or a discovered mask type name read from disk is not NFC (§3.3) | +| `R_UNIT_NOT_FOUND` | A declared imaging unit has no included image | +| `R_UNIT_NOT_DECLARED` | A discovered imaging unit is not in the file (reconciliation draft required) | +| `R_CHANNEL_COUNT_MISMATCH` | An image's channel count differs from its unit's list length | +| `R_UNIT_NOT_HOMOGENEOUS` | FOVs of one unit differ in channel count, image type, interpreted axes or header names at one C index; the message names FOV, path and C index | +| `R_HEADER_CONTRADICTION` | A header name at C*i* is not, after NFC of both sides and with no trimming (§4.3), in the confirmed `header_names` of the unit's channel at *i* | +| `R_IMAGE_TYPE_CONTRADICTS_HEADER` | A unit's declared image type (`rgb` or absent, §4.4b) contradicts the image header metadata, in either direction; pixels are never examined | +| `R_HETEROGENEOUS_LAYOUT_NOT_ENABLED` | Stage 1–2 only: the units do not all have the same ordered list; names the units; lifted at Stage 3 | +| `R_MASK_TYPE_NOT_DECLARED` | A discovered mask type has no `masks` entry | +| `R_MASK_TYPE_NAME_INVALID` | A discovered mask type name breaks the Label rules (for example `NA`, ` cells`), so it cannot be a `source_type`; refused at discovery (AQ11) | +| `R_MASK_CONTRADICTS_REPRESENTATION` | A mask declared `binary` holds more than two distinct values; raised by the mask step on each new batch, not at publication (§4.5a) | + +Within one unit, `R_UNIT_NOT_HOMOGENEOUS` is reported first and replaces +`R_CHANNEL_COUNT_MISMATCH` and `R_HEADER_CONTRADICTION` for that unit: intra-unit discrepancies +are detected before channel identity is checked. + +Undetectable by any layer: identical-shape, unnamed images with physically swapped fluorophores. +Representative previews and explicit confirmation are the guard; image-based checks are advisory. + +Required analysis-layer refusals (`A_*`): `A_SCOPE_INCOMPLETE` (a requested canonical channel or +mask is absent from part of the requested scope, **or from all of it**: one rule, one code, no +separate code for "no image has it" (owner decision SI1); the refusal names the channel or mask and +the images that lack it, as "present in k of n acquisitions" (`k` may be 0); refuse unless the user +selects and records an eligible subset; never drop FOVs, impute, re-weight technical repeats or +change biological N); +`A_MASK_NOT_INSTANCES` (a `binary` mask requested as an object set without the explicit recorded per-request conversion of §4.5c); `A_MASK_UNCHECKED` (a run uses a mask file whose representation check is `not yet checked`, §4.5a); `A_RGB_NOT_SUPPORTED` (a request on a described run includes the channels of an RGB unit, §4.4b, for a brick whose manifest does not declare `rgb_support = true`, or that does not declare it at all); `A_DRAFT_REFUSED` (§10: `RunInputs` given a draft or an unconfirmed file). Each biological repeat is +classified **fully represented**, **partially represented** or **unavailable** per request and +recorded in the run manifest. Consumers resolve channels by canonical ID per unit, never by C +index alone, and group by canonical repeat ID (`repeat_group`, §3.4) while keeping the source label +in every key. + +## 9. Serialization, round trip and caching + +### 9.1 Canonical serialization (writers) + +Writers MUST produce deterministic bytes: UTF-8, no BOM, LF, a final newline, no comments. +Order: `schema`, `schema_version`, then `conditions`, `channels`, `repeats`, `masks` (an empty +optional table is omitted), `used_ids` (kinds in the order `conditions`, `channels`, `repeats`, +`technical_repeats`; each array sorted in code point order, inline); identities are NFC. Entries of each table sorted by key in code +point order; fields in the order of §4; each table as a standard `[header]`; arrays inline; +`header_names`, `source_channels` and `used_ids` arrays sorted, unit `channels` in declared order; keys bare when +they match `[A-Za-z0-9_-]+`, else basic quoted strings; strings as TOML basic strings with the +minimal escapes. + +### 9.2 Round trip + +For every valid file F: `parse(write(parse(F)))` MUST equal `parse(F)`, and `write` MUST be +idempotent. Any field a reader would drop is refused instead (§2.4), so no confirmed semantics can +be lost silently. Reordering tables or entries, or switching between bare/quoted keys or +standard/inline/dotted tables, never changes meaning. + +### 9.3 Provenance vs cache keys + +The whole-file SHA-256 is provenance. Preprocessing SHOULD also record per-section digests (SHA-256 +of the canonical serialization of `conditions`, `channels`, `repeats`, `masks`) so that consumers +key derived products only on the sections they use. **The bytes of one section digest** are the +section's table blocks exactly as the canonical writer emits them in the file (§9.1), each block +followed by one blank line, with `display_name`, `target` and `treatment_key` left out and nothing +before or after; a section with no tables (no `masks`) is the SHA-256 of the empty byte string. +`source_folder` is part of the `conditions` and +`repeats` digests (it fixes which folder an ID names; a reassignment changes them). Digests are computed with every `display_name`, +`target` and `treatment_key` omitted (descriptive fields are never cache keys, §4.2a), and +`used_ids` is not a section: a change to any of them MUST NOT change any digest, a change to `reference` MUST NOT +invalidate pixel-derived products; a change to a unit's channel list MUST invalidate the products +that depend on that unit's channels. + +Provenance from lab tools (for example LabChronicle: repository URL, commit SHA, experiment path +and per-repeat lab-tool repeat codes) is an optional, user-supplied preprocessing input recorded +**only in the run manifest**, never in `morphobricks.toml` (`labchronicle`, `provenance` and +`source_database` are `E_FORBIDDEN_FIELD`). It is excluded from every identity, pairing and cache +key. A mapping from a lab-tool code such as `R001` to a canonical repeat ID MUST be confirmed by +the user: equal strings are not evidence, because the namespaces are unrelated. + +## 10. Reconciliation and confirmation (preprocessing behaviour) + +On every run, preprocessing compares fresh discovery with the published file and classifies each +unit **unchanged**, **new**, **removed** or **conflicting**. It never rewrites confirmed semantics +silently; edits go to a separate draft and are published atomically after validation. + +- **Publish only from the current base.** Every draft records the SHA-256 of the exact bytes of + the published `morphobricks.toml` it was derived from, or the explicit value `absent` when none + existed (a draft without it: `R_DRAFT_BASE_MISSING`). Under the output folder's lock, at the + moment of publication, preprocessing MUST compute the SHA-256 of the file currently published + (or find none) and refuse with `R_PUBLISHED_FILE_CHANGED` unless it equals the recorded base. + The lock prevents concurrent writes; the base check prevents a stale draft from replacing a newer + published file. The refused draft is never merged automatically: it is re-derived from the + current file. +- **`confirmed_on` is kept** byte-identical for every repeat whose confirmation is not reopened + (§4.4a); the draft copies it from the published file, never from the clock. + +- **N35 reopening.** The confirmation of a repeat is reopened (draft `confirmed = false`) only when + channels, layout or grouping change: a new or removed unit or member, a unit's ordered channel + list, or the pairing of folders into repeats. New FOVs (or removed ones) inside an existing unit + whose confirmed layout still holds do **not** reopen it; they are reported and recorded. A + changed `image_count` of a confirmed member is recorded as an + `image_count_changed` record (§4.6) whether or not anything else changes in the same + publication (RR3). A + changed `image_count` alone is not a reopening: a new `confirmed_on` beside only a + changed `image_count` is `R_CONFIRMED_ON_CHANGED` (§4.4a). +- **Adding a member is allowed (AQ1).** A new member folder or unit joins an existing repeat or + condition without a changed-ID refusal (§4.6); it is a grouping change, so it reopens the N35 + confirmation of that repeat, and it never changes an existing ID. Only removing or reassigning + existing members counts as a changed entity. +- A removed member is reported and removed only on user confirmation, which is the recorded + confirmation of §4.6 (without it the candidate is `R_CONFIRMED_ID_CHANGED`); every confirmed + removal of a unit, technical repeat, repeat or condition, whether or not an ID survives it, is + also written as a `removed` record (§4.6); a repeat left + without members is removed from the file, and its ID stays in `used_ids` and is never reused + (`R_USED_ID_REUSED`, `R_USED_ID_DROPPED`; §4.9). +- A new folder whose label equals an existing repeat ID is proposed for that repeat but stays + pending until confirmed. +- **Condition and technical-repeat folders (fourth round).** Discovery matches a folder to an ID + by its `source_folder`, never by the ID. A folder equal to a published `source_folder` is the + same entity. A folder not in the file is new and pending, with an ID proposed under §4.6a (the + label if allowed, else another proposal); it joins no existing ID without the user's + confirmation, which for an existing ID is a reassignment (§4.6). A published `source_folder` + with no folder on disk is a removed folder, removed only on user confirmation like a removed + member. +- A conflicting unit (`R_*`) blocks publication; no per-FOV exception is created. + +## 11. Derived `experiment_design.csv` (compatibility output) + +Preprocessing MUST derive the file (schema version 1, preprocessing contract §12) +deterministically from a published file, with the contract's byte rules: + +| CSV column | Value | +|---|---| +| `condition` | member's condition's `source_folder` (the folder label, not the condition ID) | +| `biological_repeat` | member's `source_repeat` | +| `repeat_group` | canonical repeat ID | +| `control` | `true` iff some condition's `reference` names this condition | +| `image_count` | member's `image_count` | +| `technical_repeat_count` | 0 if the only technical repeat is `single` (its `source_folder` is `single`), else the number of technical repeats | +| `confirmed` | `true` | +| `confirmed_on` | repeat's `confirmed_on` | +| `schema_version` | `1` | + +One row per member, sorted by `condition` (the source folder label), then `biological_repeat`, in +code point order. **Bytes** (preprocessing contract §12, rule `Rows`, and the `confirmed_on` +row): UTF-8, LF line endings, one header row naming the nine columns in the order of the table +above, booleans written `true` and `false`, `confirmed_on` written as the repeat's `confirmed_on` +(`YYYY-MM-DD`). The contract does not state the delimiter, the quoting, the final line terminator +or a byte-order mark, so these four are not yet normative; the golden files use a comma, no +quoting (no golden value needs it), LF after every row including the last, and no byte-order +mark, as today's writer does. The CSV carries the **source folder labels exactly as today** and no technical +repeat or condition ID, so the choice of a condition or technical repeat ID (for example `ctrl` for +the folder `control`) never changes it. + +Expected `experiment_design.csv` outputs (golden files) for `v13`, `v33` and `v34` are in +`fixtures/expected/`; a derived file MUST equal the golden file byte for byte. + +## 12. Drafts + +A draft (reconciliation proposal, unconfirmed repeats) is internal +to preprocessing and records its base SHA-256 (§10). It is never named `morphobricks.toml`. Downstream consumers (`RunInputs`) read +only the confirmed, published `morphobricks.toml` and MUST refuse a draft or any file with an +unconfirmed repeat (`A_DRAFT_REFUSED`; a published-path file with `confirmed = false` is also +`E_REPEAT_UNCONFIRMED`). diff --git a/tests/spec_fixtures/fixtures/README.md b/tests/spec_fixtures/fixtures/README.md new file mode 100644 index 0000000..58f37f8 --- /dev/null +++ b/tests/spec_fixtures/fixtures/README.md @@ -0,0 +1,490 @@ +# Fixture catalogue: experiment description v1 + +Normative test data for [experiment-v1.md](../docs/experiment-v1.md). Each file starts with a +comment header (`Fixture`, `Expected`, `Source`, description). How these become executable tests +in preprocessing is described in the [conformance plan](../docs/conformance-plan.md); the audit tool +`tools/audit/refval.py` runs them here. The records files in `records/` have no header; see +their section. + +## Rules + +- A conforming reader MUST accept every file in `valid/` and report exactly the warnings listed + (none unless stated). +- For every file in `invalid/` it MUST report exactly the listed **set** of error codes, all from + the first failing validation phase (experiment-v1 §6), and nothing from later phases. +- "Homogeneous: **no**" marks a valid file whose units have different ordered channel lists. The + spec accepts it; the Stage 1-2 runtime MUST refuse to process it with + `R_HETEROGENEOUS_LAYOUT_NOT_ENABLED` (lifted at Stage 3). +- Physical cases that need images (within-unit divergence, header contradictions, mask pixel + contradictions, reconciliation) cannot be expressed by a TOML file alone; they are listed as + scenarios P01-P35 in the [conformance plan](../docs/conformance-plan.md#3-physical-scenarios-preprocessing-owned), + built on the valid files below. +- `v18` MUST parse to the same model as `v01` (round trip); `v21` differs from `v01` only in + `display_name`, `target` and `treatment_key` values and `v16` only in `display_name` values, so + for both all four section digests equal `v01`'s (experiment-v1 §9.3). +- Every file carries the required `used_ids` table (experiment-v1 §4.9) listing at least all of its + declared condition, channel, repeat and technical repeat IDs; `i92` is the file without it. +- Every condition and every technical repeat carries the required `source_folder` (experiment-v1 + §4.6a): the exact folder label, never restricted by the reserved or forbidden names. In + fixtures that do not exercise the rule it equals the ID (the key), `single` for `single`; + `v33`, `v34` and `t10`-`t12` separate the two. The eight undecodable + or unparsable phase-1 fixtures (`i01`-`i08`) are unchanged and fail before it matters. +- `transitions/` holds thirteen files that are valid alone and are checked against a named published + predecessor: refused with an exact set of publication codes (`R_CONFIRMED_ID_CHANGED`, + `R_USED_ID_REUSED`, `R_USED_ID_DROPPED`, `R_CONFIRMED_ON_CHANGED`; experiment-v1 §4.4a, §4.6, + §4.9; a changed or reassigned `source_folder` counts, §4.6a) or, for `t06`, **accepted** (adding members is not a changed entity, §4.6). They need the + predecessor file and are not part of the single-file valid/invalid rules. Publication-base and + folder-name rules need no TOML of their own: they run on named fixtures in scenarios P17-P22. +- Rules that need images, a run or a consumer (header matching, image type against the header, + mask checks and conversions, mask type names, the FOV key, the `target` prefill, derived-CSV + dates) have no single-file TOML form: each is a scenario P23-P33 of the conformance + plan on a named fixture, listed in the coverage tables below. +- A copy of this tree vendored by a reader MUST hash to the pinned SHA's `fixtures` tree + (conformance plan §1). +- Expected `experiment_design.csv` outputs (golden files, experiment-v1 §11) are in `expected/`, + one per file listed under "Expected outputs" below; the other valid files have none yet. The fixed-SHA + consumer audit (roadmap S0.1) is done (PR #5); its questions are answered by the third round of + owner decisions (ADR 0001) and the fixtures `v28`-`v32`, `t06`-`t09`, `i100`-`i119` follow from + them; the fourth round (FR1, FR2) adds `v33`, `v34`, `t10`-`t12` and `i120`-`i127`. + +## Syntax check (6 November 2026) + +Checked with Python 3.13 `tomllib` (`tomllib.loads` on the UTF-8 decoded bytes), after the third +round of owner decisions and again after the fourth (folder labels, `source_folder`): + +- `valid/`: 34 of 34 parse. +- `transitions/`: 12 of 12 parse. +- `invalid/`: 119 of 127 parse as TOML and are invalid only by the v1 rules. The 8 phase-1 fixtures + fail deliberately: `i01_bom` (`TOMLDecodeError`, BOM read as an invalid statement), + `i02_invalid_utf8` (`UnicodeDecodeError`), `i03`-`i06` (`TOMLDecodeError`, "Cannot declare ... + twice" / "Cannot overwrite a value"), `i07`, `i08` (`TOMLDecodeError`, "Invalid value" / + "Expected newline"). + +## Matrix audit fixtures and executable validator (7 November 2026) + +The matrix audit ([`docs/audits/matrix-audit-2026-11-07.md`](../docs/audits/matrix-audit-2026-11-07.md)) +adds `v35`, `v36` and `i128`-`i130` for structures the catalogue lacked (a flat experiment, a +biological-repeat folder named like a forbidden key, `single` rules for condition folders and +members). The owner decisions of 7 November 2026 (ADR 0001, SQ6) add `i131`-`i133` (the +`single` rules for conditions) and `i134` (a duplicate key inside an inline table); the owner decision of 7 November 2026 on reserved names in `used_ids` (UR1) adds `v37` and `i135`. Its audit-only validator, [`tools/audit/refval.py`](../tools/audit/refval.py) +(`python -I tools/audit/refval.py fixtures .`), accepts 37 of 37 valid files with exactly the +listed warnings, refuses 135 of 135 invalid files with exactly the listed code set from the listed +phase, gives the listed `R_*` set for 12 of 12 transitions, and round-trips every valid and +transition file through the canonical writer. It is an audit tool, not a reader: the conformance +plan's single reference reader stays preprocessing. + +## Reference-validator cross-check (6 November 2026, roadmap S0.1) + +A reference validator (Python 3.13; now the audit tool [`tools/audit/refval.py`](../tools/audit/refval.py), not a runtime reader) implementing experiment-v1 §2-§6 phase by phase, plus the predecessor checks of §4.6, +§4.4a and §4.9 expressible from two files, was run on every fixture after the third round of +owner decisions and run again after the fourth (folder labels, `source_folder`). The validator +implements the text as written, including the new rules (`source_folder`, one folder one ID, +`used_ids.technical_repeats`, one code per broken rule, forbidden names on ID keys, the extended +reserved list, NFC of channel references, `image_type` and the RGB rules, "changed entity" as +removal or reassignment, a changed `source_folder` included). Results: + +- `valid/`: 34 of 34 accepted with exactly the listed warnings. +- `invalid/`: 127 of 127 refused with exactly the listed code set, from the listed phase only. +- `transitions/`: 12 of 12 valid alone (`t03`, `t06`, `t07`, `t08` with + `W_REPEAT_MISSING_CONDITIONS`); against their predecessors each gives exactly its listed set: + `t01` `R_CONFIRMED_ID_CHANGED`; `t02` `R_CONFIRMED_ID_CHANGED` and `R_USED_ID_DROPPED`; `t03` + `R_USED_ID_REUSED` only; `t04` `R_USED_ID_DROPPED`; `t05` `R_CONFIRMED_ON_CHANGED`; `t06` no + code (accepted); `t07`, `t08`, `t09`, `t10`, `t11`, `t12` `R_CONFIRMED_ID_CHANGED`. +- `v18` parses to the same model as `v01`; `v16` and `v21` have the same four sections as `v01` + once `display_name`, `target` and `treatment_key` are omitted. + +This is a check of the fixtures against the text, not a conformance claim for any brick. + +## Coverage of RFC #1 and its discussion + +**RFC #1, M0 fixture list (12 cases)**, from the +[RFC body](https://github.com/CellMigrationLab/MorphoBricks-Spec/issues/1): + +| # | RFC case | Fixtures | +|---|---|---| +| 1 | One control | `v01` | +| 2 | Several controls | `v02` | +| 3 | Differently named source folders paired into one canonical repeat | `v03` | +| 4 | Swapped channel order between imaging units | `v04` | +| 5 | Variable channel counts | `v05` | +| 6 | Two technical repeats of one biological repeat with different layouts | `v06` | +| 7 | One FOV diverging inside a technical repeat (invalid) | `i20` (no override can express it); physical P01, P02 | +| 8 | Instance masks | `v07` | +| 9 | Binary masks | `v08` | +| 10 | Incomplete/progressive masks | `v09` | +| 11 | Reference chains | `v10` | +| 12 | Invalid reference cycles | `i64`, `i65` | + +**Ten adversarial tests** from the independent review comment in the +[RFC #1 discussion](https://github.com/CellMigrationLab/MorphoBricks-Spec/issues/1#issuecomment-6667477114) +(5 November 2026). Most are end-to-end obligations of later stages; the fixture fixes the input: + +| # | Adversarial test | Fixtures | Obligation | +|---|---|---|---| +| 1 | Two technical repeats of different counts/orders stay ONE biological replicate | `v06` | Stage 2-3: N per repeat in normalization, embeddings, MorphoCoverage | +| 2 | One deviating FOV in a technical repeat refused with image/C-index evidence | `i20`; P01, P02 | Stage 1: `R_UNIT_NOT_HOMOGENEOUS` | +| 3 | Swapped C order never mixes ACTIN and DNA in normalization or CellPatcher | `v04`, `v06` | Stage 1 refusal; Stage 3 pooling by canonical ID | +| 4 | A homogeneous described dataset (folder-name IDs confirmed by the user) keeps the existing refusals; no parity against v0.4.0 (ADR 0002 AR10) | `v13` | Stage 1: byte-identical derived CSV (as reviewed for v0.5.0); no legacy numerical parity gate | +| 5 | `DNA` with confirmed header `DAPI` passes; unconfirmed contradiction refuses | `v11`; P03 | Stage 1: `R_HEADER_CONTRADICTION` | +| 6 | A binary region mask never becomes an instance object set | `v08` | Stage 2: `A_MASK_NOT_INSTANCES` | +| 7 | N35 confirmation survives reconciliation and reopens on discovery changes (the legacy-CSV migration is removed, ADR 0002 AR10 (9)) | `v13`, `i37`; P07 | Stage 1 reconciliation | +| 8 | Distinct source repeat names paired by the user give one canonical ID in MorphoCoverage | `v03` | Stage 2 (S2.5) | +| 9 | Missing channels in requested scope: refusal or recorded user subset, never silent drop; a channel no image has is the same refusal (SI1) | `v05`, `v17` | `A_SCOPE_INCOMPLETE`, shown at Stage 3 / gate G3 for both: their layouts are refused at Stages 1-2 (P04); owner decision GS2 for both | +| 10 | Metadata-only changes keep valid pixel-derived caches | `v01` vs `v21` | Stage 1: section digests | + +**Further required cases:** + +| Topic | Fixtures | +|---|---| +| Duplicate keys | `i03`-`i06`, `i134` (inline table) | +| Case collisions | `i48`-`i52`, `i68` (no folding when resolving), `i95` | +| NFC (refused, never normalized) | `v23`, `i78`, `i83`-`i85`; P21 | +| Display names | `v16`, `v24`, `i53`, `i76`, `i77`, `i86`-`i88` | +| Display name is not an ID | `i89`-`i91` | +| IDs never repaired, re-cased or renumbered | `v26`, `i48`-`i52`, `i55`, `i95` | +| `used_ids` registry | `v25`, `i92`-`i97`, `t03`, `t04`; P19, P20 | +| `confirmed_on` is a user fact | `i98`, `t05`; P07, P15, P22 | +| Publish only from the current base | P17, P18 (on `v01`, `v21`, `v16`) | +| Lab-tool provenance stays out of the file | `i99` | +| Unknown and forbidden fields | `i16`-`i19`; `i20`-`i26`; `i97`, `i99` | +| Implicit-conversion traps | `i07`, `i08`, `i12`-`i14`, `i19`, `i27`-`i30`, `i33`, `i34`, `i36`, `i38`, `i39`; valid side `v15` | +| Reference DAG | `i64`-`i68`, `i41`; `v10`, `v19` | +| Swapped channel order | `v04`, `v06` | +| Extra / missing channel | `v05`, `v17` (both Stage 3 / G3, GS2: P04 refuses their layouts at Stages 1-2), `v22`, `i69` | +| Within-unit divergence | `i20`; P01, P02 | +| Instance vs binary masks | `v07`, `v08`, `i42`, `i43`, `i111`, `i112`; P05, P27, P28 | +| Partial masks | `v09` | +| Immutable IDs (no renames) | `v16`, `i75`, `i82`, `t01`, `t02`; P08, P16 | +| Adding is not a change; removing or reassigning is | `t03`, `t06`, `t07`, `t08`; P07, P15 | +| Case-only correction needs a new ID; set of publication codes | `t02`, `t09`, `i95`; P16 | +| RGB images (declared type, fixed tokens, no aliases) | `v28`, `v29`, `i100`-`i106`, `i119`; P13, P23, P24 | +| Reserved names (one list for channel and mask IDs; also for `used_ids.channels`) | `i59`, `i61`, `i107`-`i109`, `i135`, `v32`, `v37` | +| Forbidden names apply to ID keys | `i113`-`i115` | +| Folder labels never restricted: condition and technical-repeat IDs separate from `source_folder` | `v33`, `v34`, `i113`, `i115`, `i120`-`i127`, `t10`-`t12`; P32, P33 | +| One folder, one ID; `single` iff no folder level (technical repeats: `i125`; conditions: `i131`-`i133`) | `i122`-`i125`, `i131`-`i133` | +| Folder change for an existing ID is a reassignment | `t10`-`t12`; P33 | +| One code per broken rule; NFC of channel references | `i116`-`i118` | +| FOV key keeps the source label, `repeat_group` added | `v03`, `v30`; P25 | +| Mask checks: when, unchecked masks, type names | `v09`; P05, P28, P29 | +| Descriptive keys prefill only | `v27`; P31 | +| Derived CSV dates (every row carries its repeat's one `confirmed_on`; no clock read) | `v13`; P30 | +| Derived CSV keeps the source folder labels | `v13`, `v33`; P32 | +| Aliases (header evidence) | `v11`, `v12`, `v31`, `i46`, `i81`, `i110`; P03, P26 | +| Versioning (offline constant, no default) | `i09`-`i15` | +| Identities | `v14`, `v15`, `v20`, `v23`, `v26`, `i54`-`i63`, `i78` | +| Target / treatment key (descriptive only) | `v01`, `v10`, `v21`, `v27`, `i71` | +| Encoding | `i01`, `i02` | + +### Owner decisions of 6 November 2026 (ADR 0001) and their fixtures + +| # | Decision | Fixtures / scenarios | +|---|---|---| +| D1 | Optional `target`, `treatment_key` (refined by Q4: descriptive only) | `v01`, `v10`, `v21`, `v27`; `i71` (non-string) | +| D2 | Channel ID = compartment name refused | `i61` | +| D3 | Unused channel is a warning | `v22` | +| D4 | Repeat ID = another repeat's source folder refused | `i75` | +| D5 | IDs immutable; `display_name` mutable and unique; no `retired_repeats` | `v16` (display names change, identity identical), `i53`, `i76`, `i77` (collisions), `i82` (no table), `t01`, `t02`; P16 | +| D6 | N35 reopens only on channel/layout/grouping change | P07, P15 | +| D7 | NFC identities (refined by Q3: non-NFC refused, never normalized; no NFC-collision code) | `v23`, `i78`, `i83`-`i85` | +| D8 | RGB refused (**superseded by AQ6**) | P13 (now: RGB accepted when declared) | +| D9 | Header names exact, case-sensitive | `v11`; P03 | +| D10 | `binary` mask with > 2 values refused | P05 | +| D11 | Drafts internal; `RunInputs` refuses them | `i37`; P14 | +| Q1 | Append-only `used_ids` per kind; no reuse, no drop | `v25`, `i92`-`i97`, `t03`, `t04`; P19, P20 | +| Q2 | Display names unique per kind after NFC + case folding | `v24`, `i86`-`i88` | +| Q3 | Reader refuses non-NFC identities | `v23`, `i78`, `i83`-`i85`; P21 | +| Q4 | `treatment_key`, `target` descriptive only | `v27`, `v21` | +| Q5 | Lab-tool links in the run manifest only | `i99` | +| A1 | Publish only from the current base SHA | P17, P18 | +| A2 | IDs never repaired, re-cased or renumbered | `v26`, `i95` | +| A3 | `confirmed_on` is a user fact | `i98`, `t05`; P22 | +| A4 | `display_name` never substitutes for an ID | `i89`-`i91` | +| A5 | Pin integrity, offline version check, no fallback rules | `i09`-`i11`; conformance plan §1-§2 | +| AQ1 | Adding a member is allowed; only removing or reassigning is a changed entity | `t03`, `t06`, `t07`, `t08`; P07, P15 | +| AQ2 | Case-only ID correction needs a new ID; publication reports every `R_*` code | `t02`, `t09`, `i95`; P16 | +| AQ3 | `target` descriptive; MorphoCoverage may only prefill, confirmed inside MorphoCoverage | `v27`; P31 | +| AQ4 | One reserved list: compartments plus MorphoRelate reserved column names | `i59`, `i61`, `i107`-`i109`, `v32` | +| UR1 | The reserved list also applies to `used_ids.channels` entries (they reserve those IDs) | `i135`, `v37` | +| AQ5 | Canonical repeat ID added as `repeat_group`; the FOV key keeps the source label | `v03`, `v30`; P25 | +| AQ6 | RGB accepted as a declared type (`image_type = "rgb"`, tokens `R`, `G`, `B`); supersedes D8 | `v28`, `v29`, `i100`-`i106`, `i119`; P13, P23, P24 | +| AQ7 | No trimming: header names exact after NFC of both sides | `v31`, `i110`; P26 | +| AQ8 | `parent_labels` masks out of scope in v1 | `i111` | +| AQ9 | Binary-as-instances only by an explicit, per-request, recorded conversion | `v08`, `i112`; P27 | +| AQ10 | Mask representation checked in the mask step; publication records "not yet checked"; unchecked masks refuse | `v09`; P05, P28 | +| AQ11 | A discovered mask type breaking the Label rules: `R_MASK_TYPE_NAME_INVALID` | P29 | +| AQ12 | Legacy groups with different `confirmed_on` take the latest date, with a note. **Superseded 10 November 2026** (ADR 0002 AR10 (9): no legacy-CSV migration) | none (P30 now tests the derived output only) | +| AQ13 | Forbidden names apply to ID keys; NFC of channel references; one code per broken rule | `i113`-`i118` | +| FR1 | Folder labels are never restricted by the reserved or forbidden names: conditions and technical repeats have a canonical ID separate from the required `source_folder`; one folder, one ID; a folder change for an existing ID is a reassignment; the derived CSV keeps the folder labels | `v33`, `v34`, `i113`, `i115`, `i120`-`i127`, `t10`-`t12`; P32, P33 | +| FR2 | Derived CSV byte-identity is guaranteed only for repeat groups with a single `confirmed_on`; a legacy group with several dates gets the latest, the migration note keeps the per-row dates. **Superseded 10 November 2026** (ADR 0002 AR10 (9): no legacy-CSV migration; every published repeat has one `confirmed_on`) | none (P30 now tests the derived output only) | +| RR1 | Record type `image_count_changed` (a confirmed member's image count changes; nothing reopens) | P34 | +| RR3 | `image_count_changed` is written for every confirmed member whose count changed, also when the publication changes other things | P34 | +| RR2 | Record type `removed` for every confirmed removal of a unit, technical repeat, repeat or condition | P35 | +| RF1 | Records file separators are `, ` and `: `; other whitespace refused | `ri25`, `ri27`, `ri30` | +| RF2 | Every records-file violation is a refusal (key set, order, escapes, duplicates, values) | `ri01`-`ri24` | +| RF3 | One code, `R_RECORDS_FILE_INVALID`, with `E_RECORDS_*` reasons | all `ri` records files | +| RF4 | Empty records file accepted; BOM and CR before LF refused; lower-case SHA hex | `rv06`, `ri13`, `ri24`, `ri26`, `ri31` | +| RF5 | `technical_repeats` not empty; `old_image_count` differs from `new_image_count` | `ri28`, `ri29` | +| RF6 | Line order within one publication stated; checked by writer tests, not by the file | P34, P35 | + +## Valid fixtures (37) + +| Fixture | Source | Expected | Homogeneous | Case | +|---|---|---|---|---| +| [`v01_single_control`](valid/v01_single_control.toml) | RFC case 1 | pass | yes | One control; trivial pairing, so canonical repeat IDs default to the folder names. | +| [`v02_multiple_controls`](valid/v02_multiple_controls.toml) | RFC case 2 | pass | yes | Two controls (DMSO, WT), each referenced by two conditions. | +| [`v03_paired_different_folder_names`](valid/v03_paired_different_folder_names.toml) | RFC case 3; adversarial 8 | pass | yes | Source folders rep1 (CTRL) and experiment_23 (DrugA) paired by the user into canonical repeat Day1; rep2/rep2_drug into Day2. MorphoCoverage must see Day1 and Day2 as the biological repeats: the FOV key keeps `rep1` / `experiment_23` in `biological_repeat` and the added `repeat_group` is Day1 (P25). | +| [`v04_swapped_channel_order`](valid/v04_swapped_channel_order.toml) | RFC case 4; adversarial 3 | pass | **no** | Same channels, opposite source order between conditions (CTRL C0=DNA, DrugA C0=ACTIN). Valid schema; heterogeneous, so Stage 1-2 runtime refuses (R_HETEROGENEOUS_LAYOUT_NOT_ENABLED). From Stage 3, ACTIN and DNA are never mixed by C index. | +| [`v05_variable_channel_counts`](valid/v05_variable_channel_counts.toml) | RFC case 5 | pass | **no** | Extra channel PHASE in DrugA/rep1 only; CTRL lacks it (missing elsewhere). Heterogeneous. | +| [`v06_technical_repeats_different_layouts`](valid/v06_technical_repeats_different_layouts.toml) | RFC case 6; adversarial 1 | pass | **no** | One biological repeat (R002) whose CTRL member has two technical repeats with different counts and orders. It stays ONE biological repeat (N = 2 repeats in total, not 3). Heterogeneous. | +| [`v07_instance_masks`](valid/v07_instance_masks.toml) | RFC case 8 | pass | yes | Two instance masks with source-channel provenance; canonical mask IDs differ from discovered source types. | +| [`v08_binary_masks`](valid/v08_binary_masks.toml) | RFC case 9; adversarial 6 | pass | yes | A binary region mask beside an instance mask. Valid; an analysis request using TumorRegion as an object set must refuse (A_MASK_NOT_INSTANCES) unless it carries an explicit, per-request, recorded conversion (P27). | +| [`v09_partial_masks`](valid/v09_partial_masks.toml) | RFC case 10 | pass | yes | Masks declared while segmentation is in progress (e.g. 0/1000 or 300/1000 FOVs). Completeness lives in the run manifest, so the file is valid at every stage; scope checks belong to the analysis layer (A_SCOPE_INCOMPLETE). | +| [`v10_reference_chain`](valid/v10_reference_chain.toml) | RFC case 11 | pass | yes | Reference chains VEC_SiCol6a1 -> VEC_SiNC -> VEC and THBS5_SiCol6a1 -> THBS5_SiNC -> THBS5; shared treatment_key across backgrounds. | +| [`v11_header_aliases`](valid/v11_header_aliases.toml) | Adversarial 5 | pass | yes | Canonical DNA confirmed with header evidence DAPI and Hoechst 33342; canonical ACTIN with an exact-match header name that still had to be confirmed. A header name outside these lists is R_HEADER_CONTRADICTION. | +| [`v12_unnamed_headers`](valid/v12_unnamed_headers.toml) | Extra: aliases | pass | yes | No header evidence at all (ImageJ TIFFs carry no channel names); the user's confirmation is the only guard. Undetectable physical swaps are a documented limit. | +| [`v13_homogeneous_legacy`](valid/v13_homogeneous_legacy.toml) | Adversarial 4 | pass | yes | A legacy homogeneous dataset: folder-name repeat IDs, one control, same layout everywhere (IDs proposed equal to the folder labels). The derived experiment_design.csv, which carries the `source_folder` labels, must be byte-identical to its golden file (a Stage 1 check); there is no numerical parity against v0.4.0 (ADR 0002 AR10). | +| [`v14_no_repeat_folders`](valid/v14_no_repeat_folders.toml) | Extra: single convention | pass | yes | No biological-repeat or technical-repeat folders: the source label is `single` (N31, one group for the whole experiment). | +| [`v15_toml_keys_are_text`](valid/v15_toml_keys_are_text.toml) | Extra: implicit-conversion traps (valid side) | pass | yes | Folder labels that YAML would retype (2026-01-01, 012, NO, 1.0, off) stay exact text in TOML keys and strings. The quoted key "1.0" is needed: a bare 1.0 would be a dotted key. | +| [`v16_display_name_change`](valid/v16_display_name_change.toml) | Owner refinement: immutable IDs | pass | yes | v01 with every `display_name` changed or added (conditions, channels, repeats). Identity is identical: all four section digests equal v01's, nothing may be invalidated. | +| [`v17_partial_channel_presence`](valid/v17_partial_channel_presence.toml) | Adversarial 9; addendum §3 | pass | **no** | PHASE exists only in CTRL/R002/acquisition_1. For a PHASE request R002 is partially represented and R001 unavailable: the analysis layer must refuse (A_SCOPE_INCOMPLETE) or record a user-selected subset, never drop or re-weight silently. A request for a channel ID that no unit of any fixture lists (the "absent from all" case, e.g. `Tubulin` against `v01`) is the same refusal with k = 0 (owner decision SI1); it needs no fixture of its own because the absent ID is a property of the request. | +| [`v18_inline_and_dotted_forms`](valid/v18_inline_and_dotted_forms.toml) | Extra: round trip | pass | yes | Same model as v01 written with inline tables, dotted keys and quoted keys; parse(v18) must equal parse(v01). | +| [`v19_warnings_unpaired`](valid/v19_warnings_unpaired.toml) | Extra: reference DAG semantics | pass + `W_REFERENCE_UNPAIRED`, `W_REPEAT_MISSING_CONDITIONS` | yes | Valid with warnings: CTRL only in rep1, DrugA only in rep2, so the repeats miss conditions (W_REPEAT_MISSING_CONDITIONS) and DrugA cannot be paired with its reference (W_REFERENCE_UNPAIRED). | +| [`v20_unicode_and_spaces`](valid/v20_unicode_and_spaces.toml) | Extra: identities | pass | yes | Condition folder labels with spaces and non-ASCII text are valid labels (quoted keys); channel and mask IDs stay tokens. | +| [`v21_metadata_only_edit`](valid/v21_metadata_only_edit.toml) | Adversarial 10 | pass | yes | v01 with only descriptive condition fields edited (display name, target removed, treatment_key changed). All four section digests equal v01's (descriptive fields are omitted, experiment-v1 9.3), so no pixel-derived product may be invalidated. | +| [`v22_unused_channel_warning`](valid/v22_unused_channel_warning.toml) | Owner decision 3 | pass + `W_UNUSED_CHANNEL` | yes | PHASE is declared but acquired by no unit: valid with a warning, never an error. | +| [`v23_nfc_identity`](valid/v23_nfc_identity.toml) | Owner decision Q3 | pass | yes | A condition key written composed (U+00E9, NFC) and referenced composed: the only form a writer emits and a reader accepts. | +| [`v24_display_name_as_typed`](valid/v24_display_name_as_typed.toml) | Owner decision Q2 | pass | yes | Display names kept as typed (one decomposed, one lower case); a channel and a condition share the display name "Drug A" because uniqueness is per kind. | +| [`v25_removed_condition_id_retained`](valid/v25_removed_condition_id_retained.toml) | Owner decision Q1 | pass | yes | v01 after DrugA was removed: DrugA stays listed in `used_ids.conditions`. Predecessor of t03 and t04 (and, with the recorded removal confirmation of experiment-v1 4.6, the removal of v01). | +| [`v26_ids_kept_as_written`](valid/v26_ids_kept_as_written.toml) | Audit B1, B2, B11 | pass | yes | IDs `cl001`, `CL1000`, `R999`, `R1000`: no width, prefix or case rule; kept exactly as written. | +| [`v27_descriptive_keys_collide`](valid/v27_descriptive_keys_collide.toml) | Owner decision Q4 | pass | yes | Identical and case-differing `target` / `treatment_key` values on different conditions: valid, no warning, no grouping. | +| [`v28_rgb_units`](valid/v28_rgb_units.toml) | Owner decision AQ6 (supersedes D8) | pass | yes | An RGB experiment (e.g. H&E): every unit declares `image_type = "rgb"` with the fixed channels R, G, B in that order and no aliases. Valid and homogeneous. The scientific meaning is left to each brick: a brick that does not declare RGB support refuses a request on these units (A_RGB_NOT_SUPPORTED, P24); preprocessing compares the declaration with the image header and never with pixels (P13, P23). The `rgb_support` manifest key of experiment-v1 §4.4b (owner decision RC1) is a property of the receiving brick, not of the file, so no TOML fixture can express it; P24 states the four manifest cases to run against this one. | +| [`v29_rgb_and_channels_mixed`](valid/v29_rgb_and_channels_mixed.toml) | Owner decision AQ6 | pass | **no** | One condition with an RGB brightfield unit and a two-channel fluorescence unit of the same biological repeat. Valid schema; heterogeneous (different lists), so the Stage 1-2 runtime refuses it (R_HETEROGENEOUS_LAYOUT_NOT_ENABLED). Both units stay ONE biological repeat. | +| [`v30_same_source_label_different_repeats`](valid/v30_same_source_label_different_repeats.toml) | Owner decision AQ5; S0.1 audit section 5 | pass | yes | Source folder label rep1 is in canonical repeat Day1 for CTRL and in Day2 for DrugA (and rep2 the other way round). Valid: no repeat ID equals another repeat's source label. The FOV key keeps the source label in `biological_repeat` and stays unique through the condition; the added `repeat_group` is Day1 or Day2, and grouping and pairing use it (experiment-v1 3.4, P25). | +| [`v31_header_alias_whitespace_and_decomposed`](valid/v31_header_alias_whitespace_and_decomposed.toml) | Owner decision AQ7 | pass | yes | Header evidence is compared exactly after NFC of both sides, with no trimming. DNA lists the aliases "DAPI", " DAPI" (leading space) and "DAPI " (trailing space): outer whitespace needs its own confirmed alias, each is a distinct entry. ACTIN lists a decomposed alias (written with a TOML escape and stored as typed), which matches the header name "Caf\u00e9 488" after NFC of both sides (P26). header_names are not identities and are not NFC-checked. | +| [`v32_reserved_name_near_misses`](valid/v32_reserved_name_near_misses.toml) | Owner decision AQ4 | pass | yes | Names next to the reserved list (experiment-v1 3.2) are valid: channels c1, t0, z2 and label, masks Patches and Labels. Only the exact names, compared case-folded, are reserved (t, z0, label_id, patch_id, ...). | +| [`v33_condition_folder_on_forbidden_list`](valid/v33_condition_folder_on_forbidden_list.toml) | Owner decision FR1 | pass | yes | The condition folder on disk is `control`, a forbidden name (experiment-v1 4.7), so it cannot be the ID. It is recorded as `source_folder` (never restricted) and the user confirmed the ID `ctrl`, with `display_name` `control`. DrugA references `ctrl`, the ID. The derived experiment_design.csv still has `control` in the `condition` column (experiment-v1 11). | +| [`v34_technical_repeat_folder_on_forbidden_list`](valid/v34_technical_repeat_folder_on_forbidden_list.toml) | Owner decision FR1 | pass | yes | Repeat rep1 has two technical-repeat folders, `files` and `overrides`, both forbidden names (experiment-v1 4.7). They are the `source_folder` values; the confirmed IDs are `acq1` and `acq2`, and the same ID carries the same folder in both members. rep2 has no technical-repeat folder level (`single`). Homogeneous. | +| [`v35_flat_experiment`](valid/v35_flat_experiment.toml) | Matrix audit 2026-11-07, case F01 | pass | yes | The experiment folder is itself the leaf: condition, repeat and technical repeat are all `single`. The condition `single` has `source_folder = "single"` ("no condition folder level", 4.2, 3.2) and is the only condition (owner decision SQ6, matrix audit MS-1). | +| [`v36_repeat_folder_forbidden_name`](valid/v36_repeat_folder_forbidden_name.toml) | Matrix audit 2026-11-07, case F16 | pass | yes | Biological-repeat folders `z` paired into repeat `R1`; `source_repeat` is never restricted. The folder label `z` is not an allowed repeat ID (`i130`), so no ID is proposed and the user names `R1` (4.6a; owner decision SQ6, matrix audit MS-2). | +| [`v37_used_ids_reserved_near_misses`](valid/v37_used_ids_reserved_near_misses.toml) | Owner decision UR1 (Spec issue 11) | pass | yes | `v32` with the retired channel IDs `Ringer` and `label2` listed in `used_ids.channels`: names next to the reserved list are valid there too (experiment-v1 4.9). | + +## Expected outputs (3) + +Golden `experiment_design.csv` files (experiment-v1 §11), byte for byte. Each is the derivation of the +named valid file by the stated rules; `tools/audit/refval.py` compares them. + +| Golden file | Derived from | Notes | +|---|---|---| +| [`v13_homogeneous_legacy.experiment_design.csv`](expected/v13_homogeneous_legacy.experiment_design.csv) | `valid/v13` | One control (`CTRL`, referenced by DrugA and DrugB); every `technical_repeat_count` 0 (only `single`). | +| [`v33_condition_folder_on_forbidden_list.experiment_design.csv`](expected/v33_condition_folder_on_forbidden_list.experiment_design.csv) | `valid/v33` | `condition` holds the folder label `control`, never the ID `ctrl`; `DrugA` sorts before `control` (code point order). | +| [`v34_technical_repeat_folder_on_forbidden_list.experiment_design.csv`](expected/v34_technical_repeat_folder_on_forbidden_list.experiment_design.csv) | `valid/v34` | Two technical repeats (folders `files`, `overrides`) give `technical_repeat_count` 2; `rep2` has only `single`, so 0. | + +## Transition fixtures (13) + +Valid alone; checked against the named published predecessor (refused with the listed set, or accepted). + +| Fixture | Predecessor | Expected | Case | +|---|---|---|---| +| [`t01_repeat_id_changed`](transitions/t01_repeat_id_changed.toml) | `valid/v01` | `R_CONFIRMED_ID_CHANGED` unless re-confirmation is recorded | Repeats rep1/rep2 (same member folders) are now Day1/Day2; rep1, rep2 stay listed in `used_ids`. | +| [`t02_channel_id_changed`](transitions/t02_channel_id_changed.toml) | `valid/v01` | `R_CONFIRMED_ID_CHANGED`, `R_USED_ID_DROPPED` | Channel ACTIN (same header evidence and positions) is now Actin, and ACTIN is no longer listed in `used_ids`. Listing it beside `Actin` would be `E_ID_CASE_COLLISION`, so a case-only ID change cannot be published in place (owner decision AQ2): it needs a new ID, `t09`. | +| [`t03_used_id_reused`](transitions/t03_used_id_reused.toml) | `valid/v25` | `R_USED_ID_REUSED` (only) | DrugA (retired in v25) is declared again for a new source folder. rep1 gains a member, which is not a changed entity (owner decision AQ1), so `R_CONFIRMED_ID_CHANGED` does not apply. Valid alone, with `W_REPEAT_MISSING_CONDITIONS` (rep2 has no DrugA member). | +| [`t04_used_id_dropped`](transitions/t04_used_id_dropped.toml) | `valid/v25` | `R_USED_ID_DROPPED` | DrugA is no longer listed in `used_ids.conditions`. | +| [`t05_confirmed_on_changed`](transitions/t05_confirmed_on_changed.toml) | `valid/v01` | `R_CONFIRMED_ON_CHANGED` (N35 not reopened) | rep1 unchanged except `confirmed_on` 2026-11-07. | +| [`t06_member_added`](transitions/t06_member_added.toml) | `valid/v01` | no R_* code: accepted (against predecessor valid/v01) | Adding is not a change of entity. Against v01: a new condition DrugB joins the existing repeats rep1 and rep2 as a member, and a new folder CTRL/rep3 (new repeat rep3) joins the existing condition CTRL. Every member folder, position and alias of v01 is still there. The confirmation of rep1 and rep2 is reopened by the new member (grouping change, experiment-v1 10) and re-confirmed on 2026-11-07, so the changed `confirmed_on` is not R_CONFIRMED_ON_CHANGED. DrugB and rep3 are new IDs, listed in `used_ids`. Valid alone with W_REPEAT_MISSING_CONDITIONS (rep3). | +| [`t07_member_removed`](transitions/t07_member_removed.toml) | `valid/v01` | R_CONFIRMED_ID_CHANGED (against predecessor valid/v01, without a recorded removal confirmation) | Removing is a change of entity. The member DrugA/rep2 of repeat rep2 is gone (rep2 reopened and re-confirmed on 2026-11-07). The removal is accepted only with the recorded user confirmation of experiment-v1 4.6 and 10; without it the candidate is refused. All IDs stay listed in `used_ids`. Valid alone with W_REPEAT_MISSING_CONDITIONS. | +| [`t08_member_reassigned`](transitions/t08_member_reassigned.toml) | `valid/v03` | R_CONFIRMED_ID_CHANGED (against predecessor valid/v03, without a recorded confirmation) | Reassigning is a change of entity. DrugA/experiment_23 was a member of Day1 in v03 and is now the member of a new repeat Day3, so Day1 lost a member it held. Accepted only with the recorded user confirmation of experiment-v1 4.6. Day3 is a new ID listed in `used_ids`. Valid alone with W_REPEAT_MISSING_CONDITIONS. | +| [`t09_case_correction_new_id`](transitions/t09_case_correction_new_id.toml) | `valid/v01` | R_CONFIRMED_ID_CHANGED (against predecessor valid/v01, accepted only with a recorded re-confirmation) | How a case-only mistake is corrected. v01's channel ACTIN was meant to be Actin; the corrected spelling cannot be used (it collides with the retired ACTIN, see t02 and i95), so the channel gets the new, non-colliding ID ActinStain at the same positions with the same header evidence, and `display_name` "Actin" carries the appearance. ACTIN stays listed in `used_ids`. The set is exactly {R_CONFIRMED_ID_CHANGED}: no R_USED_ID_DROPPED, no R_USED_ID_REUSED, no E_* code. | +| [`t10_condition_id_changed_same_folder`](transitions/t10_condition_id_changed_same_folder.toml) | `valid/v33` | R_CONFIRMED_ID_CHANGED (against predecessor valid/v33, without a recorded re-confirmation) | The folder `control` was confirmed as condition `ctrl` in v33 and now carries the ID `control_cells`. A confirmed ID never changes in place, also when the folder is the same (experiment-v1 4.6, 4.6a). `ctrl` stays listed in used_ids, so only the changed ID is refused. Valid alone. | +| [`t11_technical_repeat_id_changed_same_folder`](transitions/t11_technical_repeat_id_changed_same_folder.toml) | `valid/v34` | R_CONFIRMED_ID_CHANGED (against predecessor valid/v34, without a recorded re-confirmation) | The folder `files` was confirmed as technical repeat `acq1` in v34 and now carries the ID `acqA` in both members. A confirmed technical repeat ID never changes in place (experiment-v1 4.6, 4.6a). `acq1` stays listed in used_ids. Valid alone. | +| [`t12_condition_folder_reassigned`](transitions/t12_condition_folder_reassigned.toml) | `valid/v33` | R_CONFIRMED_ID_CHANGED (against predecessor valid/v33, without a recorded confirmation) | The ID `ctrl` held the folder `control` in v33 and now holds the folder `ctl` (a folder renamed on disk and attached to the same ID without a recorded confirmation). A folder change for an existing ID is a reassignment, so a changed entity (experiment-v1 4.6, 7). Valid alone. | +| [`t13_confirmed_on_with_image_count_only`](transitions/t13_confirmed_on_with_image_count_only.toml) | `valid/v01` | `R_CONFIRMED_ON_CHANGED` (N35 not reopened) | Against v01 only CTRL/rep1 `image_count` (12 to 17, new FOVs) and rep1 `confirmed_on` (2026-11-07) change: new FOVs do not reopen the confirmation (experiment-v1 10), so the new date is refused. | + +## Invalid fixtures (135) + +| Fixture | Source | Expected code(s) | Phase | Case | +|---|---|---|---|---| +| [`i01_bom`](invalid/i01_bom.toml) | Extra: encoding | `E_FILE_ENCODING` | 1 | The file starts with a UTF-8 byte-order mark. | +| [`i02_invalid_utf8`](invalid/i02_invalid_utf8.toml) | Extra: encoding | `E_FILE_ENCODING` | 1 | A label contains the byte 0xFF (not UTF-8). | +| [`i03_duplicate_condition_table`](invalid/i03_duplicate_condition_table.toml) | Extra: duplicate keys; audit §3.12 | `E_DUPLICATE_KEY` | 1 | [conditions.CTRL] declared twice (YAML would keep the second silently). | +| [`i04_duplicate_key_quoted_vs_bare`](invalid/i04_duplicate_key_quoted_vs_bare.toml) | Extra: duplicate keys | `E_DUPLICATE_KEY` | 1 | [conditions."CTRL"] is the same key as [conditions.CTRL]. | +| [`i05_duplicate_field`](invalid/i05_duplicate_field.toml) | Extra: duplicate keys | `E_DUPLICATE_KEY` | 1 | `display_name` written twice in one condition. | +| [`i06_duplicate_member_dotted_vs_table`](invalid/i06_duplicate_member_dotted_vs_table.toml) | Extra: duplicate keys | `E_DUPLICATE_KEY` | 1 | A member declared as a table and again by a dotted key (two members of one condition in one repeat). | +| [`i07_unquoted_bareword`](invalid/i07_unquoted_bareword.toml) | Extra: implicit conversion (YAML `NO` -> False) | `E_TOML_SYNTAX` | 1 | source_repeat = NO, unquoted. | +| [`i08_leading_zero_integer`](invalid/i08_leading_zero_integer.toml) | Extra: implicit conversion (YAML `012` -> 10) | `E_TOML_SYNTAX` | 1 | source_repeat = 012, unquoted. | +| [`i09_schema_version_2`](invalid/i09_schema_version_2.toml) | Extra: versioning | `E_SCHEMA_VERSION_UNSUPPORTED` | 2 | schema_version = 2 is refused, never read best effort. | +| [`i10_schema_version_0`](invalid/i10_schema_version_0.toml) | Extra: versioning | `E_SCHEMA_VERSION_UNSUPPORTED` | 2 | schema_version = 0. | +| [`i11_schema_version_missing`](invalid/i11_schema_version_missing.toml) | Extra: versioning | `E_SCHEMA_VERSION_MISSING` | 2 | No schema_version. | +| [`i12_schema_version_string`](invalid/i12_schema_version_string.toml) | Extra: implicit conversion | `E_TYPE` | 2 | schema_version = "1" (a string is not converted to an integer). | +| [`i13_schema_version_float`](invalid/i13_schema_version_float.toml) | Extra: implicit conversion | `E_TYPE` | 2 | schema_version = 1.0. | +| [`i14_schema_version_bool`](invalid/i14_schema_version_bool.toml) | Extra: implicit conversion | `E_TYPE` | 2 | schema_version = true (a boolean is not the integer 1). | +| [`i15_schema_id_wrong`](invalid/i15_schema_id_wrong.toml) | Extra: versioning | `E_SCHEMA_ID` | 2 | schema = "morphobricks-experiments". | +| [`i16_unknown_top_level_field`](invalid/i16_unknown_top_level_field.toml) | Extra: unknown fields | `E_UNKNOWN_FIELD` | 3 | Top-level `notes`. | +| [`i17_unknown_condition_field`](invalid/i17_unknown_condition_field.toml) | Extra: unknown fields | `E_UNKNOWN_FIELD` | 3 | Condition field `colour`. | +| [`i18_misspelled_field`](invalid/i18_misspelled_field.toml) | Extra: unknown fields | `E_MISSING_FIELD`, `E_UNKNOWN_FIELD` | 3 | `representaton` (typo) is not silently ignored; `representation` is then also missing. | +| [`i19_dotted_float_key`](invalid/i19_dotted_float_key.toml) | Extra: implicit conversion | `E_UNKNOWN_FIELD` | 3 | [conditions.1.0] is condition `1` with a sub-table `0`, not condition `1.0`. | +| [`i20_per_fov_override`](invalid/i20_per_fov_override.toml) | Extra: within-unit divergence; ADR (no per-FOV overrides) | `E_FORBIDDEN_FIELD` | 3 | A per-FOV channel override inside a technical repeat. | +| [`i21_mask_axes_shared`](invalid/i21_mask_axes_shared.toml) | Extra: masks; preprocessing #173 | `E_FORBIDDEN_FIELD` | 3 | Mask `axes = "YX"` and `shared = true` (v1 is 2-D only; a mask has its image's axes). | +| [`i22_brick_routing`](invalid/i22_brick_routing.toml) | Extra: ADR (no brick routing) | `E_FORBIDDEN_FIELD` | 3 | A channel routed to bricks. | +| [`i23_calibration_in_file`](invalid/i23_calibration_in_file.toml) | Extra: calibration stays in calibration.csv | `E_FORBIDDEN_FIELD` | 3 | pixel_size on a repeat. | +| [`i24_control_boolean`](invalid/i24_control_boolean.toml) | Extra: audit §3.3 | `E_FORBIDDEN_FIELD` | 3 | Legacy boolean `control` (derived from `reference` in v1). | +| [`i25_experiment_field`](invalid/i25_experiment_field.toml) | Extra: audit §3.4 | `E_FORBIDDEN_FIELD` | 3 | Condition `experiment` (an FDR family; MorphoCoverage configuration). | +| [`i26_source_index_field`](invalid/i26_source_index_field.toml) | Extra: RFC (order is the mapping) | `E_FORBIDDEN_FIELD` | 3 | A unit states `source_index` beside its ordered list. | +| [`i27_toml_date_source_repeat`](invalid/i27_toml_date_source_repeat.toml) | Extra: implicit conversion (YAML date) | `E_TYPE` | 3 | source_repeat = 2026-01-01 is a TOML date, not text. | +| [`i28_integer_source_repeat`](invalid/i28_integer_source_repeat.toml) | Extra: implicit conversion | `E_TYPE` | 3 | source_repeat = 12 (integer). | +| [`i29_bool_label`](invalid/i29_bool_label.toml) | Extra: implicit conversion (YAML `off`) | `E_TYPE` | 3 | display_name = false. | +| [`i30_confirmed_on_toml_date`](invalid/i30_confirmed_on_toml_date.toml) | Extra: implicit conversion | `E_TYPE` | 3 | confirmed_on = 2026-11-06 (TOML date literal, not a string). | +| [`i31_confirmed_on_bad_format`](invalid/i31_confirmed_on_bad_format.toml) | Extra: preprocessing contract §12 (spreadsheet dates) | `E_DATE_FORMAT` | 3 | confirmed_on = "06/11/2026". | +| [`i32_confirmed_on_impossible_date`](invalid/i32_confirmed_on_impossible_date.toml) | Extra | `E_DATE_FORMAT` | 3 | confirmed_on = "2026-02-30". | +| [`i33_image_count_string`](invalid/i33_image_count_string.toml) | Extra: implicit conversion | `E_TYPE` | 3 | image_count = "12". | +| [`i34_image_count_float`](invalid/i34_image_count_float.toml) | Extra: implicit conversion | `E_TYPE` | 3 | image_count = 12.0. | +| [`i35_image_count_zero`](invalid/i35_image_count_zero.toml) | Extra | `E_VALUE_RANGE` | 3 | image_count = 0. | +| [`i36_confirmed_string`](invalid/i36_confirmed_string.toml) | Extra: implicit conversion (legacy yes/no) | `E_TYPE` | 3 | confirmed = "yes". | +| [`i37_repeat_unconfirmed`](invalid/i37_repeat_unconfirmed.toml) | Adversarial 7 (N35) | `E_REPEAT_UNCONFIRMED` | 3 | A published file with an unconfirmed repeat. | +| [`i38_mixed_type_channel_list`](invalid/i38_mixed_type_channel_list.toml) | Extra: implicit conversion | `E_TYPE` | 3 | channels = ["DNA", 2]. | +| [`i39_channels_as_string`](invalid/i39_channels_as_string.toml) | Extra | `E_TYPE` | 3 | channels = "DNA,ACTIN" (no splitting of text). | +| [`i40_empty_channel_list`](invalid/i40_empty_channel_list.toml) | Extra | `E_EMPTY` | 3 | A unit with channels = []. | +| [`i41_reference_list`](invalid/i41_reference_list.toml) | Extra: reference DAG | `E_TYPE` | 3 | Two references in an array; v1 has one primary reference. | +| [`i42_representation_invalid`](invalid/i42_representation_invalid.toml) | Extra: masks | `E_ENUM` | 3 | representation = "labels". | +| [`i43_representation_missing`](invalid/i43_representation_missing.toml) | Extra: masks (never inferred from pixels) | `E_MISSING_FIELD` | 3 | A mask without representation. | +| [`i44_missing_channels_table`](invalid/i44_missing_channels_table.toml) | Extra | `E_MISSING_FIELD` | 3 | No [channels] at all. | +| [`i45_member_without_technical_repeats`](invalid/i45_member_without_technical_repeats.toml) | Extra | `E_MISSING_FIELD` | 3 | A member with no technical_repeats. | +| [`i46_header_name_empty`](invalid/i46_header_name_empty.toml) | Extra: aliases | `E_TEXT_EMPTY` | 3 | header_names = [""]. | +| [`i47_label_whitespace_only`](invalid/i47_label_whitespace_only.toml) | Extra | `E_TEXT_EMPTY` | 3 | display_name = " ". | +| [`i48_case_collision_conditions`](invalid/i48_case_collision_conditions.toml) | Extra: case collisions | `E_ID_CASE_COLLISION` | 4 | Conditions CTRL and ctrl. | +| [`i49_case_collision_channels`](invalid/i49_case_collision_channels.toml) | Extra: case collisions | `E_ID_CASE_COLLISION` | 4 | Channels DNA and dna. | +| [`i50_case_collision_source_repeats`](invalid/i50_case_collision_source_repeats.toml) | Extra: case collisions; preprocessing check_identity_case | `E_ID_CASE_COLLISION` | 4 | Source folders Rep1 (CTRL) and rep1 (DrugA): a label is spelled one way in the whole run. | +| [`i51_case_collision_repeat_ids`](invalid/i51_case_collision_repeat_ids.toml) | Extra: case collisions | `E_ID_CASE_COLLISION` | 4 | Canonical repeats rep1 and REP1. | +| [`i52_case_collision_technical_repeats`](invalid/i52_case_collision_technical_repeats.toml) | Extra: case collisions | `E_ID_CASE_COLLISION` | 4 | Technical repeats acq_1 and Acq_1 in different members. | +| [`i53_display_name_collision_conditions`](invalid/i53_display_name_collision_conditions.toml) | Owner refinement | `E_DISPLAY_NAME_COLLISION` | 5 | Two conditions with display_name "Control". | +| [`i54_channel_id_missing_text`](invalid/i54_channel_id_missing_text.toml) | Extra: identities (decision D-e) | `E_ID_TEXT` | 4 | Channel ID `NA` (tables read it as missing). | +| [`i55_condition_outer_whitespace`](invalid/i55_condition_outer_whitespace.toml) | Extra: identities (decision D-e) | `E_ID_TEXT` | 4 | Condition key " CTRL" with a leading space. | +| [`i56_source_repeat_null`](invalid/i56_source_repeat_null.toml) | Extra: identities | `E_ID_TEXT` | 4 | source_repeat = "null". | +| [`i57_channel_id_hyphen`](invalid/i57_channel_id_hyphen.toml) | Extra: identities (token rule) | `E_ID_SYNTAX` | 4 | Channel ID F-actin. | +| [`i58_channel_id_double_underscore`](invalid/i58_channel_id_double_underscore.toml) | Extra: identities (feature-column separator) | `E_ID_SYNTAX` | 4 | Channel ID DNA__2. | +| [`i59_mask_id_reserved`](invalid/i59_mask_id_reserved.toml) | Extra: MorphoFeature object-set rule | `E_ID_RESERVED` | 4 | Mask ID `Nucleus` (reserved compartment, any case). | +| [`i60_mask_id_reserved_in`](invalid/i60_mask_id_reserved_in.toml) | Extra: MorphoFeature object-set rule | `E_ID_RESERVED` | 4 | Mask ID `Cells_in_Tumor` (an `in` part). | +| [`i61_channel_id_reserved`](invalid/i61_channel_id_reserved.toml) | Extra: audit §3.8 | `E_ID_RESERVED` | 4 | Channel ID `Ring` (MorphoRelate compartment token). | +| [`i62_single_wrong_case`](invalid/i62_single_wrong_case.toml) | Extra: single convention | `E_SINGLE_MISUSE` | 4 | Technical repeat `Single` (spelled one way, so no case collision). | +| [`i63_single_not_alone`](invalid/i63_single_not_alone.toml) | Extra: single convention | `E_SINGLE_MISUSE` | 4 | Technical repeat `single` beside `acquisition_2`. | +| [`i64_reference_cycle`](invalid/i64_reference_cycle.toml) | RFC case 12 | `E_REFERENCE_CYCLE` | 5 | CTRL -> DrugA -> CTRL. | +| [`i65_reference_cycle_long`](invalid/i65_reference_cycle_long.toml) | RFC case 12 | `E_REFERENCE_CYCLE` | 5 | CTRL -> DrugB -> DrugA -> CTRL. | +| [`i66_reference_self`](invalid/i66_reference_self.toml) | Extra: reference DAG | `E_REFERENCE_SELF` | 5 | DrugA references itself. | +| [`i67_reference_unknown`](invalid/i67_reference_unknown.toml) | Extra: reference DAG | `E_REFERENCE_UNKNOWN` | 5 | Reference to undeclared DMSO. | +| [`i68_reference_case_mismatch`](invalid/i68_reference_case_mismatch.toml) | Extra: case rules (no folding on resolution) | `E_REFERENCE_UNKNOWN` | 5 | reference = "ctrl" does not resolve to CTRL. | +| [`i69_unknown_channel_in_unit`](invalid/i69_unknown_channel_in_unit.toml) | Extra: extra channel not declared | `E_UNKNOWN_CHANNEL` | 5 | A unit lists PHASE, which is not declared. | +| [`i70_duplicate_channel_in_unit`](invalid/i70_duplicate_channel_in_unit.toml) | Extra | `E_DUPLICATE_CHANNEL_IN_UNIT` | 5 | channels = ["DNA", "ACTIN", "DNA"]. | +| [`i71_target_not_text`](invalid/i71_target_not_text.toml) | Owner decision 1 | `E_TYPE` | 3 | target = 5 (plain strings only). | +| [`i72_member_unknown_condition`](invalid/i72_member_unknown_condition.toml) | Extra | `E_UNKNOWN_CONDITION` | 5 | A member for undeclared condition DrugB. | +| [`i73_condition_without_members`](invalid/i73_condition_without_members.toml) | Extra | `E_CONDITION_WITHOUT_MEMBERS` | 5 | Condition DrugB declared, no member. | +| [`i74_source_folder_in_two_repeats`](invalid/i74_source_folder_in_two_repeats.toml) | Extra: repeat pairing | `E_SOURCE_REPEAT_REUSED` | 5 | Source folder CTRL/exp7 belongs to repeats Day2 and Day3. | +| [`i75_repeat_id_swap`](invalid/i75_repeat_id_swap.toml) | Extra: renames | `E_REPEAT_ID_SHADOWS_SOURCE` | 5 | Repeat rep1 holds folder rep2 and repeat rep2 holds folder rep1 for CTRL. | +| [`i76_display_name_collision_repeats`](invalid/i76_display_name_collision_repeats.toml) | Owner refinement | `E_DISPLAY_NAME_COLLISION` | 5 | Repeats rep1 and rep2 both display as "Day 1". | +| [`i77_display_name_collision_channels`](invalid/i77_display_name_collision_channels.toml) | Owner refinement | `E_DISPLAY_NAME_COLLISION` | 5 | Channels DNA and ACTIN both display as "DAPI". | +| [`i78_non_nfc_condition_key`](invalid/i78_non_nfc_condition_key.toml) | Owner decision Q3 | `E_ID_NOT_NFC` | 4 | The condition key is decomposed "Cafe\u0301"; refused, never normalized. | +| [`i79_mask_source_channel_unknown`](invalid/i79_mask_source_channel_unknown.toml) | Extra: masks | `E_UNKNOWN_CHANNEL` | 5 | source_channels names undeclared Hoechst. | +| [`i80_mask_source_type_reused`](invalid/i80_mask_source_type_reused.toml) | Extra: masks | `E_MASK_SOURCE_TYPE_REUSED` | 5 | Two masks with source_type nuclei. | +| [`i81_header_names_duplicate`](invalid/i81_header_names_duplicate.toml) | Extra: aliases | `E_DUPLICATE_VALUE` | 5 | header_names = ["DAPI", "DAPI"]. | +| [`i82_retired_repeats_forbidden`](invalid/i82_retired_repeats_forbidden.toml) | Owner refinement | `E_FORBIDDEN_FIELD` | 3 | A `retired_repeats` table: IDs are immutable, v1 has no rename table. | +| [`i83_non_nfc_reference`](invalid/i83_non_nfc_reference.toml) | Owner decision Q3 | `E_ID_NOT_NFC` | 4 | A composed key referenced decomposed: refused in phase 4, not resolved through normalization. | +| [`i84_non_nfc_source_repeat`](invalid/i84_non_nfc_source_repeat.toml) | Owner decision Q3 | `E_ID_NOT_NFC` | 4 | A decomposed `source_repeat`. | +| [`i85_non_nfc_used_id`](invalid/i85_non_nfc_used_id.toml) | Owner decisions Q1, Q3 | `E_ID_NOT_NFC` | 4 | A decomposed retired ID in `used_ids`. | +| [`i86_display_name_collision_case`](invalid/i86_display_name_collision_case.toml) | Owner decision Q2 | `E_DISPLAY_NAME_COLLISION` | 5 | "Control" and "control". | +| [`i87_display_name_collision_casefold`](invalid/i87_display_name_collision_casefold.toml) | Owner decision Q2 | `E_DISPLAY_NAME_COLLISION` | 5 | Channels "Stra\u00dfe" and "STRASSE" (full case folding). | +| [`i88_display_name_collision_nfc_and_case`](invalid/i88_display_name_collision_nfc_and_case.toml) | Owner decision Q2 | `E_DISPLAY_NAME_COLLISION` | 5 | Repeats "Cafe\u0301 1" and "CAF\u00c9 1". | +| [`i89_reference_names_display_name`](invalid/i89_reference_names_display_name.toml) | Audit L1; owner decision A4 | `E_REFERENCE_UNKNOWN` | 5 | `reference` is the display name "Control" of CTRL. | +| [`i90_member_keyed_by_display_name`](invalid/i90_member_keyed_by_display_name.toml) | Audit L1 | `E_UNKNOWN_CONDITION` | 5 | A member keyed by the display name "Control". | +| [`i91_unit_lists_channel_display_name`](invalid/i91_unit_lists_channel_display_name.toml) | Audit L1 | `E_UNKNOWN_CHANNEL` | 5 | A unit lists the channel display name "DAPI". | +| [`i92_used_ids_missing`](invalid/i92_used_ids_missing.toml) | Owner decision Q1 | `E_MISSING_FIELD` | 3 | No `used_ids` table. | +| [`i93_used_id_not_listed`](invalid/i93_used_id_not_listed.toml) | Owner decision Q1 | `E_USED_ID_MISSING` | 5 | DrugA declared but not in `used_ids.conditions`. | +| [`i94_used_ids_duplicate`](invalid/i94_used_ids_duplicate.toml) | Owner decision Q1 | `E_DUPLICATE_VALUE` | 5 | rep1 listed twice. | +| [`i95_used_id_case_collision`](invalid/i95_used_id_case_collision.toml) | Audit B2, B11; owner decision Q1 | `E_ID_CASE_COLLISION` | 4 | Retired `ctrl` against declared CTRL. | +| [`i96_used_ids_mapping`](invalid/i96_used_ids_mapping.toml) | Owner decision Q1 | `E_TYPE` | 3 | `used_ids.repeats` is a mapping table, not a list. | +| [`i97_used_ids_unknown_kind`](invalid/i97_used_ids_unknown_kind.toml) | Owner decision Q1 | `E_UNKNOWN_FIELD` | 3 | `used_ids.masks`. | +| [`i98_confirmed_on_missing`](invalid/i98_confirmed_on_missing.toml) | Audit L8; owner decision A3 | `E_MISSING_FIELD` | 3 | No `confirmed_on`: no default exists. | +| [`i99_labchronicle_provenance`](invalid/i99_labchronicle_provenance.toml) | Owner decision Q5 | `E_FORBIDDEN_FIELD` | 3 | A top-level `labchronicle` table. | +| [`i100_image_type_unknown_value`](invalid/i100_image_type_unknown_value.toml) | Owner decision AQ6 | `E_ENUM` | 3 | `image_type = "RGB"` (upper case): the only value is exactly `rgb`. | +| [`i101_image_type_not_text`](invalid/i101_image_type_not_text.toml) | Owner decision AQ6 | `E_TYPE` | 3 | `image_type = true`: a boolean is not converted to text. | +| [`i102_rgb_unit_wrong_order`](invalid/i102_rgb_unit_wrong_order.toml) | Owner decision AQ6 | `E_RGB_UNIT_CHANNELS` | 5 | An RGB unit lists R, B, G: the order is fixed (R, G, B). | +| [`i103_rgb_unit_other_channels`](invalid/i103_rgb_unit_other_channels.toml) | Owner decision AQ6 | `E_RGB_UNIT_CHANNELS` | 5 | A unit declared `rgb` lists the fluorescence channels DNA and ACTIN: an RGB unit has exactly R, G, B. | +| [`i104_rgb_token_in_plain_unit`](invalid/i104_rgb_token_in_plain_unit.toml) | Owner decision AQ6 | `E_RGB_TOKEN_MISUSE` | 5 | Units that list R, G, B without `image_type = "rgb"`: the tokens are only for declared RGB units (and an undeclared RGB image is never inferred). | +| [`i105_rgb_token_lowercase_in_plain_unit`](invalid/i105_rgb_token_lowercase_in_plain_unit.toml) | Owner decision AQ6 | `E_RGB_TOKEN_MISUSE` | 5 | A fluorescence unit lists a channel `r`: R, G, B are reserved for RGB units in any case. | +| [`i106_rgb_channel_header_alias`](invalid/i106_rgb_channel_header_alias.toml) | Owner decision AQ6 | `E_RGB_ALIAS` | 5 | The channel R carries `header_names`: R, G, B have no aliases. | +| [`i107_channel_id_reserved_column`](invalid/i107_channel_id_reserved_column.toml) | Owner decision AQ4 | `E_ID_RESERVED` | 4 | Channel ID `label_id`, a MorphoRelate reserved column name (registry/_expand.py:41-59). | +| [`i108_channel_id_reserved_casefold`](invalid/i108_channel_id_reserved_casefold.toml) | Owner decision AQ4 | `E_ID_RESERVED` | 4 | Channel ID `T`: reserved names are compared case-folded, so `T` is the reserved `t`. | +| [`i109_mask_id_reserved_column`](invalid/i109_mask_id_reserved_column.toml) | Owner decision AQ4 | `E_ID_RESERVED` | 4 | Mask ID `patch_id`, a MorphoRelate reserved column name: one reserved list for channel and mask IDs. | +| [`i110_header_names_duplicate_after_nfc`](invalid/i110_header_names_duplicate_after_nfc.toml) | Owner decision AQ7 | `E_DUPLICATE_VALUE` | 5 | `header_names` = ["Caf\u00e9", "Cafe\u0301"]: header names are compared after NFC, so these are the same alias twice. | +| [`i111_mask_representation_parent_labels`](invalid/i111_mask_representation_parent_labels.toml) | Owner decision AQ8 | `E_ENUM` | 3 | `representation = "parent_labels"` (MorphoRelate's grouped masks): out of scope in v1. | +| [`i112_mask_conversion_field_forbidden`](invalid/i112_mask_conversion_field_forbidden.toml) | Owner decision AQ9 | `E_FORBIDDEN_FIELD` | 3 | `binary_mask_policy = "connected_components"` in a mask table: a binary-to-instances conversion is a per-request, recorded choice of the analysis request, never part of the file. | +| [`i113_forbidden_name_condition_key`](invalid/i113_forbidden_name_condition_key.toml) | Owner decision AQ13a | `E_FORBIDDEN_FIELD` | 3 | A condition ID `control`: forbidden names apply to ID keys as well as fields (reported at the condition and at its member keys). The folder `control` itself is describable under another ID (`v33`); only the key is refused. | +| [`i114_forbidden_name_channel_key`](invalid/i114_forbidden_name_channel_key.toml) | Owner decision AQ13a | `E_FORBIDDEN_FIELD` | 3 | A channel ID `files`: a forbidden name used as a channel key. | +| [`i115_forbidden_name_technical_repeat_key`](invalid/i115_forbidden_name_technical_repeat_key.toml) | Owner decision AQ13a | `E_FORBIDDEN_FIELD` | 3 | A technical repeat ID `overrides`: a forbidden name used as a technical-repeat key. The folder `overrides` is describable under another ID (`v34`). | +| [`i116_non_nfc_unit_channel`](invalid/i116_non_nfc_unit_channel.toml) | Owner decision AQ13b | `E_ID_NOT_NFC` | 4 | A unit lists "ACTIN\u0301" (decomposed). The reference is checked for NFC in phase 4 and is not reported as E_UNKNOWN_CHANNEL (phase 5). | +| [`i117_non_nfc_source_channel`](invalid/i117_non_nfc_source_channel.toml) | Owner decision AQ13b | `E_ID_NOT_NFC` | 4 | A mask's `source_channels` lists "DNA\u0301" (decomposed): NFC before resolution, so not E_UNKNOWN_CHANNEL. | +| [`i118_channel_id_whitespace_and_syntax`](invalid/i118_channel_id_whitespace_and_syntax.toml) | Owner decision AQ13c | `E_ID_TEXT`, `E_ID_SYNTAX` | 4 | Channel ID " DNA" (leading space): it breaks two rules, the Label rule (outer whitespace) and the Token expression, and gets both codes (one code per broken rule). | +| [`i119_rgb_unit_two_channels`](invalid/i119_rgb_unit_two_channels.toml) | Owner decision AQ6 | `E_RGB_UNIT_CHANNELS` | 5 | An RGB unit lists only R and G: exactly three channels are required. | +| [`i120_condition_missing_source_folder`](invalid/i120_condition_missing_source_folder.toml) | Owner decision FR1 | `E_MISSING_FIELD` | 3 | Condition DrugA has no `source_folder`: it is required, with no default (experiment-v1 4.2). | +| [`i121_technical_repeat_missing_source_folder`](invalid/i121_technical_repeat_missing_source_folder.toml) | Owner decision FR1 | `E_MISSING_FIELD` | 3 | The unit rep2/DrugA/single has no `source_folder`: required for every technical repeat, `single` included (experiment-v1 4.4). | +| [`i122_two_condition_ids_one_folder`](invalid/i122_two_condition_ids_one_folder.toml) | Owner decision FR1 | `E_SOURCE_FOLDER_REUSED` | 5 | Condition IDs CTRL and DrugA both name the folder `control`: one folder, one ID (experiment-v1 4.6a). | +| [`i123_two_technical_repeat_ids_one_folder`](invalid/i123_two_technical_repeat_ids_one_folder.toml) | Owner decision FR1 | `E_SOURCE_FOLDER_REUSED` | 5 | Technical repeat IDs acq1 and acq2 of one member both name the folder `acquisition_1` (experiment-v1 4.6a). | +| [`i124_one_technical_repeat_id_two_folders`](invalid/i124_one_technical_repeat_id_two_folders.toml) | Owner decision FR1 | `E_SOURCE_FOLDER_CONFLICT` | 5 | The technical repeat ID acq1 is declared with the folder `acquisition_1` in CTRL and `acq_1` in DrugA: one ID, one folder over the whole file (experiment-v1 4.6a). | +| [`i125_single_id_with_real_folder`](invalid/i125_single_id_with_real_folder.toml) | Owner decision FR1 | `E_SINGLE_MISUSE` | 4 | The technical repeat ID `single` names the folder `acquisition_1`: `single` means no technical-repeat folder level and is the ID of `source_folder = "single"` only (experiment-v1 3.2). | +| [`i126_used_ids_technical_repeats_missing`](invalid/i126_used_ids_technical_repeats_missing.toml) | Owner decision FR1 | `E_MISSING_FIELD` | 3 | `used_ids` has no `technical_repeats` list: required, like the other three kinds (experiment-v1 4.9). | +| [`i127_technical_repeat_id_not_in_used_ids`](invalid/i127_technical_repeat_id_not_in_used_ids.toml) | Owner decision FR1 | `E_USED_ID_MISSING` | 5 | The declared technical repeat ID `single` is not listed in `used_ids.technical_repeats` (experiment-v1 4.9). | +| [`i128_condition_folder_single_wrong_case`](invalid/i128_condition_folder_single_wrong_case.toml) | Matrix audit 2026-11-07, case F18 | `E_SINGLE_MISUSE` | 4 | A condition `source_folder` `Single` (source folders included in 3.2). | +| [`i129_source_repeat_single_not_alone`](invalid/i129_source_repeat_single_not_alone.toml) | Matrix audit 2026-11-07 | `E_SINGLE_MISUSE` | 4 | CTRL has the member `single` and the member `rep2`. | +| [`i130_repeat_id_from_forbidden_folder`](invalid/i130_repeat_id_from_forbidden_folder.toml) | Matrix audit 2026-11-07, case F16 | `E_FORBIDDEN_FIELD` | 3 | Repeat folders `z` kept as the repeat ID `z`; no ID is proposed for such a folder (4.6a), the user names it (`v36`). | +| [`i131_condition_id_single_with_real_folder`](invalid/i131_condition_id_single_with_real_folder.toml) | Owner decision SQ6, matrix audit MS-1 | `E_SINGLE_MISUSE` | 4 | The condition ID `single` names the folder `CTRL`: `single` is the ID of `source_folder = "single"` only (experiment-v1 3.2). | +| [`i132_condition_single_folder_other_id`](invalid/i132_condition_single_folder_other_id.toml) | Owner decision SQ6, matrix audit MS-1 | `E_SINGLE_MISUSE` | 4 | The condition `flat` has `source_folder = "single"`: a condition ID is `single` if and only if its folder is `single` (experiment-v1 3.2). | +| [`i133_condition_single_not_alone`](invalid/i133_condition_single_not_alone.toml) | Owner decision SQ6, matrix audit MS-1 | `E_SINGLE_MISUSE` | 4 | The condition `single` (no condition folder level) next to the condition DrugA: it must be the only condition (experiment-v1 3.2). | +| [`i134_duplicate_key_inline_table`](invalid/i134_duplicate_key_inline_table.toml) | Stage 1 design PR 0, experiment-v1 2.2 | `E_DUPLICATE_KEY` | 1 | `display_name` written twice inside the inline table of the channel DNA: a key declared twice in any spelling, inline tables included, is `E_DUPLICATE_KEY`, not `E_TOML_SYNTAX` (experiment-v1 2.2). | +| [`i135_used_id_channel_reserved`](invalid/i135_used_id_channel_reserved.toml) | Owner decision UR1 (Spec issue 11) | `E_ID_RESERVED` | 4 | The retired channel ID `Nucleus` in `used_ids.channels`: a listed ID reserves the ID, so it follows the reserved list of 3.2 (case-folded) like a declared channel ID (experiment-v1 4.9). | + +## Records fixtures (37) + +Byte-exact test data for `morphobricks.records.jsonl` (experiment-v1 §4.6, owner decisions RF1-RF6 +of 9 November 2026). Unlike the TOML fixtures a JSONL file has no comment header, so each case is +listed here. Every file is checked by `tools/audit/refrecords.py` through `refval.py fixtures`: an +`accepted` file gives no error and is byte-identical to the canonical encoding of its records; a +`refused` file gives exactly the listed reason and its first error is on the listed line. A reader +MUST accept the `rv` files and refuse the `ri` files with `R_RECORDS_FILE_INVALID`; the +`E_RECORDS_*` names are its reasons (experiment-v1 §8). + +| Fixture | Expected | Case | +|---|---|---| +| [`rv01_reconfirmation_old_bytes`](records/rv01_reconfirmation_old_bytes.jsonl) | `accepted` | Three `reconfirmation` lines exactly as written before the `removed` and `image_count_changed` types existed (a `null` `new_id`, an `old_sha256` of `absent`); the bytes are unchanged and equal the canonical encoding. | +| [`rv02_image_count_changed`](records/rv02_image_count_changed.jsonl) | `accepted` | Two `image_count_changed` lines (RR1, RR3): one unit, and two units sorted by `id`, with a space inside a `source_folder`. | +| [`rv03_removed_every_entity`](records/rv03_removed_every_entity.jsonl) | `accepted` | `removed` lines of all four entities: the entity lines (condition, repeat, technical repeat) first, then each `unit` line (RR2); a key the entity does not name is `null`. | +| [`rv04_old_and_new_mixed`](records/rv04_old_and_new_mixed.jsonl) | `accepted` | One file mixing old and new records, as a records file grows: `reconfirmation`, `removed` and `image_count_changed` lines in append order. | +| [`rv05_non_ascii_unescaped`](records/rv05_non_ascii_unescaped.jsonl) | `accepted` | Non-ASCII text written unescaped (umlaut, accent, CJK, an astral character), a U+2028 inside a string that must not end the line, and an escaped quote, backslash and tab. | +| [`rv06_empty_file`](records/rv06_empty_file.jsonl) | `accepted` | Zero bytes: no record and no torn line (RF4). | +| [`ri01_missing_final_lf`](records/ri01_missing_final_lf.jsonl) | refused: `E_RECORDS_NO_FINAL_LF` (line 2) | The last line is a complete record without its LF (a torn append). | +| [`ri02_malformed_line`](records/ri02_malformed_line.jsonl) | refused: `E_RECORDS_NOT_JSON_OBJECT` (line 2) | A line cut in the middle of an object; the lines after it are checked too. | +| [`ri03_line_not_an_object`](records/ri03_line_not_an_object.jsonl) | refused: `E_RECORDS_NOT_JSON_OBJECT` (line 2) | A line that is a JSON array. | +| [`ri04_blank_line`](records/ri04_blank_line.jsonl) | refused: `E_RECORDS_NOT_JSON_OBJECT` (line 2) | An empty line between two records. | +| [`ri05_unknown_type`](records/ri05_unknown_type.jsonl) | refused: `E_RECORDS_UNKNOWN_TYPE` (line 2) | `type` is `renamed`, none of the three. | +| [`ri06_type_missing`](records/ri06_type_missing.jsonl) | refused: `E_RECORDS_UNKNOWN_TYPE` (line 2) | No `type` key. | +| [`ri07_key_missing`](records/ri07_key_missing.jsonl) | refused: `E_RECORDS_KEY_SET` (line 2) | An `image_count_changed` line without `recorded_on`. | +| [`ri08_key_of_another_type`](records/ri08_key_of_another_type.jsonl) | refused: `E_RECORDS_KEY_SET` (line 1) | A `reconfirmation` line that also carries `recorded_on`, a key of another type. | +| [`ri09_key_order`](records/ri09_key_order.jsonl) | refused: `E_RECORDS_KEY_ORDER` (line 2) | The keys of a `reconfirmation` in the order of the text, not in code point order. | +| [`ri10_nested_key_order`](records/ri10_nested_key_order.jsonl) | refused: `E_RECORDS_KEY_ORDER` (line 2) | Inside `technical_repeats`, `source_folder` before `id`. | +| [`ri11_non_ascii_escaped`](records/ri11_non_ascii_escaped.jsonl) | refused: `E_RECORDS_NON_ASCII_ESCAPED` (line 2) | `\u00e9` written for a non-ASCII character. | +| [`ri12_invalid_utf8`](records/ri12_invalid_utf8.jsonl) | refused: `E_RECORDS_ENCODING` (line 2) | The byte 0xE9 alone (Latin-1) in a line. | +| [`ri13_byte_order_mark`](records/ri13_byte_order_mark.jsonl) | refused: `E_RECORDS_ENCODING` (line 1) | The file starts with a byte-order mark (RF4). | +| [`ri14_duplicate_key`](records/ri14_duplicate_key.jsonl) | refused: `E_RECORDS_NOT_JSON_OBJECT` (line 2) | `confirmed_on` twice in one object; the second value would silently win. | +| [`ri15_removed_unit_with_null_name`](records/ri15_removed_unit_with_null_name.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | A `unit` line with a `null` where the unit names all six keys. | +| [`ri16_removed_repeat_names_extra_key`](records/ri16_removed_repeat_names_extra_key.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | A `repeat` line with a `condition_id` that is not `null`. | +| [`ri17_removed_entity_unknown`](records/ri17_removed_entity_unknown.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `entity` is `channel`. | +| [`ri18_image_count_zero`](records/ri18_image_count_zero.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `new_image_count` is 0 (at least 1). | +| [`ri19_image_count_float`](records/ri19_image_count_float.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `new_image_count` is `15.0`, not an integer. | +| [`ri20_date_impossible`](records/ri20_date_impossible.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `confirmed_on` is `2026-02-30`. | +| [`ri21_technical_repeats_unsorted`](records/ri21_technical_repeats_unsorted.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `technical_repeats` not sorted by `id`. | +| [`ri22_id_kind_unknown`](records/ri22_id_kind_unknown.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `id_kind` is `masks`. | +| [`ri23_bad_line_after_good_new_records`](records/ri23_bad_line_after_good_new_records.jsonl) | refused: `E_RECORDS_VALUE` (line 5) | Four valid lines of three types, then an `image_count_changed` line with `old_image_count` 0; the line after it is valid. | +| [`ri24_sha256_not_hex`](records/ri24_sha256_not_hex.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `new_sha256` is 64 upper-case `B` (lower-case hex, RF4). | +| [`ri25_compact_separators`](records/ri25_compact_separators.jsonl) | refused: `E_RECORDS_FORM` (line 2) | Key order and ASCII rules met, but no space after `,` and `:` (RF1). | +| [`ri26_crlf_line_ends`](records/ri26_crlf_line_ends.jsonl) | refused: `E_RECORDS_FORM` (line 1) | A CR before the LF (RF4). | +| [`ri27_optional_ascii_escape`](records/ri27_optional_ascii_escape.jsonl) | refused: `E_RECORDS_FORM` (line 2) | `\u0031` written for the ASCII character `1`: only the minimal escapes are allowed. | +| [`ri28_technical_repeats_empty`](records/ri28_technical_repeats_empty.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | An `image_count_changed` line with `technical_repeats` `[]` (RF5). | +| [`ri29_image_count_unchanged`](records/ri29_image_count_unchanged.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `old_image_count` equals `new_image_count`: the record states no change (RF5). | +| [`ri30_extra_whitespace`](records/ri30_extra_whitespace.jsonl) | refused: `E_RECORDS_FORM` (line 2) | A space after `{` and a trailing space before the LF (RF1). | +| [`ri31_sha256_trailing_newline`](records/ri31_sha256_trailing_newline.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `new_sha256` is 64 lower-case hex digits followed by an escaped LF (`\n`): the line is canonical bytes, but the value is not 64 hex digits (RF4). Regression for a `$` anchor that matched before a final LF. | diff --git a/tests/spec_fixtures/fixtures/expected/v13_homogeneous_legacy.experiment_design.csv b/tests/spec_fixtures/fixtures/expected/v13_homogeneous_legacy.experiment_design.csv new file mode 100644 index 0000000..d7c316b --- /dev/null +++ b/tests/spec_fixtures/fixtures/expected/v13_homogeneous_legacy.experiment_design.csv @@ -0,0 +1,10 @@ +condition,biological_repeat,repeat_group,control,image_count,technical_repeat_count,confirmed,confirmed_on,schema_version +CTRL,rep1,rep1,true,10,0,true,2026-10-30,1 +CTRL,rep2,rep2,true,10,0,true,2026-10-30,1 +CTRL,rep3,rep3,true,10,0,true,2026-10-30,1 +DrugA,rep1,rep1,false,10,0,true,2026-10-30,1 +DrugA,rep2,rep2,false,10,0,true,2026-10-30,1 +DrugA,rep3,rep3,false,10,0,true,2026-10-30,1 +DrugB,rep1,rep1,false,10,0,true,2026-10-30,1 +DrugB,rep2,rep2,false,10,0,true,2026-10-30,1 +DrugB,rep3,rep3,false,10,0,true,2026-10-30,1 diff --git a/tests/spec_fixtures/fixtures/expected/v33_condition_folder_on_forbidden_list.experiment_design.csv b/tests/spec_fixtures/fixtures/expected/v33_condition_folder_on_forbidden_list.experiment_design.csv new file mode 100644 index 0000000..d5b2cd8 --- /dev/null +++ b/tests/spec_fixtures/fixtures/expected/v33_condition_folder_on_forbidden_list.experiment_design.csv @@ -0,0 +1,5 @@ +condition,biological_repeat,repeat_group,control,image_count,technical_repeat_count,confirmed,confirmed_on,schema_version +DrugA,rep1,rep1,false,11,0,true,2026-11-06,1 +DrugA,rep2,rep2,false,10,0,true,2026-11-06,1 +control,rep1,rep1,true,12,0,true,2026-11-06,1 +control,rep2,rep2,true,10,0,true,2026-11-06,1 diff --git a/tests/spec_fixtures/fixtures/expected/v34_technical_repeat_folder_on_forbidden_list.experiment_design.csv b/tests/spec_fixtures/fixtures/expected/v34_technical_repeat_folder_on_forbidden_list.experiment_design.csv new file mode 100644 index 0000000..39c2b44 --- /dev/null +++ b/tests/spec_fixtures/fixtures/expected/v34_technical_repeat_folder_on_forbidden_list.experiment_design.csv @@ -0,0 +1,5 @@ +condition,biological_repeat,repeat_group,control,image_count,technical_repeat_count,confirmed,confirmed_on,schema_version +CTRL,rep1,rep1,true,12,2,true,2026-11-06,1 +CTRL,rep2,rep2,true,10,0,true,2026-11-06,1 +DrugA,rep1,rep1,false,11,2,true,2026-11-06,1 +DrugA,rep2,rep2,false,10,0,true,2026-11-06,1 diff --git a/tests/spec_fixtures/fixtures/invalid/i01_bom.toml b/tests/spec_fixtures/fixtures/invalid/i01_bom.toml new file mode 100644 index 0000000..c1c036f --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i01_bom.toml @@ -0,0 +1,43 @@ +# Fixture i01_bom +# Expected: E_FILE_ENCODING +# Source: Extra: encoding +# The file starts with a UTF-8 byte-order mark. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +display_name = "Control" + +[conditions.DrugA] +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] diff --git a/tests/spec_fixtures/fixtures/invalid/i02_invalid_utf8.toml b/tests/spec_fixtures/fixtures/invalid/i02_invalid_utf8.toml new file mode 100644 index 0000000..24fa1d1 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i02_invalid_utf8.toml @@ -0,0 +1,43 @@ +# Fixture i02_invalid_utf8 +# Expected: E_FILE_ENCODING +# Source: Extra: encoding +# A label contains the byte 0xFF (not UTF-8). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +display_name = "Contr�l" + +[conditions.DrugA] +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] diff --git a/tests/spec_fixtures/fixtures/invalid/i03_duplicate_condition_table.toml b/tests/spec_fixtures/fixtures/invalid/i03_duplicate_condition_table.toml new file mode 100644 index 0000000..a6d8635 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i03_duplicate_condition_table.toml @@ -0,0 +1,46 @@ +# Fixture i03_duplicate_condition_table +# Expected: E_DUPLICATE_KEY +# Source: Extra: duplicate keys; audit §3.12 +# [conditions.CTRL] declared twice (YAML would keep the second silently). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +display_name = "Control" + +[conditions.DrugA] +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[conditions.CTRL] +display_name = "Control again" diff --git a/tests/spec_fixtures/fixtures/invalid/i04_duplicate_key_quoted_vs_bare.toml b/tests/spec_fixtures/fixtures/invalid/i04_duplicate_key_quoted_vs_bare.toml new file mode 100644 index 0000000..a5e3dbd --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i04_duplicate_key_quoted_vs_bare.toml @@ -0,0 +1,46 @@ +# Fixture i04_duplicate_key_quoted_vs_bare +# Expected: E_DUPLICATE_KEY +# Source: Extra: duplicate keys +# [conditions."CTRL"] is the same key as [conditions.CTRL]. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +display_name = "Control" + +[conditions.DrugA] +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[conditions."CTRL"] +target = "X" diff --git a/tests/spec_fixtures/fixtures/invalid/i05_duplicate_field.toml b/tests/spec_fixtures/fixtures/invalid/i05_duplicate_field.toml new file mode 100644 index 0000000..8ea2294 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i05_duplicate_field.toml @@ -0,0 +1,44 @@ +# Fixture i05_duplicate_field +# Expected: E_DUPLICATE_KEY +# Source: Extra: duplicate keys +# `display_name` written twice in one condition. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +display_name = "Control" +display_name = "Vehicle" + +[conditions.DrugA] +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] diff --git a/tests/spec_fixtures/fixtures/invalid/i06_duplicate_member_dotted_vs_table.toml b/tests/spec_fixtures/fixtures/invalid/i06_duplicate_member_dotted_vs_table.toml new file mode 100644 index 0000000..3d09741 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i06_duplicate_member_dotted_vs_table.toml @@ -0,0 +1,44 @@ +# Fixture i06_duplicate_member_dotted_vs_table +# Expected: E_DUPLICATE_KEY +# Source: Extra: duplicate keys +# A member declared as a table and again by a dotted key (two members of one condition in one repeat). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +display_name = "Control" + +[conditions.DrugA] +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +members.CTRL.source_repeat = "rep1b" +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] diff --git a/tests/spec_fixtures/fixtures/invalid/i07_unquoted_bareword.toml b/tests/spec_fixtures/fixtures/invalid/i07_unquoted_bareword.toml new file mode 100644 index 0000000..ff79088 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i07_unquoted_bareword.toml @@ -0,0 +1,43 @@ +# Fixture i07_unquoted_bareword +# Expected: E_TOML_SYNTAX +# Source: Extra: implicit conversion (YAML `NO` -> False) +# source_repeat = NO, unquoted. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +display_name = "Control" + +[conditions.DrugA] +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = NO +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] diff --git a/tests/spec_fixtures/fixtures/invalid/i08_leading_zero_integer.toml b/tests/spec_fixtures/fixtures/invalid/i08_leading_zero_integer.toml new file mode 100644 index 0000000..d72b78f --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i08_leading_zero_integer.toml @@ -0,0 +1,43 @@ +# Fixture i08_leading_zero_integer +# Expected: E_TOML_SYNTAX +# Source: Extra: implicit conversion (YAML `012` -> 10) +# source_repeat = 012, unquoted. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +display_name = "Control" + +[conditions.DrugA] +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = 012 +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] diff --git a/tests/spec_fixtures/fixtures/invalid/i09_schema_version_2.toml b/tests/spec_fixtures/fixtures/invalid/i09_schema_version_2.toml new file mode 100644 index 0000000..562e9ad --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i09_schema_version_2.toml @@ -0,0 +1,53 @@ +# Fixture i09_schema_version_2 +# Expected: E_SCHEMA_VERSION_UNSUPPORTED +# Source: Extra: versioning +# schema_version = 2 is refused, never read best effort. +schema = "morphobricks-experiment" +schema_version = 2 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i100_image_type_unknown_value.toml b/tests/spec_fixtures/fixtures/invalid/i100_image_type_unknown_value.toml new file mode 100644 index 0000000..16d0460 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i100_image_type_unknown_value.toml @@ -0,0 +1,54 @@ +# Fixture i100_image_type_unknown_value +# Expected: E_ENUM +# Source: Owner decision AQ6 +# `image_type = "RGB"` (upper case): the only value is exactly `rgb`. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] +image_type = "RGB" + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i101_image_type_not_text.toml b/tests/spec_fixtures/fixtures/invalid/i101_image_type_not_text.toml new file mode 100644 index 0000000..fd220af --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i101_image_type_not_text.toml @@ -0,0 +1,54 @@ +# Fixture i101_image_type_not_text +# Expected: E_TYPE +# Source: Owner decision AQ6 +# `image_type = true`: a boolean is not converted to text. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] +image_type = true + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i102_rgb_unit_wrong_order.toml b/tests/spec_fixtures/fixtures/invalid/i102_rgb_unit_wrong_order.toml new file mode 100644 index 0000000..ef4cd46 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i102_rgb_unit_wrong_order.toml @@ -0,0 +1,78 @@ +# Fixture i102_rgb_unit_wrong_order +# Expected: E_RGB_UNIT_CHANNELS +# Source: Owner decision AQ6 +# An RGB unit lists R, B, G: the order is fixed (R, G, B). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.R] +display_name = "Red" + +[channels.G] +display_name = "Green" + +[channels.B] +display_name = "Blue" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["R", "B", "G"] +image_type = "rgb" + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 8 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 8 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["B", "G", "R"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i103_rgb_unit_other_channels.toml b/tests/spec_fixtures/fixtures/invalid/i103_rgb_unit_other_channels.toml new file mode 100644 index 0000000..5ef1b2e --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i103_rgb_unit_other_channels.toml @@ -0,0 +1,54 @@ +# Fixture i103_rgb_unit_other_channels +# Expected: E_RGB_UNIT_CHANNELS +# Source: Owner decision AQ6 +# A unit declared `rgb` lists the fluorescence channels DNA and ACTIN: an RGB unit has exactly R, G, B. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] +image_type = "rgb" + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i104_rgb_token_in_plain_unit.toml b/tests/spec_fixtures/fixtures/invalid/i104_rgb_token_in_plain_unit.toml new file mode 100644 index 0000000..52dfb6b --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i104_rgb_token_in_plain_unit.toml @@ -0,0 +1,74 @@ +# Fixture i104_rgb_token_in_plain_unit +# Expected: E_RGB_TOKEN_MISUSE +# Source: Owner decision AQ6 +# Units that list R, G, B without `image_type = "rgb"`: the tokens are only for declared RGB units (and an undeclared RGB image is never inferred). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.R] +display_name = "Red" + +[channels.G] +display_name = "Green" + +[channels.B] +display_name = "Blue" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 8 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 8 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["B", "G", "R"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i105_rgb_token_lowercase_in_plain_unit.toml b/tests/spec_fixtures/fixtures/invalid/i105_rgb_token_lowercase_in_plain_unit.toml new file mode 100644 index 0000000..45815dd --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i105_rgb_token_lowercase_in_plain_unit.toml @@ -0,0 +1,53 @@ +# Fixture i105_rgb_token_lowercase_in_plain_unit +# Expected: E_RGB_TOKEN_MISUSE +# Source: Owner decision AQ6 +# A fluorescence unit lists a channel `r`: R, G, B are reserved for RGB units in any case. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.r] +display_name = "Red fluorophore" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "r"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "r"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["DNA", "r"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i106_rgb_channel_header_alias.toml b/tests/spec_fixtures/fixtures/invalid/i106_rgb_channel_header_alias.toml new file mode 100644 index 0000000..f22098b --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i106_rgb_channel_header_alias.toml @@ -0,0 +1,79 @@ +# Fixture i106_rgb_channel_header_alias +# Expected: E_RGB_ALIAS +# Source: Owner decision AQ6 +# The channel R carries `header_names`: R, G, B have no aliases. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.R] +display_name = "Red" +header_names = ["Red"] + +[channels.G] +display_name = "Green" + +[channels.B] +display_name = "Blue" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 8 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 8 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["B", "G", "R"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i107_channel_id_reserved_column.toml b/tests/spec_fixtures/fixtures/invalid/i107_channel_id_reserved_column.toml new file mode 100644 index 0000000..33fc21d --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i107_channel_id_reserved_column.toml @@ -0,0 +1,53 @@ +# Fixture i107_channel_id_reserved_column +# Expected: E_ID_RESERVED +# Source: Owner decision AQ4 +# Channel ID `label_id`, a MorphoRelate reserved column name (registry/_expand.py:41-59). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.label_id] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "label_id"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "label_id"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["DNA", "label_id"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i108_channel_id_reserved_casefold.toml b/tests/spec_fixtures/fixtures/invalid/i108_channel_id_reserved_casefold.toml new file mode 100644 index 0000000..5de2097 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i108_channel_id_reserved_casefold.toml @@ -0,0 +1,53 @@ +# Fixture i108_channel_id_reserved_casefold +# Expected: E_ID_RESERVED +# Source: Owner decision AQ4 +# Channel ID `T`: reserved names are compared case-folded, so `T` is the reserved `t`. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.T] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "T"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "T"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["DNA", "T"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i109_mask_id_reserved_column.toml b/tests/spec_fixtures/fixtures/invalid/i109_mask_id_reserved_column.toml new file mode 100644 index 0000000..410e419 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i109_mask_id_reserved_column.toml @@ -0,0 +1,53 @@ +# Fixture i109_mask_id_reserved_column +# Expected: E_ID_RESERVED +# Source: Owner decision AQ4 +# Mask ID `patch_id`, a MorphoRelate reserved column name: one reserved list for channel and mask IDs. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.patch_id] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i10_schema_version_0.toml b/tests/spec_fixtures/fixtures/invalid/i10_schema_version_0.toml new file mode 100644 index 0000000..12afc2a --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i10_schema_version_0.toml @@ -0,0 +1,53 @@ +# Fixture i10_schema_version_0 +# Expected: E_SCHEMA_VERSION_UNSUPPORTED +# Source: Extra: versioning +# schema_version = 0. +schema = "morphobricks-experiment" +schema_version = 0 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i110_header_names_duplicate_after_nfc.toml b/tests/spec_fixtures/fixtures/invalid/i110_header_names_duplicate_after_nfc.toml new file mode 100644 index 0000000..fbc2f61 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i110_header_names_duplicate_after_nfc.toml @@ -0,0 +1,53 @@ +# Fixture i110_header_names_duplicate_after_nfc +# Expected: E_DUPLICATE_VALUE +# Source: Owner decision AQ7 +# `header_names` = ["Caf\u00e9", "Cafe\u0301"]: header names are compared after NFC, so these are the same alias twice. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["Caf\u00e9", "Cafe\u0301"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i111_mask_representation_parent_labels.toml b/tests/spec_fixtures/fixtures/invalid/i111_mask_representation_parent_labels.toml new file mode 100644 index 0000000..9dab554 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i111_mask_representation_parent_labels.toml @@ -0,0 +1,53 @@ +# Fixture i111_mask_representation_parent_labels +# Expected: E_ENUM +# Source: Owner decision AQ8 +# `representation = "parent_labels"` (MorphoRelate's grouped masks): out of scope in v1. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "parent_labels" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i112_mask_conversion_field_forbidden.toml b/tests/spec_fixtures/fixtures/invalid/i112_mask_conversion_field_forbidden.toml new file mode 100644 index 0000000..14eaf89 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i112_mask_conversion_field_forbidden.toml @@ -0,0 +1,54 @@ +# Fixture i112_mask_conversion_field_forbidden +# Expected: E_FORBIDDEN_FIELD +# Source: Owner decision AQ9 +# `binary_mask_policy = "connected_components"` in a mask table: a binary-to-instances conversion is a per-request, recorded choice of the analysis request, never part of the file. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "binary" +binary_mask_policy = "connected_components" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i113_forbidden_name_condition_key.toml b/tests/spec_fixtures/fixtures/invalid/i113_forbidden_name_condition_key.toml new file mode 100644 index 0000000..8cebf21 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i113_forbidden_name_condition_key.toml @@ -0,0 +1,53 @@ +# Fixture i113_forbidden_name_condition_key +# Expected: E_FORBIDDEN_FIELD +# Source: Owner decision AQ13a +# A condition ID `control`: forbidden names apply to ID keys as well as fields (reported at the condition and at its member keys). The folder `control` itself is describable under another ID (v33); only the key is refused. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.control] +source_folder = "control" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "control" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.control] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.control.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["control", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i114_forbidden_name_channel_key.toml b/tests/spec_fixtures/fixtures/invalid/i114_forbidden_name_channel_key.toml new file mode 100644 index 0000000..22dd259 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i114_forbidden_name_channel_key.toml @@ -0,0 +1,53 @@ +# Fixture i114_forbidden_name_channel_key +# Expected: E_FORBIDDEN_FIELD +# Source: Owner decision AQ13a +# A channel ID `files`: a forbidden name used as a channel key. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.files] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "files"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "files"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["DNA", "files"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i115_forbidden_name_technical_repeat_key.toml b/tests/spec_fixtures/fixtures/invalid/i115_forbidden_name_technical_repeat_key.toml new file mode 100644 index 0000000..c3b66b4 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i115_forbidden_name_technical_repeat_key.toml @@ -0,0 +1,53 @@ +# Fixture i115_forbidden_name_technical_repeat_key +# Expected: E_FORBIDDEN_FIELD +# Source: Owner decision AQ13a +# A technical repeat ID `overrides`: a forbidden name used as a technical-repeat key. The folder `overrides` is describable under another ID (v34). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.overrides] +source_folder = "overrides" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["overrides", "single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i116_non_nfc_unit_channel.toml b/tests/spec_fixtures/fixtures/invalid/i116_non_nfc_unit_channel.toml new file mode 100644 index 0000000..c50e93b --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i116_non_nfc_unit_channel.toml @@ -0,0 +1,53 @@ +# Fixture i116_non_nfc_unit_channel +# Expected: E_ID_NOT_NFC +# Source: Owner decision AQ13b +# A unit lists "ACTIN\u0301" (decomposed). The reference is checked for NFC in phase 4 and is not reported as E_UNKNOWN_CHANNEL (phase 5). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN\u0301"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i117_non_nfc_source_channel.toml b/tests/spec_fixtures/fixtures/invalid/i117_non_nfc_source_channel.toml new file mode 100644 index 0000000..720eb3c --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i117_non_nfc_source_channel.toml @@ -0,0 +1,53 @@ +# Fixture i117_non_nfc_source_channel +# Expected: E_ID_NOT_NFC +# Source: Owner decision AQ13b +# A mask's `source_channels` lists "DNA\u0301" (decomposed): NFC before resolution, so not E_UNKNOWN_CHANNEL. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA\u0301"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i118_channel_id_whitespace_and_syntax.toml b/tests/spec_fixtures/fixtures/invalid/i118_channel_id_whitespace_and_syntax.toml new file mode 100644 index 0000000..e1b7661 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i118_channel_id_whitespace_and_syntax.toml @@ -0,0 +1,53 @@ +# Fixture i118_channel_id_whitespace_and_syntax +# Expected: E_ID_TEXT, E_ID_SYNTAX +# Source: Owner decision AQ13c +# Channel ID " DNA" (leading space): it breaks two rules, the Label rule (outer whitespace) and the Token expression, and gets both codes (one code per broken rule). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels." DNA"] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = [" DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = [" DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = [" DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = [" DNA", "ACTIN"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i119_rgb_unit_two_channels.toml b/tests/spec_fixtures/fixtures/invalid/i119_rgb_unit_two_channels.toml new file mode 100644 index 0000000..b3b695f --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i119_rgb_unit_two_channels.toml @@ -0,0 +1,78 @@ +# Fixture i119_rgb_unit_two_channels +# Expected: E_RGB_UNIT_CHANNELS +# Source: Owner decision AQ6 +# An RGB unit lists only R and G: exactly three channels are required. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.R] +display_name = "Red" + +[channels.G] +display_name = "Green" + +[channels.B] +display_name = "Blue" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["R", "G"] +image_type = "rgb" + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 8 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 8 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["B", "G", "R"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i11_schema_version_missing.toml b/tests/spec_fixtures/fixtures/invalid/i11_schema_version_missing.toml new file mode 100644 index 0000000..ea2fbca --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i11_schema_version_missing.toml @@ -0,0 +1,52 @@ +# Fixture i11_schema_version_missing +# Expected: E_SCHEMA_VERSION_MISSING +# Source: Extra: versioning +# No schema_version. +schema = "morphobricks-experiment" + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i120_condition_missing_source_folder.toml b/tests/spec_fixtures/fixtures/invalid/i120_condition_missing_source_folder.toml new file mode 100644 index 0000000..1ac91a7 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i120_condition_missing_source_folder.toml @@ -0,0 +1,70 @@ +# Fixture i120_condition_missing_source_folder +# Expected: E_MISSING_FIELD +# Source: Owner decision of 6 Nov 2026 (fourth round) +# Condition DrugA has no `source_folder`: it is required, with no default (experiment-v1 4.2). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i121_technical_repeat_missing_source_folder.toml b/tests/spec_fixtures/fixtures/invalid/i121_technical_repeat_missing_source_folder.toml new file mode 100644 index 0000000..42a9ee6 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i121_technical_repeat_missing_source_folder.toml @@ -0,0 +1,70 @@ +# Fixture i121_technical_repeat_missing_source_folder +# Expected: E_MISSING_FIELD +# Source: Owner decision of 6 Nov 2026 (fourth round) +# The unit rep2/DrugA/single has no `source_folder`: required for every technical repeat, `single` included (experiment-v1 4.4). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i122_two_condition_ids_one_folder.toml b/tests/spec_fixtures/fixtures/invalid/i122_two_condition_ids_one_folder.toml new file mode 100644 index 0000000..b46dd28 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i122_two_condition_ids_one_folder.toml @@ -0,0 +1,71 @@ +# Fixture i122_two_condition_ids_one_folder +# Expected: E_SOURCE_FOLDER_REUSED +# Source: Owner decision of 6 Nov 2026 (fourth round) +# Condition IDs CTRL and DrugA both name the folder `control`: one folder, one ID (experiment-v1 4.6a). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "control" +display_name = "Control" + +[conditions.DrugA] +source_folder = "control" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i123_two_technical_repeat_ids_one_folder.toml b/tests/spec_fixtures/fixtures/invalid/i123_two_technical_repeat_ids_one_folder.toml new file mode 100644 index 0000000..262821d --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i123_two_technical_repeat_ids_one_folder.toml @@ -0,0 +1,75 @@ +# Fixture i123_two_technical_repeat_ids_one_folder +# Expected: E_SOURCE_FOLDER_REUSED +# Source: Owner decision of 6 Nov 2026 (fourth round) +# Technical repeat IDs acq1 and acq2 of one member both name the folder `acquisition_1` (experiment-v1 4.6a). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.acq1] +source_folder = "acquisition_1" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.CTRL.technical_repeats.acq2] +source_folder = "acquisition_1" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["acq1", "acq2", "single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i124_one_technical_repeat_id_two_folders.toml b/tests/spec_fixtures/fixtures/invalid/i124_one_technical_repeat_id_two_folders.toml new file mode 100644 index 0000000..aec57a2 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i124_one_technical_repeat_id_two_folders.toml @@ -0,0 +1,71 @@ +# Fixture i124_one_technical_repeat_id_two_folders +# Expected: E_SOURCE_FOLDER_CONFLICT +# Source: Owner decision of 6 Nov 2026 (fourth round) +# The technical repeat ID acq1 is declared with the folder `acquisition_1` in CTRL and `acq_1` in DrugA: one ID, one folder over the whole file (experiment-v1 4.6a). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.acq1] +source_folder = "acquisition_1" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.acq1] +source_folder = "acq_1" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["acq1", "single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i125_single_id_with_real_folder.toml b/tests/spec_fixtures/fixtures/invalid/i125_single_id_with_real_folder.toml new file mode 100644 index 0000000..95bea64 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i125_single_id_with_real_folder.toml @@ -0,0 +1,71 @@ +# Fixture i125_single_id_with_real_folder +# Expected: E_SINGLE_MISUSE +# Source: Owner decision of 6 Nov 2026 (fourth round) +# The technical repeat ID `single` names the folder `acquisition_1`: `single` means no technical-repeat folder level and is the ID of `source_folder = "single"` only (experiment-v1 3.2). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "acquisition_1" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i126_used_ids_technical_repeats_missing.toml b/tests/spec_fixtures/fixtures/invalid/i126_used_ids_technical_repeats_missing.toml new file mode 100644 index 0000000..767cfb2 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i126_used_ids_technical_repeats_missing.toml @@ -0,0 +1,70 @@ +# Fixture i126_used_ids_technical_repeats_missing +# Expected: E_MISSING_FIELD +# Source: Owner decision of 6 Nov 2026 (fourth round) +# `used_ids` has no `technical_repeats` list: required, like the other three kinds (experiment-v1 4.9). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] diff --git a/tests/spec_fixtures/fixtures/invalid/i127_technical_repeat_id_not_in_used_ids.toml b/tests/spec_fixtures/fixtures/invalid/i127_technical_repeat_id_not_in_used_ids.toml new file mode 100644 index 0000000..51470d4 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i127_technical_repeat_id_not_in_used_ids.toml @@ -0,0 +1,71 @@ +# Fixture i127_technical_repeat_id_not_in_used_ids +# Expected: E_USED_ID_MISSING +# Source: Owner decision of 6 Nov 2026 (fourth round) +# The declared technical repeat ID `single` is not listed in `used_ids.technical_repeats` (experiment-v1 4.9). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.acq1] +source_folder = "acquisition_1" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["acq1"] diff --git a/tests/spec_fixtures/fixtures/invalid/i128_condition_folder_single_wrong_case.toml b/tests/spec_fixtures/fixtures/invalid/i128_condition_folder_single_wrong_case.toml new file mode 100644 index 0000000..e35b5cd --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i128_condition_folder_single_wrong_case.toml @@ -0,0 +1,46 @@ +# Fixture i128_condition_folder_single_wrong_case +# Expected: E_SINGLE_MISUSE +# Source: Matrix audit 2026-11-07 (case F18) +# A condition `source_folder` `Single`: a source folder whose case-folded form is `single` must be exactly `single` (3.2); preprocessing refuses the folder at discovery (`reserved_folder_name`). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "Single" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" + +[channels.ACTIN] +header_names = ["Actin"] + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 2 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 2 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i129_source_repeat_single_not_alone.toml b/tests/spec_fixtures/fixtures/invalid/i129_source_repeat_single_not_alone.toml new file mode 100644 index 0000000..e66b695 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i129_source_repeat_single_not_alone.toml @@ -0,0 +1,58 @@ +# Fixture i129_source_repeat_single_not_alone +# Expected: E_SINGLE_MISUSE +# Source: Matrix audit 2026-11-07 (case F17 variant) +# CTRL has the member `single` (no biological-repeat folder) and also the member `rep2`: a member with `source_repeat = "single"` must be the only member of its condition (3.2). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" + +[channels.ACTIN] +header_names = ["Actin"] + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 2 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.single] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.single.members.CTRL] +source_repeat = "single" +image_count = 2 + +[repeats.single.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.single.members.DrugA] +source_repeat = "single" +image_count = 2 + +[repeats.single.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep2", "single"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i12_schema_version_string.toml b/tests/spec_fixtures/fixtures/invalid/i12_schema_version_string.toml new file mode 100644 index 0000000..6a84ab0 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i12_schema_version_string.toml @@ -0,0 +1,53 @@ +# Fixture i12_schema_version_string +# Expected: E_TYPE +# Source: Extra: implicit conversion +# schema_version = "1" (a string is not converted to an integer). +schema = "morphobricks-experiment" +schema_version = "1" + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i130_repeat_id_from_forbidden_folder.toml b/tests/spec_fixtures/fixtures/invalid/i130_repeat_id_from_forbidden_folder.toml new file mode 100644 index 0000000..522f667 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i130_repeat_id_from_forbidden_folder.toml @@ -0,0 +1,46 @@ +# Fixture i130_repeat_id_from_forbidden_folder +# Expected: E_FORBIDDEN_FIELD +# Source: Matrix audit 2026-11-07 (case F16) +# The biological-repeat folders `z` kept as the default repeat ID `z`: a forbidden name as an ID key (4.7). No proposal is made for such a folder (4.6a): the user names the repeat ID, and the folders are describable under another ID (v36). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" + +[channels.ACTIN] +header_names = ["Actin"] + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[repeats.z] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.z.members.CTRL] +source_repeat = "z" +image_count = 2 + +[repeats.z.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.z.members.DrugA] +source_repeat = "z" +image_count = 2 + +[repeats.z.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["z"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i131_condition_id_single_with_real_folder.toml b/tests/spec_fixtures/fixtures/invalid/i131_condition_id_single_with_real_folder.toml new file mode 100644 index 0000000..23708fd --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i131_condition_id_single_with_real_folder.toml @@ -0,0 +1,34 @@ +# Fixture i131_condition_id_single_with_real_folder +# Expected: E_SINGLE_MISUSE +# Source: Owner decision SQ6 (7 Nov 2026; matrix audit MS-1) +# The condition ID `single` names the folder `CTRL`: `single` means no condition folder level and is the ID of `source_folder = "single"` only (experiment-v1 3.2). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.single] +source_folder = "CTRL" + +[channels.ACTIN] +header_names = ["Actin"] + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[repeats.single] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.single.members.single] +source_repeat = "single" +image_count = 4 + +[repeats.single.members.single.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["single"] +channels = ["ACTIN", "DNA"] +repeats = ["single"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i132_condition_single_folder_other_id.toml b/tests/spec_fixtures/fixtures/invalid/i132_condition_single_folder_other_id.toml new file mode 100644 index 0000000..dd8e6cf --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i132_condition_single_folder_other_id.toml @@ -0,0 +1,34 @@ +# Fixture i132_condition_single_folder_other_id +# Expected: E_SINGLE_MISUSE +# Source: Owner decision SQ6 (7 Nov 2026; matrix audit MS-1) +# The condition `flat` names `source_folder = "single"`: a condition ID is `single` if and only if its folder is `single` (experiment-v1 3.2). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.flat] +source_folder = "single" + +[channels.ACTIN] +header_names = ["Actin"] + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[repeats.single] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.single.members.flat] +source_repeat = "single" +image_count = 4 + +[repeats.single.members.flat.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["flat"] +channels = ["ACTIN", "DNA"] +repeats = ["single"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i133_condition_single_not_alone.toml b/tests/spec_fixtures/fixtures/invalid/i133_condition_single_not_alone.toml new file mode 100644 index 0000000..cae41be --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i133_condition_single_not_alone.toml @@ -0,0 +1,45 @@ +# Fixture i133_condition_single_not_alone +# Expected: E_SINGLE_MISUSE +# Source: Owner decision SQ6 (7 Nov 2026; matrix audit MS-1) +# The condition `single` (no condition folder level) is declared next to the condition DrugA: a condition whose `source_folder` is `single` must be the only condition of the file (experiment-v1 3.2). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.DrugA] +source_folder = "DrugA" + +[conditions.single] +source_folder = "single" + +[channels.ACTIN] +header_names = ["Actin"] + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[repeats.single] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.single.members.DrugA] +source_repeat = "single" +image_count = 4 + +[repeats.single.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.single.members.single] +source_repeat = "single" +image_count = 4 + +[repeats.single.members.single.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["DrugA", "single"] +channels = ["ACTIN", "DNA"] +repeats = ["single"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i134_duplicate_key_inline_table.toml b/tests/spec_fixtures/fixtures/invalid/i134_duplicate_key_inline_table.toml new file mode 100644 index 0000000..01fc0e6 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i134_duplicate_key_inline_table.toml @@ -0,0 +1,70 @@ +# Fixture i134_duplicate_key_inline_table +# Expected: E_DUPLICATE_KEY +# Source: Stage 1 design PR 0 (7 Nov 2026); experiment-v1 2.2 +# `display_name` written twice inside the inline table of the channel DNA: a key declared twice "in any spelling TOML treats as the same key (bare vs quoted, table vs dotted vs inline)" is `E_DUPLICATE_KEY` (experiment-v1 2.2), not `E_TOML_SYNTAX`. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels] +DNA = { display_name = "DAPI", display_name = "DNA", header_names = ["DAPI"] } + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i135_used_id_channel_reserved.toml b/tests/spec_fixtures/fixtures/invalid/i135_used_id_channel_reserved.toml new file mode 100644 index 0000000..a2640d4 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i135_used_id_channel_reserved.toml @@ -0,0 +1,71 @@ +# Fixture i135_used_id_channel_reserved +# Expected: E_ID_RESERVED +# Source: Owner decision of 7 Nov 2026 (MorphoBricks-Spec issue 11) +# The retired channel ID "Nucleus" in used_ids.channels is on the reserved list (experiment-v1 3.2, 4.9): a listed ID reserves the ID, so it follows the same list as a declared channel ID, case-folded. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA", "Nucleus"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i13_schema_version_float.toml b/tests/spec_fixtures/fixtures/invalid/i13_schema_version_float.toml new file mode 100644 index 0000000..83dd5c0 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i13_schema_version_float.toml @@ -0,0 +1,53 @@ +# Fixture i13_schema_version_float +# Expected: E_TYPE +# Source: Extra: implicit conversion +# schema_version = 1.0. +schema = "morphobricks-experiment" +schema_version = 1.0 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i14_schema_version_bool.toml b/tests/spec_fixtures/fixtures/invalid/i14_schema_version_bool.toml new file mode 100644 index 0000000..cc9d47a --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i14_schema_version_bool.toml @@ -0,0 +1,53 @@ +# Fixture i14_schema_version_bool +# Expected: E_TYPE +# Source: Extra: implicit conversion +# schema_version = true (a boolean is not the integer 1). +schema = "morphobricks-experiment" +schema_version = true + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i15_schema_id_wrong.toml b/tests/spec_fixtures/fixtures/invalid/i15_schema_id_wrong.toml new file mode 100644 index 0000000..16be838 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i15_schema_id_wrong.toml @@ -0,0 +1,53 @@ +# Fixture i15_schema_id_wrong +# Expected: E_SCHEMA_ID +# Source: Extra: versioning +# schema = "morphobricks-experiments". +schema = "morphobricks-experiments" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i16_unknown_top_level_field.toml b/tests/spec_fixtures/fixtures/invalid/i16_unknown_top_level_field.toml new file mode 100644 index 0000000..be500c1 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i16_unknown_top_level_field.toml @@ -0,0 +1,54 @@ +# Fixture i16_unknown_top_level_field +# Expected: E_UNKNOWN_FIELD +# Source: Extra: unknown fields +# Top-level `notes`. +schema = "morphobricks-experiment" +schema_version = 1 +notes = "pilot" + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i17_unknown_condition_field.toml b/tests/spec_fixtures/fixtures/invalid/i17_unknown_condition_field.toml new file mode 100644 index 0000000..a187efb --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i17_unknown_condition_field.toml @@ -0,0 +1,54 @@ +# Fixture i17_unknown_condition_field +# Expected: E_UNKNOWN_FIELD +# Source: Extra: unknown fields +# Condition field `colour`. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" +colour = "grey" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i18_misspelled_field.toml b/tests/spec_fixtures/fixtures/invalid/i18_misspelled_field.toml new file mode 100644 index 0000000..5912fde --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i18_misspelled_field.toml @@ -0,0 +1,53 @@ +# Fixture i18_misspelled_field +# Expected: E_MISSING_FIELD, E_UNKNOWN_FIELD +# Source: Extra: unknown fields +# `representaton` (typo) is not silently ignored; `representation` is then also missing. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representaton = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i19_dotted_float_key.toml b/tests/spec_fixtures/fixtures/invalid/i19_dotted_float_key.toml new file mode 100644 index 0000000..033ebaa --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i19_dotted_float_key.toml @@ -0,0 +1,59 @@ +# Fixture i19_dotted_float_key +# Expected: E_UNKNOWN_FIELD +# Source: Extra: implicit conversion +# [conditions.1.0] is condition `1` with a sub-table `0`, not condition `1.0`. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[conditions.1] +source_folder = "1" + +[conditions.1.0] +display_name = "one point zero" + +[used_ids] +conditions = ["1", "CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i20_per_fov_override.toml b/tests/spec_fixtures/fixtures/invalid/i20_per_fov_override.toml new file mode 100644 index 0000000..6ddba25 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i20_per_fov_override.toml @@ -0,0 +1,56 @@ +# Fixture i20_per_fov_override +# Expected: E_FORBIDDEN_FIELD +# Source: Extra: within-unit divergence; ADR (no per-FOV overrides) +# A per-FOV channel override inside a technical repeat. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[repeats.rep1.members.DrugA.technical_repeats.single.fovs.FOV003] +channels = ["ACTIN", "DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i21_mask_axes_shared.toml b/tests/spec_fixtures/fixtures/invalid/i21_mask_axes_shared.toml new file mode 100644 index 0000000..a6c66e5 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i21_mask_axes_shared.toml @@ -0,0 +1,55 @@ +# Fixture i21_mask_axes_shared +# Expected: E_FORBIDDEN_FIELD +# Source: Extra: masks; preprocessing #173 +# Mask `axes = "YX"` and `shared = true` (v1 is 2-D only; a mask has its image's axes). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] +axes = "YX" +shared = true + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i22_brick_routing.toml b/tests/spec_fixtures/fixtures/invalid/i22_brick_routing.toml new file mode 100644 index 0000000..e335cd3 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i22_brick_routing.toml @@ -0,0 +1,54 @@ +# Fixture i22_brick_routing +# Expected: E_FORBIDDEN_FIELD +# Source: Extra: ADR (no brick routing) +# A channel routed to bricks. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +bricks = ["MorphoRelate", "fiberfeatures"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i23_calibration_in_file.toml b/tests/spec_fixtures/fixtures/invalid/i23_calibration_in_file.toml new file mode 100644 index 0000000..c790b79 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i23_calibration_in_file.toml @@ -0,0 +1,54 @@ +# Fixture i23_calibration_in_file +# Expected: E_FORBIDDEN_FIELD +# Source: Extra: calibration stays in calibration.csv +# pixel_size on a repeat. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" +pixel_size = 0.325 + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i24_control_boolean.toml b/tests/spec_fixtures/fixtures/invalid/i24_control_boolean.toml new file mode 100644 index 0000000..363b086 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i24_control_boolean.toml @@ -0,0 +1,54 @@ +# Fixture i24_control_boolean +# Expected: E_FORBIDDEN_FIELD +# Source: Extra: audit §3.3 +# Legacy boolean `control` (derived from `reference` in v1). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" +control = true + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i25_experiment_field.toml b/tests/spec_fixtures/fixtures/invalid/i25_experiment_field.toml new file mode 100644 index 0000000..7068f8d --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i25_experiment_field.toml @@ -0,0 +1,54 @@ +# Fixture i25_experiment_field +# Expected: E_FORBIDDEN_FIELD +# Source: Extra: audit §3.4 +# Condition `experiment` (an FDR family; MorphoCoverage configuration). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +experiment = "screen_1" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i26_source_index_field.toml b/tests/spec_fixtures/fixtures/invalid/i26_source_index_field.toml new file mode 100644 index 0000000..5b3cf7c --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i26_source_index_field.toml @@ -0,0 +1,54 @@ +# Fixture i26_source_index_field +# Expected: E_FORBIDDEN_FIELD +# Source: Extra: RFC (order is the mapping) +# A unit states `source_index` beside its ordered list. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] +source_index = [0, 1] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i27_toml_date_source_repeat.toml b/tests/spec_fixtures/fixtures/invalid/i27_toml_date_source_repeat.toml new file mode 100644 index 0000000..2c6a517 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i27_toml_date_source_repeat.toml @@ -0,0 +1,53 @@ +# Fixture i27_toml_date_source_repeat +# Expected: E_TYPE +# Source: Extra: implicit conversion (YAML date) +# source_repeat = 2026-01-01 is a TOML date, not text. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = 2026-01-01 +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i28_integer_source_repeat.toml b/tests/spec_fixtures/fixtures/invalid/i28_integer_source_repeat.toml new file mode 100644 index 0000000..8eaa987 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i28_integer_source_repeat.toml @@ -0,0 +1,53 @@ +# Fixture i28_integer_source_repeat +# Expected: E_TYPE +# Source: Extra: implicit conversion +# source_repeat = 12 (integer). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = 12 +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i29_bool_label.toml b/tests/spec_fixtures/fixtures/invalid/i29_bool_label.toml new file mode 100644 index 0000000..5bb7553 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i29_bool_label.toml @@ -0,0 +1,53 @@ +# Fixture i29_bool_label +# Expected: E_TYPE +# Source: Extra: implicit conversion (YAML `off`) +# display_name = false. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = false + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i30_confirmed_on_toml_date.toml b/tests/spec_fixtures/fixtures/invalid/i30_confirmed_on_toml_date.toml new file mode 100644 index 0000000..f5ef381 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i30_confirmed_on_toml_date.toml @@ -0,0 +1,53 @@ +# Fixture i30_confirmed_on_toml_date +# Expected: E_TYPE +# Source: Extra: implicit conversion +# confirmed_on = 2026-11-06 (TOML date literal, not a string). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = 2026-11-06 + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i31_confirmed_on_bad_format.toml b/tests/spec_fixtures/fixtures/invalid/i31_confirmed_on_bad_format.toml new file mode 100644 index 0000000..da11ed2 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i31_confirmed_on_bad_format.toml @@ -0,0 +1,53 @@ +# Fixture i31_confirmed_on_bad_format +# Expected: E_DATE_FORMAT +# Source: Extra: preprocessing contract §12 (spreadsheet dates) +# confirmed_on = "06/11/2026". +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "06/11/2026" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i32_confirmed_on_impossible_date.toml b/tests/spec_fixtures/fixtures/invalid/i32_confirmed_on_impossible_date.toml new file mode 100644 index 0000000..149a3c1 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i32_confirmed_on_impossible_date.toml @@ -0,0 +1,53 @@ +# Fixture i32_confirmed_on_impossible_date +# Expected: E_DATE_FORMAT +# Source: Extra +# confirmed_on = "2026-02-30". +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-02-30" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i33_image_count_string.toml b/tests/spec_fixtures/fixtures/invalid/i33_image_count_string.toml new file mode 100644 index 0000000..c490cfc --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i33_image_count_string.toml @@ -0,0 +1,53 @@ +# Fixture i33_image_count_string +# Expected: E_TYPE +# Source: Extra: implicit conversion +# image_count = "12". +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = "12" + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i34_image_count_float.toml b/tests/spec_fixtures/fixtures/invalid/i34_image_count_float.toml new file mode 100644 index 0000000..fc69be6 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i34_image_count_float.toml @@ -0,0 +1,53 @@ +# Fixture i34_image_count_float +# Expected: E_TYPE +# Source: Extra: implicit conversion +# image_count = 12.0. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12.0 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i35_image_count_zero.toml b/tests/spec_fixtures/fixtures/invalid/i35_image_count_zero.toml new file mode 100644 index 0000000..9bd2bb7 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i35_image_count_zero.toml @@ -0,0 +1,53 @@ +# Fixture i35_image_count_zero +# Expected: E_VALUE_RANGE +# Source: Extra +# image_count = 0. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 0 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i36_confirmed_string.toml b/tests/spec_fixtures/fixtures/invalid/i36_confirmed_string.toml new file mode 100644 index 0000000..43fbf28 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i36_confirmed_string.toml @@ -0,0 +1,53 @@ +# Fixture i36_confirmed_string +# Expected: E_TYPE +# Source: Extra: implicit conversion (legacy yes/no) +# confirmed = "yes". +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = "yes" +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i37_repeat_unconfirmed.toml b/tests/spec_fixtures/fixtures/invalid/i37_repeat_unconfirmed.toml new file mode 100644 index 0000000..67e0c5f --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i37_repeat_unconfirmed.toml @@ -0,0 +1,53 @@ +# Fixture i37_repeat_unconfirmed +# Expected: E_REPEAT_UNCONFIRMED +# Source: Adversarial 7 (N35) +# A published file with an unconfirmed repeat. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = false +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i38_mixed_type_channel_list.toml b/tests/spec_fixtures/fixtures/invalid/i38_mixed_type_channel_list.toml new file mode 100644 index 0000000..6748cdf --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i38_mixed_type_channel_list.toml @@ -0,0 +1,53 @@ +# Fixture i38_mixed_type_channel_list +# Expected: E_TYPE +# Source: Extra: implicit conversion +# channels = ["DNA", 2]. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", 2] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i39_channels_as_string.toml b/tests/spec_fixtures/fixtures/invalid/i39_channels_as_string.toml new file mode 100644 index 0000000..7c4f653 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i39_channels_as_string.toml @@ -0,0 +1,53 @@ +# Fixture i39_channels_as_string +# Expected: E_TYPE +# Source: Extra +# channels = "DNA,ACTIN" (no splitting of text). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = "DNA,ACTIN" + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i40_empty_channel_list.toml b/tests/spec_fixtures/fixtures/invalid/i40_empty_channel_list.toml new file mode 100644 index 0000000..8ec01c9 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i40_empty_channel_list.toml @@ -0,0 +1,53 @@ +# Fixture i40_empty_channel_list +# Expected: E_EMPTY +# Source: Extra +# A unit with channels = []. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = [] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i41_reference_list.toml b/tests/spec_fixtures/fixtures/invalid/i41_reference_list.toml new file mode 100644 index 0000000..d532ea6 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i41_reference_list.toml @@ -0,0 +1,53 @@ +# Fixture i41_reference_list +# Expected: E_TYPE +# Source: Extra: reference DAG +# Two references in an array; v1 has one primary reference. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = ["CTRL", "DrugA"] + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i42_representation_invalid.toml b/tests/spec_fixtures/fixtures/invalid/i42_representation_invalid.toml new file mode 100644 index 0000000..1749b67 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i42_representation_invalid.toml @@ -0,0 +1,53 @@ +# Fixture i42_representation_invalid +# Expected: E_ENUM +# Source: Extra: masks +# representation = "labels". +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "labels" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i43_representation_missing.toml b/tests/spec_fixtures/fixtures/invalid/i43_representation_missing.toml new file mode 100644 index 0000000..17895a2 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i43_representation_missing.toml @@ -0,0 +1,52 @@ +# Fixture i43_representation_missing +# Expected: E_MISSING_FIELD +# Source: Extra: masks (never inferred from pixels) +# A mask without representation. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i44_missing_channels_table.toml b/tests/spec_fixtures/fixtures/invalid/i44_missing_channels_table.toml new file mode 100644 index 0000000..529d843 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i44_missing_channels_table.toml @@ -0,0 +1,46 @@ +# Fixture i44_missing_channels_table +# Expected: E_MISSING_FIELD +# Source: Extra +# No [channels] at all. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i45_member_without_technical_repeats.toml b/tests/spec_fixtures/fixtures/invalid/i45_member_without_technical_repeats.toml new file mode 100644 index 0000000..310f679 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i45_member_without_technical_repeats.toml @@ -0,0 +1,50 @@ +# Fixture i45_member_without_technical_repeats +# Expected: E_MISSING_FIELD +# Source: Extra +# A member with no technical_repeats. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i46_header_name_empty.toml b/tests/spec_fixtures/fixtures/invalid/i46_header_name_empty.toml new file mode 100644 index 0000000..ae047c9 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i46_header_name_empty.toml @@ -0,0 +1,53 @@ +# Fixture i46_header_name_empty +# Expected: E_TEXT_EMPTY +# Source: Extra: aliases +# header_names = [""]. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = [""] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i47_label_whitespace_only.toml b/tests/spec_fixtures/fixtures/invalid/i47_label_whitespace_only.toml new file mode 100644 index 0000000..8b0b156 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i47_label_whitespace_only.toml @@ -0,0 +1,53 @@ +# Fixture i47_label_whitespace_only +# Expected: E_TEXT_EMPTY +# Source: Extra +# display_name = " ". +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = " " + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i48_case_collision_conditions.toml b/tests/spec_fixtures/fixtures/invalid/i48_case_collision_conditions.toml new file mode 100644 index 0000000..421f0ad --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i48_case_collision_conditions.toml @@ -0,0 +1,64 @@ +# Fixture i48_case_collision_conditions +# Expected: E_ID_CASE_COLLISION +# Source: Extra: case collisions +# Conditions CTRL and ctrl. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[conditions.ctrl] +source_folder = "ctrl" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[repeats.rep1.members.ctrl] +source_repeat = "rep1" +image_count = 4 + +[repeats.rep1.members.ctrl.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA", "ctrl"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i49_case_collision_channels.toml b/tests/spec_fixtures/fixtures/invalid/i49_case_collision_channels.toml new file mode 100644 index 0000000..5bdc78b --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i49_case_collision_channels.toml @@ -0,0 +1,55 @@ +# Fixture i49_case_collision_channels +# Expected: E_ID_CASE_COLLISION +# Source: Extra: case collisions +# Channels DNA and dna. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[channels.dna] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN", "dna"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA", "dna"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i50_case_collision_source_repeats.toml b/tests/spec_fixtures/fixtures/invalid/i50_case_collision_source_repeats.toml new file mode 100644 index 0000000..9185793 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i50_case_collision_source_repeats.toml @@ -0,0 +1,53 @@ +# Fixture i50_case_collision_source_repeats +# Expected: E_ID_CASE_COLLISION +# Source: Extra: case collisions; preprocessing check_identity_case +# Source folders Rep1 (CTRL) and rep1 (DrugA): a label is spelled one way in the whole run. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "Rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i51_case_collision_repeat_ids.toml b/tests/spec_fixtures/fixtures/invalid/i51_case_collision_repeat_ids.toml new file mode 100644 index 0000000..33ab79a --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i51_case_collision_repeat_ids.toml @@ -0,0 +1,65 @@ +# Fixture i51_case_collision_repeat_ids +# Expected: E_ID_CASE_COLLISION +# Source: Extra: case collisions +# Canonical repeats rep1 and REP1. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[repeats.REP1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.REP1.members.CTRL] +source_repeat = "rep2" +image_count = 4 + +[repeats.REP1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["REP1", "rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i52_case_collision_technical_repeats.toml b/tests/spec_fixtures/fixtures/invalid/i52_case_collision_technical_repeats.toml new file mode 100644 index 0000000..571c026 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i52_case_collision_technical_repeats.toml @@ -0,0 +1,53 @@ +# Fixture i52_case_collision_technical_repeats +# Expected: E_ID_CASE_COLLISION +# Source: Extra: case collisions +# Technical repeats acq_1 and Acq_1 in different members. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.acq_1] +source_folder = "acq_1" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.Acq_1] +source_folder = "Acq_1" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["Acq_1", "acq_1"] diff --git a/tests/spec_fixtures/fixtures/invalid/i53_display_name_collision_conditions.toml b/tests/spec_fixtures/fixtures/invalid/i53_display_name_collision_conditions.toml new file mode 100644 index 0000000..81c3f54 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i53_display_name_collision_conditions.toml @@ -0,0 +1,71 @@ +# Fixture i53_display_name_collision_conditions +# Expected: E_DISPLAY_NAME_COLLISION +# Source: Owner refinement (6 Nov 2026) +# Conditions CTRL and DrugA both have display_name "Control". Display names are unique within their kind, though never identity. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Control" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i54_channel_id_missing_text.toml b/tests/spec_fixtures/fixtures/invalid/i54_channel_id_missing_text.toml new file mode 100644 index 0000000..e5d14b5 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i54_channel_id_missing_text.toml @@ -0,0 +1,53 @@ +# Fixture i54_channel_id_missing_text +# Expected: E_ID_TEXT +# Source: Extra: identities (decision D-e) +# Channel ID `NA` (tables read it as missing). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.NA] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "NA"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "NA"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["DNA", "NA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i55_condition_outer_whitespace.toml b/tests/spec_fixtures/fixtures/invalid/i55_condition_outer_whitespace.toml new file mode 100644 index 0000000..3e65240 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i55_condition_outer_whitespace.toml @@ -0,0 +1,53 @@ +# Fixture i55_condition_outer_whitespace +# Expected: E_ID_TEXT +# Source: Extra: identities (decision D-e) +# Condition key " CTRL" with a leading space. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions." CTRL"] +source_folder = " CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = " CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members." CTRL"] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members." CTRL".technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = [" CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i56_source_repeat_null.toml b/tests/spec_fixtures/fixtures/invalid/i56_source_repeat_null.toml new file mode 100644 index 0000000..80e9998 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i56_source_repeat_null.toml @@ -0,0 +1,53 @@ +# Fixture i56_source_repeat_null +# Expected: E_ID_TEXT +# Source: Extra: identities +# source_repeat = "null". +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "null" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i57_channel_id_hyphen.toml b/tests/spec_fixtures/fixtures/invalid/i57_channel_id_hyphen.toml new file mode 100644 index 0000000..f35fd79 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i57_channel_id_hyphen.toml @@ -0,0 +1,53 @@ +# Fixture i57_channel_id_hyphen +# Expected: E_ID_SYNTAX +# Source: Extra: identities (token rule) +# Channel ID F-actin. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.F-actin] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "F-actin"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "F-actin"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["DNA", "F-actin"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i58_channel_id_double_underscore.toml b/tests/spec_fixtures/fixtures/invalid/i58_channel_id_double_underscore.toml new file mode 100644 index 0000000..8edd2ca --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i58_channel_id_double_underscore.toml @@ -0,0 +1,53 @@ +# Fixture i58_channel_id_double_underscore +# Expected: E_ID_SYNTAX +# Source: Extra: identities (feature-column separator) +# Channel ID DNA__2. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.DNA__2] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "DNA__2"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "DNA__2"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["DNA", "DNA__2"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i59_mask_id_reserved.toml b/tests/spec_fixtures/fixtures/invalid/i59_mask_id_reserved.toml new file mode 100644 index 0000000..a6def80 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i59_mask_id_reserved.toml @@ -0,0 +1,53 @@ +# Fixture i59_mask_id_reserved +# Expected: E_ID_RESERVED +# Source: Extra: MorphoFeature object-set rule +# Mask ID `Nucleus` (reserved compartment, any case). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nucleus] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i60_mask_id_reserved_in.toml b/tests/spec_fixtures/fixtures/invalid/i60_mask_id_reserved_in.toml new file mode 100644 index 0000000..550c13d --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i60_mask_id_reserved_in.toml @@ -0,0 +1,53 @@ +# Fixture i60_mask_id_reserved_in +# Expected: E_ID_RESERVED +# Source: Extra: MorphoFeature object-set rule +# Mask ID `Cells_in_Tumor` (an `in` part). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Cells_in_Tumor] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i61_channel_id_reserved.toml b/tests/spec_fixtures/fixtures/invalid/i61_channel_id_reserved.toml new file mode 100644 index 0000000..ebffee3 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i61_channel_id_reserved.toml @@ -0,0 +1,53 @@ +# Fixture i61_channel_id_reserved +# Expected: E_ID_RESERVED +# Source: Extra: audit §3.8 +# Channel ID `Ring` (MorphoRelate compartment token). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.Ring] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "Ring"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "Ring"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["DNA", "Ring"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i62_single_wrong_case.toml b/tests/spec_fixtures/fixtures/invalid/i62_single_wrong_case.toml new file mode 100644 index 0000000..adf113a --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i62_single_wrong_case.toml @@ -0,0 +1,53 @@ +# Fixture i62_single_wrong_case +# Expected: E_SINGLE_MISUSE +# Source: Extra: single convention +# Technical repeat `Single` (spelled one way, so no case collision). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.Single] +source_folder = "Single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.Single] +source_folder = "Single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["Single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i63_single_not_alone.toml b/tests/spec_fixtures/fixtures/invalid/i63_single_not_alone.toml new file mode 100644 index 0000000..4f781fe --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i63_single_not_alone.toml @@ -0,0 +1,57 @@ +# Fixture i63_single_not_alone +# Expected: E_SINGLE_MISUSE +# Source: Extra: single convention +# Technical repeat `single` beside `acquisition_2`. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[repeats.rep1.members.CTRL.technical_repeats.acquisition_2] +source_folder = "acquisition_2" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["acquisition_2", "single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i64_reference_cycle.toml b/tests/spec_fixtures/fixtures/invalid/i64_reference_cycle.toml new file mode 100644 index 0000000..5bef982 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i64_reference_cycle.toml @@ -0,0 +1,54 @@ +# Fixture i64_reference_cycle +# Expected: E_REFERENCE_CYCLE +# Source: RFC case 12 +# CTRL -> DrugA -> CTRL. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" +reference = "DrugA" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i65_reference_cycle_long.toml b/tests/spec_fixtures/fixtures/invalid/i65_reference_cycle_long.toml new file mode 100644 index 0000000..520ebf3 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i65_reference_cycle_long.toml @@ -0,0 +1,66 @@ +# Fixture i65_reference_cycle_long +# Expected: E_REFERENCE_CYCLE +# Source: RFC case 12 +# CTRL -> DrugB -> DrugA -> CTRL. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" +reference = "DrugB" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[conditions.DrugB] +source_folder = "DrugB" +reference = "DrugA" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[repeats.rep1.members.DrugB] +source_repeat = "rep1" +image_count = 4 + +[repeats.rep1.members.DrugB.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA", "DrugB"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i66_reference_self.toml b/tests/spec_fixtures/fixtures/invalid/i66_reference_self.toml new file mode 100644 index 0000000..e815aa0 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i66_reference_self.toml @@ -0,0 +1,53 @@ +# Fixture i66_reference_self +# Expected: E_REFERENCE_SELF +# Source: Extra: reference DAG +# DrugA references itself. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "DrugA" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i67_reference_unknown.toml b/tests/spec_fixtures/fixtures/invalid/i67_reference_unknown.toml new file mode 100644 index 0000000..9da25e6 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i67_reference_unknown.toml @@ -0,0 +1,53 @@ +# Fixture i67_reference_unknown +# Expected: E_REFERENCE_UNKNOWN +# Source: Extra: reference DAG +# Reference to undeclared DMSO. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "DMSO" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i68_reference_case_mismatch.toml b/tests/spec_fixtures/fixtures/invalid/i68_reference_case_mismatch.toml new file mode 100644 index 0000000..162aa89 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i68_reference_case_mismatch.toml @@ -0,0 +1,53 @@ +# Fixture i68_reference_case_mismatch +# Expected: E_REFERENCE_UNKNOWN +# Source: Extra: case rules (no folding on resolution) +# reference = "ctrl" does not resolve to CTRL. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "ctrl" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i69_unknown_channel_in_unit.toml b/tests/spec_fixtures/fixtures/invalid/i69_unknown_channel_in_unit.toml new file mode 100644 index 0000000..739d9a2 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i69_unknown_channel_in_unit.toml @@ -0,0 +1,53 @@ +# Fixture i69_unknown_channel_in_unit +# Expected: E_UNKNOWN_CHANNEL +# Source: Extra: extra channel not declared +# A unit lists PHASE, which is not declared. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN", "PHASE"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i70_duplicate_channel_in_unit.toml b/tests/spec_fixtures/fixtures/invalid/i70_duplicate_channel_in_unit.toml new file mode 100644 index 0000000..04fd2ff --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i70_duplicate_channel_in_unit.toml @@ -0,0 +1,53 @@ +# Fixture i70_duplicate_channel_in_unit +# Expected: E_DUPLICATE_CHANNEL_IN_UNIT +# Source: Extra +# channels = ["DNA", "ACTIN", "DNA"]. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN", "DNA"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i71_target_not_text.toml b/tests/spec_fixtures/fixtures/invalid/i71_target_not_text.toml new file mode 100644 index 0000000..3cfdc32 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i71_target_not_text.toml @@ -0,0 +1,71 @@ +# Fixture i71_target_not_text +# Expected: E_TYPE +# Source: Owner decision 1 (6 Nov 2026) +# target = 5: `target` and `treatment_key` are plain strings, never converted. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = 5 +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i72_member_unknown_condition.toml b/tests/spec_fixtures/fixtures/invalid/i72_member_unknown_condition.toml new file mode 100644 index 0000000..4a8e811 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i72_member_unknown_condition.toml @@ -0,0 +1,61 @@ +# Fixture i72_member_unknown_condition +# Expected: E_UNKNOWN_CONDITION +# Source: Extra +# A member for undeclared condition DrugB. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[repeats.rep1.members.DrugB] +source_repeat = "rep1" +image_count = 4 + +[repeats.rep1.members.DrugB.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i73_condition_without_members.toml b/tests/spec_fixtures/fixtures/invalid/i73_condition_without_members.toml new file mode 100644 index 0000000..acdf859 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i73_condition_without_members.toml @@ -0,0 +1,57 @@ +# Fixture i73_condition_without_members +# Expected: E_CONDITION_WITHOUT_MEMBERS +# Source: Extra +# Condition DrugB declared, no member. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[conditions.DrugB] +source_folder = "DrugB" +reference = "DrugA" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA", "DrugB"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i74_source_folder_in_two_repeats.toml b/tests/spec_fixtures/fixtures/invalid/i74_source_folder_in_two_repeats.toml new file mode 100644 index 0000000..b3cab04 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i74_source_folder_in_two_repeats.toml @@ -0,0 +1,77 @@ +# Fixture i74_source_folder_in_two_repeats +# Expected: E_SOURCE_REPEAT_REUSED +# Source: Extra: repeat pairing +# Source folder CTRL/exp7 belongs to repeats Day2 and Day3. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[repeats.Day2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.Day2.members.CTRL] +source_repeat = "exp7" +image_count = 6 + +[repeats.Day2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.Day3] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.Day3.members.CTRL] +source_repeat = "exp7" +image_count = 6 + +[repeats.Day3.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["Day2", "Day3", "rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i75_repeat_id_swap.toml b/tests/spec_fixtures/fixtures/invalid/i75_repeat_id_swap.toml new file mode 100644 index 0000000..48cd571 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i75_repeat_id_swap.toml @@ -0,0 +1,65 @@ +# Fixture i75_repeat_id_swap +# Expected: E_REPEAT_ID_SHADOWS_SOURCE +# Source: Extra: renames +# Repeat rep1 holds folder rep2 and repeat rep2 holds folder rep1 for CTRL. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep2" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep1" +image_count = 7 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i76_display_name_collision_repeats.toml b/tests/spec_fixtures/fixtures/invalid/i76_display_name_collision_repeats.toml new file mode 100644 index 0000000..7910828 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i76_display_name_collision_repeats.toml @@ -0,0 +1,73 @@ +# Fixture i76_display_name_collision_repeats +# Expected: E_DISPLAY_NAME_COLLISION +# Source: Owner refinement (6 Nov 2026) +# Repeats rep1 and rep2 both display as "Day 1". +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +display_name = "Day 1" +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +display_name = "Day 1" +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i77_display_name_collision_channels.toml b/tests/spec_fixtures/fixtures/invalid/i77_display_name_collision_channels.toml new file mode 100644 index 0000000..863bf4d --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i77_display_name_collision_channels.toml @@ -0,0 +1,71 @@ +# Fixture i77_display_name_collision_channels +# Expected: E_DISPLAY_NAME_COLLISION +# Source: Owner refinement (6 Nov 2026) +# Channels DNA and ACTIN both display as "DAPI". +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "DAPI" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i78_non_nfc_condition_key.toml b/tests/spec_fixtures/fixtures/invalid/i78_non_nfc_condition_key.toml new file mode 100644 index 0000000..b34e948 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i78_non_nfc_condition_key.toml @@ -0,0 +1,71 @@ +# Fixture i78_non_nfc_condition_key +# Expected: E_ID_NOT_NFC +# Source: Owner decisions of 6 Nov 2026 (Q3) +# The condition key is "Cafe\u0301" (decomposed). A reader refuses it, reporting each non-NFC identity (key, references, members, used_ids); it never normalizes, so there is no NFC collision case. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions."Cafe\u0301"] +source_folder = "Cafe\u0301" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "Cafe\u0301" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members."Cafe\u0301"] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members."Cafe\u0301".technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members."Cafe\u0301"] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members."Cafe\u0301".technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["Cafe\u0301", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i79_mask_source_channel_unknown.toml b/tests/spec_fixtures/fixtures/invalid/i79_mask_source_channel_unknown.toml new file mode 100644 index 0000000..f47f10d --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i79_mask_source_channel_unknown.toml @@ -0,0 +1,53 @@ +# Fixture i79_mask_source_channel_unknown +# Expected: E_UNKNOWN_CHANNEL +# Source: Extra: masks +# source_channels names undeclared Hoechst. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["Hoechst"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i80_mask_source_type_reused.toml b/tests/spec_fixtures/fixtures/invalid/i80_mask_source_type_reused.toml new file mode 100644 index 0000000..3909d60 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i80_mask_source_type_reused.toml @@ -0,0 +1,57 @@ +# Fixture i80_mask_source_type_reused +# Expected: E_MASK_SOURCE_TYPE_REUSED +# Source: Extra: masks +# Two masks with source_type nuclei. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[masks.Nuclei2] +source_type = "nuclei" +representation = "binary" + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i81_header_names_duplicate.toml b/tests/spec_fixtures/fixtures/invalid/i81_header_names_duplicate.toml new file mode 100644 index 0000000..05c3347 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i81_header_names_duplicate.toml @@ -0,0 +1,53 @@ +# Fixture i81_header_names_duplicate +# Expected: E_DUPLICATE_VALUE +# Source: Extra: aliases +# header_names = ["DAPI", "DAPI"]. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI", "DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i82_retired_repeats_forbidden.toml b/tests/spec_fixtures/fixtures/invalid/i82_retired_repeats_forbidden.toml new file mode 100644 index 0000000..e658556 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i82_retired_repeats_forbidden.toml @@ -0,0 +1,75 @@ +# Fixture i82_retired_repeats_forbidden +# Expected: E_FORBIDDEN_FIELD +# Source: Owner refinement (6 Nov 2026) +# A `retired_repeats` rename table: IDs are immutable and are never renamed, so v1 has no such table. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[retired_repeats.rep0] +retired_on = "2026-11-01" +replaced_by = "rep1" + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i83_non_nfc_reference.toml b/tests/spec_fixtures/fixtures/invalid/i83_non_nfc_reference.toml new file mode 100644 index 0000000..3db8ad5 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i83_non_nfc_reference.toml @@ -0,0 +1,71 @@ +# Fixture i83_non_nfc_reference +# Expected: E_ID_NOT_NFC +# Source: Owner decisions of 6 Nov 2026 (Q3) +# The condition key is composed but DrugA references it decomposed. Not E_REFERENCE_UNKNOWN: phase 4 stops first, and the reader does not normalize to find a match. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions."Caf\u00e9"] +source_folder = "Caf\u00e9" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "Cafe\u0301" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members."Caf\u00e9"] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members."Caf\u00e9".technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members."Caf\u00e9"] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members."Caf\u00e9".technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["Caf\u00e9", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i84_non_nfc_source_repeat.toml b/tests/spec_fixtures/fixtures/invalid/i84_non_nfc_source_repeat.toml new file mode 100644 index 0000000..23295be --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i84_non_nfc_source_repeat.toml @@ -0,0 +1,71 @@ +# Fixture i84_non_nfc_source_repeat +# Expected: E_ID_NOT_NFC +# Source: Owner decisions of 6 Nov 2026 (Q3) +# source_repeat "re\u0301p1" is decomposed: source identities are refused too, never normalized. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "re\u0301p1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i85_non_nfc_used_id.toml b/tests/spec_fixtures/fixtures/invalid/i85_non_nfc_used_id.toml new file mode 100644 index 0000000..6097180 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i85_non_nfc_used_id.toml @@ -0,0 +1,71 @@ +# Fixture i85_non_nfc_used_id +# Expected: E_ID_NOT_NFC +# Source: Owner decisions of 6 Nov 2026 (Q1, Q3) +# A retired condition ID listed in used_ids is decomposed: the registry holds identities and is held to the same NFC rule. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA", "Cafe\u0301"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i86_display_name_collision_case.toml b/tests/spec_fixtures/fixtures/invalid/i86_display_name_collision_case.toml new file mode 100644 index 0000000..7b808db --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i86_display_name_collision_case.toml @@ -0,0 +1,71 @@ +# Fixture i86_display_name_collision_case +# Expected: E_DISPLAY_NAME_COLLISION +# Source: Owner decisions of 6 Nov 2026 (Q2) +# Condition display names "Control" and "control" collide: uniqueness is decided after NFC and Unicode case folding. Names are still kept as typed. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "control" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i87_display_name_collision_casefold.toml b/tests/spec_fixtures/fixtures/invalid/i87_display_name_collision_casefold.toml new file mode 100644 index 0000000..4c49fb5 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i87_display_name_collision_casefold.toml @@ -0,0 +1,71 @@ +# Fixture i87_display_name_collision_casefold +# Expected: E_DISPLAY_NAME_COLLISION +# Source: Owner decisions of 6 Nov 2026 (Q2) +# Channel display names "Stra\u00dfe" and "STRASSE" collide because full case folding maps the sharp s to "ss" (a lower-case-only comparison would miss it). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "Stra\u00dfe" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "STRASSE" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i88_display_name_collision_nfc_and_case.toml b/tests/spec_fixtures/fixtures/invalid/i88_display_name_collision_nfc_and_case.toml new file mode 100644 index 0000000..ad1b626 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i88_display_name_collision_nfc_and_case.toml @@ -0,0 +1,73 @@ +# Fixture i88_display_name_collision_nfc_and_case +# Expected: E_DISPLAY_NAME_COLLISION +# Source: Owner decisions of 6 Nov 2026 (Q2) +# Repeat display names "Cafe\u0301 1" (decomposed) and "CAF\u00c9 1" (composed, upper case) collide only when NFC and case folding are both applied. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +display_name = "Cafe\u0301 1" +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +display_name = "CAF\u00c9 1" +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i89_reference_names_display_name.toml b/tests/spec_fixtures/fixtures/invalid/i89_reference_names_display_name.toml new file mode 100644 index 0000000..a266cd9 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i89_reference_names_display_name.toml @@ -0,0 +1,71 @@ +# Fixture i89_reference_names_display_name +# Expected: E_REFERENCE_UNKNOWN +# Source: LabChronicle audit L1; Owner decisions of 6 Nov 2026 +# DrugA references "Control", which is the display_name of CTRL, not a condition ID. A display name never substitutes for an ID, even though it is unique. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "Control" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i90_member_keyed_by_display_name.toml b/tests/spec_fixtures/fixtures/invalid/i90_member_keyed_by_display_name.toml new file mode 100644 index 0000000..1e6d6fe --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i90_member_keyed_by_display_name.toml @@ -0,0 +1,71 @@ +# Fixture i90_member_keyed_by_display_name +# Expected: E_UNKNOWN_CONDITION +# Source: LabChronicle audit L1 +# A member is keyed by the display name "Control" instead of the condition ID CTRL (CTRL still has a member in rep2). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members."Control"] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members."Control".technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i91_unit_lists_channel_display_name.toml b/tests/spec_fixtures/fixtures/invalid/i91_unit_lists_channel_display_name.toml new file mode 100644 index 0000000..19cf8bf --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i91_unit_lists_channel_display_name.toml @@ -0,0 +1,71 @@ +# Fixture i91_unit_lists_channel_display_name +# Expected: E_UNKNOWN_CHANNEL +# Source: LabChronicle audit L1 +# A unit lists "DAPI", the display_name of channel DNA, instead of the channel ID. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DAPI", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i92_used_ids_missing.toml b/tests/spec_fixtures/fixtures/invalid/i92_used_ids_missing.toml new file mode 100644 index 0000000..fcc0c9f --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i92_used_ids_missing.toml @@ -0,0 +1,65 @@ +# Fixture i92_used_ids_missing +# Expected: E_MISSING_FIELD +# Source: Owner decisions of 6 Nov 2026 (Q1) +# The required used_ids table is absent. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] diff --git a/tests/spec_fixtures/fixtures/invalid/i93_used_id_not_listed.toml b/tests/spec_fixtures/fixtures/invalid/i93_used_id_not_listed.toml new file mode 100644 index 0000000..78c3cba --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i93_used_id_not_listed.toml @@ -0,0 +1,71 @@ +# Fixture i93_used_id_not_listed +# Expected: E_USED_ID_MISSING +# Source: Owner decisions of 6 Nov 2026 (Q1) +# DrugA is declared but absent from used_ids.conditions: every declared ID MUST be listed. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i94_used_ids_duplicate.toml b/tests/spec_fixtures/fixtures/invalid/i94_used_ids_duplicate.toml new file mode 100644 index 0000000..d8bb23d --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i94_used_ids_duplicate.toml @@ -0,0 +1,71 @@ +# Fixture i94_used_ids_duplicate +# Expected: E_DUPLICATE_VALUE +# Source: Owner decisions of 6 Nov 2026 (Q1) +# rep1 is listed twice in used_ids.repeats. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i95_used_id_case_collision.toml b/tests/spec_fixtures/fixtures/invalid/i95_used_id_case_collision.toml new file mode 100644 index 0000000..a49157d --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i95_used_id_case_collision.toml @@ -0,0 +1,71 @@ +# Fixture i95_used_id_case_collision +# Expected: E_ID_CASE_COLLISION +# Source: LabChronicle audit B2, B11; Owner decisions of 6 Nov 2026 (Q1) +# The retired ID "ctrl" in used_ids collides ignoring case with the declared CTRL: a retired ID cannot return re-cased. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA", "ctrl"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i96_used_ids_mapping.toml b/tests/spec_fixtures/fixtures/invalid/i96_used_ids_mapping.toml new file mode 100644 index 0000000..09df3b7 --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i96_used_ids_mapping.toml @@ -0,0 +1,74 @@ +# Fixture i96_used_ids_mapping +# Expected: E_TYPE +# Source: Owner decisions of 6 Nov 2026 (Q1) +# used_ids.repeats is a table mapping old to new IDs. The registry is a list of IDs only; a mapping would be a rename table (see i82). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +technical_repeats = ["single"] + +[used_ids.repeats] +rep1 = "Day1" +rep2 = "Day2" diff --git a/tests/spec_fixtures/fixtures/invalid/i97_used_ids_unknown_kind.toml b/tests/spec_fixtures/fixtures/invalid/i97_used_ids_unknown_kind.toml new file mode 100644 index 0000000..ed03d1f --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i97_used_ids_unknown_kind.toml @@ -0,0 +1,72 @@ +# Fixture i97_used_ids_unknown_kind +# Expected: E_UNKNOWN_FIELD +# Source: Owner decisions of 6 Nov 2026 (Q1) +# used_ids has the kinds conditions, channels and repeats only; masks are not immutable identities in v1. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] +masks = ["Nuclei"] diff --git a/tests/spec_fixtures/fixtures/invalid/i98_confirmed_on_missing.toml b/tests/spec_fixtures/fixtures/invalid/i98_confirmed_on_missing.toml new file mode 100644 index 0000000..c28b1ff --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i98_confirmed_on_missing.toml @@ -0,0 +1,70 @@ +# Fixture i98_confirmed_on_missing +# Expected: E_MISSING_FIELD +# Source: LabChronicle audit L8, B5 +# confirmed_on is absent from rep1. It is a user fact: a reader supplies no default (not the read date, not the write date, not another repeat's date). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/invalid/i99_labchronicle_provenance.toml b/tests/spec_fixtures/fixtures/invalid/i99_labchronicle_provenance.toml new file mode 100644 index 0000000..9780a0e --- /dev/null +++ b/tests/spec_fixtures/fixtures/invalid/i99_labchronicle_provenance.toml @@ -0,0 +1,75 @@ +# Fixture i99_labchronicle_provenance +# Expected: E_FORBIDDEN_FIELD +# Source: Owner decisions of 6 Nov 2026 (Q5) +# A top-level labchronicle table. LabChronicle links live in the run manifest provenance (preprocessing) only. A forbidden table is reported once at its name; its contents are not examined. +schema = "morphobricks-experiment" +schema_version = 1 + +[labchronicle] +repository = "https://example.org/lab/database" +repeat_code = "R001" + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/records/ri01_missing_final_lf.jsonl b/tests/spec_fixtures/fixtures/records/ri01_missing_final_lf.jsonl new file mode 100644 index 0000000..922c9b0 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri01_missing_final_lf.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} \ No newline at end of file diff --git a/tests/spec_fixtures/fixtures/records/ri02_malformed_line.jsonl b/tests/spec_fixtures/fixtures/records/ri02_malformed_line.jsonl new file mode 100644 index 0000000..4a3c174 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri02_malformed_line.jsonl @@ -0,0 +1,3 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"type": "removed", "entity": +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri03_line_not_an_object.jsonl b/tests/spec_fixtures/fixtures/records/ri03_line_not_an_object.jsonl new file mode 100644 index 0000000..e243c71 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri03_line_not_an_object.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +["reconfirmation"] diff --git a/tests/spec_fixtures/fixtures/records/ri04_blank_line.jsonl b/tests/spec_fixtures/fixtures/records/ri04_blank_line.jsonl new file mode 100644 index 0000000..fae05b3 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri04_blank_line.jsonl @@ -0,0 +1,3 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} + +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri05_unknown_type.jsonl b/tests/spec_fixtures/fixtures/records/ri05_unknown_type.jsonl new file mode 100644 index 0000000..83f1812 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri05_unknown_type.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "renamed"} diff --git a/tests/spec_fixtures/fixtures/records/ri06_type_missing.jsonl b/tests/spec_fixtures/fixtures/records/ri06_type_missing.jsonl new file mode 100644 index 0000000..9c08592 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri06_type_missing.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7"} diff --git a/tests/spec_fixtures/fixtures/records/ri07_key_missing.jsonl b/tests/spec_fixtures/fixtures/records/ri07_key_missing.jsonl new file mode 100644 index 0000000..fa8868f --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri07_key_missing.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri08_key_of_another_type.jsonl b/tests/spec_fixtures/fixtures/records/ri08_key_of_another_type.jsonl new file mode 100644 index 0000000..4859140 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri08_key_of_another_type.jsonl @@ -0,0 +1 @@ +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "recorded_on": "2026-11-08", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri09_key_order.jsonl b/tests/spec_fixtures/fixtures/records/ri09_key_order.jsonl new file mode 100644 index 0000000..d63e47e --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri09_key_order.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"type": "reconfirmation", "id_kind": "conditions", "old_id": "DrugA", "new_id": null, "confirmed_on": "2026-11-08", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf"} diff --git a/tests/spec_fixtures/fixtures/records/ri10_nested_key_order.jsonl b/tests/spec_fixtures/fixtures/records/ri10_nested_key_order.jsonl new file mode 100644 index 0000000..39ee817 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri10_nested_key_order.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"source_folder": "single", "id": "single"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri11_non_ascii_escaped.jsonl b/tests/spec_fixtures/fixtures/records/ri11_non_ascii_escaped.jsonl new file mode 100644 index 0000000..8b2d681 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri11_non_ascii_escaped.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Caf\u00e9", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri12_invalid_utf8.jsonl b/tests/spec_fixtures/fixtures/records/ri12_invalid_utf8.jsonl new file mode 100644 index 0000000..dc5649b --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri12_invalid_utf8.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Caf�", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri13_byte_order_mark.jsonl b/tests/spec_fixtures/fixtures/records/ri13_byte_order_mark.jsonl new file mode 100644 index 0000000..df93143 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri13_byte_order_mark.jsonl @@ -0,0 +1 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri14_duplicate_key.jsonl b/tests/spec_fixtures/fixtures/records/ri14_duplicate_key.jsonl new file mode 100644 index 0000000..1cfd89c --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri14_duplicate_key.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "confirmed_on": "2026-11-09", "confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri15_removed_unit_with_null_name.jsonl b/tests/spec_fixtures/fixtures/records/ri15_removed_unit_with_null_name.jsonl new file mode 100644 index 0000000..40b5416 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri15_removed_unit_with_null_name.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "unit", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeat_id": "single", "technical_repeat_source_folder": null, "type": "removed"} diff --git a/tests/spec_fixtures/fixtures/records/ri16_removed_repeat_names_extra_key.jsonl b/tests/spec_fixtures/fixtures/records/ri16_removed_repeat_names_extra_key.jsonl new file mode 100644 index 0000000..b0968a6 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri16_removed_repeat_names_extra_key.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": null, "confirmed_on": "2026-11-09", "entity": "repeat", "repeat_id": "rep1", "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} diff --git a/tests/spec_fixtures/fixtures/records/ri17_removed_entity_unknown.jsonl b/tests/spec_fixtures/fixtures/records/ri17_removed_entity_unknown.jsonl new file mode 100644 index 0000000..223f04e --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri17_removed_entity_unknown.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": null, "condition_source_folder": null, "confirmed_on": "2026-11-09", "entity": "channel", "repeat_id": "rep1", "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} diff --git a/tests/spec_fixtures/fixtures/records/ri18_image_count_zero.jsonl b/tests/spec_fixtures/fixtures/records/ri18_image_count_zero.jsonl new file mode 100644 index 0000000..b5421dc --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri18_image_count_zero.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 0, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri19_image_count_float.jsonl b/tests/spec_fixtures/fixtures/records/ri19_image_count_float.jsonl new file mode 100644 index 0000000..1f92bed --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri19_image_count_float.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15.0, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri20_date_impossible.jsonl b/tests/spec_fixtures/fixtures/records/ri20_date_impossible.jsonl new file mode 100644 index 0000000..518c68f --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri20_date_impossible.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-02-30", "entity": "unit", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeat_id": "single", "technical_repeat_source_folder": "single", "type": "removed"} diff --git a/tests/spec_fixtures/fixtures/records/ri21_technical_repeats_unsorted.jsonl b/tests/spec_fixtures/fixtures/records/ri21_technical_repeats_unsorted.jsonl new file mode 100644 index 0000000..a197bc3 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri21_technical_repeats_unsorted.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "new_image_count": 4, "old_image_count": 9, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "acquisition_2", "source_folder": "a"}, {"id": "acquisition_1", "source_folder": "b"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri22_id_kind_unknown.jsonl b/tests/spec_fixtures/fixtures/records/ri22_id_kind_unknown.jsonl new file mode 100644 index 0000000..177c67e --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri22_id_kind_unknown.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "masks", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri23_bad_line_after_good_new_records.jsonl b/tests/spec_fixtures/fixtures/records/ri23_bad_line_after_good_new_records.jsonl new file mode 100644 index 0000000..f027c9c --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri23_bad_line_after_good_new_records.jsonl @@ -0,0 +1,6 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "condition", "repeat_id": null, "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "unit", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeat_id": "single", "technical_repeat_source_folder": "single", "type": "removed"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "new_image_count": 4, "old_image_count": 0, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "acquisition_1", "source_folder": "acq 1"}, {"id": "acquisition_2", "source_folder": "acq 2"}], "type": "image_count_changed"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri24_sha256_not_hex.jsonl b/tests/spec_fixtures/fixtures/records/ri24_sha256_not_hex.jsonl new file mode 100644 index 0000000..1b3c6c0 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri24_sha256_not_hex.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri25_compact_separators.jsonl b/tests/spec_fixtures/fixtures/records/ri25_compact_separators.jsonl new file mode 100644 index 0000000..13f0b4f --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri25_compact_separators.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on":"2026-11-08","id_kind":"conditions","new_id":null,"new_sha256":"9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf","old_id":"DrugA","old_sha256":"f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7","type":"reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri26_crlf_line_ends.jsonl b/tests/spec_fixtures/fixtures/records/ri26_crlf_line_ends.jsonl new file mode 100644 index 0000000..0c9d0e6 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri26_crlf_line_ends.jsonl @@ -0,0 +1 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri27_optional_ascii_escape.jsonl b/tests/spec_fixtures/fixtures/records/ri27_optional_ascii_escape.jsonl new file mode 100644 index 0000000..fbce527 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri27_optional_ascii_escape.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day\u0031", "old_sha256": "absent", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri28_technical_repeats_empty.jsonl b/tests/spec_fixtures/fixtures/records/ri28_technical_repeats_empty.jsonl new file mode 100644 index 0000000..e7635f2 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri28_technical_repeats_empty.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri29_image_count_unchanged.jsonl b/tests/spec_fixtures/fixtures/records/ri29_image_count_unchanged.jsonl new file mode 100644 index 0000000..02e4d90 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri29_image_count_unchanged.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 10, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri30_extra_whitespace.jsonl b/tests/spec_fixtures/fixtures/records/ri30_extra_whitespace.jsonl new file mode 100644 index 0000000..8492bf3 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri30_extra_whitespace.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{ "condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type" : "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri31_sha256_trailing_newline.jsonl b/tests/spec_fixtures/fixtures/records/ri31_sha256_trailing_newline.jsonl new file mode 100644 index 0000000..dabba5e --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri31_sha256_trailing_newline.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf\n", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/rv01_reconfirmation_old_bytes.jsonl b/tests/spec_fixtures/fixtures/records/rv01_reconfirmation_old_bytes.jsonl new file mode 100644 index 0000000..491df32 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/rv01_reconfirmation_old_bytes.jsonl @@ -0,0 +1,3 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-07", "id_kind": "channels", "new_id": "Nuclei", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DAPI", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/rv02_image_count_changed.jsonl b/tests/spec_fixtures/fixtures/records/rv02_image_count_changed.jsonl new file mode 100644 index 0000000..e033a31 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/rv02_image_count_changed.jsonl @@ -0,0 +1,2 @@ +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "new_image_count": 4, "old_image_count": 9, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "acquisition_1", "source_folder": "acq 1"}, {"id": "acquisition_2", "source_folder": "acq 2"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/rv03_removed_every_entity.jsonl b/tests/spec_fixtures/fixtures/records/rv03_removed_every_entity.jsonl new file mode 100644 index 0000000..cf8e575 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/rv03_removed_every_entity.jsonl @@ -0,0 +1,5 @@ +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "condition", "repeat_id": null, "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} +{"condition_id": null, "condition_source_folder": null, "confirmed_on": "2026-11-09", "entity": "repeat", "repeat_id": "rep1", "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} +{"condition_id": null, "condition_source_folder": null, "confirmed_on": "2026-11-09", "entity": "technical_repeat", "repeat_id": null, "source_repeat": null, "technical_repeat_id": "acquisition_2", "technical_repeat_source_folder": "acquisition_2", "type": "removed"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "unit", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeat_id": "single", "technical_repeat_source_folder": "single", "type": "removed"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "unit", "repeat_id": "rep2", "source_repeat": "rep2", "technical_repeat_id": "single", "technical_repeat_source_folder": "single", "type": "removed"} diff --git a/tests/spec_fixtures/fixtures/records/rv04_old_and_new_mixed.jsonl b/tests/spec_fixtures/fixtures/records/rv04_old_and_new_mixed.jsonl new file mode 100644 index 0000000..3bba809 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/rv04_old_and_new_mixed.jsonl @@ -0,0 +1,7 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-07", "id_kind": "channels", "new_id": "Nuclei", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DAPI", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "condition", "repeat_id": null, "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "unit", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeat_id": "single", "technical_repeat_source_folder": "single", "type": "removed"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "new_image_count": 4, "old_image_count": 9, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "acquisition_1", "source_folder": "acq 1"}, {"id": "acquisition_2", "source_folder": "acq 2"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/rv05_non_ascii_unescaped.jsonl b/tests/spec_fixtures/fixtures/records/rv05_non_ascii_unescaped.jsonl new file mode 100644 index 0000000..a0adebe --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/rv05_non_ascii_unescaped.jsonl @@ -0,0 +1,4 @@ +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": "Kontrolle", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Zellkörper-Ctrl", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} +{"condition_id": "対照", "condition_source_folder": "対照/𝔸", "confirmed_on": "2026-11-09", "entity": "unit", "repeat_id": "日1", "source_repeat": "実験 1", "technical_repeat_id": "single", "technical_repeat_source_folder": "single", "type": "removed"} +{"condition_id": "Café", "condition_source_folder": "Café", "new_image_count": 4, "old_image_count": 3, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "ligne
deux"}], "type": "image_count_changed"} +{"condition_id": null, "condition_source_folder": null, "confirmed_on": "2026-11-09", "entity": "repeat", "repeat_id": "say \"hi\" \\ back\ttab", "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} diff --git a/tests/spec_fixtures/fixtures/records/rv06_empty_file.jsonl b/tests/spec_fixtures/fixtures/records/rv06_empty_file.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/tests/spec_fixtures/fixtures/transitions/t01_repeat_id_changed.toml b/tests/spec_fixtures/fixtures/transitions/t01_repeat_id_changed.toml new file mode 100644 index 0000000..c37b95e --- /dev/null +++ b/tests/spec_fixtures/fixtures/transitions/t01_repeat_id_changed.toml @@ -0,0 +1,71 @@ +# Fixture t01_repeat_id_changed +# Expected: R_CONFIRMED_ID_CHANGED (against predecessor valid/v01, without a recorded re-confirmation) +# Source: Owner refinement (6 Nov 2026) +# Valid on its own, but the repeat with members CTRL/rep1 and DrugA/rep1 was confirmed as `rep1` in v01 and is now `Day1` (likewise rep2 -> Day2). A confirmed ID is never changed in place; display_name is the field for a friendlier name. Accepted only with an explicit, recorded re-confirmation (e.g. a mistyped ID). rep1 and rep2 stay listed in used_ids (append-only, experiment-v1 4.9), so the candidate is refused only for the changed ID. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.Day1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.Day1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.Day1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.Day1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.Day1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.Day2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.Day2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.Day2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.Day2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.Day2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["Day1", "Day2", "rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/transitions/t02_channel_id_changed.toml b/tests/spec_fixtures/fixtures/transitions/t02_channel_id_changed.toml new file mode 100644 index 0000000..5a04f48 --- /dev/null +++ b/tests/spec_fixtures/fixtures/transitions/t02_channel_id_changed.toml @@ -0,0 +1,71 @@ +# Fixture t02_channel_id_changed +# Expected: R_CONFIRMED_ID_CHANGED, R_USED_ID_DROPPED (against predecessor valid/v01, without a recorded re-confirmation) +# Source: Owner refinement (6 Nov 2026) +# Valid on its own, but channel ACTIN of v01 (same header evidence, same unit positions) is now `Actin`. Confirmed channel IDs are immutable; only display_name may change. ACTIN is also no longer listed in used_ids (R_USED_ID_DROPPED); keeping it listed beside `Actin` would make the file itself invalid (E_ID_CASE_COLLISION, experiment-v1 4.9), so a case-only change of a confirmed ID can never be published. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.Actin] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "Actin"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "Actin"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "Actin"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "Actin"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["Actin", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/transitions/t03_used_id_reused.toml b/tests/spec_fixtures/fixtures/transitions/t03_used_id_reused.toml new file mode 100644 index 0000000..8efd1c7 --- /dev/null +++ b/tests/spec_fixtures/fixtures/transitions/t03_used_id_reused.toml @@ -0,0 +1,61 @@ +# Fixture t03_used_id_reused +# Expected: R_USED_ID_REUSED (against predecessor valid/v25) +# Source: Owner decisions of 6 Nov 2026 (Q1); LabChronicle audit L2 +# Valid on its own, but DrugA was retired in v25 (listed in used_ids, not declared) and is declared again here for a new source folder. A listed ID is never reused, even by the same folder returning: it needs a new ID. Repeat rep1 gains the member DrugA/new_folder_1; adding a member to an existing repeat is not a changed entity (experiment-v1 4.6, owner decision AQ1), so the set is exactly {R_USED_ID_REUSED}, not R_CONFIRMED_ID_CHANGED. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "new_folder_1" +image_count = 4 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/transitions/t04_used_id_dropped.toml b/tests/spec_fixtures/fixtures/transitions/t04_used_id_dropped.toml new file mode 100644 index 0000000..1f98f12 --- /dev/null +++ b/tests/spec_fixtures/fixtures/transitions/t04_used_id_dropped.toml @@ -0,0 +1,48 @@ +# Fixture t04_used_id_dropped +# Expected: R_USED_ID_DROPPED (against predecessor valid/v25) +# Source: Owner decisions of 6 Nov 2026 (Q1) +# Valid on its own, but v25 listed DrugA in used_ids.conditions and this file no longer does. used_ids is append-only: an entry is never removed. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/transitions/t05_confirmed_on_changed.toml b/tests/spec_fixtures/fixtures/transitions/t05_confirmed_on_changed.toml new file mode 100644 index 0000000..511adf0 --- /dev/null +++ b/tests/spec_fixtures/fixtures/transitions/t05_confirmed_on_changed.toml @@ -0,0 +1,71 @@ +# Fixture t05_confirmed_on_changed +# Expected: R_CONFIRMED_ON_CHANGED (against predecessor valid/v01, N35 not reopened) +# Source: LabChronicle audit L8; Owner decisions of 6 Nov 2026 +# Valid on its own, but rep1 has the same members, units and channel lists as in v01 and confirmed_on moved from 2026-11-06 to 2026-11-07. confirmed_on is the date of the user's confirmation: it is kept byte-identical unless the confirmation was reopened and the user confirmed again. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-07" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/transitions/t06_member_added.toml b/tests/spec_fixtures/fixtures/transitions/t06_member_added.toml new file mode 100644 index 0000000..1803586 --- /dev/null +++ b/tests/spec_fixtures/fixtures/transitions/t06_member_added.toml @@ -0,0 +1,104 @@ +# Fixture t06_member_added +# Expected: no R_* code: accepted (against predecessor valid/v01) +# Source: Owner decision AQ1 +# Adding is not a change of entity. Against v01: a new condition DrugB joins the existing repeats rep1 and rep2 as a member, and a new folder CTRL/rep3 (new repeat rep3) joins the existing condition CTRL. Every member folder, position and alias of v01 is still there. The confirmation of rep1 and rep2 is reopened by the new member (grouping change, experiment-v1 10) and re-confirmed on 2026-11-07, so the changed `confirmed_on` is not R_CONFIRMED_ON_CHANGED. DrugB and rep3 are new IDs, listed in `used_ids`. Valid alone with W_REPEAT_MISSING_CONDITIONS (rep3). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[conditions.DrugB] +source_folder = "DrugB" +display_name = "Drug B" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-07" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugB] +source_repeat = "rep1_b" +image_count = 9 + +[repeats.rep1.members.DrugB.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-07" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugB] +source_repeat = "rep2_b" +image_count = 8 + +[repeats.rep2.members.DrugB.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep3] +confirmed = true +confirmed_on = "2026-11-07" + +[repeats.rep3.members.CTRL] +source_repeat = "rep3" +image_count = 10 + +[repeats.rep3.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA", "DrugB"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2", "rep3"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/transitions/t07_member_removed.toml b/tests/spec_fixtures/fixtures/transitions/t07_member_removed.toml new file mode 100644 index 0000000..c67eb93 --- /dev/null +++ b/tests/spec_fixtures/fixtures/transitions/t07_member_removed.toml @@ -0,0 +1,63 @@ +# Fixture t07_member_removed +# Expected: R_CONFIRMED_ID_CHANGED (against predecessor valid/v01, without a recorded removal confirmation) +# Source: Owner decision AQ1 +# Removing is a change of entity. The member DrugA/rep2 of repeat rep2 is gone (rep2 reopened and re-confirmed on 2026-11-07). The removal is accepted only with the recorded user confirmation of experiment-v1 4.6 and 10; without it the candidate is refused. All IDs stay listed in `used_ids`. Valid alone with W_REPEAT_MISSING_CONDITIONS. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-07" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/transitions/t08_member_reassigned.toml b/tests/spec_fixtures/fixtures/transitions/t08_member_reassigned.toml new file mode 100644 index 0000000..e6fb7e1 --- /dev/null +++ b/tests/spec_fixtures/fixtures/transitions/t08_member_reassigned.toml @@ -0,0 +1,69 @@ +# Fixture t08_member_reassigned +# Expected: R_CONFIRMED_ID_CHANGED (against predecessor valid/v03, without a recorded confirmation) +# Source: Owner decision AQ1 +# Reassigning is a change of entity. DrugA/experiment_23 was a member of Day1 in v03 and is now the member of a new repeat Day3, so Day1 lost a member it held. Accepted only with the recorded user confirmation of experiment-v1 4.6. Day3 is a new ID listed in `used_ids`. Valid alone with W_REPEAT_MISSING_CONDITIONS. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.Day1] +confirmed = true +confirmed_on = "2026-11-07" + +[repeats.Day1.members.CTRL] +source_repeat = "rep1" +image_count = 9 + +[repeats.Day1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.Day2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.Day2.members.CTRL] +source_repeat = "rep2" +image_count = 9 + +[repeats.Day2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.Day2.members.DrugA] +source_repeat = "rep2_drug" +image_count = 8 + +[repeats.Day2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.Day3] +confirmed = true +confirmed_on = "2026-11-07" + +[repeats.Day3.members.DrugA] +source_repeat = "experiment_23" +image_count = 7 + +[repeats.Day3.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["Day1", "Day2", "Day3"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/transitions/t09_case_correction_new_id.toml b/tests/spec_fixtures/fixtures/transitions/t09_case_correction_new_id.toml new file mode 100644 index 0000000..4a9984a --- /dev/null +++ b/tests/spec_fixtures/fixtures/transitions/t09_case_correction_new_id.toml @@ -0,0 +1,71 @@ +# Fixture t09_case_correction_new_id +# Expected: R_CONFIRMED_ID_CHANGED (against predecessor valid/v01, accepted only with a recorded re-confirmation) +# Source: Owner decision AQ2 +# How a case-only mistake is corrected. v01's channel ACTIN was meant to be Actin; the corrected spelling cannot be used (it collides with the retired ACTIN, see t02 and i95), so the channel gets the new, non-colliding ID ActinStain at the same positions with the same header evidence, and `display_name` "Actin" carries the appearance. ACTIN stays listed in `used_ids`. The set is exactly {R_CONFIRMED_ID_CHANGED}: no R_USED_ID_DROPPED, no R_USED_ID_REUSED, no E_* code. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ActinStain] +display_name = "Actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ActinStain"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ActinStain"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ActinStain"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ActinStain"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "ActinStain", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/transitions/t10_condition_id_changed_same_folder.toml b/tests/spec_fixtures/fixtures/transitions/t10_condition_id_changed_same_folder.toml new file mode 100644 index 0000000..32db910 --- /dev/null +++ b/tests/spec_fixtures/fixtures/transitions/t10_condition_id_changed_same_folder.toml @@ -0,0 +1,71 @@ +# Fixture t10_condition_id_changed_same_folder +# Expected: R_CONFIRMED_ID_CHANGED (against predecessor valid/v33, without a recorded re-confirmation) +# Source: Owner decision of 6 Nov 2026 (fourth round) +# The folder `control` was confirmed as condition `ctrl` in v33 and now carries the ID `control_cells`. A confirmed ID never changes in place, also when the folder is the same (experiment-v1 4.6, 4.6a). `ctrl` stays listed in used_ids, so only the changed ID is refused. Valid alone. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.control_cells] +source_folder = "control" +display_name = "control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "control_cells" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.control_cells] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.control_cells.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.control_cells] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.control_cells.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["DrugA", "control_cells", "ctrl"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/transitions/t11_technical_repeat_id_changed_same_folder.toml b/tests/spec_fixtures/fixtures/transitions/t11_technical_repeat_id_changed_same_folder.toml new file mode 100644 index 0000000..7135b1f --- /dev/null +++ b/tests/spec_fixtures/fixtures/transitions/t11_technical_repeat_id_changed_same_folder.toml @@ -0,0 +1,81 @@ +# Fixture t11_technical_repeat_id_changed_same_folder +# Expected: R_CONFIRMED_ID_CHANGED (against predecessor valid/v34, without a recorded re-confirmation) +# Source: Owner decision of 6 Nov 2026 (fourth round) +# The folder `files` was confirmed as technical repeat `acq1` in v34 and now carries the ID `acqA` in both members. A confirmed technical repeat ID never changes in place (experiment-v1 4.6, 4.6a). `acq1` stays listed in used_ids. Valid alone. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.acqA] +source_folder = "files" +display_name = "files" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.CTRL.technical_repeats.acq2] +source_folder = "overrides" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.acqA] +source_folder = "files" +display_name = "files" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA.technical_repeats.acq2] +source_folder = "overrides" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["acq1", "acq2", "acqA", "single"] diff --git a/tests/spec_fixtures/fixtures/transitions/t12_condition_folder_reassigned.toml b/tests/spec_fixtures/fixtures/transitions/t12_condition_folder_reassigned.toml new file mode 100644 index 0000000..0a71e03 --- /dev/null +++ b/tests/spec_fixtures/fixtures/transitions/t12_condition_folder_reassigned.toml @@ -0,0 +1,71 @@ +# Fixture t12_condition_folder_reassigned +# Expected: R_CONFIRMED_ID_CHANGED (against predecessor valid/v33, without a recorded confirmation) +# Source: Owner decision of 6 Nov 2026 (fourth round) +# The ID `ctrl` held the folder `control` in v33 and now holds the folder `ctl` (a folder renamed on disk and attached to the same ID without a recorded confirmation). A folder change for an existing ID is a reassignment, so a changed entity (experiment-v1 4.6, 7). Valid alone. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.ctrl] +source_folder = "ctl" +display_name = "control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "ctrl" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.ctrl] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.ctrl.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.ctrl] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.ctrl.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["DrugA", "ctrl"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/transitions/t13_confirmed_on_with_image_count_only.toml b/tests/spec_fixtures/fixtures/transitions/t13_confirmed_on_with_image_count_only.toml new file mode 100644 index 0000000..8f60851 --- /dev/null +++ b/tests/spec_fixtures/fixtures/transitions/t13_confirmed_on_with_image_count_only.toml @@ -0,0 +1,71 @@ +# Fixture t13_confirmed_on_with_image_count_only +# Expected: R_CONFIRMED_ON_CHANGED (against predecessor valid/v01, N35 not reopened) +# Source: Preprocessing PR 188 (Stage 1 publication); experiment-v1 10 +# Valid on its own. Against v01, rep1 has the same members, units and channel lists and only two things change: CTRL/rep1 `image_count` 12 to 17 (new FOVs in an existing unit) and `confirmed_on` 2026-11-06 to 2026-11-07. New FOVs do not reopen the N35 confirmation (experiment-v1 10), so the new date is refused: a changed `image_count` alone never makes a new `confirmed_on` acceptable. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-07" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 17 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v01_single_control.toml b/tests/spec_fixtures/fixtures/valid/v01_single_control.toml new file mode 100644 index 0000000..5e35961 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v01_single_control.toml @@ -0,0 +1,71 @@ +# Fixture v01_single_control +# Expected: PASS +# Source: RFC case 1 +# One control; trivial pairing, so canonical repeat IDs default to the folder names. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v02_multiple_controls.toml b/tests/spec_fixtures/fixtures/valid/v02_multiple_controls.toml new file mode 100644 index 0000000..cac26e7 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v02_multiple_controls.toml @@ -0,0 +1,92 @@ +# Fixture v02_multiple_controls +# Expected: PASS +# Source: RFC case 2 +# Two controls (DMSO, WT), each referenced by two conditions. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.DMSO] +source_folder = "DMSO" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "DMSO" + +[conditions.DrugB] +source_folder = "DrugB" +reference = "DMSO" + +[conditions.WT] +source_folder = "WT" + +[conditions.KO1] +source_folder = "KO1" +reference = "WT" + +[conditions.KO2] +source_folder = "KO2" +reference = "WT" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.DMSO] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.DMSO.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugB] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.DrugB.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.KO1] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.KO1.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.KO2] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.KO2.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.WT] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.WT.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["DMSO", "DrugA", "DrugB", "KO1", "KO2", "WT"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v03_paired_different_folder_names.toml b/tests/spec_fixtures/fixtures/valid/v03_paired_different_folder_names.toml new file mode 100644 index 0000000..988fb46 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v03_paired_different_folder_names.toml @@ -0,0 +1,65 @@ +# Fixture v03_paired_different_folder_names +# Expected: PASS +# Source: RFC case 3; adversarial 8 +# Source folders rep1 (CTRL) and experiment_23 (DrugA) paired by the user into canonical repeat Day1; rep2/rep2_drug into Day2. MorphoCoverage must see Day1 and Day2 as the biological repeats. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.Day1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.Day1.members.CTRL] +source_repeat = "rep1" +image_count = 9 + +[repeats.Day1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.Day1.members.DrugA] +source_repeat = "experiment_23" +image_count = 7 + +[repeats.Day1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.Day2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.Day2.members.CTRL] +source_repeat = "rep2" +image_count = 9 + +[repeats.Day2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.Day2.members.DrugA] +source_repeat = "rep2_drug" +image_count = 8 + +[repeats.Day2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["Day1", "Day2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v04_swapped_channel_order.toml b/tests/spec_fixtures/fixtures/valid/v04_swapped_channel_order.toml new file mode 100644 index 0000000..8599e9b --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v04_swapped_channel_order.toml @@ -0,0 +1,45 @@ +# Fixture v04_swapped_channel_order +# Expected: PASS +# Source: RFC case 4; adversarial 3 +# Same channels, opposite source order between conditions (CTRL C0=DNA, DrugA C0=ACTIN). Valid schema; heterogeneous, so Stage 1-2 runtime refuses (R_HETEROGENEOUS_LAYOUT_NOT_ENABLED). From Stage 3, ACTIN and DNA are never mixed by C index. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 6 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 6 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["ACTIN", "DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v05_variable_channel_counts.toml b/tests/spec_fixtures/fixtures/valid/v05_variable_channel_counts.toml new file mode 100644 index 0000000..8a90d03 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v05_variable_channel_counts.toml @@ -0,0 +1,48 @@ +# Fixture v05_variable_channel_counts +# Expected: PASS +# Source: RFC case 5 +# Extra channel PHASE in DrugA/rep1 only; CTRL lacks it (missing elsewhere). Heterogeneous. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[channels.PHASE] +display_name = "Phase contrast" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 6 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 6 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN", "PHASE"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA", "PHASE"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v06_technical_repeats_different_layouts.toml b/tests/spec_fixtures/fixtures/valid/v06_technical_repeats_different_layouts.toml new file mode 100644 index 0000000..0c9676d --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v06_technical_repeats_different_layouts.toml @@ -0,0 +1,75 @@ +# Fixture v06_technical_repeats_different_layouts +# Expected: PASS +# Source: RFC case 6; adversarial 1 +# One biological repeat (R002) whose CTRL member has two technical repeats with different counts and orders. It stays ONE biological repeat (N = 2 repeats in total, not 3). Heterogeneous. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[channels.PAXILLIN] +display_name = "Paxillin" + +[channels.PHASE] +display_name = "Phase contrast" + +[repeats.R001] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.R001.members.CTRL] +source_repeat = "rep1" +image_count = 5 + +[repeats.R001.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN", "PAXILLIN"] + +[repeats.R001.members.DrugA] +source_repeat = "experiment_23" +image_count = 5 + +[repeats.R001.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["ACTIN", "DNA", "PAXILLIN"] + +[repeats.R002] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.R002.members.CTRL] +source_repeat = "rep2" +image_count = 9 + +[repeats.R002.members.CTRL.technical_repeats.acquisition_1] +source_folder = "acquisition_1" +channels = ["DNA", "ACTIN", "PAXILLIN", "PHASE"] + +[repeats.R002.members.CTRL.technical_repeats.acquisition_2] +source_folder = "acquisition_2" +channels = ["ACTIN", "DNA", "PAXILLIN"] + +[repeats.R002.members.DrugA] +source_repeat = "rep2_drug" +image_count = 5 + +[repeats.R002.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN", "PAXILLIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA", "PAXILLIN", "PHASE"] +repeats = ["R001", "R002"] +technical_repeats = ["acquisition_1", "acquisition_2", "single"] diff --git a/tests/spec_fixtures/fixtures/valid/v07_instance_masks.toml b/tests/spec_fixtures/fixtures/valid/v07_instance_masks.toml new file mode 100644 index 0000000..95be34f --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v07_instance_masks.toml @@ -0,0 +1,44 @@ +# Fixture v07_instance_masks +# Expected: PASS +# Source: RFC case 8 +# Two instance masks with source-channel provenance; canonical mask IDs differ from discovered source types. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 4 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[masks.Cells] +source_type = "cells" +representation = "instances" +source_channels = ["ACTIN", "DNA"] +display_name = "Whole cells" + +[used_ids] +conditions = ["CTRL"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v08_binary_masks.toml b/tests/spec_fixtures/fixtures/valid/v08_binary_masks.toml new file mode 100644 index 0000000..d0a6af8 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v08_binary_masks.toml @@ -0,0 +1,42 @@ +# Fixture v08_binary_masks +# Expected: PASS +# Source: RFC case 9; adversarial 6 +# A binary region mask beside an instance mask. Valid; an analysis request using TumorRegion as an object set must refuse (A_MASK_NOT_INSTANCES). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 4 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[masks.TumorRegion] +source_type = "tumor_region" +representation = "binary" + +[used_ids] +conditions = ["CTRL"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v09_partial_masks.toml b/tests/spec_fixtures/fixtures/valid/v09_partial_masks.toml new file mode 100644 index 0000000..86375f3 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v09_partial_masks.toml @@ -0,0 +1,50 @@ +# Fixture v09_partial_masks +# Expected: PASS +# Source: RFC case 10 +# Masks declared while segmentation is in progress (e.g. 0/1000 or 300/1000 FOVs). Completeness lives in the run manifest, so the file is valid at every stage; scope checks belong to the analysis layer (A_SCOPE_INCOMPLETE). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 500 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 500 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v10_reference_chain.toml b/tests/spec_fixtures/fixtures/valid/v10_reference_chain.toml new file mode 100644 index 0000000..7daae9d --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v10_reference_chain.toml @@ -0,0 +1,96 @@ +# Fixture v10_reference_chain +# Expected: PASS +# Source: RFC case 11 +# Reference chains VEC_SiCol6a1 -> VEC_SiNC -> VEC and THBS5_SiCol6a1 -> THBS5_SiNC -> THBS5; shared treatment_key across backgrounds. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.VEC] +source_folder = "VEC" + +[conditions.VEC_SiNC] +source_folder = "VEC_SiNC" +reference = "VEC" + +[conditions.VEC_SiCol6a1] +source_folder = "VEC_SiCol6a1" +reference = "VEC_SiNC" +target = "COL6A1" +treatment_key = "siCol6a1" + +[conditions.THBS5] +source_folder = "THBS5" + +[conditions.THBS5_SiNC] +source_folder = "THBS5_SiNC" +reference = "THBS5" + +[conditions.THBS5_SiCol6a1] +source_folder = "THBS5_SiCol6a1" +reference = "THBS5_SiNC" +target = "COL6A1" +treatment_key = "siCol6a1" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.THBS5] +source_repeat = "rep1" +image_count = 6 + +[repeats.rep1.members.THBS5.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.THBS5_SiCol6a1] +source_repeat = "rep1" +image_count = 6 + +[repeats.rep1.members.THBS5_SiCol6a1.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.THBS5_SiNC] +source_repeat = "rep1" +image_count = 6 + +[repeats.rep1.members.THBS5_SiNC.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.VEC] +source_repeat = "rep1" +image_count = 6 + +[repeats.rep1.members.VEC.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.VEC_SiCol6a1] +source_repeat = "rep1" +image_count = 6 + +[repeats.rep1.members.VEC_SiCol6a1.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.VEC_SiNC] +source_repeat = "rep1" +image_count = 6 + +[repeats.rep1.members.VEC_SiNC.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["THBS5", "THBS5_SiCol6a1", "THBS5_SiNC", "VEC", "VEC_SiCol6a1", "VEC_SiNC"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v11_header_aliases.toml b/tests/spec_fixtures/fixtures/valid/v11_header_aliases.toml new file mode 100644 index 0000000..122f3ff --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v11_header_aliases.toml @@ -0,0 +1,38 @@ +# Fixture v11_header_aliases +# Expected: PASS +# Source: Adversarial 5 +# Canonical DNA confirmed with header evidence DAPI and Hoechst 33342; canonical ACTIN with an exact-match header name that still had to be confirmed. A header name outside these lists is R_HEADER_CONTRADICTION. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[channels.DNA] +display_name = "Nuclei stain" +header_names = ["DAPI", "Hoechst 33342"] + +[channels.ACTIN] +header_names = ["ACTIN"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 4 + +[repeats.rep1.members.CTRL.technical_repeats.acquisition_1] +source_folder = "acquisition_1" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.CTRL.technical_repeats.acquisition_2] +source_folder = "acquisition_2" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["acquisition_1", "acquisition_2"] diff --git a/tests/spec_fixtures/fixtures/valid/v12_unnamed_headers.toml b/tests/spec_fixtures/fixtures/valid/v12_unnamed_headers.toml new file mode 100644 index 0000000..dfb2096 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v12_unnamed_headers.toml @@ -0,0 +1,32 @@ +# Fixture v12_unnamed_headers +# Expected: PASS +# Source: Extra: aliases +# No header evidence at all (ImageJ TIFFs carry no channel names); the user's confirmation is the only guard. Undetectable physical swaps are a documented limit. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[channels.DNA] + +[channels.ACTIN] +header_names = [] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 4 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v13_homogeneous_legacy.toml b/tests/spec_fixtures/fixtures/valid/v13_homogeneous_legacy.toml new file mode 100644 index 0000000..2fe5053 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v13_homogeneous_legacy.toml @@ -0,0 +1,113 @@ +# Fixture v13_homogeneous_legacy +# Expected: PASS +# Source: Adversarial 4 +# A legacy homogeneous dataset: folder-name repeat IDs, one control, same layout everywhere (IDs proposed equal to the folder labels). The derived experiment_design.csv, which carries the `source_folder` labels, must be byte-identical to today's; numerical outputs must keep parity. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" + +[conditions.DrugB] +source_folder = "DrugB" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-10-30" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugB] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugB.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-10-30" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugB] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugB.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep3] +confirmed = true +confirmed_on = "2026-10-30" + +[repeats.rep3.members.CTRL] +source_repeat = "rep3" +image_count = 10 + +[repeats.rep3.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep3.members.DrugA] +source_repeat = "rep3" +image_count = 10 + +[repeats.rep3.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep3.members.DrugB] +source_repeat = "rep3" +image_count = 10 + +[repeats.rep3.members.DrugB.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA", "DrugB"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2", "rep3"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v14_no_repeat_folders.toml b/tests/spec_fixtures/fixtures/valid/v14_no_repeat_folders.toml new file mode 100644 index 0000000..6d6fa28 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v14_no_repeat_folders.toml @@ -0,0 +1,45 @@ +# Fixture v14_no_repeat_folders +# Expected: PASS +# Source: Extra: single convention +# No biological-repeat or technical-repeat folders: the source label is `single` (N31, one group for the whole experiment). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.single] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.single.members.CTRL] +source_repeat = "single" +image_count = 20 + +[repeats.single.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.single.members.DrugA] +source_repeat = "single" +image_count = 18 + +[repeats.single.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["single"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v15_toml_keys_are_text.toml b/tests/spec_fixtures/fixtures/valid/v15_toml_keys_are_text.toml new file mode 100644 index 0000000..00e945c --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v15_toml_keys_are_text.toml @@ -0,0 +1,45 @@ +# Fixture v15_toml_keys_are_text +# Expected: PASS +# Source: Extra: implicit-conversion traps (valid side) +# Folder labels that YAML would retype (2026-01-01, 012, NO, 1.0, off) stay exact text in TOML keys and strings. The quoted key "1.0" is needed: a bare 1.0 would be a dotted key. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.NO] +source_folder = "NO" + +[conditions."1.0"] +source_folder = "1.0" +reference = "NO" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.2026-01-01] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.2026-01-01.members.NO] +source_repeat = "2026-01-01" +image_count = 3 + +[repeats.2026-01-01.members.NO.technical_repeats.012] +source_folder = "012" +channels = ["DNA", "ACTIN"] + +[repeats.2026-01-01.members."1.0"] +source_repeat = "2026-01-01" +image_count = 3 + +[repeats.2026-01-01.members."1.0".technical_repeats.off] +source_folder = "off" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["1.0", "NO"] +channels = ["ACTIN", "DNA"] +repeats = ["2026-01-01"] +technical_repeats = ["012", "off"] diff --git a/tests/spec_fixtures/fixtures/valid/v16_display_name_change.toml b/tests/spec_fixtures/fixtures/valid/v16_display_name_change.toml new file mode 100644 index 0000000..9c52701 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v16_display_name_change.toml @@ -0,0 +1,73 @@ +# Fixture v16_display_name_change +# Expected: PASS +# Source: Owner refinement (6 Nov 2026): immutable IDs +# v01 with every display_name changed or added (conditions, channels, repeats). All IDs, references, units and members are identical, so identity is identical: the conditions, channels, repeats and masks section digests (display_name omitted, experiment-v1 9.3) equal v01's and nothing may be invalidated. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Vehicle control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Saracatinib 1 uM" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "Nuclei (Hoechst)" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "Actin (phalloidin)" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +display_name = "Day 1" +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +display_name = "Day 2" +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v17_partial_channel_presence.toml b/tests/spec_fixtures/fixtures/valid/v17_partial_channel_presence.toml new file mode 100644 index 0000000..08d3f28 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v17_partial_channel_presence.toml @@ -0,0 +1,51 @@ +# Fixture v17_partial_channel_presence +# Expected: PASS +# Source: Adversarial 9; addendum §3 +# PHASE exists only in CTRL/R002/acquisition_1. For a PHASE request R002 is partially represented and R001 unavailable: the analysis layer must refuse (A_SCOPE_INCOMPLETE) or record a user-selected subset, never drop or re-weight silently. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[channels.PHASE] + +[repeats.R001] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.R001.members.CTRL] +source_repeat = "rep1" +image_count = 5 + +[repeats.R001.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.R002] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.R002.members.CTRL] +source_repeat = "rep2" +image_count = 8 + +[repeats.R002.members.CTRL.technical_repeats.acquisition_1] +source_folder = "acquisition_1" +channels = ["DNA", "ACTIN", "PHASE"] + +[repeats.R002.members.CTRL.technical_repeats.acquisition_2] +source_folder = "acquisition_2" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL"] +channels = ["ACTIN", "DNA", "PHASE"] +repeats = ["R001", "R002"] +technical_repeats = ["acquisition_1", "acquisition_2", "single"] diff --git a/tests/spec_fixtures/fixtures/valid/v18_inline_and_dotted_forms.toml b/tests/spec_fixtures/fixtures/valid/v18_inline_and_dotted_forms.toml new file mode 100644 index 0000000..93c283b --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v18_inline_and_dotted_forms.toml @@ -0,0 +1,37 @@ +# Fixture v18_inline_and_dotted_forms +# Expected: PASS +# Source: Extra: round trip +# Same model as v01 written with inline tables, dotted keys and quoted keys; parse(v18) must equal parse(v01). +schema = "morphobricks-experiment" +schema_version = 1 + +conditions.CTRL = { source_folder = "CTRL", display_name = "Control" } +conditions."DrugA" = { source_folder = "DrugA", display_name = "Drug A", reference = "CTRL", target = "SRC", treatment_key = "saracatinib" } + +[channels] +DNA = { display_name = "DAPI", header_names = ["DAPI"] } +ACTIN = { display_name = "F-actin", header_names = ["Phalloidin-488"] } + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" +members.CTRL = { source_repeat = "rep2", image_count = 10, technical_repeats = { single = { source_folder = "single", channels = ["DNA", "ACTIN"] } } } +members.DrugA = { source_repeat = "rep2", image_count = 10, technical_repeats.single.source_folder = "single", technical_repeats.single.channels = ["DNA", "ACTIN"] } + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" +members.DrugA.source_repeat = "rep1" +members.DrugA.image_count = 11 +members.DrugA.technical_repeats.single.source_folder = "single" +members.DrugA.technical_repeats.single.channels = ["DNA", "ACTIN"] +members.CTRL.source_repeat = "rep1" +members.CTRL.image_count = 12 +members.CTRL.technical_repeats.single.source_folder = "single" +members.CTRL.technical_repeats.single.channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v19_warnings_unpaired.toml b/tests/spec_fixtures/fixtures/valid/v19_warnings_unpaired.toml new file mode 100644 index 0000000..216d060 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v19_warnings_unpaired.toml @@ -0,0 +1,49 @@ +# Fixture v19_warnings_unpaired +# Expected: PASS; warnings W_REFERENCE_UNPAIRED, W_REPEAT_MISSING_CONDITIONS +# Source: Extra: reference DAG semantics +# Valid with warnings: CTRL only in rep1, DrugA only in rep2, so the repeats miss conditions (W_REPEAT_MISSING_CONDITIONS) and DrugA cannot be paired with its reference (W_REFERENCE_UNPAIRED). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 5 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 5 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v20_unicode_and_spaces.toml b/tests/spec_fixtures/fixtures/valid/v20_unicode_and_spaces.toml new file mode 100644 index 0000000..7bbacc7 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v20_unicode_and_spaces.toml @@ -0,0 +1,46 @@ +# Fixture v20_unicode_and_spaces +# Expected: PASS +# Source: Extra: identities +# Condition folder labels with spaces and non-ASCII text are valid labels (quoted keys); channel and mask IDs stay tokens. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions."Wild type"] +source_folder = "Wild type" +display_name = "Wild type (WT)" + +[conditions."Knock-out β1"] +source_folder = "Knock-out β1" +reference = "Wild type" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats."Réplica_1"] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats."Réplica_1".members."Wild type"] +source_repeat = "Réplica_1" +image_count = 4 + +[repeats."Réplica_1".members."Wild type".technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats."Réplica_1".members."Knock-out β1"] +source_repeat = "Réplica_1" +image_count = 4 + +[repeats."Réplica_1".members."Knock-out β1".technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["Knock-out \u03b21", "Wild type"] +channels = ["ACTIN", "DNA"] +repeats = ["R\u00e9plica_1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v21_metadata_only_edit.toml b/tests/spec_fixtures/fixtures/valid/v21_metadata_only_edit.toml new file mode 100644 index 0000000..81f5364 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v21_metadata_only_edit.toml @@ -0,0 +1,70 @@ +# Fixture v21_metadata_only_edit +# Expected: PASS +# Source: Adversarial 10 +# v01 with only descriptive condition fields edited (display name, target removed, treatment_key changed). All four section digests equal v01's (display_name, target and treatment_key are omitted, experiment-v1 9.3), so no pixel-derived product may be invalidated. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Vehicle (DMSO 0.1%)" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +treatment_key = "AZD0530" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v22_unused_channel_warning.toml b/tests/spec_fixtures/fixtures/valid/v22_unused_channel_warning.toml new file mode 100644 index 0000000..0770728 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v22_unused_channel_warning.toml @@ -0,0 +1,55 @@ +# Fixture v22_unused_channel_warning +# Expected: PASS + W_UNUSED_CHANNEL +# Source: Owner decision 3 (6 Nov 2026) +# PHASE is declared but no unit acquires it. The file is valid; the reader reports the warning W_UNUSED_CHANNEL (a declared channel can be pre-confirmed or a unit excluded). It is never an error. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[channels.PHASE] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 10 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" +source_channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA", "PHASE"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v23_nfc_identity.toml b/tests/spec_fixtures/fixtures/valid/v23_nfc_identity.toml new file mode 100644 index 0000000..0a6d7ad --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v23_nfc_identity.toml @@ -0,0 +1,71 @@ +# Fixture v23_nfc_identity +# Expected: PASS +# Source: Owner decisions of 6 Nov 2026 (Q3) +# A condition key written composed (U+00E9, NFC) and referenced composed. A writer emits only this form and a reader accepts only this form: any decomposed spelling of an identity is E_ID_NOT_NFC (i78, i83-i85), never normalized. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions."Caf\u00e9"] +source_folder = "Caf\u00e9" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "Caf\u00e9" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members."Caf\u00e9"] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members."Caf\u00e9".technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members."Caf\u00e9"] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members."Caf\u00e9".technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["Caf\u00e9", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v24_display_name_as_typed.toml b/tests/spec_fixtures/fixtures/valid/v24_display_name_as_typed.toml new file mode 100644 index 0000000..feeaff5 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v24_display_name_as_typed.toml @@ -0,0 +1,73 @@ +# Fixture v24_display_name_as_typed +# Expected: PASS +# Source: Owner decisions of 6 Nov 2026 (Q2) +# Display names are kept as typed: one decomposed (U+0301) and one lower-case, neither normalized. The channel ACTIN and the condition DrugA both display as "Drug A": uniqueness is per kind, so this is valid. Collisions are decided after NFC + case folding within one kind (i53, i76, i77, i86-i88). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Cafe\u0301 control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "Drug A" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +display_name = "Day 1" +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +display_name = "day 2" +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v25_removed_condition_id_retained.toml b/tests/spec_fixtures/fixtures/valid/v25_removed_condition_id_retained.toml new file mode 100644 index 0000000..fc47bf7 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v25_removed_condition_id_retained.toml @@ -0,0 +1,48 @@ +# Fixture v25_removed_condition_id_retained +# Expected: PASS +# Source: Owner decisions of 6 Nov 2026 (Q1) +# v01 after the user confirmed the removal of every DrugA member and the condition. DrugA is no longer declared but stays in used_ids.conditions: removing an entity keeps its ID listed. This is the published predecessor of t03 and t04. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v26_ids_kept_as_written.toml b/tests/spec_fixtures/fixtures/valid/v26_ids_kept_as_written.toml new file mode 100644 index 0000000..b20d76d --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v26_ids_kept_as_written.toml @@ -0,0 +1,71 @@ +# Fixture v26_ids_kept_as_written +# Expected: PASS +# Source: Owner decisions of 6 Nov 2026 (A2); LabChronicle audit B1, B2, B11 +# IDs are opaque text with no width, prefix, case or number rule: cl001 (lower case), CL1000 (four digits after a three-digit width), R999 and R1000 are all valid and a reader or writer MUST keep each exactly as written, never padding, re-casing or renumbering it. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.cl001] +source_folder = "cl001" +display_name = "Control" + +[conditions.CL1000] +source_folder = "CL1000" +display_name = "Drug A" +reference = "cl001" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.R999] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.R999.members.cl001] +source_repeat = "rep1" +image_count = 12 + +[repeats.R999.members.cl001.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.R999.members.CL1000] +source_repeat = "rep1" +image_count = 11 + +[repeats.R999.members.CL1000.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.R1000] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.R1000.members.cl001] +source_repeat = "rep2" +image_count = 10 + +[repeats.R1000.members.cl001.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.R1000.members.CL1000] +source_repeat = "rep2" +image_count = 10 + +[repeats.R1000.members.CL1000.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CL1000", "cl001"] +channels = ["ACTIN", "DNA"] +repeats = ["R1000", "R999"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v27_descriptive_keys_collide.toml b/tests/spec_fixtures/fixtures/valid/v27_descriptive_keys_collide.toml new file mode 100644 index 0000000..3ba59e3 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v27_descriptive_keys_collide.toml @@ -0,0 +1,108 @@ +# Fixture v27_descriptive_keys_collide +# Expected: PASS +# Source: Owner decisions of 6 Nov 2026 (Q4) +# target and treatment_key are descriptive only. DrugA and DrugC carry identical values, DrugB the same values in another case: no collision, no warning, no grouping. A consumer MUST NOT group, join, pair or key a cache on them. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" +target = "SRC" +treatment_key = "AZD0530" + +[conditions.DrugB] +source_folder = "DrugB" +reference = "CTRL" +target = "src" +treatment_key = "azd0530" + +[conditions.DrugC] +source_folder = "DrugC" +reference = "CTRL" +target = "SRC" +treatment_key = "AZD0530" + +[channels.DNA] +header_names = ["DAPI"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 5 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 5 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA"] + +[repeats.rep1.members.DrugB] +source_repeat = "rep1" +image_count = 5 + +[repeats.rep1.members.DrugB.technical_repeats.single] +source_folder = "single" +channels = ["DNA"] + +[repeats.rep1.members.DrugC] +source_repeat = "rep1" +image_count = 5 + +[repeats.rep1.members.DrugC.technical_repeats.single] +source_folder = "single" +channels = ["DNA"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 5 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 5 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA"] + +[repeats.rep2.members.DrugB] +source_repeat = "rep2" +image_count = 5 + +[repeats.rep2.members.DrugB.technical_repeats.single] +source_folder = "single" +channels = ["DNA"] + +[repeats.rep2.members.DrugC] +source_repeat = "rep2" +image_count = 5 + +[repeats.rep2.members.DrugC.technical_repeats.single] +source_folder = "single" +channels = ["DNA"] + +[used_ids] +conditions = ["CTRL", "DrugA", "DrugB", "DrugC"] +channels = ["DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v28_rgb_units.toml b/tests/spec_fixtures/fixtures/valid/v28_rgb_units.toml new file mode 100644 index 0000000..693a5a3 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v28_rgb_units.toml @@ -0,0 +1,78 @@ +# Fixture v28_rgb_units +# Expected: PASS +# Source: Owner decision AQ6 (supersedes D8) +# An RGB experiment (e.g. H&E): every unit declares `image_type = "rgb"` with the fixed channels R, G, B in that order and no aliases. Valid and homogeneous. The scientific meaning is left to each brick: a brick that does not declare RGB support refuses a request on these units (A_RGB_NOT_SUPPORTED, P24); preprocessing compares the declaration with the image header and never with pixels (P13, P23). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" + +[channels.R] +display_name = "Red" + +[channels.G] +display_name = "Green" + +[channels.B] +display_name = "Blue" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 8 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 8 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 8 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["R", "G", "B"] +image_type = "rgb" + +[masks.Nuclei] +source_type = "nuclei" +representation = "instances" + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["B", "G", "R"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v29_rgb_and_channels_mixed.toml b/tests/spec_fixtures/fixtures/valid/v29_rgb_and_channels_mixed.toml new file mode 100644 index 0000000..ae9195f --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v29_rgb_and_channels_mixed.toml @@ -0,0 +1,46 @@ +# Fixture v29_rgb_and_channels_mixed +# Expected: PASS +# Source: Owner decision AQ6 +# One condition with an RGB brightfield unit and a two-channel fluorescence unit of the same biological repeat. Valid schema; heterogeneous (different lists), so the Stage 1-2 runtime refuses it (R_HETEROGENEOUS_LAYOUT_NOT_ENABLED). Both units stay ONE biological repeat. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[channels.R] + +[channels.G] + +[channels.B] + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 6 + +[repeats.rep1.members.CTRL.technical_repeats.brightfield] +source_folder = "brightfield" +channels = ["R", "G", "B"] +image_type = "rgb" + +[repeats.rep1.members.CTRL.technical_repeats.fluorescence] +source_folder = "fluorescence" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL"] +channels = ["ACTIN", "B", "DNA", "G", "R"] +repeats = ["rep1"] +technical_repeats = ["brightfield", "fluorescence"] diff --git a/tests/spec_fixtures/fixtures/valid/v30_same_source_label_different_repeats.toml b/tests/spec_fixtures/fixtures/valid/v30_same_source_label_different_repeats.toml new file mode 100644 index 0000000..3b77cc9 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v30_same_source_label_different_repeats.toml @@ -0,0 +1,65 @@ +# Fixture v30_same_source_label_different_repeats +# Expected: PASS +# Source: Owner decision AQ5; S0.1 audit section 5 +# Source folder label rep1 is in canonical repeat Day1 for CTRL and in Day2 for DrugA (and rep2 the other way round). Valid: no repeat ID equals another repeat's source label. The FOV key keeps the source label in `biological_repeat` and stays unique through the condition; the added `repeat_group` is Day1 or Day2, and grouping and pairing use it (experiment-v1 3.4, P25). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" + +[channels.DNA] +display_name = "DAPI" + +[channels.ACTIN] +display_name = "F-actin" + +[repeats.Day1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.Day1.members.CTRL] +source_repeat = "rep1" +image_count = 9 + +[repeats.Day1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.Day1.members.DrugA] +source_repeat = "rep2" +image_count = 9 + +[repeats.Day1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.Day2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.Day2.members.CTRL] +source_repeat = "rep2" +image_count = 9 + +[repeats.Day2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.Day2.members.DrugA] +source_repeat = "rep1" +image_count = 9 + +[repeats.Day2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["Day1", "Day2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v31_header_alias_whitespace_and_decomposed.toml b/tests/spec_fixtures/fixtures/valid/v31_header_alias_whitespace_and_decomposed.toml new file mode 100644 index 0000000..0bea169 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v31_header_alias_whitespace_and_decomposed.toml @@ -0,0 +1,33 @@ +# Fixture v31_header_alias_whitespace_and_decomposed +# Expected: PASS +# Source: Owner decision AQ7 +# Header evidence is compared exactly after NFC of both sides, with no trimming. DNA lists the aliases "DAPI", " DAPI" (leading space) and "DAPI " (trailing space): outer whitespace needs its own confirmed alias, each is a distinct entry. ACTIN lists a decomposed alias (written with a TOML escape and stored as typed), which matches the header name "Caf\u00e9 488" after NFC of both sides (P26). header_names are not identities and are not NFC-checked. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[channels.DNA] +header_names = ["DAPI", " DAPI", "DAPI "] + +[channels.ACTIN] +header_names = ["Cafe\u0301 488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 4 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v32_reserved_name_near_misses.toml b/tests/spec_fixtures/fixtures/valid/v32_reserved_name_near_misses.toml new file mode 100644 index 0000000..ad5e502 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v32_reserved_name_near_misses.toml @@ -0,0 +1,43 @@ +# Fixture v32_reserved_name_near_misses +# Expected: PASS +# Source: Owner decision AQ4 +# Names next to the reserved list (experiment-v1 3.2) are valid: channels c1, t0, z2 and label, masks Patches and Labels. Only the exact names, compared case-folded, are reserved (t, z0, label_id, patch_id, ...). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[channels.c1] + +[channels.t0] + +[channels.z2] + +[channels.label] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 4 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["c1", "t0", "z2", "label"] + +[masks.Patches] +source_type = "patches" +representation = "instances" + +[masks.Labels] +source_type = "labels" +representation = "binary" + +[used_ids] +conditions = ["CTRL"] +channels = ["c1", "label", "t0", "z2"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v33_condition_folder_on_forbidden_list.toml b/tests/spec_fixtures/fixtures/valid/v33_condition_folder_on_forbidden_list.toml new file mode 100644 index 0000000..0617c7f --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v33_condition_folder_on_forbidden_list.toml @@ -0,0 +1,71 @@ +# Fixture v33_condition_folder_on_forbidden_list +# Expected: PASS +# Source: Owner decision of 6 Nov 2026 (fourth round: folder labels never restricted) +# The condition folder on disk is `control`, a forbidden name (experiment-v1 4.7), so it cannot be the ID. It is recorded as `source_folder` (never restricted) and the user confirmed the ID `ctrl`, with `display_name` `control`. DrugA references `ctrl`, the ID. The derived experiment_design.csv still has `control` in the `condition` column (experiment-v1 11). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.ctrl] +source_folder = "control" +display_name = "control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "ctrl" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.ctrl] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.ctrl.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.ctrl] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.ctrl.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["DrugA", "ctrl"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v34_technical_repeat_folder_on_forbidden_list.toml b/tests/spec_fixtures/fixtures/valid/v34_technical_repeat_folder_on_forbidden_list.toml new file mode 100644 index 0000000..b94a78e --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v34_technical_repeat_folder_on_forbidden_list.toml @@ -0,0 +1,81 @@ +# Fixture v34_technical_repeat_folder_on_forbidden_list +# Expected: PASS +# Source: Owner decision of 6 Nov 2026 (fourth round: folder labels never restricted) +# Repeat rep1 has two technical-repeat folders, `files` and `overrides`, both forbidden names (experiment-v1 4.7). They are the `source_folder` values; the confirmed IDs are `acq1` and `acq2`, and the same ID carries the same folder in both members. rep2 has no technical-repeat folder level (`single`). Homogeneous. +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" +display_name = "Control" + +[conditions.DrugA] +source_folder = "DrugA" +display_name = "Drug A" +reference = "CTRL" +target = "SRC" +treatment_key = "saracatinib" + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[channels.ACTIN] +display_name = "F-actin" +header_names = ["Phalloidin-488"] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 12 + +[repeats.rep1.members.CTRL.technical_repeats.acq1] +source_folder = "files" +display_name = "files" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.CTRL.technical_repeats.acq2] +source_folder = "overrides" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA] +source_repeat = "rep1" +image_count = 11 + +[repeats.rep1.members.DrugA.technical_repeats.acq1] +source_folder = "files" +display_name = "files" +channels = ["DNA", "ACTIN"] + +[repeats.rep1.members.DrugA.technical_repeats.acq2] +source_folder = "overrides" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.CTRL] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 10 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA"] +repeats = ["rep1", "rep2"] +technical_repeats = ["acq1", "acq2", "single"] diff --git a/tests/spec_fixtures/fixtures/valid/v35_flat_experiment.toml b/tests/spec_fixtures/fixtures/valid/v35_flat_experiment.toml new file mode 100644 index 0000000..eb3ee8e --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v35_flat_experiment.toml @@ -0,0 +1,34 @@ +# Fixture v35_flat_experiment +# Expected: PASS +# Source: Matrix audit 2026-11-07 (case F01) +# An experiment folder that is itself the leaf: no condition, biological-repeat or technical-repeat folder. Preprocessing labels all three levels `single` (contract 1). The condition `single` has `source_folder = "single"` ("no condition folder level", 3.2, 4.2, 4.6a) and is the only condition; see i131-i133 for the refused misuses (owner decision SQ6, matrix audit MS-1). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.single] +source_folder = "single" + +[channels.ACTIN] +header_names = ["Actin"] + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[repeats.single] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.single.members.single] +source_repeat = "single" +image_count = 4 + +[repeats.single.members.single.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["single"] +channels = ["ACTIN", "DNA"] +repeats = ["single"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v36_repeat_folder_forbidden_name.toml b/tests/spec_fixtures/fixtures/valid/v36_repeat_folder_forbidden_name.toml new file mode 100644 index 0000000..6a5e930 --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v36_repeat_folder_forbidden_name.toml @@ -0,0 +1,67 @@ +# Fixture v36_repeat_folder_forbidden_name +# Expected: PASS +# Source: Matrix audit 2026-11-07 (case F16) +# Biological-repeat folders named `z` (a forbidden ID key, 4.7) in both conditions are paired into the repeat `R1`; `source_repeat` is never restricted. The proposed repeat ID would be the folder label `z`, which is not allowed (a forbidden name, 4.7; i130), so preprocessing makes no proposal and the user names the repeat ID `R1` (4.6a; owner decision SQ6, matrix audit MS-2). The condition folder `control` is the condition `ctrl` (as v33). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.DrugA] +source_folder = "DrugA" +reference = "ctrl" + +[conditions.ctrl] +source_folder = "control" +display_name = "control" + +[channels.ACTIN] +header_names = ["Actin"] + +[channels.DNA] +display_name = "DAPI" +header_names = ["DAPI"] + +[repeats.R1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.R1.members.DrugA] +source_repeat = "z" +image_count = 2 + +[repeats.R1.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.R1.members.ctrl] +source_repeat = "z" +image_count = 2 + +[repeats.R1.members.ctrl.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep2.members.DrugA] +source_repeat = "rep2" +image_count = 2 + +[repeats.rep2.members.DrugA.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[repeats.rep2.members.ctrl] +source_repeat = "rep2" +image_count = 2 + +[repeats.rep2.members.ctrl.technical_repeats.single] +source_folder = "single" +channels = ["DNA", "ACTIN"] + +[used_ids] +conditions = ["DrugA", "ctrl"] +channels = ["ACTIN", "DNA"] +repeats = ["R1", "rep2"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/fixtures/valid/v37_used_ids_reserved_near_misses.toml b/tests/spec_fixtures/fixtures/valid/v37_used_ids_reserved_near_misses.toml new file mode 100644 index 0000000..19c8a7f --- /dev/null +++ b/tests/spec_fixtures/fixtures/valid/v37_used_ids_reserved_near_misses.toml @@ -0,0 +1,43 @@ +# Fixture v37_used_ids_reserved_near_misses +# Expected: PASS +# Source: Owner decision of 7 Nov 2026 (MorphoBricks-Spec issue 11) +# Retired channel IDs next to the reserved list are valid in used_ids.channels (experiment-v1 4.9): Ringer and label2 are listed but not declared. Only the exact reserved names, compared case-folded, are refused there as for declared IDs (i135). +schema = "morphobricks-experiment" +schema_version = 1 + +[conditions.CTRL] +source_folder = "CTRL" + +[channels.c1] + +[channels.t0] + +[channels.z2] + +[channels.label] + +[repeats.rep1] +confirmed = true +confirmed_on = "2026-11-06" + +[repeats.rep1.members.CTRL] +source_repeat = "rep1" +image_count = 4 + +[repeats.rep1.members.CTRL.technical_repeats.single] +source_folder = "single" +channels = ["c1", "t0", "z2", "label"] + +[masks.Patches] +source_type = "patches" +representation = "instances" + +[masks.Labels] +source_type = "labels" +representation = "binary" + +[used_ids] +conditions = ["CTRL"] +channels = ["Ringer", "c1", "label", "label2", "t0", "z2"] +repeats = ["rep1"] +technical_repeats = ["single"] diff --git a/tests/spec_fixtures/tools/audit/refrecords.py b/tests/spec_fixtures/tools/audit/refrecords.py new file mode 100644 index 0000000..ef06a1e --- /dev/null +++ b/tests/spec_fixtures/tools/audit/refrecords.py @@ -0,0 +1,333 @@ +"""AUDIT TOOL ONLY: reference validator and canonical encoder of `morphobricks.records.jsonl`. + +This is not a runtime package and no brick may import it. It implements, and only, the records +rules of docs/experiment-v1.md section 4.6: the three typed record forms (`reconfirmation`, +`image_count_changed`, `removed`, with RR3) and the byte rules of the file. It sits beside +`refval.py` (which stays the validator of `morphobricks.toml`) and is run by `refval.py fixtures`. + +A refused file is `R_RECORDS_FILE_INVALID` (section 8, owner decision RF3); the `E_RECORDS_*` +names below are its reasons, named by this tool (owner decisions RF1-RF5 of 9 November 2026). + +Usage: + python -I refrecords.py check FILE [--json] +""" + +from __future__ import annotations + +import json +import re +import sys +from datetime import date +from pathlib import Path + +DATE = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}") # matched with fullmatch +SHA256 = re.compile(r"[0-9a-f]{64}") # lower-case hex (RF4); matched with fullmatch, not `$` +ID_KINDS = ("conditions", "channels", "repeats", "technical_repeats") +ENTITIES = ("unit", "technical_repeat", "repeat", "condition") +RECONFIRMATION = ("type", "id_kind", "old_id", "new_id", "confirmed_on", "old_sha256", "new_sha256") +IMAGE_COUNT = ( + "type", + "condition_id", + "condition_source_folder", + "repeat_id", + "source_repeat", + "technical_repeats", + "old_image_count", + "new_image_count", + "recorded_on", +) +REMOVED_NAMES = ( + "condition_id", + "condition_source_folder", + "repeat_id", + "source_repeat", + "technical_repeat_id", + "technical_repeat_source_folder", +) +REMOVED = ("type", "entity", *REMOVED_NAMES, "confirmed_on") +KEY_SETS = { + "reconfirmation": frozenset(RECONFIRMATION), + "image_count_changed": frozenset(IMAGE_COUNT), + "removed": frozenset(REMOVED), +} +# the keys a `removed` line names, per entity; every other name key is null (section 4.6) +NAMED = { + "unit": REMOVED_NAMES, + "technical_repeat": ("technical_repeat_id", "technical_repeat_source_folder"), + "repeat": ("repeat_id",), + "condition": ("condition_id", "condition_source_folder"), +} +UNIT_KEYS = ("id", "source_folder") +ESCAPE = re.compile(r'\\(?:u([0-9a-fA-F]{4})|.)', re.DOTALL) + + +class Report: + """Errors of one records file: (reason, line number, detail).""" + + def __init__(self) -> None: + self.errors: list[tuple[str, int, str]] = [] + self.records: list[dict] = [] + + def e(self, code: str, line: int, detail: str = "") -> None: + self.errors.append((code, line, detail)) + + @property + def codes(self) -> set[str]: + return {c for c, _, _ in self.errors} + + def as_dict(self) -> dict: + return { + "records": len(self.records), + "errors": [list(map(str, x)) for x in self.errors], + "error_codes": sorted(self.codes), + } + + +# canonical encoder -------------------------------------------------------------------------- + + +def encode_record(record: dict) -> bytes: + """Return the canonical bytes of one record (section 4.6, RF1): keys in code point order, + non-ASCII unescaped, the separators `, ` and `: `, the minimal string escapes, one LF. This is + Python's `json.dumps(ensure_ascii=False, sort_keys=True)`, which wrote every line so far. + """ + return (json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n").encode("utf-8") + + +def encode_records(records: list[dict]) -> bytes: + """Return the bytes of a records file holding exactly these records, in order.""" + return b"".join(encode_record(r) for r in records) + + +# parsing ------------------------------------------------------------------------------------ + + +class _Duplicate(ValueError): + pass + + +def _pairs(items: list[tuple[str, object]]) -> dict: + out: dict = {} + for key, value in items: + if key in out: + raise _Duplicate(key) + out[key] = value + return out + + +def _constant(name: str) -> object: + raise ValueError(f"{name} is not JSON") + + +def _sorted_keys(value: object) -> bool: + """True when every object nested in ``value`` lists its keys in code point order.""" + if isinstance(value, dict): + keys = list(value) + return keys == sorted(keys) and all(_sorted_keys(v) for v in value.values()) + if isinstance(value, list): + return all(_sorted_keys(v) for v in value) + return True + + +def _escaped_non_ascii(line: str) -> bool: + """True when the line writes a non-ASCII character as a `\\uXXXX` escape.""" + return any(m.group(1) and int(m.group(1), 16) >= 0x80 for m in ESCAPE.finditer(line)) + + +# values ------------------------------------------------------------------------------------- + + +def _text(v: object) -> bool: + return type(v) is str and not any(0xD800 <= ord(c) <= 0xDFFF for c in v) + + +def _date(v: object) -> bool: + if type(v) is not str or not DATE.fullmatch(v): + return False + try: + date.fromisoformat(v) + except ValueError: + return False + return True + + +def _count(v: object) -> bool: + return type(v) is int and v >= 1 + + +def _bad_reconfirmation(rec: dict) -> list[str]: + bad = [] + if rec["id_kind"] not in ID_KINDS: + bad.append("id_kind") + if not _text(rec["old_id"]): + bad.append("old_id") + if rec["new_id"] is not None and not _text(rec["new_id"]): + bad.append("new_id") + if not _date(rec["confirmed_on"]): + bad.append("confirmed_on") + old = rec["old_sha256"] + if not (old == "absent" or (type(old) is str and SHA256.fullmatch(old))): + bad.append("old_sha256") + if not (type(rec["new_sha256"]) is str and SHA256.fullmatch(rec["new_sha256"])): + bad.append("new_sha256") + return bad + + +def _bad_removed(rec: dict) -> list[str]: + if rec["entity"] not in ENTITIES: + return ["entity"] + bad = [] + named = NAMED[rec["entity"]] + for key in REMOVED_NAMES: + if key in named and not _text(rec[key]) or key not in named and rec[key] is not None: + bad.append(key) + if not _date(rec["confirmed_on"]): + bad.append("confirmed_on") + return bad + + +def _bad_units(units: object) -> bool: + if type(units) is not list: + return True + ids = [] + for unit in units: + if type(unit) is not dict or set(unit) != set(UNIT_KEYS): + return True + if not (_text(unit["id"]) and _text(unit["source_folder"])): + return True + ids.append(unit["id"]) + return ids != sorted(ids) # sorted by id in code point order (ties cannot occur in a member) + + +def _bad_image_count(rec: dict) -> list[str]: + names = ("condition_id", "condition_source_folder", "repeat_id", "source_repeat") + bad = [k for k in names if not _text(rec[k])] + if _bad_units(rec["technical_repeats"]) or not rec["technical_repeats"]: + bad.append("technical_repeats") # an array of {id, source_folder}, sorted, not empty + bad += [k for k in ("old_image_count", "new_image_count") if not _count(rec[k])] + if not bad and rec["old_image_count"] == rec["new_image_count"]: + bad.append("new_image_count") # the record states a change (RF5) + if not _date(rec["recorded_on"]): + bad.append("recorded_on") + return bad + + +CHECKS = { + "reconfirmation": _bad_reconfirmation, + "removed": _bad_removed, + "image_count_changed": _bad_image_count, +} + + +def _check_line(raw: bytes, number: int, r: Report) -> None: + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + r.e("E_RECORDS_ENCODING", number, "not UTF-8") + return + try: + rec = json.loads(text, object_pairs_hook=_pairs, parse_constant=_constant) + except _Duplicate as err: + r.e("E_RECORDS_NOT_JSON_OBJECT", number, f"key {err} twice") + return + except ValueError as err: + r.e("E_RECORDS_NOT_JSON_OBJECT", number, str(err)) + return + if type(rec) is not dict: + r.e("E_RECORDS_NOT_JSON_OBJECT", number, type(rec).__name__) + return + kind = rec.get("type") + if type(kind) is not str or kind not in KEY_SETS: + r.e("E_RECORDS_UNKNOWN_TYPE", number, repr(kind)) + return + if set(rec) != KEY_SETS[kind]: + odd = sorted(set(rec) ^ KEY_SETS[kind]) + r.e("E_RECORDS_KEY_SET", number, f"{kind}: differs in {odd}") + return + before = len(r.errors) + bad = CHECKS[kind](rec) + if bad: + r.e("E_RECORDS_VALUE", number, f"{kind}: {bad}") + if not _sorted_keys(rec): + r.e("E_RECORDS_KEY_ORDER", number) + if _escaped_non_ascii(text): + r.e("E_RECORDS_NON_ASCII_ESCAPED", number) + if len(r.errors) > before: + return + r.records.append(rec) + if raw + b"\n" != encode_record(rec): + r.e("E_RECORDS_FORM", number, "not the canonical bytes (whitespace, CR, optional escape)") + return + + +def validate_records(data: bytes) -> Report: + """Validate the bytes of a `morphobricks.records.jsonl`; every line is checked.""" + r = Report() + if not data: + return r # an empty file holds no record and is accepted (RF4) + if data.startswith(b"\xef\xbb\xbf"): + r.e("E_RECORDS_ENCODING", 1, "byte-order mark") # RF4 + parts = data.split(b"\n") # LF only: U+2028, U+0085 and CR do not end a line + if parts[-1] != b"": + r.e("E_RECORDS_NO_FINAL_LF", len(parts), "last line lacks its LF") + else: + parts = parts[:-1] + for number, raw in enumerate(parts, start=1): + if number == 1 and data.startswith(b"\xef\xbb\xbf"): + continue + _check_line(raw, number, r) + return r + + +CODE = re.compile(r"\b(E_RECORDS_[A-Z_]+)\b") +ROW = re.compile(r"\| \[`(r[vi]\d+[^`]*)`\]\([^)]+\) \|([^|]*)\|") + + +def run_fixtures(root: Path) -> tuple[int, int]: + """Check every file of fixtures/records against its row of fixtures/README.md. + + Returns (files checked, mismatches). A file with no row, or a row with no file, is a mismatch. + """ + fx = root / "fixtures" + readme = (fx / "README.md").read_text(encoding="utf-8") + rows = {m.group(1): m.group(2).strip() for m in map(ROW.match, readme.splitlines()) if m} + files = {f.stem: f for f in sorted((fx / "records").glob("*.jsonl"))} + bad = 0 + for name in sorted(set(rows) ^ set(files)): + bad += 1 + print(f"NO-CATALOGUE-ROW-OR-FILE records/{name}") + for name in sorted(set(rows) & set(files)): + data = files[name].read_bytes() + r = validate_records(data) + cell = rows[name] + codes = set(CODE.findall(cell)) + if name.startswith("rv"): + # an accepted file is the canonical encoding of the records it holds + ok = not r.errors and encode_records(r.records) == data + want = "accepted" + else: + line = re.search(r"\(line (\d+)\)", cell) + ok = r.codes == codes and bool(line) and r.errors[0][1] == int(line.group(1)) + want = f"refused {sorted(codes)} first at line {line and line.group(1)}" + if not ok: + bad += 1 + print(f"MISMATCH records/{name}: got errors={r.errors[:3]}; expected {want}") + return len(files), bad + + +def main(argv: list[str]) -> int: + if len(argv) >= 2 and argv[0] == "check": + r = validate_records(Path(argv[1]).read_bytes()) + if "--json" in argv: + print(json.dumps(r.as_dict(), ensure_ascii=False)) + else: + for e in r.errors: + print("ERROR", *e) + print(len(r.records), "records", "OK" if not r.errors else "REFUSED") + return 1 if r.errors else 0 + print(__doc__) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/spec_fixtures/tools/audit/refval.py b/tests/spec_fixtures/tools/audit/refval.py new file mode 100644 index 0000000..0ea8309 --- /dev/null +++ b/tests/spec_fixtures/tools/audit/refval.py @@ -0,0 +1,944 @@ +"""AUDIT TOOL ONLY: throwaway reference validator for MorphoBricks experiment description v1. + +This is not a runtime package and no brick may import it. It implements the normative text of +docs/experiment-v1.md (phases 1-5 of section 6 and the warnings), the canonical writer of +section 9.1, the section digests of section 9.3 and the predecessor checks of sections 4.4a, +4.6 and 4.9 that two TOML files can express. The records file of section 4.6 is `refrecords.py`; +`fixtures` runs its goldens too. Where the text is ambiguous the choice made here is +marked "AMBIGUITY" and reported in docs/audits/matrix-audit-2026-11-07.md. + +Usage: + python -I refval.py check FILE [--json] + python -I refval.py fixtures SPEC_ROOT # run the fixture catalogue + python -I refval.py transition OLD NEW +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import re +import sys +import tomllib +import unicodedata +from datetime import date +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) # -I drops the script folder +import refrecords # noqa: E402 (records of section 4.6; a sibling audit module) + +SENTINELS = {"", "na", "nan", "none", "null", "n/a", ""} +TOKEN = re.compile(r"^[A-Za-z][A-Za-z0-9]*(?:_[A-Za-z0-9]+)*$") +DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +RESERVED = { + "whole", "cell", "nucleus", "cytoplasm", "ring", "expanded_nucleus", + "condition", "biological_repeat", "repeat", "technical_repeat", "position_id", + "image_file", "image_path", "mask_file", "patch_file", "patch_id", "label_id", + "z", "z0", "z1", "c", "t", +} +FORBIDDEN = { + "axes", "shared", "plane_scope", "z", "projection", "pixel_size", "calibration", "fovs", + "files", "paths", "overrides", "exclude", "fingerprints", "bricks", "routes", "features", + "normalization", "statistics", "contrasts", "binary_mask_policy", "mask_conversion", + "control", "roles", "experiment", "source_index", "channel_count", "retired_repeats", + "replaced_by", "labchronicle", "provenance", "source_database", +} +TOP = ("schema", "schema_version", "conditions", "channels", "repeats", "masks", "used_ids") +COND = ("source_folder", "display_name", "reference", "target", "treatment_key") +CHAN = ("display_name", "header_names") +REP = ("display_name", "confirmed", "confirmed_on", "members") +MEM = ("source_repeat", "image_count", "technical_repeats") +UNIT = ("source_folder", "display_name", "channels", "image_type") +MASK = ("source_type", "representation", "source_channels", "display_name") +USED = ("conditions", "channels", "repeats", "technical_repeats") +REPRESENTATIONS = ("instances", "binary") + + +class Report: + def __init__(self) -> None: + self.errors: list[tuple[str, str, object]] = [] + self.warnings: list[tuple[str, str, object]] = [] + self.phase = 0 + + def e(self, code: str, path: str, value: object = None) -> None: + self.errors.append((code, path, value)) + + def w(self, code: str, path: str, value: object = None) -> None: + self.warnings.append((code, path, value)) + + @property + def codes(self) -> set[str]: + return {c for c, _, _ in self.errors} + + @property + def warning_codes(self) -> set[str]: + return {c for c, _, _ in self.warnings} + + def as_dict(self) -> dict: + return { + "phase": self.phase, + "errors": [list(map(str, x)) for x in self.errors], + "warnings": [list(map(str, x)) for x in self.warnings], + "error_codes": sorted(self.codes), + "warning_codes": sorted(self.warning_codes), + } + + +def nfc(s: str) -> str: + return unicodedata.normalize("NFC", s) + + +def is_text(x: object) -> bool: + return type(x) is str + + +def is_table(x: object) -> bool: + return type(x) is dict + + +def kpath(base: str, key: str) -> str: + return f"{base}.{key}" if re.fullmatch(r"[A-Za-z0-9_-]+", key) else f'{base}."{key}"' + + +# ------------------------------------------------------------------ phase 1: decode +def phase1(data: bytes, r: Report): + if data.startswith(b"\xef\xbb\xbf"): + r.e("E_FILE_ENCODING", "", "byte-order mark") + return None + try: + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + r.e("E_FILE_ENCODING", "", str(exc)) + return None + try: + return tomllib.loads(text) + except tomllib.TOMLDecodeError as exc: + msg = str(exc) + dup = ("Cannot declare" in msg) or ("Cannot overwrite" in msg) or ("twice" in msg) or ("Duplicate" in msg) + r.e("E_DUPLICATE_KEY" if dup else "E_TOML_SYNTAX", "", msg) + return None + + +# ------------------------------------------------------------------ phase 2: header +def phase2(doc: dict, r: Report) -> None: + if "schema" not in doc: + r.e("E_SCHEMA_ID", "schema") + elif not is_text(doc["schema"]): + r.e("E_TYPE", "schema", doc["schema"]) + elif doc["schema"] != "morphobricks-experiment": + r.e("E_SCHEMA_ID", "schema", doc["schema"]) + if "schema_version" not in doc: + r.e("E_SCHEMA_VERSION_MISSING", "schema_version") + elif type(doc["schema_version"]) is not int: + r.e("E_TYPE", "schema_version", doc["schema_version"]) + elif doc["schema_version"] != 1: + r.e("E_SCHEMA_VERSION_UNSUPPORTED", "schema_version", doc["schema_version"]) + + +# ------------------------------------------------------------------ phase 3: structure +def _fields(tbl: dict, allowed, path: str, r: Report) -> None: + for k in tbl: + if k in FORBIDDEN: + r.e("E_FORBIDDEN_FIELD", kpath(path, k), k) + elif k not in allowed: + r.e("E_UNKNOWN_FIELD", kpath(path, k), k) + + +def _required(tbl: dict, keys, path: str, r: Report) -> None: + for k in keys: + if k not in tbl: + r.e("E_MISSING_FIELD", kpath(path, k)) + + +def _text(tbl: dict, k: str, path: str, r: Report, nonempty: bool = False) -> None: + if k not in tbl: + return + v = tbl[k] + if not is_text(v): + r.e("E_TYPE", kpath(path, k), v) + elif nonempty and v.strip() == "": + r.e("E_TEXT_EMPTY", kpath(path, k), v) + + +def _text_array(tbl: dict, k: str, path: str, r: Report, nonempty_items: bool = False, + min1: bool = False) -> None: + if k not in tbl: + return + v = tbl[k] + p = kpath(path, k) + if type(v) is not list: + r.e("E_TYPE", p, v) + return + if min1 and not v: + r.e("E_EMPTY", p) + for i, x in enumerate(v): + if not is_text(x): + r.e("E_TYPE", f"{p}[{i}]", x) + elif nonempty_items and x.strip() == "": + r.e("E_TEXT_EMPTY", f"{p}[{i}]", x) + + +def _id_table(parent: dict, k: str, path: str, r: Report, required: bool = True, + min1: bool = True): + """Return [(id, table)] of an ID-keyed table; forbidden ID keys are reported and skipped.""" + p = kpath(path, k) + if k not in parent: + if required: + r.e("E_MISSING_FIELD", p) + return [] + t = parent[k] + if not is_table(t): + r.e("E_TYPE", p, t) + return [] + if min1 and not t: + r.e("E_EMPTY", p) + out = [] + for key, v in t.items(): + if key in FORBIDDEN: + r.e("E_FORBIDDEN_FIELD", kpath(p, key), key) + continue + if not is_table(v): + r.e("E_TYPE", kpath(p, key), v) + continue + out.append((key, v)) + return out + + +def phase3(doc: dict, r: Report) -> None: + _fields(doc, TOP, "", r) + _required(doc, ("conditions", "channels", "repeats", "used_ids"), "", r) + for cid, c in _id_table(doc, "conditions", "", r): + p = kpath("conditions", cid) + _fields(c, COND, p, r) + _required(c, ("source_folder",), p, r) + _text(c, "source_folder", p, r) + _text(c, "display_name", p, r, nonempty=True) + _text(c, "reference", p, r) + _text(c, "target", p, r, nonempty=True) + _text(c, "treatment_key", p, r, nonempty=True) + for chid, ch in _id_table(doc, "channels", "", r): + p = kpath("channels", chid) + _fields(ch, CHAN, p, r) + _text(ch, "display_name", p, r, nonempty=True) + _text_array(ch, "header_names", p, r, nonempty_items=True) + for rid, rep in _id_table(doc, "repeats", "", r): + p = kpath("repeats", rid) + _fields(rep, REP, p, r) + _required(rep, ("confirmed", "confirmed_on", "members"), p, r) + _text(rep, "display_name", p, r, nonempty=True) + if "confirmed" in rep: + if type(rep["confirmed"]) is not bool: + r.e("E_TYPE", p + ".confirmed", rep["confirmed"]) + elif rep["confirmed"] is not True: + r.e("E_REPEAT_UNCONFIRMED", p + ".confirmed", False) + if "confirmed_on" in rep: + v = rep["confirmed_on"] + if not is_text(v): + r.e("E_TYPE", p + ".confirmed_on", v) + else: + ok = bool(DATE.fullmatch(v)) + if ok: + try: + date.fromisoformat(v) + except ValueError: + ok = False + if not ok: + r.e("E_DATE_FORMAT", p + ".confirmed_on", v) + for cid, m in _id_table(rep, "members", p, r): + mp = kpath(p + ".members", cid) + _fields(m, MEM, mp, r) + _required(m, ("source_repeat", "image_count", "technical_repeats"), mp, r) + _text(m, "source_repeat", mp, r) + if "image_count" in m: + v = m["image_count"] + if type(v) is not int: + r.e("E_TYPE", mp + ".image_count", v) + elif v < 1: + r.e("E_VALUE_RANGE", mp + ".image_count", v) + for tid, u in _id_table(m, "technical_repeats", mp, r): + up = kpath(mp + ".technical_repeats", tid) + _fields(u, UNIT, up, r) + _required(u, ("source_folder", "channels"), up, r) + _text(u, "source_folder", up, r) + _text(u, "display_name", up, r, nonempty=True) + _text_array(u, "channels", up, r, min1=True) + if "image_type" in u: + v = u["image_type"] + if not is_text(v): + r.e("E_TYPE", up + ".image_type", v) + elif v != "rgb": + r.e("E_ENUM", up + ".image_type", v) + for mid, mk in _id_table(doc, "masks", "", r, required=False, min1=False): + p = kpath("masks", mid) + _fields(mk, MASK, p, r) + _required(mk, ("source_type", "representation"), p, r) + _text(mk, "source_type", p, r) + _text(mk, "display_name", p, r, nonempty=True) + _text_array(mk, "source_channels", p, r) + if "representation" in mk and is_text(mk["representation"]): + if mk["representation"] not in REPRESENTATIONS: + r.e("E_ENUM", p + ".representation", mk["representation"]) + elif "representation" in mk: + r.e("E_TYPE", p + ".representation", mk["representation"]) + if "used_ids" in doc: + u = doc["used_ids"] + if not is_table(u): + r.e("E_TYPE", "used_ids", u) + else: + _fields(u, USED, "used_ids", r) + _required(u, USED, "used_ids", r) + for k in USED: + _text_array(u, k, "used_ids", r, min1=True) + + +# ------------------------------------------------------------------ helpers over a valid structure +def members(doc: dict): + for rid, rep in doc.get("repeats", {}).items(): + for cid, m in rep.get("members", {}).items(): + yield rid, cid, m + + +def units(doc: dict): + for rid, cid, m in members(doc): + for tid, u in m.get("technical_repeats", {}).items(): + yield rid, cid, tid, u + + +def upath(rid: str, cid: str, tid: str) -> str: + return kpath(kpath(kpath("repeats", rid) + ".members", cid) + ".technical_repeats", tid) + + +# ------------------------------------------------------------------ phase 4: identity +def _label(v: str, path: str, r: Report) -> None: + if v != v.strip() or v.casefold() in SENTINELS: + r.e("E_ID_TEXT", path, v) + + +def _nfc(v: str, path: str, r: Report) -> None: + if nfc(v) != v: + r.e("E_ID_NOT_NFC", path, v) + + +def _token(v: str, path: str, r: Report) -> None: + _label(v, path, r) + if not TOKEN.fullmatch(v): + r.e("E_ID_SYNTAX", path, v) + + +def _single(v: str, path: str, r: Report) -> None: + if v.casefold() == "single" and v != "single": + r.e("E_SINGLE_MISUSE", path, v) + + +def phase4(doc: dict, r: Report) -> None: + scopes: dict[str, list[tuple[str, str]]] = {k: [] for k in ( + "conditions", "condition_folders", "repeats", "source_repeats", "technical_repeats", + "technical_folders", "channels", "masks", "mask_types")} + + def label_id(v, path, scope): + _label(v, path, r) + _nfc(v, path, r) + _single(v, path, r) + scopes[scope].append((v, path)) + + for cid, c in doc["conditions"].items(): + p = kpath("conditions", cid) + label_id(cid, p, "conditions") + label_id(c["source_folder"], p + ".source_folder", "condition_folders") + if (cid == "single") != (c["source_folder"] == "single"): + r.e("E_SINGLE_MISUSE", p + ".source_folder", c["source_folder"]) + elif c["source_folder"] == "single" and len(doc["conditions"]) > 1: + r.e("E_SINGLE_MISUSE", p + ".source_folder", "single is not the only condition") + if "reference" in c: + _nfc(c["reference"], p + ".reference", r) # a reference is NFC-checked only + for chid in doc["channels"]: + p = kpath("channels", chid) + _token(chid, p, r) + _nfc(chid, p, r) + _single(chid, p, r) + if chid.casefold() in RESERVED: + r.e("E_ID_RESERVED", p, chid) + scopes["channels"].append((chid, p)) + for rid, rep in doc["repeats"].items(): + p = kpath("repeats", rid) + label_id(rid, p, "repeats") + cond_members: dict[str, list[tuple[str, str]]] = {} + for rid, cid, m in members(doc): + mp = kpath(kpath("repeats", rid) + ".members", cid) + _nfc(cid, mp, r) # a member key is a reference to a condition ID + label_id(m["source_repeat"], mp + ".source_repeat", "source_repeats") + cond_members.setdefault(cid, []).append((m["source_repeat"], mp)) + tids = list(m["technical_repeats"]) + for tid, u in m["technical_repeats"].items(): + up = upath(rid, cid, tid) + label_id(tid, up, "technical_repeats") + label_id(u["source_folder"], up + ".source_folder", "technical_folders") + if tid == "single" and len(tids) > 1: + r.e("E_SINGLE_MISUSE", up, "single is not the only technical repeat") + if (tid == "single") != (u["source_folder"] == "single"): + r.e("E_SINGLE_MISUSE", up + ".source_folder", u["source_folder"]) + for i, ch in enumerate(u["channels"]): + _nfc(ch, f"{up}.channels[{i}]", r) + for cid, lst in cond_members.items(): + if any(s == "single" for s, _ in lst) and len(lst) > 1: + for s, mp in lst: + if s == "single": + r.e("E_SINGLE_MISUSE", mp + ".source_repeat", "single is not the only member") + for mid, mk in doc.get("masks", {}).items(): + p = kpath("masks", mid) + _token(mid, p, r) + _nfc(mid, p, r) + _single(mid, p, r) + parts = mid.casefold().split("_") + if mid.casefold() in RESERVED or "in" in parts: + r.e("E_ID_RESERVED", p, mid) + scopes["masks"].append((mid, p)) + label_id(mk["source_type"], p + ".source_type", "mask_types") + for i, ch in enumerate(mk.get("source_channels", [])): + _nfc(ch, f"{p}.source_channels[{i}]", r) + for kind in USED: + for i, v in enumerate(doc["used_ids"][kind]): + p = f"used_ids.{kind}[{i}]" + if kind == "channels": + # Owner decision of 7 Nov 2026 (Spec issue 11): a listed channel ID reserves it, + # so the reserved list of 3.2 applies as to a declared channel ID (4.9). + _token(v, p, r) + _nfc(v, p, r) + if v.casefold() in RESERVED: + r.e("E_ID_RESERVED", p, v) + else: + _label(v, p, r) + _nfc(v, p, r) + _single(v, p, r) + scopes[kind].append((v, p)) + for scope, items in scopes.items(): + by_fold: dict[str, set[str]] = {} + for v, _ in items: + by_fold.setdefault(nfc(v).casefold(), set()).add(nfc(v)) + for v, p in items: + if len(by_fold[nfc(v).casefold()]) > 1: + r.e("E_ID_CASE_COLLISION", p, v) + + +# ------------------------------------------------------------------ phase 5: semantics +def dn_key(x: str) -> str: + return nfc(nfc(x).casefold()) + + +def phase5(doc: dict, r: Report) -> None: + conds = doc["conditions"] + chans = doc["channels"] + for rid, cid, m in members(doc): + if cid not in conds: + r.e("E_UNKNOWN_CONDITION", kpath(kpath("repeats", rid) + ".members", cid), cid) + for rid, cid, tid, u in units(doc): + up = upath(rid, cid, tid) + seen = set() + for i, ch in enumerate(u["channels"]): + if ch not in chans: + r.e("E_UNKNOWN_CHANNEL", f"{up}.channels[{i}]", ch) + if ch in seen: + r.e("E_DUPLICATE_CHANNEL_IN_UNIT", f"{up}.channels[{i}]", ch) + seen.add(ch) + if u.get("image_type") == "rgb": + if u["channels"] != ["R", "G", "B"]: + r.e("E_RGB_UNIT_CHANNELS", up + ".channels", u["channels"]) + else: + for i, ch in enumerate(u["channels"]): + if ch.casefold() in ("r", "g", "b"): + r.e("E_RGB_TOKEN_MISUSE", f"{up}.channels[{i}]", ch) + for chid, ch in chans.items(): + hn = ch.get("header_names", []) + if chid.casefold() in ("r", "g", "b") and hn: + r.e("E_RGB_ALIAS", kpath("channels", chid) + ".header_names", hn) + if len({nfc(x) for x in hn}) != len(hn): + r.e("E_DUPLICATE_VALUE", kpath("channels", chid) + ".header_names", hn) + for mid, mk in doc.get("masks", {}).items(): + sc = mk.get("source_channels", []) + for i, ch in enumerate(sc): + if ch not in chans: + r.e("E_UNKNOWN_CHANNEL", f"{kpath('masks', mid)}.source_channels[{i}]", ch) + if len(set(sc)) != len(sc): + r.e("E_DUPLICATE_VALUE", kpath("masks", mid) + ".source_channels", sc) + # display names + groups: dict[str, list[tuple[str, str, str]]] = {k: [] for k in + ("c", "ch", "r", "t", "m")} + for k, t in (("c", conds), ("ch", chans), ("r", doc["repeats"]), + ("m", doc.get("masks", {}))): + for i, v in t.items(): + if "display_name" in v: + groups[k].append((i, v["display_name"], kpath(k, i))) + for rid, cid, tid, u in units(doc): + if "display_name" in u: + groups["t"].append((tid, u["display_name"], upath(rid, cid, tid))) + for items in groups.values(): + owners: dict[str, set[str]] = {} + for owner, name, _ in items: + owners.setdefault(dn_key(name), set()).add(owner) + for owner, name, p in items: + if len(owners[dn_key(name)]) > 1: + r.e("E_DISPLAY_NAME_COLLISION", p + ".display_name", name) + # used_ids + used = doc["used_ids"] + declared = { + "conditions": set(conds), "channels": set(chans), "repeats": set(doc["repeats"]), + "technical_repeats": {tid for _, _, tid, _ in units(doc)}, + } + for kind in USED: + lst = used[kind] + if len(set(lst)) != len(lst): + r.e("E_DUPLICATE_VALUE", f"used_ids.{kind}", lst) + for i in sorted(declared[kind] - set(lst)): + r.e("E_USED_ID_MISSING", f"used_ids.{kind}", i) + # one folder, one ID + folder_ids: dict[str, set[str]] = {} + for cid, c in conds.items(): + folder_ids.setdefault(c["source_folder"], set()).add(cid) + for f, ids in folder_ids.items(): + if len(ids) > 1: + r.e("E_SOURCE_FOLDER_REUSED", "conditions", (f, sorted(ids))) + tfolder_ids: dict[str, set[str]] = {} + tid_folders: dict[str, set[str]] = {} + for rid, cid, tid, u in units(doc): + tfolder_ids.setdefault(u["source_folder"], set()).add(tid) + tid_folders.setdefault(tid, set()).add(u["source_folder"]) + for f, ids in tfolder_ids.items(): + if len(ids) > 1: + r.e("E_SOURCE_FOLDER_REUSED", "technical_repeats", (f, sorted(ids))) + for tid, fs in tid_folders.items(): + if len(fs) > 1: + r.e("E_SOURCE_FOLDER_CONFLICT", tid, sorted(fs)) + # conditions with members + member_conds = {cid for _, cid, _ in members(doc)} + for cid in conds: + if cid not in member_conds: + r.e("E_CONDITION_WITHOUT_MEMBERS", kpath("conditions", cid), cid) + # source repeats + owner: dict[tuple[str, str], set[str]] = {} + for rid, cid, m in members(doc): + owner.setdefault((cid, m["source_repeat"]), set()).add(rid) + for key, rids in owner.items(): + if len(rids) > 1: + r.e("E_SOURCE_REPEAT_REUSED", "repeats", (key, sorted(rids))) + for rid in doc["repeats"]: + for rid2, cid, m in members(doc): + if rid2 != rid and nfc(m["source_repeat"]).casefold() == nfc(rid).casefold(): + r.e("E_REPEAT_ID_SHADOWS_SOURCE", + kpath(kpath("repeats", rid2) + ".members", cid), (rid, m["source_repeat"])) + types: dict[str, set[str]] = {} + for mid, mk in doc.get("masks", {}).items(): + types.setdefault(mk["source_type"], set()).add(mid) + for t, ids in types.items(): + if len(ids) > 1: + r.e("E_MASK_SOURCE_TYPE_REUSED", "masks", (t, sorted(ids))) + # reference graph + edges = {} + for cid, c in conds.items(): + if "reference" in c: + ref = c["reference"] + p = kpath("conditions", cid) + ".reference" + if ref == cid: + r.e("E_REFERENCE_SELF", p, ref) + elif ref not in conds: + r.e("E_REFERENCE_UNKNOWN", p, ref) + else: + edges[cid] = ref + reported = set() + for start in edges: + seen = [] + cur = start + while cur in edges and cur not in seen: + seen.append(cur) + cur = edges[cur] + if cur in seen: + cyc = frozenset(seen[seen.index(cur):]) + if cyc not in reported: + reported.add(cyc) + r.e("E_REFERENCE_CYCLE", "conditions", sorted(cyc)) + + +def warnings(doc: dict, r: Report) -> None: + used = {ch for _, _, _, u in units(doc) for ch in u["channels"]} + for chid in doc["channels"]: + if chid not in used: + r.w("W_UNUSED_CHANNEL", kpath("channels", chid), chid) + conds = set(doc["conditions"]) + rep_conds = {rid: set(rep["members"]) for rid, rep in doc["repeats"].items()} + for rid, cs in rep_conds.items(): + if cs != conds: + r.w("W_REPEAT_MISSING_CONDITIONS", kpath("repeats", rid), sorted(conds - cs)) + for cid, c in doc["conditions"].items(): + if "reference" in c: + if not any(cid in cs and c["reference"] in cs for cs in rep_conds.values()): + r.w("W_REFERENCE_UNPAIRED", kpath("conditions", cid), c["reference"]) + + +def validate(data: bytes) -> tuple[Report, dict | None]: + r = Report() + doc = phase1(data, r) + if r.errors: + r.phase = 1 + return r, None + for n, fn in ((2, phase2), (3, phase3), (4, phase4), (5, phase5)): + fn(doc, r) + if r.errors: + r.phase = n + return r, doc + warnings(doc, r) + return r, doc + + +# ------------------------------------------------------------------ canonical writer (section 9.1) +def _key(k: str) -> str: + return k if re.fullmatch(r"[A-Za-z0-9_-]+", k) else _str(k) + + +def _str(s: str) -> str: + out = ['"'] + esc = {'"': '\\"', "\\": "\\\\", "\b": "\\b", "\t": "\\t", "\n": "\\n", "\f": "\\f", + "\r": "\\r"} + for ch in s: + if ch in esc: + out.append(esc[ch]) + elif ord(ch) < 0x20 or ord(ch) == 0x7F: + out.append(f"\\u{ord(ch):04X}") + else: + out.append(ch) + out.append('"') + return "".join(out) + + +def _val(v) -> str: + if type(v) is bool: + return "true" if v else "false" + if type(v) is int: + return str(v) + if type(v) is str: + return _str(v) + if type(v) is list: + return "[" + ", ".join(_val(x) for x in v) + "]" + raise TypeError(type(v)) + + +def _block(header: str, tbl: dict, order, sort_arrays=()) -> list[str]: + lines = [f"[{header}]"] + for k in order: + if k in tbl and not is_table(tbl[k]): + v = tbl[k] + if k in sort_arrays: + v = sorted(v) + lines.append(f"{k} = {_val(v)}") + return lines + [""] + + +def section_lines(doc: dict, sec: str) -> list[str]: + """The table blocks of one section, each followed by a blank line (section 9.1).""" + out: list[str] = [] + if sec == "conditions": + for cid in sorted(doc["conditions"]): + out += _block("conditions." + _key(cid), doc["conditions"][cid], COND) + elif sec == "channels": + for chid in sorted(doc["channels"]): + out += _block("channels." + _key(chid), doc["channels"][chid], CHAN, + ("header_names",)) + elif sec == "repeats": + for rid in sorted(doc["repeats"]): + rep = doc["repeats"][rid] + rh = "repeats." + _key(rid) + out += _block(rh, rep, REP) + for cid in sorted(rep["members"]): + m = rep["members"][cid] + mh = rh + ".members." + _key(cid) + out += _block(mh, m, MEM) + for tid in sorted(m["technical_repeats"]): + out += _block(mh + ".technical_repeats." + _key(tid), + m["technical_repeats"][tid], UNIT) + else: + for mid in sorted(doc.get("masks", {})): + out += _block("masks." + _key(mid), doc["masks"][mid], MASK, ("source_channels",)) + return out + + +def write(doc: dict) -> bytes: + """Canonical bytes of a valid document (experiment-v1 section 9.1).""" + out = [f"schema = {_str(doc['schema'])}", f"schema_version = {doc['schema_version']}", ""] + for sec in ("conditions", "channels", "repeats", "masks"): + out += section_lines(doc, sec) + out.append("[used_ids]") + for k in USED: + out.append(f"{k} = {_val(sorted(doc['used_ids'][k]))}") + return ("\n".join(out) + "\n").encode("utf-8") + + +def model(doc): + """Parsed document with the arrays whose order is not semantic sorted (section 9.2).""" + def walk(x, key=None): + if is_table(x): + return {k: walk(v, k) for k, v in x.items()} + if type(x) is list: + return sorted(x) if key in ("header_names", "source_channels") else list(x) + return x + d = walk(doc) + d["used_ids"] = {k: sorted(v) for k, v in d.get("used_ids", {}).items()} + return d + + +# ------------------------------------------------------------------ digests (section 9.3) +DESCRIPTIVE = {"display_name", "target", "treatment_key"} + + +def _strip(x): + if is_table(x): + return {k: _strip(v) for k, v in x.items() if k not in DESCRIPTIVE} + if type(x) is list: + return [_strip(v) for v in x] + return x + + +def section_digests(doc: dict) -> dict[str, str]: + """SHA-256 of one section's table blocks as the canonical writer emits them (section 9.3). + + The blocks of the stripped section (no `display_name`, `target`, `treatment_key`), each + followed by one blank line, nothing before or after; an absent section is the empty bytes. + """ + s = _strip(doc) + return {sec: hashlib.sha256("".join(f"{line}\n" for line in section_lines(s, sec)) + .encode("utf-8")).hexdigest() + for sec in ("conditions", "channels", "repeats", "masks")} + + +# ------------------------------------------------------------------ derived CSV (section 11) +CSV_COLUMNS = ("condition", "biological_repeat", "repeat_group", "control", "image_count", + "technical_repeat_count", "confirmed", "confirmed_on", "schema_version") + + +def derive_csv(doc: dict) -> bytes: + """The derived experiment_design.csv of a valid document (experiment-v1 section 11). + + AMBIGUITY: the contract (section 12) fixes UTF-8, LF, the header row, the column order, + `true`/`false` and the sort; it does not state the delimiter, the quoting, a final line + terminator or a byte-order mark. Here: comma, minimal quoting, LF after every row including + the last, no byte-order mark (as today's writer does). + """ + referenced = {c["reference"] for c in doc["conditions"].values() if "reference" in c} + rows = [] + for rid, cid, m in members(doc): + tr = m["technical_repeats"] + single = len(tr) == 1 and next(iter(tr.values()))["source_folder"] == "single" + rows.append([doc["conditions"][cid]["source_folder"], m["source_repeat"], rid, + "true" if cid in referenced else "false", str(m["image_count"]), + "0" if single else str(len(tr)), "true", + doc["repeats"][rid]["confirmed_on"], "1"]) + rows.sort(key=lambda r: (r[0], r[1])) + buf = io.StringIO() + w = csv.writer(buf, lineterminator="\n") + w.writerow(CSV_COLUMNS) + w.writerows(rows) + return buf.getvalue().encode("utf-8") + + +# ------------------------------------------------------------------ transitions (R_*) +def transition(old: dict, new: dict) -> set[str]: + codes: set[str] = set() + for k in USED: + ou, nu = set(old["used_ids"][k]), set(new["used_ids"][k]) + if ou - nu: + codes.add("R_USED_ID_DROPPED") + old_declared = (set(old[k]) if k != "technical_repeats" + else {t for _, _, t, _ in units(old)}) + new_declared = (set(new[k]) if k != "technical_repeats" + else {t for _, _, t, _ in units(new)}) + if (ou - old_declared) & new_declared: + codes.add("R_USED_ID_REUSED") + + def rep_folders(doc, rid): + return frozenset((c, m["source_repeat"]) for c, m in doc["repeats"][rid]["members"].items()) + + def cond_folders(doc, cid): + return {m["source_repeat"] for _, c, m in members(doc) if c == cid} + + def tech(doc): + out: dict[str, tuple[set, set]] = {} + for rid, cid, tid, u in units(doc): + src = doc["repeats"][rid]["members"][cid]["source_repeat"] + f, ms = out.setdefault(tid, (set(), set())) + f.add(u["source_folder"]) + ms.add((cid, src)) + return out + + def positions(doc): + pos: dict[str, set] = {} + for rid, cid, tid, u in units(doc): + src = doc["repeats"][rid]["members"][cid]["source_repeat"] + for i, ch in enumerate(u["channels"]): + pos.setdefault(ch, set()).add((cid, src, tid, i)) + return pos + + # (a) same member folders, other repeat ID + old_by_f = {rep_folders(old, r): r for r in old["repeats"]} + for rid in new["repeats"]: + f = rep_folders(new, rid) + if f in old_by_f and old_by_f[f] != rid: + codes.add("R_CONFIRMED_ID_CHANGED") + # (b) channel ID replaced at every position with the same header evidence + op, np_ = positions(old), positions(new) + for ch, ps in op.items(): + if ch in new["channels"]: + continue + for nch, nps in np_.items(): + if nch not in old["channels"] and nps == ps and \ + sorted(old["channels"][ch].get("header_names", [])) == \ + sorted(new["channels"][nch].get("header_names", [])): + codes.add("R_CONFIRMED_ID_CHANGED") + # (b2) same source_folder, other condition / technical repeat ID + old_cf = {c["source_folder"]: cid for cid, c in old["conditions"].items()} + for cid, c in new["conditions"].items(): + if c["source_folder"] in old_cf and old_cf[c["source_folder"]] != cid: + codes.add("R_CONFIRMED_ID_CHANGED") + ot, nt = tech(old), tech(new) + old_tf = {f: tid for tid, (fs, _) in ot.items() for f in fs} + for tid, (fs, _) in nt.items(): + for f in fs: + if f in old_tf and old_tf[f] != tid: + codes.add("R_CONFIRMED_ID_CHANGED") + # (c) an ID declared in both names a changed entity (something removed or reassigned) + for rid in set(old["repeats"]) & set(new["repeats"]): + if not rep_folders(old, rid) <= rep_folders(new, rid): + codes.add("R_CONFIRMED_ID_CHANGED") + for cid in set(old["conditions"]) & set(new["conditions"]): + if old["conditions"][cid]["source_folder"] != new["conditions"][cid]["source_folder"] \ + or not cond_folders(old, cid) <= cond_folders(new, cid): + codes.add("R_CONFIRMED_ID_CHANGED") + for tid in set(ot) & set(nt): + if ot[tid][0] != nt[tid][0] or not ot[tid][1] <= nt[tid][1]: + codes.add("R_CONFIRMED_ID_CHANGED") + for ch in set(old["channels"]) & set(new["channels"]): + if not op.get(ch, set()) <= np_.get(ch, set()) or \ + not set(old["channels"][ch].get("header_names", [])) <= \ + set(new["channels"][ch].get("header_names", [])): + codes.add("R_CONFIRMED_ID_CHANGED") + # confirmed_on of a repeat whose confirmation is not reopened (section 10: members, units and + # channel lists unchanged; a changed image_count alone does not reopen it) + def layout(rep): + return {c: {k: v for k, v in _strip(m).items() if k != "image_count"} + for c, m in rep["members"].items()} + + for rid in set(old["repeats"]) & set(new["repeats"]): + o, n = old["repeats"][rid], new["repeats"][rid] + if layout(o) == layout(n) and o["confirmed_on"] != n["confirmed_on"]: + codes.add("R_CONFIRMED_ON_CHANGED") + return codes + + +# ------------------------------------------------------------------ fixture catalogue +CODE = re.compile(r"\b([EWR]_[A-Z_]+)\b") + + +def run_fixtures(root: Path) -> int: + fx = root / "fixtures" + readme = (fx / "README.md").read_text(encoding="utf-8") + rows = {} + for line in readme.splitlines(): + m = re.match(r"\| \[`([vit]\d+[^`]*)`\]\([^)]+\) \|(.*)", line) + if m: + rows[m.group(1)] = [c.strip() for c in m.group(2).split("|")] + bad = 0 + docs = {} + counts = {} + for sub in ("valid", "invalid", "transitions"): + for f in sorted((fx / sub).glob("*.toml"), key=lambda p: int(re.match(r"\D(\d+)", p.stem)[1])): + data = f.read_bytes() + r, doc = validate(data) + docs[f.stem.split("_")[0]] = doc + row = rows.get(f.stem) + if row is None: + print(f"NO-CATALOGUE-ROW {sub}/{f.stem}") + bad += 1 + continue + if sub == "invalid": + exp = set(CODE.findall(row[1])) + ok = r.codes == exp and not r.warnings and str(r.phase) == row[2] + got = f"phase={r.phase} {sorted(r.codes)}" + want = f"phase={row[2]} {sorted(exp)}" + else: + cell = row[1] if sub == "valid" else row[2] + expw = {c for c in CODE.findall(cell) if c.startswith("W_")} + ok = not r.errors and r.warning_codes == expw + got = f"errors={sorted(r.codes)} warnings={sorted(r.warning_codes)}" + want = f"warnings={sorted(expw)}" + counts[sub] = counts.get(sub, 0) + 1 + if not ok: + bad += 1 + print(f"MISMATCH {sub}/{f.stem}: got {got}; expected {want}") + for e in r.errors[:5]: + print(" ", e) + for f in sorted((fx / "transitions").glob("*.toml")): + row = rows[f.stem] + pred = re.search(r"`valid/(v\d+)`|valid/(v\d+)", row[0] + " " + row[1] + " " + row[2]) + pred = pred.group(1) or pred.group(2) + exp = {c for c in CODE.findall(row[1]) if c.startswith("R_")} + got = transition(docs[pred], docs[f.stem.split("_")[0]]) + if got != exp: + bad += 1 + print(f"MISMATCH transition {f.stem} vs {pred}: got {sorted(got)} expected {sorted(exp)}") + counts["transition-checks"] = counts.get("transition-checks", 0) + 1 + # derived CSV golden files (section 11): each is listed in the README and equals the derivation + for f in sorted((fx / "expected").glob("*.csv")): + key = f.stem.split("_")[0] + if f"`{f.name}`" not in readme: + bad += 1 + print(f"NO-CATALOGUE-ROW expected/{f.name}") + if key not in docs or derive_csv(docs[key]) != f.read_bytes(): + bad += 1 + print(f"MISMATCH expected/{f.name}: not the derivation of {key}") + counts["expected"] = counts.get("expected", 0) + 1 + # round trip and digests + for k, d in docs.items(): + if k.startswith("v") or k.startswith("t"): + once = write(d) + if model(tomllib.loads(once.decode())) != model(d) or \ + write(tomllib.loads(once.decode())) != once: + bad += 1 + print(f"MISMATCH round trip {k}") + if docs["v18"] != docs["v01"]: + bad += 1 + print("MISMATCH v18 model != v01") + for k in ("v16", "v21"): + if section_digests(docs[k]) != section_digests(docs["v01"]): + bad += 1 + print(f"MISMATCH {k} digests != v01") + n_records, bad_records = refrecords.run_fixtures(root) # the records goldens (section 4.6) + bad += bad_records + counts["records"] = n_records + print(f"checked {counts}; round trip on {sum(1 for k in docs if k[0] in 'vt')} files; " + f"mismatches: {bad}") + return bad + + +def main(argv: list[str]) -> int: + if len(argv) >= 2 and argv[0] == "check": + r, _ = validate(Path(argv[1]).read_bytes()) + if "--json" in argv: + print(json.dumps(r.as_dict(), ensure_ascii=False)) + else: + for e in r.errors: + print("ERROR", *e) + for w in r.warnings: + print("WARN ", *w) + print("phase", r.phase, "OK" if not r.errors else "REFUSED") + return 1 if r.errors else 0 + if len(argv) == 2 and argv[0] == "fixtures": + return 1 if run_fixtures(Path(argv[1])) else 0 + if len(argv) == 3 and argv[0] == "transition": + old = tomllib.loads(Path(argv[1]).read_text(encoding="utf-8")) + new = tomllib.loads(Path(argv[2]).read_text(encoding="utf-8")) + print(sorted(transition(old, new))) + return 0 + print(__doc__) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/test_checks.py b/tests/test_checks.py index 83db69d..3237df0 100644 --- a/tests/test_checks.py +++ b/tests/test_checks.py @@ -67,6 +67,8 @@ def summaries_of(design, seed=0): SINGLE, histograms, position_id="p", + # IDs that look like indices are legal Tokens; the checks go by ID. + channels=tuple(f"C{c}" for c in range(len(histograms))), ) ) return image_summaries(records) @@ -88,7 +90,7 @@ def test_a_shifted_image_is_an_outlier_in_its_median_and_high_percentile(): design["control", "2024_05_02"][3] *= 1.6 issues = intensity_checks(summaries_of(design)) shifted = path("control", "2024_05_02", 3) - assert found(issues) == [("image_intensity_outlier", (shifted,), 0)] * 2 + assert found(issues) == [("image_intensity_outlier", (shifted,), "C0")] * 2 median, high = issues assert median.message.startswith(f"Image {shifted}, channel C0: its median intensity ") assert "the median of the other 5 images of condition 'control', biological repeat " in ( @@ -124,7 +126,7 @@ def test_a_shifted_repeat_is_reported_for_its_condition_only(): design = balanced() design["control", "2024_05_03"] = [level * 1.5 for level in design["control", "2024_05_03"]] issues = intensity_checks(summaries_of(design)) - assert found(issues) == [("repeat_intensity_shift", ("control/2024_05_03",), 0)] + assert found(issues) == [("repeat_intensity_shift", ("control/2024_05_03",), "C0")] assert issues[0].message.startswith( "Biological repeat '2024_05_03' of condition 'control', channel C0: the median of its " "6 image medians, " @@ -152,7 +154,7 @@ def test_groups_below_min_images_are_skipped(): issues = intensity_checks(summaries, IntensityCheckSettings(min_images=4)) assert ( found(issues, "image_intensity_outlier") - == [("image_intensity_outlier", (path("control", "2024_05_01", 1),), 0)] * 2 + == [("image_intensity_outlier", (path("control", "2024_05_01", 1),), "C0")] * 2 ) @@ -165,8 +167,8 @@ def test_repeat_check_needs_min_images_in_the_other_repeats(): issues = intensity_checks(summaries, IntensityCheckSettings(min_images=2)) # With two repeats, each is compared with the other, so both are reported. assert found(issues) == [ - ("repeat_intensity_shift", ("control/2024_05_01",), 0), - ("repeat_intensity_shift", ("control/2024_05_02",), 0), + ("repeat_intensity_shift", ("control/2024_05_01",), "C0"), + ("repeat_intensity_shift", ("control/2024_05_02",), "C0"), ] @@ -188,7 +190,7 @@ def test_saturated_images(): design = {("control", "rep1"): [many, few, image(100, rng)]} summaries = summaries_of(design) issues = intensity_checks(summaries) - assert found(issues) == [("saturated_image", (path("control", "rep1", 0),), 0)] + assert found(issues) == [("saturated_image", (path("control", "rep1", 0),), "C0")] assert issues[0].message == ( f"Image {path('control', 'rep1', 0)}, channel C0: 5.08 % of its pixels are at the maximum " "of its pixel type, 255 (uint8), above the limit of 1 %." @@ -203,8 +205,8 @@ def test_constant_images_have_no_spread(): design["control", "rep1"].append(image(100, np.random.default_rng(3))) issues = intensity_checks(summaries_of(design)) assert found(issues) == [ - ("no_intensity_spread", (path("control", "rep1", 0),), 0), - ("no_intensity_spread", (path("control", "rep1", 1),), 0), + ("no_intensity_spread", (path("control", "rep1", 0),), "C0"), + ("no_intensity_spread", (path("control", "rep1", 1),), "C0"), ] assert issues[0].message == ( f"Image {path('control', 'rep1', 0)}, channel C0: its 1st and 99.8th percentiles are " @@ -219,7 +221,7 @@ def test_each_channel_is_checked_separately(): design["treated", "2024_05_02"][4] = (first, second * 0.4) # only channel 1 is dimmer issues = intensity_checks(summaries_of(design)) shifted = path("treated", "2024_05_02", 4) - assert found(issues) == [("image_intensity_outlier", (shifted,), 1)] * 2 + assert found(issues) == [("image_intensity_outlier", (shifted,), "C1")] * 2 assert all(", channel C1: " in issue.message for issue in issues) @@ -271,6 +273,7 @@ def test_checks_match_a_direct_implementation(): SINGLE, np.dtype(np.uint16), (1.0, 99.8), + ("C0", "C1"), tuple(map(float, median)), tuple(map(float, low)), tuple(map(float, high)), @@ -321,7 +324,7 @@ def beyond(value, others): ] if len(others) >= settings.min_images and beyond(np.median(own), others): shifts.append(("repeat_intensity_shift", (f"{condition}/{repeat}",), c)) - return outliers + shifts + saturated + spread + return [(code, items, f"C{c}") for code, items, c in outliers + shifts + saturated + spread] def test_settings_defaults_and_round_trip(): @@ -366,7 +369,7 @@ def test_unknown_settings_are_refused(): def test_other_warnings_have_no_channel(): issue = Issue("warning", "repeat_missing_conditions", "message") assert issue.channel is None - assert replace(issue, channel=1) != issue + assert replace(issue, channel="DNA") != issue def _direct(path_, channels, percentile=(1.0, 99.8)): @@ -379,6 +382,7 @@ def _direct(path_, channels, percentile=(1.0, 99.8)): technical_repeat="", dtype=np.dtype(np.uint8), percentile=percentile, + channels=tuple(f"C{c}" for c in range(channels)), median=values, p_low=values, p_high=tuple(v + 5 for v in values), @@ -393,15 +397,24 @@ def test_summary_with_mismatched_channel_tuples_is_refused(): with pytest.raises(IncompatibleSummariesError, match=r"a\.tif.*p_high: 1"): replace(good, p_high=(1.0,)) with pytest.raises(IncompatibleSummariesError, match=r"a\.tif"): - replace(good, median=(), p_low=(), p_high=(), saturated_fraction=(), exact=()) + replace(good, channels=(), median=(), p_low=(), p_high=(), saturated_fraction=(), exact=()) + with pytest.raises(IncompatibleSummariesError, match=r"a\.tif.*channels: 1"): + replace(good, channels=("C0",)) + with pytest.raises(IncompatibleSummariesError, match="not in sorted order"): + replace(good, channels=("C1", "C0")) @pytest.mark.parametrize("order", [(1, 2), (2, 1)]) -def test_summaries_with_different_channel_counts_are_refused(order): - """Issue #56: one- and two-channel images in either order, naming both images.""" - summaries = [_direct(f"img_{i}.tif", n) for i, n in enumerate(order * 3)] - with pytest.raises(IncompatibleSummariesError, match=r"1 channel\(s\).*2 channel\(s\)"): - intensity_checks(summaries) +def test_summaries_with_different_channels_are_compared_by_channel_id(order): + """ADR 0003 replaces N43: a channel is compared among the images that have it.""" + summaries = [] + for i, n in enumerate(order * 3): + s = _direct(f"img_{i}.tif", n) + level = (10.0, 11.0, 12.0, 10.5, 11.5, 1000.0)[i] # img_5 is far from the others in C0 + summaries.append(replace(s, median=(level,) * n, p_low=(level,) * n)) + found_ = found(intensity_checks(summaries)) + # C1 is only in three images, below min_images: not compared, and never taken as zero. + assert found_ == [("image_intensity_outlier", ("img_5.tif",), "C0")] @pytest.mark.parametrize("odd_first", [True, False]) diff --git a/tests/test_csv_io.py b/tests/test_csv_io.py index 06a3fcf..acb7232 100644 --- a/tests/test_csv_io.py +++ b/tests/test_csv_io.py @@ -6,19 +6,31 @@ import pytest from morphobricks_normalization import ( + ChannelMeasurementError, + ChannelMeasurementFailure, PlaneRecord, compute_plane_histogram, csv_io, make_plane_id, + read_plane_histogram_table, read_plane_histograms, write_plane_histograms, ) -from morphobricks_normalization.csv_io import METADATA_COLUMNS, CsvFormatError, bin_column +from morphobricks_normalization.csv_io import ( + METADATA_COLUMNS, + CsvFormatError, + bin_column, + header_channels, + table_header, +) rng = np.random.default_rng(7) DATA = Path(__file__).parent / "data" -# Contract §6.1 (N42): __C[_], CamelCase words. -PER_CHANNEL_NAME = re.compile(r"[A-Z][A-Za-z]*_[A-Z][A-Za-z]*_C(0|[1-9][0-9]*)(_[0-9]{5})?") +# Contract §6.1 (N42 as amended by ADR 0003 HV2): __[_]. +TOKEN = r"[A-Za-z][A-Za-z0-9]*(?:_[A-Za-z0-9]+)*" +PER_CHANNEL_NAME = re.compile(rf"[A-Z][A-Za-z]*_[A-Z][A-Za-z]*_{TOKEN}") +IDS = ("DNA", "ACTIN") # source order C0, C1; sorted they are ACTIN, DNA +UNIVERSE = ("DNA", "ACTIN", "PHASE") def record( @@ -27,12 +39,21 @@ def record( channels=None, bin_count=None, position_id="p", + ids=None, **coords, ): if channels is None: channels = [rng.integers(0, 4000, (16, 16)).astype(np.uint16) for _ in range(2)] + if ids is None: + ids = IDS[: len(channels)] if len(channels) <= 2 else UNIVERSE[: len(channels)] histograms = [compute_plane_histogram(pixels, bin_count) for pixels in channels] - return PlaneRecord(path, *labels, histograms=histograms, position_id=position_id, **coords) + return PlaneRecord( + path, *labels, histograms=histograms, position_id=position_id, channels=ids, **coords + ) + + +def write(records, path, channel_ids=UNIVERSE, **kwargs): + return write_plane_histograms(records, path, channel_ids=channel_ids, **kwargs) def assert_same(a, b): @@ -47,6 +68,7 @@ def assert_same(a, b): "plane_id", "image_file", "channel_count", + "channels", ): assert getattr(a, field) == getattr(b, field) for ha, hb in zip(a.histograms, b.histograms, strict=True): @@ -65,7 +87,7 @@ def test_integer_round_trip(tmp_path): record(channels=three), record("single/single/single/b.tif", ("single",) * 3, channels=three[::-1], z=0, t=5), ] - path = write_plane_histograms(records, tmp_path / "plane_histograms.csv") + path = write(records, tmp_path / "plane_histograms.csv") for a, b in zip(records, read_plane_histograms(path), strict=True): assert_same(a, b) @@ -83,7 +105,7 @@ def test_float_round_trip_keeps_full_precision(tmp_path): bin_count=128, ), ] - path = write_plane_histograms(original, tmp_path / "t.csv") + path = write(original, tmp_path / "t.csv") for a, b in zip(original, read_plane_histograms(path), strict=True): assert_same(a, b) @@ -91,12 +113,12 @@ def test_float_round_trip_keeps_full_precision(tmp_path): @pytest.mark.parametrize("label", ["NA", "nan", "None", "001", "TRUE", "1e3", "a,b", 'say "hi"']) def test_labels_are_kept_as_text(tmp_path, label): original = record(f"{label}/img.tif", (label, "single", "single")) - (back,) = read_plane_histograms(write_plane_histograms([original], tmp_path / "t.csv")) + (back,) = read_plane_histograms(write([original], tmp_path / "t.csv")) assert back.condition == label and back.source_relative_path == f"{label}/img.tif" def test_header_follows_the_contract(tmp_path): - path = write_plane_histograms([record(bin_count=3)], tmp_path / "t.csv") + path = write([record(bin_count=3)], tmp_path / "t.csv") header = path.read_text(encoding="utf-8").splitlines()[0].split(",") assert tuple(header[: len(METADATA_COLUMNS)]) == METADATA_COLUMNS assert header[:9] == [ @@ -110,47 +132,54 @@ def test_header_follows_the_contract(tmp_path): "source_relative_path", "plane_id", ] - assert "c" not in header - per_channel = header[len(METADATA_COLUMNS) :] + assert "c" not in header and "failed_channels" not in header + # HV1: one presence column right after the plane columns. + assert header[len(METADATA_COLUMNS)] == "channels" + per_channel = header[len(METADATA_COLUMNS) + 1 :] expected = [] - for c in (0, 1): # grouped by channel: features, then bins + for c in ("ACTIN", "DNA", "PHASE"): # every ID of the experiment, sorted; features, bins expected += [ - f"Intensity_MinIntensity_C{c}", - f"Intensity_MaxIntensity_C{c}", - f"Histogram_ValidPixelCount_C{c}", - f"Histogram_BinningMin_C{c}", - f"Histogram_BinningMax_C{c}", - f"Histogram_UnderflowCount_C{c}", - f"Histogram_UnderflowSum_C{c}", - f"Histogram_UnderflowSumOfSquares_C{c}", - f"Histogram_OverflowCount_C{c}", - f"Histogram_OverflowSum_C{c}", - f"Histogram_OverflowSumOfSquares_C{c}", + f"Intensity_MinIntensity_{c}", + f"Intensity_MaxIntensity_{c}", + f"Histogram_ValidPixelCount_{c}", + f"Histogram_BinningMin_{c}", + f"Histogram_BinningMax_{c}", + f"Histogram_UnderflowCount_{c}", + f"Histogram_UnderflowMean_{c}", + f"Histogram_UnderflowStd_{c}", + f"Histogram_OverflowCount_{c}", + f"Histogram_OverflowMean_{c}", + f"Histogram_OverflowStd_{c}", ] - expected += [f"Histogram_Count_C{c}_{i:05d}" for i in range(3)] + expected += [f"Histogram_Count_{c}_{i:05d}" for i in range(3)] assert per_channel == expected assert all(PER_CHANNEL_NAME.fullmatch(name) for name in per_channel) + assert header == table_header(["PHASE", "DNA", "ACTIN"], 3) -def test_rows_hold_every_channel_side_by_side(tmp_path): +def test_rows_hold_every_channel_side_by_side_by_id(tmp_path): channels = [np.array([[3, 5], [5, 9]], np.uint16), np.array([[0, 0], [1, 2]], np.uint16)] - path = write_plane_histograms([record(channels=channels, bin_count=8, z=2)], tmp_path / "t.csv") + path = write([record(channels=channels, bin_count=8, z=2)], tmp_path / "t.csv") with open(path, newline="", encoding="utf-8") as fh: (row,) = csv.DictReader(fh) assert row["plane_id"] == "control/bio_rep1/tech_rep1/img.tif::z=2,t=0" assert (row["dtype"], row["bit_depth"], row["channel_count"]) == ("uint16", "16", "2") - assert (row["total_pixel_count"], row["bin_count"], row["schema_version"]) == ("4", "8", "4") - assert (row["Intensity_MinIntensity_C0"], row["Intensity_MaxIntensity_C0"]) == ("3", "9") - assert (row["Histogram_BinningMin_C1"], row["Histogram_BinningMax_C1"]) == ("0", "2") - assert row["Histogram_ValidPixelCount_C1"] == "4" + assert (row["total_pixel_count"], row["bin_count"], row["schema_version"]) == ("4", "8", "5") + # Source order in the presence column: C0 is DNA, C1 is ACTIN. + assert row["channels"] == "DNA;ACTIN" + assert (row["Intensity_MinIntensity_DNA"], row["Intensity_MaxIntensity_DNA"]) == ("3", "9") + assert (row["Histogram_BinningMin_ACTIN"], row["Histogram_BinningMax_ACTIN"]) == ("0", "2") + assert row["Histogram_ValidPixelCount_ACTIN"] == "4" # The min-max range leaves no pixel outside the bins (contract §1.4). assert [ - row[f"Histogram_{tail}{what}_C0"] + row[f"Histogram_{tail}{what}_DNA"] for tail in ("Underflow", "Overflow") - for what in ("Count", "Sum", "SumOfSquares") + for what in ("Count", "Mean", "Std") ] == ["0", "0.0", "0.0"] * 2 - assert [row[bin_column(0, i)] for i in range(8)] == ["1", "0", "2", "0", "0", "0", "1", "0"] - assert [row[bin_column(1, i)] for i in range(3)] == ["2", "1", "1"] + assert [row[bin_column("DNA", i)] for i in range(8)] == ["1", "0", "2", "0", "0", "0", "1", "0"] + assert [row[bin_column("ACTIN", i)] for i in range(3)] == ["2", "1", "1"] + # PHASE is in the experiment but absent from this plane: every cell blank, never zero. + assert all(row[name] == "" for name in row if name.endswith("_PHASE") or "_PHASE_" in name) def test_plane_id_has_both_coordinates_and_no_channel(tmp_path): @@ -163,73 +192,129 @@ def test_plane_id_has_both_coordinates_and_no_channel(tmp_path): def test_mixed_bin_counts_are_rejected(tmp_path): with pytest.raises(ValueError, match="same bin count"): - write_plane_histograms([record(bin_count=10), record(bin_count=20)], tmp_path / "t.csv") + write([record(bin_count=10), record(bin_count=20)], tmp_path / "t.csv") -def test_images_with_different_channel_counts_are_not_written(tmp_path): +def test_images_with_different_channels_are_written_with_blank_absent_cells(tmp_path): + # ADR 0003 HV1 replaces N43: rows may have different channels. one = [rng.integers(0, 10, (4, 4)).astype(np.uint16)] - planes = [record("a/1.tif"), record("a/2.tif", channels=one), record("a/2.tif", channels=one)] - with pytest.raises(ValueError) as error: - write_plane_histograms(planes, tmp_path / "t.csv") - message = str(error.value) - assert "These images have different numbers of channels" in message - assert "(1 channel(s): a/2.tif; 2 channel(s): a/1.tif)" in message - assert "Every image of an experiment must have the same channels" in message + planes = [record("a/1.tif"), record("a/2.tif", channels=one, ids=("PHASE",))] + table = read_plane_histogram_table(write(planes, tmp_path / "t.csv")) + assert table.channel_ids == ("ACTIN", "DNA", "PHASE") + assert [r.channels for r in table.records] == [("DNA", "ACTIN"), ("PHASE",)] + for a, b in zip(planes, table.records, strict=True): + assert_same(a, b) + + +def test_a_record_channel_outside_the_experiment_is_refused(tmp_path): + with pytest.raises(ValueError, match=r"not in the experiment's channel IDs.*PAX_488"): + write([record(ids=("DNA", "PAX_488"))], tmp_path / "t.csv") assert not (tmp_path / "t.csv").exists() +@pytest.mark.parametrize("ids", [[], ["DNA", "DNA"], ["0"], [0, 1], "DNA"]) +def test_the_experiment_channel_ids_must_be_ids(tmp_path, ids): + with pytest.raises(ValueError): + write([record()], tmp_path / "t.csv", channel_ids=ids) + + +def test_planes_of_one_image_with_different_channels_are_refused(tmp_path): + planes = [record("a/1.tif", t=0), record("a/1.tif", ids=("ACTIN", "DNA"), t=1)] + with pytest.raises(ValueError, match=r"a/1\.tif \(DNA;ACTIN in .*ACTIN;DNA in"): + write(planes, tmp_path / "t.csv") + + +def test_a_failed_measurement_writes_nothing(tmp_path): + # HV3: the table is not written, and the run is told which plane and channel failed. + path = tmp_path / "t.csv" + failure = ChannelMeasurementFailure(record("a/2.tif").plane_id, "ACTIN", "read error") + with pytest.raises(ChannelMeasurementError) as error: + write([record("a/1.tif"), record("a/2.tif")], path, failures=[failure]) + assert error.value.failures == (failure,) and error.value.removed is None + assert "plane a/2.tif::z=0,t=0, channel ACTIN: read error" in str(error.value) + assert "plane_histograms.csv was not written" in str(error.value) + assert "earlier run" not in str(error.value) + assert list(tmp_path.iterdir()) == [] + + +def test_a_failed_measurement_removes_an_earlier_table_so_none_looks_current(tmp_path): + # Open question 5 of PR #90: an earlier run's table left in place would read as this + # run's, so the writer removes it and says so; nothing else in the folder is touched. + path = write([record()], tmp_path / "t.csv") + other = tmp_path / "other.csv" + other.write_text("kept", encoding="utf-8") + failure = ChannelMeasurementFailure(record("a/2.tif").plane_id, "ACTIN", "read error") + with pytest.raises(ChannelMeasurementError) as error: + write([record("a/1.tif"), record("a/2.tif")], path, failures=[failure]) + assert error.value.removed == str(path) and error.value.failures == (failure,) + assert f"The table of an earlier run, {path}, was removed" in str(error.value) + assert [p.name for p in tmp_path.iterdir()] == ["other.csv"] + assert other.read_text(encoding="utf-8") == "kept" + + def test_the_channels_of_a_plane_share_type_bins_and_size(): a = compute_plane_histogram(np.arange(16, dtype=np.uint16).reshape(4, 4)) - with pytest.raises(ValueError, match="channel 1 has pixel type uint8, but channel 0"): + two = {"position_id": "p", "channels": ("DNA", "ACTIN")} + with pytest.raises(ValueError, match="channel ACTIN has pixel type uint8, but channel DNA"): PlaneRecord( - "p.tif", - "c", - "r", - "t", - (a, compute_plane_histogram(np.zeros(16, np.uint8))), - position_id="p", + "p.tif", "c", "r", "t", (a, compute_plane_histogram(np.zeros(16, np.uint8))), **two ) - with pytest.raises(ValueError, match="channel 1 has bin count 4"): + with pytest.raises(ValueError, match="channel ACTIN has bin count 4"): PlaneRecord( - "p.tif", - "c", - "r", - "t", - (a, compute_plane_histogram(np.zeros(16, a.dtype), 4)), - position_id="p", + "p.tif", "c", "r", "t", (a, compute_plane_histogram(np.zeros(16, a.dtype), 4)), **two ) - with pytest.raises(ValueError, match="channel 1 has number of pixels 9"): + with pytest.raises(ValueError, match="channel ACTIN has number of pixels 9"): PlaneRecord( - "p.tif", - "c", - "r", - "t", - (a, compute_plane_histogram(np.zeros(9, a.dtype))), - position_id="p", + "p.tif", "c", "r", "t", (a, compute_plane_histogram(np.zeros(9, a.dtype))), **two ) with pytest.raises(ValueError, match="has no channels"): - PlaneRecord("p.tif", "c", "r", "t", (), position_id="p") - single = PlaneRecord("p.tif", "c", "r", "t", a, position_id="p") + PlaneRecord("p.tif", "c", "r", "t", (), position_id="p", channels=()) + single = PlaneRecord("p.tif", "c", "r", "t", a, position_id="p", channels=("DNA",)) assert single.histograms == (a,) and single.channel_count == 1 - with pytest.raises(ValueError, match="has 1 channel"): - single.channel_histogram(1) + assert single.channel_histogram("DNA") is a + with pytest.raises(ValueError, match="channel ACTIN is absent from it"): + single.channel_histogram("ACTIN") + with pytest.raises(ValueError, match="never by its position"): + single.channel_histogram(0) + + +@pytest.mark.parametrize( + "ids, match", + [ + (("DNA",), "2 histogram\\(s\\) but 1 channel ID"), + (("DNA", "DNA"), "listed more than once"), + (("DNA", "C 1"), "is not a canonical channel ID"), + ((0, 1), "never by its position"), + ("DNA", "expected a list of channel IDs"), + ], +) +def test_a_record_needs_one_channel_id_per_histogram(ids, match): + a = compute_plane_histogram(np.arange(16, dtype=np.uint16)) + with pytest.raises(ValueError, match=match): + PlaneRecord("p.tif", "c", "r", "t", (a, a), position_id="p", channels=ids) + + +def test_channels_have_no_default_and_are_keyword_only(): + histogram = compute_plane_histogram(np.arange(6)) + with pytest.raises(TypeError, match="channels"): + PlaneRecord("c/a.tif", "c", "r", "t", histogram, position_id="p") def test_failed_write_leaves_previous_file_untouched(tmp_path, monkeypatch): - path = write_plane_histograms([record()], tmp_path / "t.csv") + path = write([record()], tmp_path / "t.csv") before = path.read_bytes() calls = iter([False, True]) - def failing_row(rec): + def failing_row(rec, ids): if next(calls): raise RuntimeError("simulated failure") - return original_row(rec) + return original_row(rec, ids) original_row = csv_io._row monkeypatch.setattr(csv_io, "_row", failing_row) with pytest.raises(RuntimeError): # Two different planes: one plane twice is refused before writing (issue #43). - write_plane_histograms([record("a/1.tif"), record("a/2.tif")], path) + write([record("a/1.tif"), record("a/2.tif")], path) assert path.read_bytes() == before assert [p.name for p in tmp_path.iterdir()] == ["t.csv"] # no temporary files left @@ -248,7 +333,7 @@ def edit_table(path, changes, row_index=0): def table_with(tmp_path, column, value): """Write a one-row table, then set ``column`` of its row to ``value``.""" - path = write_plane_histograms([record(bin_count=2)], tmp_path / "t.csv") + path = write([record(bin_count=2)], tmp_path / "t.csv") return edit_table(path, {column: value}) @@ -265,37 +350,135 @@ def test_unknown_schema_version_is_rejected(tmp_path): @pytest.mark.parametrize( "column, value", [ - ("Histogram_ValidPixelCount_C1", "1"), - ("Histogram_Count_C0_00001", "999"), - ("Intensity_MaxIntensity_C1", "1"), + ("Histogram_ValidPixelCount_ACTIN", "1"), + ("Histogram_Count_DNA_00001", "999"), + ("Intensity_MaxIntensity_ACTIN", "1"), ], ) def test_inconsistent_channel_columns_are_rejected(tmp_path, column, value): - channel = re.search(r"_C([0-9]+)", column)[1] + channel = "ACTIN" if "ACTIN" in column else "DNA" with pytest.raises(CsvFormatError, match=f"bin counts of channel {channel} are inconsistent"): read_plane_histograms(table_with(tmp_path, column, value)) -def test_rows_with_another_channel_count_are_rejected(tmp_path): - planes = [record("a/1.tif"), record("a/2.tif"), record("a/3.tif")] - path = write_plane_histograms(planes, tmp_path / "t.csv") - edit_table(path, {"channel_count": "3"}, row_index=1) - with pytest.raises(CsvFormatError) as error: +@pytest.mark.parametrize( + "column, value", + [ + ("Histogram_ValidPixelCount_PHASE", "0"), # a fabricated zero + ("Histogram_Count_PHASE_00000", "0"), + ("Intensity_MinIntensity_PHASE", "nan"), # a fabricated NaN + ("Histogram_UnderflowMean_PHASE", "0.0"), + ], +) +def test_a_value_in_an_absent_channel_is_refused(tmp_path, column, value): + with pytest.raises(CsvFormatError, match=f"line 2: channel PHASE is absent .*{column}"): + read_plane_histograms(table_with(tmp_path, column, value)) + + +def test_a_listed_channel_with_blank_cells_is_refused(tmp_path): + # Listing a channel claims a measured histogram; blank counts are not one (HV3). + path = write([record(bin_count=2)], tmp_path / "t.csv") + columns = csv_io.channel_columns("ACTIN", 2) + edit_table(path, dict.fromkeys(columns, "")) + with pytest.raises(CsvFormatError, match="line 2"): read_plane_histograms(path) - message = str(error.value) - assert "Its images have different numbers of channels" in message - assert "(2 channel(s): a/1.tif, a/3.tif; 3 channel(s): a/2.tif)" in message - assert "Every image of an experiment must have the same channels" in message -def test_unexpected_channel_columns_are_rejected(tmp_path): - path = write_plane_histograms([record(bin_count=2)], tmp_path / "t.csv") +@pytest.mark.parametrize( + "value, match", + [ + ("", "channels cell is blank"), + ("DNA;DNA", "listed more than once"), + ("DNA;PAX", "lists PAX, which the header does not have"), + ("DNA", "channel_count is '2', but its channels cell lists 1"), + ("DNA ;ACTIN", "is not a canonical channel ID"), + ("DNA,ACTIN", "is not a canonical channel ID"), + ], +) +def test_a_bad_presence_cell_is_refused(tmp_path, value, match): + with pytest.raises(CsvFormatError, match=f"line 2: .*{match}"): + read_plane_histograms(table_with(tmp_path, "channels", value)) + + +def test_listing_an_absent_channel_as_present_is_refused(tmp_path): + # PHASE's cells are blank, so claiming it present (with a matching count) is refused. + path = table_with(tmp_path, "channels", "DNA;ACTIN;PHASE") + edit_table(path, {"channel_count": "3"}) + with pytest.raises(CsvFormatError, match="line 2"): + read_plane_histograms(path) + + +def test_rows_of_one_image_with_different_channels_are_refused(tmp_path): + planes = [record("a/1.tif", t=0), record("a/1.tif", t=1)] + path = write(planes, tmp_path / "t.csv") + with open(path, newline="", encoding="utf-8") as fh: + header, *rows = list(csv.reader(fh)) + # Swap the presence list of the second plane and its column groups to match: each row is + # valid on its own, but one image cannot have two channel orders. + rows[1][header.index("channels")] = "ACTIN;DNA" + with open(path, "w", newline="", encoding="utf-8") as fh: + csv.writer(fh).writerows([header, *rows]) + with pytest.raises(CsvFormatError, match="planes list different channels"): + read_plane_histograms(path) + + +@pytest.mark.parametrize( + "old, new", + [ + ("Histogram_BinningMax_ACTIN", "Histogram_BinningMax_ACTINX"), # broken group + (",channels,", ",presence,"), # no presence column + ], +) +def test_unexpected_channel_columns_are_rejected(tmp_path, old, new): + path = write([record(bin_count=2)], tmp_path / "t.csv") text = path.read_text(encoding="utf-8") - path.write_text(text.replace("Histogram_BinningMax_C1", "Histogram_BinningMax_C2"), "utf-8") - with pytest.raises(CsvFormatError, match="unexpected per-channel column names"): + path.write_text(text.replace(old, new, 1), "utf-8") + with pytest.raises(CsvFormatError, match="does not have the per-channel columns of schema"): read_plane_histograms(path) +def test_header_channels_follow_the_id_token_rule(): + # HV2: IDs come from Histogram_ValidPixelCount_, the bin index from the last _NNNNN, + # even when an ID contains "_" or ends in five digits. + ids = ["PAX_488", "DNA", "DNA_00001", "A_B_C"] + assert header_channels(table_header(ids, 4)) == (tuple(sorted(ids)), 4) + header = table_header(ids, 4) + assert bin_column("DNA", 1) in header and bin_column("DNA_00001", 1) in header + assert header.index(bin_column("DNA", 1)) < header.index("Histogram_ValidPixelCount_DNA_00001") + unsorted = table_header(["B"], 2) + csv_io.channel_columns("A", 2) + assert header_channels(unsorted) is None + missing = [name for name in table_header(["A"], 2) if name != "Histogram_BinningMin_A"] + assert header_channels(missing) is None + + +def test_a_version_4_table_is_refused_naming_version_5_and_the_fix(tmp_path): + # HV4: a hard cut; the table is recomputed, never converted. + path = tmp_path / "plane_histograms.csv" + with open(path, "w", newline="", encoding="utf-8") as fh: + csv.writer(fh).writerows( + [ + [*METADATA_COLUMNS, "Histogram_ValidPixelCount_C0"], + ["p", "img.tif", "0", "0", "c", "r", "t", "img.tif", "img.tif::z=0,t=0"] + + ["uint8", "8", "1", "4", "2", "4", "4"], + ] + ) + with pytest.raises(CsvFormatError) as error: + read_plane_histograms(path) + message = str(error.value) + assert message == ( + f"{path}: This plane_histograms.csv was written with schema version 4, its channels " + "named by their index in the image (C0, C1, ...). This version reads schema version 5 " + "only, with every channel named by its canonical channel ID from the experiment " + "description; old tables are not converted. Run the preprocessing brick again on this " + "experiment: it recomputes the table (schema version 5)." + ) + + +def test_no_converter_from_version_4_exists(): + names = {name for name in dir(csv_io) if not name.startswith("__")} + assert not {n for n in names if "convert" in n.lower() or "v4" in n.lower()} + + @pytest.mark.parametrize("version", ["2", "1"]) def test_tables_with_one_row_per_channel_are_refused(tmp_path, version): # Only the schema_version value is read: the layout of such a table is not (and a table @@ -310,11 +493,23 @@ def test_tables_with_one_row_per_channel_are_refused(tmp_path, version): ) with pytest.raises(CsvFormatError) as error: read_plane_histograms(path) - assert str(error.value) == ( + assert str(error.value).startswith( f"{path}: This plane_histograms.csv was written with schema version {version}, one row " - "per channel. Run the preprocessing brick again on this experiment: it rewrites the " - "table with the channels in columns (schema version 4)." + "per channel. This version reads schema version 5 only" ) + assert str(error.value).endswith("it recomputes the table (schema version 5).") + + +def test_writing_is_byte_deterministic_and_shards_share_one_header(tmp_path): + planes = [record(f"a/{i}.tif") for i in range(4)] + one = write(planes, tmp_path / "one.csv").read_bytes() + assert write(planes, tmp_path / "two.csv").read_bytes() == one + # Two shards given the same experiment IDs have the same header; their rows, merged in + # path order, give the serial table byte for byte. + first = write(planes[2:], tmp_path / "s1.csv").read_bytes().splitlines(keepends=True) + second = write(planes[:2], tmp_path / "s2.csv").read_bytes().splitlines(keepends=True) + assert first[0] == second[0] == one.splitlines(keepends=True)[0] + assert b"".join([second[0], *second[1:], *first[1:]]) == one def test_a_table_without_a_known_version_or_the_columns_is_refused_without_a_fix_guess(tmp_path): @@ -353,19 +548,19 @@ def test_derived_plane_columns_must_match_the_row(tmp_path, column, value, impli def test_a_record_without_a_position_id_is_refused(value): histogram = compute_plane_histogram(np.arange(6)) with pytest.raises(ValueError, match=r"c/a\.tif.*no position_id.*preprocessing brick"): - PlaneRecord("c/a.tif", "c", "r", "t", histogram, position_id=value) + PlaneRecord("c/a.tif", "c", "r", "t", histogram, position_id=value, channels=("DNA",)) def test_position_id_has_no_default_and_is_keyword_only(): histogram = compute_plane_histogram(np.arange(6)) with pytest.raises(TypeError, match="position_id"): - PlaneRecord("c/a.tif", "c", "r", "t", histogram) + PlaneRecord("c/a.tif", "c", "r", "t", histogram, channels=("DNA",)) with pytest.raises(TypeError): - PlaneRecord("c/a.tif", "c", "r", "t", histogram, 0, 0, "p") + PlaneRecord("c/a.tif", "c", "r", "t", histogram, 0, 0, "p", channels=("DNA",)) def test_a_row_with_a_blank_position_id_is_refused_not_filled(tmp_path): - path = write_plane_histograms([record()], tmp_path / "t.csv") + path = write([record()], tmp_path / "t.csv") with open(path, newline="", encoding="utf-8") as fh: header, *rows = list(csv.reader(fh)) for row in rows: @@ -378,5 +573,5 @@ def test_a_row_with_a_blank_position_id_is_refused_not_filled(tmp_path): def test_position_id_round_trips_as_written(tmp_path): original = record(position_id="NA") - (back,) = read_plane_histograms(write_plane_histograms([original], tmp_path / "t.csv")) + (back,) = read_plane_histograms(write([original], tmp_path / "t.csv")) assert back.position_id == "NA" diff --git a/tests/test_fit_goldens.py b/tests/test_fit_goldens.py index 9ecc25f..63d951a 100644 --- a/tests/test_fit_goldens.py +++ b/tests/test_fit_goldens.py @@ -10,6 +10,9 @@ Here the same pixels go through the current code (histograms, the ``plane_histograms.csv`` writer and reader, scope groups, ``fit``) and every parameter must equal the stored number. +Since schema version 5 the channels are named by canonical channel ID (ADR 0003): channel ``c`` +is ``ch``, whose sorted order is the index order, so the stored per-index numbers line up +with the parameters' channels and every channel is present in every plane. Parameters built from a mean or standard deviation are sums over the pooled values (``np.dot``, weighted ``np.bincount``), whose last bits depend on the NumPy build and the @@ -35,6 +38,7 @@ scope_groups, write_plane_histograms, ) +from simulation import channel_ids DATA = Path(__file__).parent / "data" DATASETS = ("uint16", "float32") @@ -74,10 +78,13 @@ def table_records(name, folder): histograms=[channels[c] for c in sorted(channels)], t=t, position_id=path.rsplit("/", 1)[-1].rsplit(".", 1)[0], + channels=channel_ids(len(channels)), ) for (path, t), (labels, channels) in by_plane.items() ] - table = write_plane_histograms(records, Path(folder) / f"golden_{name}.csv") + table = write_plane_histograms( + records, Path(folder) / f"golden_{name}.csv", channel_ids=channel_ids(2) + ) return read_plane_histograms(table) @@ -97,7 +104,7 @@ def test_fitted_parameters_equal_the_golden_numbers_at_every_scope(name, folder) group = entry["group"] if settings.scope == "patch": stack = np.stack([p for _, path, c, t, p in planes(name) if path == group and t == 0]) - statistics, options = stack, {"channel_axis": 0} + statistics, options = stack, {"channel_axis": 0, "channels": channel_ids(2)} else: key = json.dumps(entry["settings"], sort_keys=True) if key not in groups_by_settings: @@ -112,6 +119,7 @@ def test_fitted_parameters_equal_the_golden_numbers_at_every_scope(name, folder) else: parameters = fit(statistics, settings, group=group, **options) where = (entry["settings"], group) + assert parameters.channels == channel_ids(len(entry["offset"])), where if settings.method in SUMMED_METHODS: for value, stored in (("offset", entry["offset"]), ("scale", entry["scale"])): np.testing.assert_array_max_ulp( diff --git a/tests/test_fitting.py b/tests/test_fitting.py index 93d5def..4859e7b 100644 --- a/tests/test_fitting.py +++ b/tests/test_fitting.py @@ -5,6 +5,7 @@ import pytest from morphobricks_normalization import ( + ChannelScopeIncompleteError, MethodNotAllowedError, NormalizationParameters, NormalizationSettings, @@ -23,7 +24,7 @@ ) from morphobricks_normalization.fitting import COLUMNS, offset_column, scale_column from morphobricks_normalization.settings import FLUORESCENCE_METHODS -from simulation import cached_design +from simulation import cached_design, channel_id, channel_ids METHOD_SETTINGS = [ NormalizationSettings(method="minmax"), @@ -102,7 +103,7 @@ def test_numbers_are_those_of_the_existing_methods(settings): _, records = cached_design(seed=11, dtype=np.float32, bin_count=512) parameters = fit(records, settings) for c in (0, 1): - expected = functions[settings.method](PooledHistogram(records, c)) + expected = functions[settings.method](PooledHistogram(records, channel_id(c))) assert (parameters.offset[c], parameters.scale[c]) == (expected.offset, expected.scale) assert parameters.method == expected.method assert not parameters.exact @@ -121,10 +122,14 @@ def test_image_scope_from_pixels_equals_image_scope_from_the_file(method): for time in range(2) ] ) - from_pixels = fit(stack, settings, group=path, channel_axis=1) + ids = channel_ids(3) + from_pixels = fit(stack, settings, group=path, channel_axis=1, channels=ids) from_file = fit(members, settings, group=path) assert from_pixels == from_file - assert fit(stack, settings, group=path, channel_axis=-3) == from_file + assert fit(stack, settings, group=path, channel_axis=-3, channels=ids) == from_file + # The array's channels in another order, named by ID: the same parameters. + swapped = stack[:, ::-1] + assert fit(swapped, settings, group=path, channel_axis=1, channels=ids[::-1]) == from_file @pytest.mark.parametrize("settings", METHOD_SETTINGS, ids=lambda s: s.method_name) @@ -132,13 +137,16 @@ def test_patch_scope_from_pixels(settings): settings = with_scope(settings, scope="patch") rng = np.random.default_rng(13) patch = rng.integers(100, 4000, size=(16, 16, 2)).astype(np.uint16) - parameters = fit(patch, settings, group="control/r1/img.tif::7", channel_axis=2) + parameters = fit( + patch, settings, group="control/r1/img.tif::7", channel_axis=2, channels=channel_ids(2) + ) assert_matches_direct(parameters, [patch[..., 0], patch[..., 1]], settings) assert parameters.group == "control/r1/img.tif::7" and parameters.scope == "patch" # Pixels selected by a mask: one row of values per channel. mask = rng.random((16, 16)) > 0.5 selected = np.stack([patch[..., c][mask] for c in range(2)]) - assert_matches_direct(fit(selected, settings, channel_axis=0), list(selected), settings) + fitted = fit(selected, settings, channel_axis=0, channels=channel_ids(2)) + assert_matches_direct(fitted, list(selected), settings) def test_per_time_point_groups_fit_each_time_point(): @@ -152,25 +160,33 @@ def test_per_time_point_groups_fit_each_time_point(): def test_apply_matches_the_formula_per_channel(): - parameters = Parameters("z-score", "image", "a.tif", True, (10.0, -3.5), (2.0, 0.25)) + parameters = Parameters( + "z-score", "image", "a.tif", True, (10.0, -3.5), (2.0, 0.25), ("ACTIN", "DNA") + ) rng = np.random.default_rng(15) pixels = rng.normal(0, 50, size=(2, 5, 6)) pixels[1, 0, 0] = np.nan original = pixels.copy() - result = parameters.apply(pixels, channel_axis=0) + result = parameters.apply(pixels, channel_axis=0, channels=["ACTIN", "DNA"]) expected = np.stack([(pixels[0] - 10.0) / 2.0, (pixels[1] + 3.5) / 0.25]).astype(np.float32) np.testing.assert_array_equal(result, expected) assert result.dtype == np.float32 and np.isnan(result[1, 0, 0]) np.testing.assert_array_equal(pixels, original) # input never modified - moved = parameters.apply(np.moveaxis(pixels, 0, -1), channel_axis=-1) + moved = parameters.apply(np.moveaxis(pixels, 0, -1), channel_axis=-1, channels=["ACTIN", "DNA"]) np.testing.assert_array_equal(moved, np.moveaxis(expected, 0, -1)) + # Each entry is normalized with its own ID's parameters, never by position. + swapped = parameters.apply(pixels[::-1], channel_axis=0, channels=["DNA", "ACTIN"]) + np.testing.assert_array_equal(swapped, expected[::-1]) + only_dna = parameters.apply(pixels[1:], channel_axis=0, channels=["DNA"]) + np.testing.assert_array_equal(only_dna, expected[1:]) + np.testing.assert_array_equal(parameters.apply(pixels[1], channels=["DNA"]), expected[1]) def test_apply_with_one_channel_equals_the_existing_apply(): pooled = PooledHistogram([compute_plane_histogram(np.arange(1000, dtype=np.uint16))]) existing = robust_z_score(pooled) parameters = Parameters( - existing.method, "patch", None, True, (existing.offset,), (existing.scale,) + existing.method, "patch", None, True, (existing.offset,), (existing.scale,), ("DNA",) ) pixels = np.random.default_rng(16).integers(0, 1000, (8, 8)).astype(np.uint16) np.testing.assert_array_equal(parameters.apply(pixels), existing.apply(pixels)) @@ -179,22 +195,33 @@ def test_apply_with_one_channel_equals_the_existing_apply(): def test_apply_refuses_a_wrong_channel_axis(): - parameters = Parameters("min-max", "image", None, True, (0.0, 1.0), (1.0, 2.0)) - with pytest.raises(ValueError, match="give channel_axis"): + parameters = Parameters("min-max", "image", None, True, (0.0, 1.0), (1.0, 2.0), ("A", "B")) + with pytest.raises(ValueError, match="give channels"): parameters.apply(np.zeros((2, 4))) + with pytest.raises(ValueError, match="give channels"): + parameters.apply(np.zeros((2, 4)), channel_axis=0) with pytest.raises(ValueError, match="has 3 channel"): - parameters.apply(np.zeros((3, 4)), channel_axis=0) + parameters.apply(np.zeros((3, 4)), channel_axis=0, channels=["A", "B"]) with pytest.raises(ValueError, match="outside"): - parameters.apply(np.zeros((2, 4)), channel_axis=2) + parameters.apply(np.zeros((2, 4)), channel_axis=2, channels=["A", "B"]) + with pytest.raises(ValueError, match="not C; normalize those channels"): + parameters.apply(np.zeros((2, 4)), channel_axis=0, channels=["A", "C"]) + with pytest.raises(ValueError, match="never by its position"): + parameters.apply(np.zeros((2, 4)), channel_axis=0, channels=[0, 1]) + with pytest.raises(ValueError, match="more than once"): + parameters.apply(np.zeros((2, 4)), channel_axis=0, channels=["A", "A"]) VALUES = [0.1, 1 / 3, 1e-300, 123456789.123456789, -2.5e-7, 65535.0, math.pi] -CHANNEL_COLUMN = re.compile(r"Normalization_(Offset|Scale)_C(0|[1-9][0-9]*)") +CHANNEL_COLUMN = re.compile(r"Normalization_(Offset|Scale)_[A-Za-z][A-Za-z0-9]*(?:_[A-Za-z0-9]+)*") -@pytest.mark.parametrize("channels", [(0,), (0, 1), (0, 1, 2, 3), (2,), (1, 3)]) +@pytest.mark.parametrize( + "channels", + [("DNA",), ("ACTIN", "DNA"), ("A", "B", "C", "D"), ("PAX_488",), ("C0", "PAX_488_2")], +) def test_columns_round_trip_exactly(channels): - rng = np.random.default_rng(len(channels) + sum(channels)) + rng = np.random.default_rng(len(channels) + sum(map(len, channels))) offset = tuple(rng.choice(VALUES, len(channels))) scale = tuple(abs(v) for v in rng.choice(VALUES, len(channels))) for group in (None, "control/r1/img.tif::5::t=2"): @@ -209,8 +236,8 @@ def test_columns_round_trip_exactly(channels): settings = NormalizationSettings(scope="patch") assert Parameters.from_columns(columns, settings) == parameters assert columns["normalization_exact"] == "false" - assert columns[f"Normalization_Offset_C{channels[0]}"] == repr(float(offset[0])) - assert columns[f"Normalization_Scale_C{channels[-1]}"] == repr(float(scale[-1])) + assert columns[f"Normalization_Offset_{channels[0]}"] == repr(float(offset[0])) + assert columns[f"Normalization_Scale_{channels[-1]}"] == repr(float(scale[-1])) def test_columns_of_a_fit_round_trip_exactly(): @@ -223,27 +250,34 @@ def test_columns_of_a_fit_round_trip_exactly(): assert columns["normalization_method"] == "robust z-score" assert columns["normalization_scope"] == "biological_repeat" assert columns["normalization_group"] == key[0] - assert parameters.channels == (0, 1, 2) + assert parameters.channels == channel_ids(3) per_channel = [name for name in columns if CHANNEL_COLUMN.fullmatch(name)] assert per_channel == [ - f"Normalization_{feature}_C{c}" for c in range(3) for feature in ("Offset", "Scale") + f"Normalization_{feature}_{c}" + for c in channel_ids(3) + for feature in ("Offset", "Scale") ] assert Parameters.from_columns(columns, settings) == parameters def test_a_single_channel_patch_fills_only_its_own_channel(): - parameters = Parameters("z-score", "image", "a.tif", True, (1.0, 2.0, 3.0), (4.0, 5.0, 6.0)) - second = parameters.select([1]) - assert (second.channels, second.offset, second.scale) == ((1,), (2.0,), (5.0,)) + ids = ("ACTIN", "DNA", "PAX_488") + parameters = Parameters( + "z-score", "image", "a.tif", True, (1.0, 2.0, 3.0), (4.0, 5.0, 6.0), ids + ) + second = parameters.select(["DNA"]) + assert (second.channels, second.offset, second.scale) == (("DNA",), (2.0,), (5.0,)) np.testing.assert_array_equal(second.apply(np.array([2.0, 7.0])), [0.0, 1.0]) - assert parameters.select([2, 0]).channels == (0, 2) - with pytest.raises(ValueError, match=r"channel\(s\) \[3\] cannot be selected"): - parameters.select([3]) + assert parameters.select(["PAX_488", "ACTIN"]).channels == ("ACTIN", "PAX_488") + with pytest.raises(ValueError, match=r"channel\(s\) PHASE cannot be selected"): + parameters.select(["PHASE"]) + with pytest.raises(ValueError, match="never by its position"): + parameters.select([1]) # A table of single-channel patches of every channel: each row has the columns # of every channel, blank except for its own (as a CSV or pandas reader gives them). settings = NormalizationSettings(method="zscore") header = list(parameters.to_columns()) - for c in range(3): + for c in ids: written = parameters.select([c]).to_columns() row = {name: written.get(name, "") for name in header} assert Parameters.from_columns(row, settings) == parameters.select([c]) @@ -257,42 +291,64 @@ def test_from_columns_reads_numbers_and_booleans_as_a_table_reader_gives_them(): "normalization_scope": "image", "normalization_group": float("nan"), # blank cell read by pandas "normalization_exact": True, - "Normalization_Offset_C0": 12.5, - "Normalization_Scale_C0": np.float64(0.1), - "Normalization_Offset_C1": float("nan"), - "Normalization_Scale_C1": None, + "normalization_parameters_version": np.int64(2), + "Normalization_Offset_DNA": 12.5, + "Normalization_Scale_DNA": np.float64(0.1), + "Normalization_Offset_ACTIN": float("nan"), + "Normalization_Scale_ACTIN": None, "image_file": "a.tif", - "Normalization_Offset_C2_extra": "ignored", + "Normalization_Offset_Extra_": "ignored", # not a channel ID: ignored } parameters = Parameters.from_columns(row, NormalizationSettings(method="zscore")) - assert parameters == Parameters("z-score", "image", None, True, (12.5,), (0.1,)) + assert parameters == Parameters("z-score", "image", None, True, (12.5,), (0.1,), ("DNA",)) + row["normalization_parameters_version"] = 2.0 # a column with blanks, read by pandas + assert Parameters.from_columns(row, NormalizationSettings(method="zscore")) == parameters + + +@pytest.mark.parametrize("version", [None, "", "1", 1, 3.0, True, "2.0"]) +def test_from_columns_refuses_parameter_columns_of_another_version(version): + # Open question 4 of PR #90: the version-4 column Normalization_Offset_C0 is also a valid + # channel-ID column (C0), so the columns carry their own version and an old row is refused. + settings = NormalizationSettings(method="minmax") + row = Parameters("min-max", "image", "a", True, (1.0,), (2.0,), ("C0",)).to_columns() + assert row["normalization_parameters_version"] == "2" + if version is None: + del row["normalization_parameters_version"] + message = "has no normalization_parameters_version column, so it was written by" + else: + row["normalization_parameters_version"] = version + message = "parameter columns (has|of version 2 only)" + with pytest.raises(ValueError, match=message): + Parameters.from_columns(row, settings) @pytest.mark.parametrize( "change, message", [ ({"normalization_scope": None}, "normalization_scope"), - ({"Normalization_Scale_C0": None}, "Column Normalization_Scale_C0 is blank or missing"), - ({"Normalization_Offset_C1": ""}, "Column Normalization_Offset_C1 is blank or missing"), - ({"Normalization_Offset_C0": "1.0;2.0"}, "Normalization_Offset_C0 cannot be read"), - ({"Normalization_Scale_C1": "0.0"}, "Normalization_Scale_C1 must hold a positive"), - ({"Normalization_Offset_C0": "nan"}, "Normalization_Offset_C0 must hold a finite"), + ({"Normalization_Scale_A": None}, "Column Normalization_Scale_A is blank or missing"), + ({"Normalization_Offset_B": ""}, "Column Normalization_Offset_B is blank or missing"), + ({"Normalization_Offset_A": "1.0;2.0"}, "Normalization_Offset_A cannot be read"), + ({"Normalization_Scale_B": "0.0"}, "Normalization_Scale_B must hold a positive"), + ({"Normalization_Offset_A": "nan"}, "Normalization_Offset_A must hold a finite"), ({"normalization_exact": "maybe"}, "normalization_exact must be 'true' or 'false'"), ({"normalization_method": "min-max"}, "normalization_method is 'min-max'"), ( - {name: None for name in ("Normalization_Offset_C0", "Normalization_Scale_C0")} - | {name: None for name in ("Normalization_Offset_C1", "Normalization_Scale_C1")}, - "Normalization_Offset_C0 and Normalization_Scale_C0 are missing", + {name: None for name in ("Normalization_Offset_A", "Normalization_Scale_A")} + | {name: None for name in ("Normalization_Offset_B", "Normalization_Scale_B")}, + "no Normalization_Offset_ and Normalization_Scale_ columns", ), ( - {name: "" for name in ("Normalization_Offset_C0", "Normalization_Scale_C0")} - | {name: "" for name in ("Normalization_Offset_C1", "Normalization_Scale_C1")}, + {name: "" for name in ("Normalization_Offset_A", "Normalization_Scale_A")} + | {name: "" for name in ("Normalization_Offset_B", "Normalization_Scale_B")}, "column of this row is blank", ), ], ) def test_from_columns_names_the_column_it_cannot_read(change, message): - parameters = Parameters("percentile 1-99.8", "image", "a", True, (1.0, 3.0), (2.0, 4.0)) + parameters = Parameters( + "percentile 1-99.8", "image", "a", True, (1.0, 3.0), (2.0, 4.0), ("A", "B") + ) columns = parameters.to_columns() for name, value in change.items(): if value is None: @@ -304,10 +360,14 @@ def test_from_columns_names_the_column_it_cannot_read(change, message): def test_parameters_refuse_channels_that_do_not_match_the_values(): - with pytest.raises(ValueError, match="one channel index per offset"): - Parameters("min-max", "image", None, True, (0.0, 1.0), (1.0, 1.0), channels=(0,)) - with pytest.raises(ValueError, match="increasing order"): - Parameters("min-max", "image", None, True, (0.0, 1.0), (1.0, 1.0), channels=(1, 0)) + with pytest.raises(ValueError, match="one channel ID per offset"): + Parameters("min-max", "image", None, True, (0.0, 1.0), (1.0, 1.0), channels=("A",)) + with pytest.raises(ValueError, match="sorted order"): + Parameters("min-max", "image", None, True, (0.0, 1.0), (1.0, 1.0), channels=("B", "A")) + with pytest.raises(ValueError, match="never by its position"): + Parameters("min-max", "image", None, True, (0.0, 1.0), (1.0, 1.0), channels=(0, 1)) + with pytest.raises(TypeError, match="channels"): + Parameters("min-max", "image", None, True, (0.0,), (1.0,)) def test_approximate_parameters_are_reported(): @@ -319,68 +379,91 @@ def test_approximate_parameters_are_reported(): def test_planes_without_finite_pixels_are_left_out_and_reported(): + one = {"position_id": "p", "channels": ("DNA",)} good = PlaneRecord( - "c/r/a.tif", "c", "r", "single", compute_plane_histogram(np.arange(10.0)), position_id="p" + "c/r/a.tif", "c", "r", "single", compute_plane_histogram(np.arange(10.0)), **one ) empty = PlaneRecord( - "c/r/b.tif", - "c", - "r", - "single", - compute_plane_histogram(np.full(4, np.nan)), - position_id="p", + "c/r/b.tif", "c", "r", "single", compute_plane_histogram(np.full(4, np.nan)), **one ) parameters = fit([good, empty], NormalizationSettings(method="minmax")) assert (parameters.offset, parameters.scale) == ((0.0,), (9.0,)) codes = [issue.code for issue in parameters.issues] assert "planes_without_finite_pixels" in codes issue = parameters.issues[codes.index("planes_without_finite_pixels")] - assert issue.items == (f"{empty.plane_id} (channel 0)",) - with pytest.raises(NoValidPixelsError, match="Group 'x', channel 0"): + assert issue.items == (f"{empty.plane_id} (channel DNA)",) + with pytest.raises(NoValidPixelsError, match="Group 'x', channel DNA"): fit([empty], NormalizationSettings(), group="x") def test_a_channel_without_finite_pixels_in_one_plane_is_left_out_of_that_channel_only(): finite = compute_plane_histogram(np.arange(10.0)) empty = compute_plane_histogram(np.full(10, np.nan)) + two = {"position_id": "p", "channels": ("DNA", "ACTIN")} planes = [ - PlaneRecord("c/r/a.tif", "c", "r", "single", (finite, empty), t=0, position_id="p"), - PlaneRecord("c/r/a.tif", "c", "r", "single", (finite, finite), t=1, position_id="p"), + PlaneRecord("c/r/a.tif", "c", "r", "single", (finite, empty), t=0, **two), + PlaneRecord("c/r/a.tif", "c", "r", "single", (finite, finite), t=1, **two), ] parameters = fit(planes, NormalizationSettings(method="minmax"), group="c/r/a.tif") assert (parameters.offset, parameters.scale) == ((0.0, 0.0), (9.0, 9.0)) issue = {issue.code: issue for issue in parameters.issues}["planes_without_finite_pixels"] - assert issue.items == ("c/r/a.tif::z=0,t=0 (channel 1)",) - with pytest.raises(NoValidPixelsError, match="Group 'g', channel 1"): + assert issue.items == ("c/r/a.tif::z=0,t=0 (channel ACTIN)",) + with pytest.raises(NoValidPixelsError, match="Group 'g', channel ACTIN"): fit(planes[:1], NormalizationSettings(), group="g") -def test_fit_refuses_rows_with_different_channel_counts(): - one = compute_plane_histogram(np.arange(10, dtype=np.uint8)) +def test_fit_pools_each_channel_over_the_rows_that_have_it(): + # ADR 0003 replaces N43: rows with different channels are fitted by channel ID. + a = compute_plane_histogram(np.arange(10, dtype=np.uint8)) + b = compute_plane_histogram(np.arange(100, 110, dtype=np.uint8)) rows = [ - PlaneRecord("c/r/a.tif", "c", "r", "single", (one, one), position_id="p"), - PlaneRecord("c/r/b.tif", "c", "r", "single", (one,), position_id="p"), + PlaneRecord( + "c/r/a.tif", "c", "r", "single", (a, b), position_id="p", channels=("DNA", "PHASE") + ), + PlaneRecord("c/r/b.tif", "c", "r", "single", (a,), position_id="p", channels=("DNA",)), ] - with pytest.raises(ValueError) as error: - fit(rows, NormalizationSettings(), group="r") - message = str(error.value) - assert "group 'r' have different numbers of channels" in message - assert "1 channel(s): c/r/b.tif; 2 channel(s): c/r/a.tif" in message - assert "Every image of an experiment must have the same channels" in message + with pytest.raises(ChannelScopeIncompleteError, match="channel PHASE is present in 1 of 2"): + fit(rows, NormalizationSettings(method="minmax"), group="r") # refused by default + subset = NormalizationSettings(method="minmax", partial_channels="planes_with_channel") + parameters = fit(rows, subset, group="r") + assert parameters.channels == ("DNA", "PHASE") + assert (parameters.offset, parameters.scale) == ((0.0, 100.0), (9.0, 9.0)) + (issue,) = [i for i in parameters.issues if i.code == "channel_partially_present"] + assert (issue.channel, issue.items) == ("PHASE", ("c/r/b.tif::z=0,t=0",)) + assert "absent from 1 of the 2 planes of group 'r'" in issue.message + only = fit(rows, NormalizationSettings(method="minmax"), group="r", channels=["DNA"]) + assert only.channels == ("DNA",) and only.issues == () + with pytest.raises(ValueError, match="No plane of group 'r' has channel\\(s\\) ACTIN"): + fit(rows, NormalizationSettings(), group="r", channels=["ACTIN"]) + with pytest.raises(ValueError, match="never by its position"): + fit(rows, NormalizationSettings(), group="r", channels=[0]) def test_zero_spread_names_the_group_and_channel(): constant = np.full((2, 4, 4), 7, np.uint16) constant[0, 0, 0] = 8 - with pytest.raises(ValueError, match="Group 'p', channel 1: The min-max normalization"): + with pytest.raises(ValueError, match="Group 'p', channel DNA: The min-max normalization"): fit( constant, NormalizationSettings(method="minmax", scope="patch"), group="p", channel_axis=0, + channels=["ACTIN", "DNA"], ) +def test_a_pixel_fit_needs_the_channel_ids_of_the_array(): + pixels = np.arange(24, dtype=np.uint16).reshape(2, 12) + settings = NormalizationSettings(scope="patch") + with pytest.raises(ValueError, match="Give channels"): + fit(pixels, settings, channel_axis=0) + with pytest.raises(ValueError, match="2 channel\\(s\\) along axis 0, but 1"): + fit(pixels, settings, channel_axis=0, channels=["DNA"]) + with pytest.raises(ValueError, match="1 channel\\(s\\) without a channel axis, but 2"): + fit(pixels, settings, channels=["DNA", "ACTIN"]) + assert fit(pixels[0], settings, channels=["DNA"]).channels == ("DNA",) + + def test_biological_repeat_scope_needs_rows_not_pixels(): with pytest.raises(MethodNotAllowedError, match="needs the preprocessing brick"): fit(np.arange(10, dtype=np.uint8), NormalizationSettings(scope="biological_repeat")) @@ -416,7 +499,7 @@ def test_from_columns_refuses_rows_of_other_percentiles_with_a_colliding_short_n True, (1.0,) * channels, (2.0,) * channels, - tuple(range(channels)), + ("B", "G", "R") if image_type == "rgb" else ("DNA",), ) columns = parameters.to_columns() assert Parameters.from_columns(columns, written) == parameters @@ -426,3 +509,15 @@ def test_from_columns_refuses_rows_of_other_percentiles_with_a_colliding_short_n columns["normalization_method"] = columns["normalization_method"].replace("1.000001", "1") with pytest.raises(ValueError, match="Column normalization_method is .*'.*percentile 1-99.8'"): Parameters.from_columns(columns, written) + + +def test_a_single_text_is_never_split_into_channel_ids(): + # "DNA" is one ID, not the IDs D, N and A. + pixels = np.arange(36, dtype=np.uint16).reshape(3, 12) + with pytest.raises(ValueError, match="expected a list of channel IDs"): + fit(pixels, NormalizationSettings(scope="patch"), channel_axis=0, channels="DNA") + parameters = Parameters("min-max", "image", None, True, (0.0,), (1.0,), ("DNA",)) + with pytest.raises(ValueError, match="expected a list of channel IDs"): + parameters.select("DNA") + with pytest.raises(ValueError, match="expected a list of channel IDs"): + parameters.apply(pixels[0], channels="DNA") diff --git a/tests/test_heterogeneous_channels.py b/tests/test_heterogeneous_channels.py new file mode 100644 index 0000000..f0d4bb8 --- /dev/null +++ b/tests/test_heterogeneous_channels.py @@ -0,0 +1,433 @@ +"""Stage 3 (roadmap S3.2, ADR 0003): statistics and fits by canonical channel ID on described trees. + +The owner's case first: three biological repeats share three stains, and +Biological_repeat_2 has a fourth, EXTRA_STAIN. Then the Spec fixtures ``v04`` (same stains, +other source order), ``v17`` (partial presence), ``v22`` (a declared channel no unit has), +``v05``/``v06`` (heterogeneous counts and technical repeats), one discrepant field of view, an +injected measurement failure, a channel ID with ``_``, and byte determinism. + +Tolerances (documented here, no comparison with an older release, ADR 0002 AR10): the pixels +are uint8 with exact histograms, so min-max, percentile and robust z-score parameters equal +the formulas on the raw pixels exactly (tolerance 0); z-score parameters are sums and are +compared with ``rel=1e-12``. Fits of the same pixels under two layouts are compared with +``==`` (bit for bit). +""" + +import csv + +import numpy as np +import pytest + +from described import FIXTURES, fixture, images, parse, pixels_of, record, records, renamed +from morphobricks_normalization import ( + ChannelMeasurementError, + ChannelMeasurementFailure, + ChannelScopeIncompleteError, + NormalizationSettings, + Parameters, + PooledHistogram, + fit, + group_name, + image_summaries, + percentile_range, + read_plane_histogram_table, + scope_groups, + summary_columns, + write_plane_histograms, +) + +OWNER_CASE = ( + """ +schema = "morphobricks-experiment" +schema_version = 1 +[conditions.CTRL] +source_folder = "CTRL" +[conditions.DrugA] +source_folder = "DrugA" +reference = "CTRL" +[channels.DNA] +[channels.ACTIN] +[channels.TUBULIN] +[channels.EXTRA_STAIN] +""" + + "".join( + f""" +[repeats.Biological_repeat_{r}] +confirmed = true +confirmed_on = "2026-11-10" +""" + + "".join( + f""" +[repeats.Biological_repeat_{r}.members.{c}] +source_repeat = "Biological_repeat_{r}" +image_count = 2 +[repeats.Biological_repeat_{r}.members.{c}.technical_repeats.single] +source_folder = "single" +channels = {channels} +""" + for c in ("CTRL", "DrugA") + ) + for r, channels in ( + (1, '["DNA", "ACTIN", "TUBULIN"]'), + (2, '["DNA", "ACTIN", "TUBULIN", "EXTRA_STAIN"]'), + (3, '["DNA", "ACTIN", "TUBULIN"]'), + ) + ) + + """ +[used_ids] +conditions = ["CTRL", "DrugA"] +channels = ["ACTIN", "DNA", "EXTRA_STAIN", "TUBULIN"] +repeats = ["Biological_repeat_1", "Biological_repeat_2", "Biological_repeat_3"] +technical_repeats = ["single"] +""" +) +SHARED = ("ACTIN", "DNA", "TUBULIN") +SUBSET = {"partial_channels": "planes_with_channel"} # the user's recorded choice +METHODS = ("minmax", "percentile", "zscore", "robust_zscore") + + +def direct(x, method): + """Contract §3.2 on raw pixels (inverse-CDF percentiles).""" + if method == "minmax": + return x.min(), x.max() - x.min() + if method == "percentile": + low, high = (np.percentile(x, q, method="inverted_cdf") for q in (1.0, 99.8)) + return low, high - low + if method == "zscore": + return x.mean(), x.std() + median = np.percentile(x, 50, method="inverted_cdf") + return median, np.percentile(np.abs(x - median), 50, method="inverted_cdf") * 1.482602218505602 + + +def assert_direct(parameters, channel, pixels, method): + k = parameters.channels.index(channel) + offset, scale = direct(pixels, method) + if method == "zscore": + assert parameters.offset[k] == pytest.approx(offset, rel=1e-12) + assert parameters.scale[k] == pytest.approx(scale, rel=1e-12) + else: + assert (parameters.offset[k], parameters.scale[k]) == (offset, scale) + + +def in_repeat(group): + """Select the images of one canonical repeat, for :func:`pixels_of`.""" + return lambda unit, path, t: unit.repeat == group + + +def table(tmp_path, description, rows, name="plane_histograms.csv"): + path = write_plane_histograms(rows, tmp_path / name, channel_ids=description.channel_ids) + return path, read_plane_histogram_table(path) + + +def repeat_fits(description, rows, method="percentile", **options): + settings = NormalizationSettings(method=method, scope="biological_repeat", **options) + groups, _ = scope_groups(rows, settings, description.repeat_groups()) + return {group_name(k): fit(v, settings, group=group_name(k)) for k, v in groups.items()} + + +# --- The owner's case: an extra stain in one biological repeat --------------------------------- + + +@pytest.fixture(scope="module") +def owner(tmp_path_factory): + description = parse(OWNER_CASE) + path, read = table(tmp_path_factory.mktemp("owner"), description, records(description)) + return description, path, read.records + + +def test_owner_case_table_lists_every_id_and_leaves_the_extra_stain_blank_elsewhere(owner): + description, path, rows = owner + assert read_plane_histogram_table(path).channel_ids == (*SHARED[:2], "EXTRA_STAIN", "TUBULIN") + assert len(rows) == 3 * 2 * 2 * 2 # one row per plane: repeats x conditions x images x T + with open(path, newline="", encoding="utf-8") as fh: + for row in csv.DictReader(fh): + extra = [ + v for k, v in row.items() if k.endswith("_EXTRA_STAIN") or "_EXTRA_STAIN_" in k + ] + if row["biological_repeat"] == "Biological_repeat_2": + assert row["channels"] == "DNA;ACTIN;TUBULIN;EXTRA_STAIN" + assert all(v != "" for v in extra) + else: + assert row["channels"] == "DNA;ACTIN;TUBULIN" + assert set(extra) == {""} # blank, never a zero or NaN histogram + + +def test_the_header_lists_the_declared_channel_ids_not_retired_ones(tmp_path): + # Open question 1 of PR #90: the caller passes the description's declared channels (its + # [channels] table), not used_ids.channels, which keeps retired IDs no image can have. + used = 'channels = ["ACTIN", "DNA", "EXTRA_STAIN", "TUBULIN"]' + assert used in OWNER_CASE + description = parse(OWNER_CASE.replace(used, used.replace('"TUBULIN"', '"OLD", "TUBULIN"'))) + assert description.channel_ids == ("DNA", "ACTIN", "TUBULIN", "EXTRA_STAIN") + _, read = table(tmp_path, description, records(description)) + assert read.channel_ids == ("ACTIN", "DNA", "EXTRA_STAIN", "TUBULIN") # sorted, no OLD + + +@pytest.mark.parametrize("method", METHODS) +def test_owner_case_shared_stains_pool_across_repeats_extra_only_within_its_repeat(owner, method): + description, _, rows = owner + refused = NormalizationSettings(method=method, scope="biological_repeat") + with pytest.raises( + ChannelScopeIncompleteError, match="channel EXTRA_STAIN is present in 8 of 24" + ) as e: + fit(rows, refused, group="all") # by default a channel some planes lack is refused + assert list(e.value.channels) == ["EXTRA_STAIN"] + assert {p.split("/")[1] for p in e.value.channels["EXTRA_STAIN"]} == { + "Biological_repeat_1", + "Biological_repeat_3", + } + assert fit(rows, refused, group="all", channels=SHARED).channels == SHARED + settings = NormalizationSettings(method=method, scope="biological_repeat", **SUBSET) + pooled = fit(rows, settings, group="all") # every repeat pooled (a global fit), opted in + assert pooled.channels == ("ACTIN", "DNA", "EXTRA_STAIN", "TUBULIN") + for channel in SHARED: + assert_direct(pooled, channel, pixels_of(description, channel), method) + in_repeat_2 = pixels_of(description, "EXTRA_STAIN") + assert_direct(pooled, "EXTRA_STAIN", in_repeat_2, method) + rep2 = [r for r in rows if r.biological_repeat == "Biological_repeat_2"] + extra_alone = fit(rep2, settings, group="all", channels=["EXTRA_STAIN"]) + assert pooled.select(["EXTRA_STAIN"]) == extra_alone # bit for bit + (warning,) = [i for i in pooled.issues if i.code == "channel_partially_present"] + assert warning.channel == "EXTRA_STAIN" + assert {r.split("/")[1] for r in warning.items} == { + "Biological_repeat_1", + "Biological_repeat_3", + } + + +@pytest.mark.parametrize("method", METHODS) +def test_owner_case_repeat_scope_fits_the_channels_each_repeat_has(owner, method): + description, _, rows = owner + fits = repeat_fits(description, rows, method) + assert {g: p.channels for g, p in fits.items()} == { + "Biological_repeat_1": SHARED, + "Biological_repeat_2": ("ACTIN", "DNA", "EXTRA_STAIN", "TUBULIN"), + "Biological_repeat_3": SHARED, + } + for group, parameters in fits.items(): + assert [i.code for i in parameters.issues] == [] # every plane of a repeat has them + for channel in parameters.channels: + where = in_repeat(group) + assert_direct(parameters, channel, pixels_of(description, channel, where), method) + + +def test_owner_case_planes_with_every_channel_fit_bit_for_bit_like_the_shared_layout(owner): + # Evidence for "all channels present gives today's numbers": the shared stains' fits from + # the heterogeneous table equal those of a homogeneous table of the same pixels. + description, _, rows = owner + homogeneous = [ + record(unit, path, t, pixels[: len(SHARED)], channels=unit.channels[:3]) + for unit, path, t, pixels in images(description) + ] + for method in METHODS: + for options in ({}, {"pooling": "conditions"}): + a = repeat_fits(description, rows, method, **options) + b = repeat_fits(description, homogeneous, method, **options) + for group in a: + assert a[group].select(SHARED) == b[group] + + +def test_owner_case_parameter_columns_are_named_by_id_and_blank_where_absent(owner): + description, _, rows = owner + fits = repeat_fits(description, rows) + header = {name for p in fits.values() for name in p.to_columns()} + assert "Normalization_Offset_EXTRA_STAIN" in header + assert not any("_C0" in name or "_C3" in name for name in header) + settings = NormalizationSettings(scope="biological_repeat") + for parameters in fits.values(): + written = parameters.to_columns() + row = {name: written.get(name, "") for name in sorted(header)} + assert Parameters.from_columns(row, settings) == parameters + + +def test_owner_case_summary_columns_hold_the_extra_stain_only_where_acquired(owner): + description, _, rows = owner + columns = summary_columns(description.channel_ids) + assert "Intensity_MedianIntensity_EXTRA_STAIN" in columns + for summary in image_summaries(rows): + has_it = summary.biological_repeat == "Biological_repeat_2" + assert ("Histogram_Exact_EXTRA_STAIN" in summary.to_row()) == has_it + + +# --- v04: same stains, other source order ------------------------------------------------------ + + +def test_v04_swapped_order_gives_the_same_results_per_id_as_the_unswapped_layout(tmp_path): + swapped = fixture("v04_swapped_channel_order") + assert {u.condition: u.channels for u in swapped.units} == { + "CTRL": ("DNA", "ACTIN"), + "DrugA": ("ACTIN", "DNA"), + } + straight = parse(_unswapped_v04_text()) # the same description, DrugA as DNA, ACTIN + _, a = table(tmp_path, swapped, records(swapped), "a.csv") + _, b = table(tmp_path, straight, records(straight), "b.csv") + for method in METHODS: + for scope in ("biological_repeat", "image"): + settings = NormalizationSettings(method=method, scope=scope) + ga, _ = scope_groups(a.records, settings) + gb, _ = scope_groups(b.records, settings) + assert list(ga) == list(gb) + for key in ga: + assert fit(ga[key], settings) == fit(gb[key], settings) # bit for bit + assert image_summaries(a.records) == image_summaries(b.records) + # By C index the two stains of DrugA would be pooled with the other stain of CTRL. + by_index = percentile_range(PooledHistogram([r.histograms[0] for r in a.records]), 1, 99.8) + by_id = fit(a.records, NormalizationSettings()).select(["DNA"]) + assert (by_index.offset, by_index.scale) != (by_id.offset[0], by_id.scale[0]) + + +def _unswapped_v04_text(): + text = (FIXTURES / "v04_swapped_channel_order.toml").read_text(encoding="utf-8") + section = "[repeats.rep1.members.DrugA.technical_repeats.single]" + head, tail = text.split(section) + swapped = tail.replace('channels = ["ACTIN", "DNA"]', 'channels = ["DNA", "ACTIN"]', 1) + return head + section + swapped + + +# --- v17: partial channel presence ------------------------------------------------------------- + + +def test_v17_partial_presence_is_refused_by_default(tmp_path): + description = fixture("v17_partial_channel_presence") + _, read = table(tmp_path, description, records(description)) + settings = NormalizationSettings(scope="biological_repeat") + groups, _ = scope_groups(read.records, settings, description.repeat_groups()) + assert fit(groups[("R001",)], settings, group="R001").channels == ("ACTIN", "DNA") + with pytest.raises( + ChannelScopeIncompleteError, + match="A_SCOPE_INCOMPLETE: in group 'R002', channel PHASE is present in", + ) as e: + fit(groups[("R002",)], settings, group="R002") + assert {item.split("/")[2] for item in e.value.channels["PHASE"]} == {"acquisition_2"} + fitted = fit(groups[("R002",)], settings, group="R002", channels=["ACTIN", "DNA"]) + assert fitted.channels == ("ACTIN", "DNA") and fitted.issues == () + + +def test_v17_partial_presence_is_fitted_where_present_when_selected_and_reported(tmp_path): + description = fixture("v17_partial_channel_presence") + path, read = table(tmp_path, description, records(description)) + fits = repeat_fits(description, read.records, **SUBSET) + assert fits["R001"].channels == ("ACTIN", "DNA") and fits["R001"].issues == () + assert fits["R002"].channels == ("ACTIN", "DNA", "PHASE") + (warning,) = fits["R002"].issues + assert (warning.code, warning.channel) == ("channel_partially_present", "PHASE") + assert {item.split("/")[2] for item in warning.items} == {"acquisition_2"} + + def where(unit, path, t): + return unit.technical_repeat == "acquisition_1" + + assert_direct(fits["R002"], "PHASE", pixels_of(description, "PHASE", where), "percentile") + settings = NormalizationSettings(scope="biological_repeat", **SUBSET) + groups, _ = scope_groups(read.records, settings, description.repeat_groups()) + with pytest.raises(ValueError, match="No plane of group 'R001' has channel\\(s\\) PHASE"): + fit(groups[("R001",)], settings, group="R001", channels=["PHASE"]) + + +# --- v22: a declared channel that no unit has (a missing optional channel) --------------------- + + +def test_v22_a_channel_no_image_has_is_in_the_header_blank_and_never_fitted(tmp_path): + description = fixture("v22_unused_channel_warning") + path, read = table(tmp_path, description, records(description)) + assert read.channel_ids == ("ACTIN", "DNA", "PHASE") + with open(path, newline="", encoding="utf-8") as fh: + rows = list(csv.DictReader(fh)) + assert {v for row in rows for k, v in row.items() if k.endswith("_PHASE")} == {""} + assert fit(read.records, NormalizationSettings()).channels == ("ACTIN", "DNA") + with pytest.raises(ValueError, match="never fitted from zeros"): + fit(read.records, NormalizationSettings(), channels=["PHASE"]) + + +# --- v05, v06: heterogeneous counts and technical repeats -------------------------------------- + + +@pytest.mark.parametrize( + "name", ["v05_variable_channel_counts", "v06_technical_repeats_different_layouts"] +) +def test_heterogeneous_layouts_keep_one_row_per_plane_and_one_group_per_repeat(tmp_path, name): + description = fixture(name) + rows = records(description) + _, read = table(tmp_path, description, rows) + assert len(read.records) == len(rows) # N41: the row is the plane + with pytest.raises(ChannelScopeIncompleteError): # PHASE in part of a repeat: refused + repeat_fits(description, read.records) + fits = repeat_fits(description, read.records, **SUBSET) + assert sorted(fits) == sorted({u.repeat for u in description.units}) # never per acquisition + for group, parameters in fits.items(): + for channel in parameters.channels: + where = in_repeat(group) + assert_direct(parameters, channel, pixels_of(description, channel, where), "percentile") + + +def test_a_channel_id_with_an_underscore_names_its_columns(tmp_path): + description = renamed(fixture("v06_technical_repeats_different_layouts"), "PAXILLIN", "PAX_488") + path, read = table(tmp_path, description, records(description)) + assert "PAX_488" in read.channel_ids + header = path.read_text(encoding="utf-8").splitlines()[0].split(",") + assert ( + "Histogram_ValidPixelCount_PAX_488" in header and "Histogram_Count_PAX_488_00255" in header + ) + parameters = repeat_fits(description, read.records, **SUBSET)["R001"] + assert "Normalization_Scale_PAX_488" in parameters.to_columns() + + +# --- One discrepant field of view, a failed measurement, determinism --------------------------- + + +def test_a_single_field_of_view_without_a_channel_is_named_and_never_zero_filled(tmp_path): + description = fixture("v13_homogeneous_legacy") + rows = [] + for unit, path, t, pixels in images(description): + odd = path == "DrugA/rep2/img_1.tif" # acquired without ACTIN + rows.append(record(unit, path, t, pixels[:1] if odd else pixels, ("DNA",) if odd else None)) + _, read = table(tmp_path, description, rows) + with pytest.raises(ChannelScopeIncompleteError, match="channel ACTIN is present in 10 of 12"): + repeat_fits(description, read.records) + fits = repeat_fits(description, read.records, **SUBSET) + (warning,) = fits["rep2"].issues + assert warning.channel == "ACTIN" + assert warning.items == ("DrugA/rep2/img_1.tif::z=0,t=0", "DrugA/rep2/img_1.tif::z=0,t=1") + + def where(unit, path, t): + return unit.repeat == "rep2" and path != "DrugA/rep2/img_1.tif" + + assert_direct(fits["rep2"], "ACTIN", pixels_of(description, "ACTIN", where), "percentile") + assert fits["rep1"].issues == fits["rep3"].issues == () + + +def test_an_injected_measurement_failure_writes_no_table_and_names_plane_and_channel(tmp_path): + description = fixture("v06_technical_repeats_different_layouts") + rows = records(description) + failure = ChannelMeasurementFailure(rows[5].plane_id, rows[5].channels[1], "read error") + with pytest.raises(ChannelMeasurementError, match=f"plane {rows[5].plane_id}, channel"): + write_plane_histograms( + rows, + tmp_path / "plane_histograms.csv", + channel_ids=description.channel_ids, + failures=[failure], + ) + assert list(tmp_path.iterdir()) == [] + + +def test_tables_are_byte_identical_serially_in_shards_and_on_resume(tmp_path): + description = fixture("v06_technical_repeats_different_layouts") + rows = sorted(records(description), key=lambda r: (r.source_relative_path, r.z, r.t)) + serial = table(tmp_path, description, rows, "serial.csv")[0].read_bytes() + assert table(tmp_path, description, rows, "again.csv")[0].read_bytes() == serial + by_file = {} + for row in rows: + by_file.setdefault(row.source_relative_path, []).append(row) + shards = {} # one per file, written in reverse order (a parallel run's completion order) + for k, (path, members) in enumerate(reversed(list(by_file.items()))): + shard = table(tmp_path, description, members, f"shard{k}.csv")[0] + shards[path] = shard.read_bytes().splitlines(keepends=True) + assert len({lines[0] for lines in shards.values()}) == 1 # every shard got every ID + # Merged by the final row order (path, then z and t within each staged file). + merged = [line for path in sorted(shards) for line in shards[path][1:]] + assert b"".join([shards[path][0], *merged]) == serial + # A resumed run: half the shards from before, half written again, the same bytes. + resumed = {**shards, **{p: shards[p] for p in sorted(shards)[::2]}} + assert ( + b"".join([shards[path][0], *[ln for p in sorted(resumed) for ln in resumed[p][1:]]]) + == serial + ) diff --git a/tests/test_histogram.py b/tests/test_histogram.py index 27ef002..2bb482e 100644 --- a/tests/test_histogram.py +++ b/tests/test_histogram.py @@ -386,7 +386,7 @@ def test_adjacent_64_bit_integers_sharing_a_float_are_not_exact(): assert not hist.is_exact and hist.max_representative_error == 1.0 pooled = PooledHistogram([hist]) assert pooled.counts.tolist() == [3] and pooled.max_representative_error == 1.0 - record = PlaneRecord("a.tif", "c", "r", "t", [hist], position_id="a") + record = PlaneRecord("a.tif", "c", "r", "t", [hist], position_id="a", channels=["DNA"]) settings = NormalizationSettings(method="minmax", scope="image") with pytest.raises(ValueError, match="zero"): # 2**53 + 1 - 2**53 rounds to 0 in float64 fit([record], settings, group="a.tif") @@ -394,7 +394,13 @@ def test_adjacent_64_bit_integers_sharing_a_float_are_not_exact(): parameters = fit( [ PlaneRecord( - "a.tif", "c", "r", "t", [compute_plane_histogram(plane, 256)], position_id="a" + "a.tif", + "c", + "r", + "t", + [compute_plane_histogram(plane, 256)], + position_id="a", + channels=["DNA"], ) ], settings, @@ -426,8 +432,10 @@ def test_wide_64_bit_bound_covers_every_pixel_exactly(dtype, bins): def test_64_bit_bound_survives_the_statistics_file(tmp_path): plane = np.array([[2**53 + 1, 2**53 + 3, 2**53 + 4]], dtype=np.uint64) hist = compute_plane_histogram(plane, 256) - record = PlaneRecord("a.tif", "c", "r", "t", [hist], position_id="a") - (back,) = read_plane_histograms(write_plane_histograms([record], tmp_path / "p.csv")) + record = PlaneRecord("a.tif", "c", "r", "t", [hist], position_id="a", channels=["DNA"]) + (back,) = read_plane_histograms( + write_plane_histograms([record], tmp_path / "p.csv", channel_ids=["DNA"]) + ) read = back.histograms[0] assert read.pixel_min == 2**53 + 1 and read.pixel_max == 2**53 + 4 assert read.max_representative_error == hist.max_representative_error == 1.0 diff --git a/tests/test_integrity.py b/tests/test_integrity.py index 3efd5d5..6b0b09d 100644 --- a/tests/test_integrity.py +++ b/tests/test_integrity.py @@ -19,7 +19,9 @@ image_summaries, read_plane_histograms, scope_groups, - write_plane_histograms, +) +from morphobricks_normalization import ( + write_plane_histograms as write_table, ) from morphobricks_normalization.csv_io import CsvFormatError from morphobricks_normalization.integrity import LIMIT_NAMES @@ -221,24 +223,24 @@ def test_a_plane_without_valid_pixels_has_no_limits(): @pytest.mark.parametrize( "tail, message", [ - ((2, float("nan"), 1.0), "the sum must be a number"), - ((2, 1.0, -1.0), "sum of squares 0 or more"), - ((2, 1.0, float("nan")), "sum of squares 0 or more"), - ((2, float("inf"), 4.0), "infinite sum"), - ((2, 10.0, 1.0), "variance would be negative"), + ((2, float("nan"), 1.0), "the mean must be finite"), + ((2, float("inf"), 1.0), "the mean must be finite"), + ((2, 1.0, -1.0), "standard deviation a finite number 0 or above"), + ((2, 1.0, float("nan")), "standard deviation a finite number 0 or above"), + ((2, 1.0, float("inf")), "standard deviation a finite number 0 or above"), ((-1, 0.0, 0.0), "between 0 and 2"), - ((0, 1.0, 0.0), "empty underflow or overflow must have zero sums"), + ((0, 1.0, 0.0), "empty underflow or overflow must have mean and std 0"), + ((0, 0.0, 1.0), "empty underflow or overflow must have mean and std 0"), ], ) -def test_impossible_tail_sums_are_refused(tail, message): +def test_impossible_tail_moments_are_refused(tail, message): with pytest.raises(ValueError, match=message): HistogramTail(*tail) def test_signed_and_constant_tails_are_accepted(): - assert HistogramTail(2, -6.0, 18.0).mean == -3.0 - third = 0.1 + 0.2 # rounding of the sums must not refuse a constant tail - assert HistogramTail(3, 3 * third, 3 * third * third).count == 3 + assert HistogramTail(2, -3.0, 0.0).mean == -3.0 + assert HistogramTail(3, 0.1 + 0.2, 0.0).count == 3 def robust_plane_histogram(dtype=np.float64, outliers=(-1e6, 1e6)): @@ -253,10 +255,11 @@ def test_tails_must_fit_their_range(): good = robust_plane_histogram() assert not good.underflow.is_empty and not good.overflow.is_empty lo, above = good.underflow, good.binning_min + 1.0 + half_range = (good.binning_min - good.pixel_min) / 2 for bad in ( - HistogramTail(lo.count, lo.count * above, lo.count * above * above), # above the limit - HistogramTail(lo.count, lo.sum, lo.sum_of_squares * 10), # values beyond the minimum - HistogramTail(lo.count, lo.sum, float("inf")), # cannot overflow in this range + HistogramTail(lo.count, above, lo.std), # mean above the limit + HistogramTail(lo.count, good.pixel_min - 1.0, lo.std), # mean below the minimum + HistogramTail(lo.count, lo.mean, half_range * 1.001), # wider than its range ): with pytest.raises(ValueError, match="which values in that range cannot have"): PlaneHistogram( @@ -274,11 +277,13 @@ def test_tails_must_fit_their_range(): ) -@pytest.mark.filterwarnings("ignore:overflow encountered:RuntimeWarning") # issue #33 -def test_a_sum_of_squares_that_overflows_float64_is_accepted_where_possible(): - # Values near 1e200 overflow their squares (issue #33): legitimate, not corruption. +def test_tails_of_values_whose_squares_overflow_float64_stay_finite(): + # Values near 1e200 overflowed the raw sums of squares of the draft format (issue #33); + # the centred moments are exact and finite. result = robust_plane_histogram(outliers=(-1e200, 1e200)) - assert result.underflow.sum_of_squares == np.inf and result.overflow.count >= 1 + assert result.overflow.count >= 1 + for tail in (result.underflow, result.overflow): + assert np.isfinite([tail.mean, tail.std]).all() and tail.std > 1e198 # --- The statistics file (#43, #44, #45, #46, #47, #48) ---------------------------------------- @@ -287,7 +292,12 @@ def test_a_sum_of_squares_that_overflows_float64_is_accepted_where_possible(): def record(path="c/r/a.tif", histograms=None, **coords): if histograms is None: histograms = [compute_plane_histogram(np.array([[0, 1], [1, 255]], np.uint8), 2)] - return PlaneRecord(path, "c", "r", "t", histograms, **coords, position_id="p") + # A channel named C0: a legal channel ID (ADR 0003), so the columns keep their old names. + return PlaneRecord(path, "c", "r", "t", histograms, **coords, position_id="p", channels=("C0",)) + + +def write_plane_histograms(records, path): + return write_table(records, path, channel_ids=("C0",)) def edited(tmp_path, records, changes, row=0): @@ -323,7 +333,7 @@ def edited(tmp_path, records, changes, row=0): ) def test_impossible_counts_and_limits_in_the_file_are_refused(tmp_path, changes, message): path = edited(tmp_path, [record()], changes) - with pytest.raises(CsvFormatError, match=f"line 2: the bin counts of channel 0 .*{message}"): + with pytest.raises(CsvFormatError, match=f"line 2: the bin counts of channel C0 .*{message}"): read_plane_histograms(path) @@ -332,7 +342,7 @@ def test_a_single_blank_limit_in_the_file_is_refused(tmp_path, column): # The default record's limits are min-max, so a blank extremum equals its binning limit # and was silently filled in from it before #47 was fixed. path = edited(tmp_path, [record()], {column: ""}) - message = f"line 2: the bin counts of channel 0 .*{column} is blank, but the channel has 4" + message = f"line 2: the bin counts of channel C0 .*{column} is blank, but the channel has 4" with pytest.raises(CsvFormatError, match=message): read_plane_histograms(path) @@ -357,21 +367,20 @@ def test_wrapping_counts_in_the_file_are_refused(tmp_path): @pytest.mark.parametrize( "column, value", [ - ("Histogram_UnderflowSum_C0", "nan"), - ("Histogram_UnderflowSumOfSquares_C0", "-1"), - ("Histogram_UnderflowSumOfSquares_C0", "inf"), - ("Histogram_OverflowSum_C0", "inf"), - ("Histogram_OverflowSum_C0", "0.0"), + ("Histogram_UnderflowMean_C0", "nan"), + ("Histogram_UnderflowStd_C0", "-1"), + ("Histogram_UnderflowStd_C0", "inf"), + ("Histogram_OverflowMean_C0", "inf"), + ("Histogram_OverflowMean_C0", "0.0"), ], ) def test_impossible_sums_of_a_populated_tail_are_refused(tmp_path, column, value): path = edited(tmp_path, [record(histograms=[robust_plane_histogram()])], {column: value}) - with pytest.raises(CsvFormatError, match="line 2: the bin counts of channel 0"): + with pytest.raises(CsvFormatError, match="line 2: the bin counts of channel C0"): read_plane_histograms(path) -@pytest.mark.filterwarnings("ignore:overflow encountered:RuntimeWarning") # issue #33 -def test_populated_tails_and_overflowed_squares_round_trip(tmp_path): +def test_populated_tails_of_huge_values_round_trip(tmp_path): records = [ record("c/r/a.tif", [robust_plane_histogram()]), record("c/r/b.tif", [robust_plane_histogram(outliers=(-1e200, 1e200))]), diff --git a/tests/test_parameter_checks.py b/tests/test_parameter_checks.py index de68752..44a952c 100644 --- a/tests/test_parameter_checks.py +++ b/tests/test_parameter_checks.py @@ -22,6 +22,8 @@ def parameters(offset=(0.0,), scale=(1.0,), channels=None, scope="image", group=None): + if channels is None: + channels = tuple(f"ch{c}" for c in range(len(offset))) return Parameters("z-score", scope, group, True, offset, scale, channels=channels) @@ -30,7 +32,7 @@ def parameters(offset=(0.0,), scale=(1.0,), channels=None, scope="image", group= @pytest.mark.parametrize("value", BAD_NUMBERS) def test_parameters_refuse_a_non_finite_offset(value): - with pytest.raises(ValueError, match="Channel 1: the normalization offset must be a finite"): + with pytest.raises(ValueError, match="Channel ch1: the normalization offset must be a finite"): parameters(offset=(0.0, value), scale=(1.0, 1.0)) with pytest.raises(ValueError, match="The min-max method: the normalization offset"): NormalizationParameters("min-max", value, 1.0) @@ -38,7 +40,7 @@ def test_parameters_refuse_a_non_finite_offset(value): @pytest.mark.parametrize("value", [*BAD_NUMBERS, 0.0, -0.0, -2.5]) def test_parameters_refuse_a_scale_that_is_not_finite_and_positive(value): - with pytest.raises(ValueError, match="Channel 0: the normalization scale must be a finite"): + with pytest.raises(ValueError, match="Channel ch0: the normalization scale must be a finite"): parameters(scale=(value,)) with pytest.raises(ValueError, match="normalization scale must be a finite number above 0"): NormalizationParameters("z-score", 0.0, value) @@ -54,7 +56,7 @@ def test_small_scales_and_non_finite_pixels_are_kept(): def test_accepted_parameters_round_trip_through_their_columns(): - original = parameters(offset=(1.5, -2.0), scale=(1e-300, 7.0), channels=(0, 3)) + original = parameters(offset=(1.5, -2.0), scale=(1e-300, 7.0), channels=("DNA", "PAX_488")) assert ( Parameters.from_columns(original.to_columns(), NormalizationSettings(method="zscore")) == original @@ -64,12 +66,24 @@ def test_accepted_parameters_round_trip_through_their_columns(): def test_fitted_parameters_are_valid_and_a_zero_spread_keeps_its_message(): plane = np.arange(64, dtype=np.uint16).reshape(8, 8) record = PlaneRecord( - "c/r/a.tif", "c", "r", SINGLE, [compute_plane_histogram(plane)], position_id="p" + "c/r/a.tif", + "c", + "r", + SINGLE, + [compute_plane_histogram(plane)], + position_id="p", + channels=("DNA",), ) fitted = fit([record], NormalizationSettings(method="zscore")) assert all(math.isfinite(o) for o in fitted.offset) and fitted.scale[0] > 0 flat = PlaneRecord( - "c/r/b.tif", "c", "r", SINGLE, [compute_plane_histogram(plane * 0)], position_id="p" + "c/r/b.tif", + "c", + "r", + SINGLE, + [compute_plane_histogram(plane * 0)], + position_id="p", + channels=("DNA",), ) with pytest.raises(ValueError, match="intensity spread is zero"): fit([flat], NormalizationSettings(method="zscore")) @@ -117,22 +131,30 @@ def test_the_scope_check_keeps_group_and_time_point_as_written(): assert Parameters.from_columns(written.to_columns(), settings) == written -# --- Channel indices (#38) --------------------------------------------------------------------- +# --- Channel IDs (#38; ADR 0003 HV2) ----------------------------------------------------------- -@pytest.mark.parametrize("channel", [1.9, 1.0, True, np.bool_(True), "1", -1, None]) -def test_channel_indices_are_never_truncated(channel): - with pytest.raises(ValueError, match="A channel index is a whole number counted from 0"): - parameters(offset=(0.0, 10.0), scale=(1.0, 1.0), channels=(0, channel)) - with pytest.raises(ValueError, match="A channel index is a whole number counted from 0"): +@pytest.mark.parametrize("channel", [1.9, 1, np.int64(1), True, np.bool_(True), None]) +def test_channel_indices_are_refused_not_read_as_ids(channel): + with pytest.raises(ValueError, match="never by its position in the image"): + parameters(offset=(0.0, 10.0), scale=(1.0, 1.0), channels=("A", channel)) + with pytest.raises(ValueError, match="never by its position in the image"): parameters(offset=(0.0, 10.0), scale=(1.0, 1.0)).select([channel]) -def test_integer_channel_indices_are_accepted(): - p = parameters(offset=(0.0, 10.0), scale=(1.0, 2.0), channels=(np.int64(0), np.uint8(4))) - assert p.channels == (0, 4) and all(type(c) is int for c in p.channels) - selected = p.select(c for c in [np.int32(4)]) - assert (selected.channels, selected.offset, selected.scale) == ((4,), (10.0,), (2.0,)) +@pytest.mark.parametrize( + "channel", ["1", "", " DNA", "DNA ", "dna-1", "D__NA", "_DNA", "DNA_", "A;B"] +) +def test_text_that_is_not_a_token_is_refused(channel): + with pytest.raises(ValueError, match="is not a canonical channel ID"): + parameters(offset=(0.0,), scale=(1.0,), channels=(channel,)) + + +def test_channel_ids_with_underscores_are_accepted_and_kept_as_written(): + p = parameters(offset=(0.0, 10.0), scale=(1.0, 2.0), channels=("PAX_488", "dna")) + assert p.channels == ("PAX_488", "dna") + selected = p.select(c for c in ["dna"]) + assert (selected.channels, selected.offset, selected.scale) == (("dna",), (10.0,), (2.0,)) # --- Channels without finite pixels (#57) ------------------------------------------------------ @@ -141,7 +163,17 @@ def test_integer_channel_indices_are_accepted(): def summaries_with(*channels): histograms = [compute_plane_histogram(np.asarray(c, np.float32), 100) for c in channels] return image_summaries( - [PlaneRecord("c/r/img.tif", "c", "r", SINGLE, histograms, position_id="p")] + [ + PlaneRecord( + "c/r/img.tif", + "c", + "r", + SINGLE, + histograms, + position_id="p", + channels=("DNA", "PHASE", "ACTIN")[: len(histograms)], + ) + ] ) @@ -149,15 +181,16 @@ def summaries_with(*channels): def test_a_channel_without_finite_pixels_is_not_exact_and_is_reported(missing): good = np.arange(16, dtype=np.float32).reshape(4, 4) (summary,) = summaries_with(good, np.full((4, 4), missing)) + assert summary.channels == ("DNA", "PHASE") assert summary.exact == (False, None) - assert summary.to_row()["Histogram_Exact_C1"] is None + assert summary.to_row()["Histogram_Exact_PHASE"] is None (issue,) = unchecked_channels([summary]) assert (issue.code, issue.channel, issue.items) == ( "channel_without_finite_pixels", - 1, + "PHASE", ("c/r/img.tif",), ) - assert "channel C1: it has no finite pixels" in issue.message + assert "channel PHASE: it has no finite pixels" in issue.message assert intensity_checks([summary]) == [] # the four checks of §7.2 are unchanged diff --git a/tests/test_pooling.py b/tests/test_pooling.py index 4333506..0cf43af 100644 --- a/tests/test_pooling.py +++ b/tests/test_pooling.py @@ -1,3 +1,5 @@ +from fractions import Fraction + import numpy as np import pytest @@ -98,18 +100,30 @@ def test_percentile_limits(): pooled.percentile(101) -def test_opposing_infinite_tail_sums_are_a_named_refusal(): - # Float64 pixels at both ends overflow the tails' sums to +inf and -inf; the mean then - # needs a refusal that says why, not a bare error from math.fsum. +def test_tails_at_both_float64_limits_have_a_finite_exact_mean(): + # Float64 pixels at both ends overflowed the draft format's raw tail sums to +inf and + # -inf (a named refusal then). Their tails' means are finite, and the pooled mean is + # found by exact power-of-two scaling. plane = np.full((80, 80), 1.0) plane.ravel()[:5] = -1.7e308 plane.ravel()[5:10] = 1.7e308 plane.ravel()[10:] = np.linspace(0, 10, 6390) histogram = compute_plane_histogram(plane, settings=ROBUST) - assert histogram.underflow.sum == -np.inf and histogram.overflow.sum == np.inf - with pytest.raises(ValueError, match="opposite sign") as caught: - _ = PooledHistogram([histogram]).mean - assert "tail 1 of 2" in str(caught.value) and "tail 2 of 2" in str(caught.value) + assert histogram.underflow.mean < -1e307 and histogram.overflow.mean > 1e307 + pooled = PooledHistogram([histogram]) + stored = [ + (c, Fraction(v)) for c, v in zip(pooled.bin_counts, pooled.bin_values.tolist(), strict=True) + ] + stored += [(c, Fraction(m)) for c, m, _ in pooled.tails] + oracle = sum(c * v for c, v in stored) / pooled.pixel_count # exact, from what is stored + # Each count * value product is rounded once (as for the bins): within that bound. + bound = sum(abs(c * v) for c, v in stored) / pooled.pixel_count * Fraction(2) ** -52 + assert abs(Fraction(pooled.mean) - oracle) <= bound + # From the pixels: as close as float64 tail means allow (each within half a step of + # 1.7e308, and the two tails cancel), never inf or nan. + exact = sum(Fraction(v) for v in plane.ravel().tolist()) / plane.size + assert abs(pooled.mean - float(exact)) <= 1e-15 * 1.7e308 + assert np.isfinite(pooled.std) @pytest.mark.parametrize("q", [0, 25, 50, 75, 100]) @@ -158,7 +172,7 @@ def test_percentiles_from_pixels_near_float64_limits_are_finite_and_exact(q, lim def exact_std(values, weights, tails=()): - """Population standard deviation of weighted values and (count, sum, squares) tails. + """Population standard deviation of weighted values and (count, mean, std) tails. Every float is taken at its exact value (Fraction) and the square root in 40 digits, so nothing here can overflow or round like the package's float64 arithmetic. @@ -171,10 +185,10 @@ def exact_std(values, weights, tails=()): tails = [tuple(Fraction(x) for x in tail) for tail in tails] total = sum(weights) + sum(c for c, _, _ in tails) mean = ( - sum(w * v for w, v in zip(weights, values, strict=True)) + sum(s for _, s, _ in tails) + sum(w * v for w, v in zip(weights, values, strict=True)) + sum(c * m for c, m, _ in tails) ) / total squares = sum(w * (v - mean) ** 2 for w, v in zip(weights, values, strict=True)) - squares += sum(s2 - s * s / c + c * (s / c - mean) ** 2 for c, s, s2 in tails) + squares += sum(c * s * s + c * (m - mean) ** 2 for c, m, s in tails) variance = squares / total with localcontext() as context: context.prec = 40 @@ -215,25 +229,26 @@ def test_std_with_large_tails_is_found_by_scaling(): plane = np.random.default_rng(7).normal(0, 1, 10000) * 1e153 settings = HistogramSettings(histogram_range="robust", robust_quantiles=(0.05, 99.95)) hist = compute_plane_histogram(plane, 256, settings=settings) - assert hist.has_tails and np.isfinite(hist.overflow.sum_of_squares) + assert hist.has_tails and np.isfinite(hist.overflow.std) pooled = PooledHistogram([hist]) expected = exact_std(pooled.bin_values, pooled.bin_counts, pooled.tails) assert pooled.std == pytest.approx(expected, rel=1e-12, abs=0) assert pooled.std == pytest.approx(np.std(plane / 1e153) * 1e153, rel=1e-4) -def test_tail_whose_squares_overflowed_is_a_named_refusal(tmp_path): +def test_tails_whose_squares_overflow_float64_give_the_exact_std(tmp_path): + # The draft format's raw sums of squares overflowed here and the z-score was refused; + # centred moments keep the spread, through the statistics file. plane = np.random.default_rng(1).normal(0, 1, 10000) * 1e150 plane[:5] = (1e200, 2e200, -1e200, 3e200, -2e200) - with np.errstate(over="ignore"): # the tail's squares overflow as the file is written - hist = compute_plane_histogram(plane, 256, settings=ROBUST) - assert hist.overflow.sum_of_squares == np.inf - record = PlaneRecord("a.tif", "c", "r", "t", [hist], position_id="a") - (back,) = read_plane_histograms(write_plane_histograms([record], tmp_path / "p.csv")) + hist = compute_plane_histogram(plane, 256, settings=ROBUST) + record = PlaneRecord("a.tif", "c", "r", "t", [hist], position_id="a", channels=["DNA"]) + (back,) = read_plane_histograms( + write_plane_histograms([record], tmp_path / "p.csv", channel_ids=["DNA"]) + ) pooled = PooledHistogram([back.histograms[0]]) - assert np.isfinite(pooled.mean) and np.isfinite(pooled.percentile(50)) - message = "pooled: tail 1 of 2 has sum of squares inf, tail 2 of 2 has sum of squares inf" - with pytest.raises(ValueError, match=message): - pooled.summary() # the standard deviation; mean and percentiles stay available - with pytest.raises(ValueError, match="percentile or robust z-score"): - fit([back], NormalizationSettings(method="zscore", scope="image"), group="a.tif") + expected = exact_std(pooled.bin_values, pooled.bin_counts, pooled.tails) + assert pooled.std == pytest.approx(expected, rel=1e-12, abs=0) + assert pooled.std == pytest.approx(np.std(plane / 1e150) * 1e150, rel=1e-6) + parameters = fit([back], NormalizationSettings(method="zscore", scope="image"), group="a") + assert parameters.scale[0] == pooled.std diff --git a/tests/test_pooling_weights.py b/tests/test_pooling_weights.py index 1611248..759c8c1 100644 --- a/tests/test_pooling_weights.py +++ b/tests/test_pooling_weights.py @@ -13,7 +13,7 @@ ) from morphobricks_normalization.normalization import NORMAL_CONSISTENCY from morphobricks_normalization.settings import FLUORESCENCE_METHODS -from simulation import cached_design +from simulation import cached_design, channel_ids def repeat_settings(method, pooling): @@ -23,7 +23,13 @@ def repeat_settings(method, pooling): def plane(condition, pixels, repeat="r1", name="img.tif"): histogram = compute_plane_histogram(np.asarray(pixels)) return PlaneRecord( - f"{condition}/{repeat}/{name}", condition, repeat, "single", histogram, position_id="p" + f"{condition}/{repeat}/{name}", + condition, + repeat, + "single", + histogram, + position_id="p", + channels=("DNA",), ) @@ -121,7 +127,10 @@ def test_both_weights_agree_at_the_image_and_patch_scopes(method, scope): by_pixels = NormalizationSettings(method=method, scope=scope, pooling="pixels") by_conditions = NormalizationSettings(method=method, scope=scope, pooling="conditions") assert by_pixels == by_conditions # the pooling weight has no effect at these scopes - assert fit(stack, by_conditions, channel_axis=0) == fit(stack, by_pixels, channel_axis=0) + ids = channel_ids(2) + assert fit(stack, by_conditions, channel_axis=0, channels=ids) == fit( + stack, by_pixels, channel_axis=0, channels=ids + ) records = [r for r in all_records if r.source_relative_path == path] if scope == "image": assert fit(records, by_conditions) == fit(records, by_pixels) diff --git a/tests/test_range_check_settings.py b/tests/test_range_check_settings.py index 075120d..d81c474 100644 --- a/tests/test_range_check_settings.py +++ b/tests/test_range_check_settings.py @@ -33,21 +33,28 @@ def _narrow(**changes): ], ) def test_range_check_issues_refuse_mixed_settings_in_either_order(change): - pair = [("A", 0, _narrow()), ("B", 0, _narrow(**change))] + pair = [("A", "DNA", _narrow()), ("B", "DNA", _narrow(**change))] for order in (pair, pair[::-1]): - with pytest.raises(ValueError, match="channel C0.*A.*B|channel C0.*B.*A") as refused: + with pytest.raises(ValueError, match="channel DNA.*A.*B|channel DNA.*B.*A") as refused: range_check_issues(order) assert type(refused.value).__name__ == "IncompatibleRangeChecksError" # the same settings in another channel are not mixed - assert len(range_check_issues([("A", 0, _narrow()), ("B", 1, _narrow(**change))])) == 2 + pairs = [("A", "DNA", _narrow()), ("B", "ACTIN", _narrow(**change))] + assert [i.channel for i in range_check_issues(pairs)] == ["ACTIN", "DNA"] -@pytest.mark.parametrize("channel", [True, 1.0, 1.5, "1", -1, None]) -def test_range_check_issues_refuse_malformed_channels(channel): - with pytest.raises(ValueError, match="whole number counted from 0"): +@pytest.mark.parametrize("channel", [True, 1.0, 1.5, 0, np.int64(2), None]) +def test_range_check_issues_refuse_channel_indices(channel): + with pytest.raises(ValueError, match="never by its position"): range_check_issues([("A", channel, _narrow())]) -def test_range_check_issues_accept_numpy_integer_channels(): - (issue,) = range_check_issues([("A", np.int64(2), _narrow())]) - assert issue.channel == 2 and type(issue.channel) is int +@pytest.mark.parametrize("channel", ["1", "", "DNA "]) +def test_range_check_issues_refuse_text_that_is_not_an_id(channel): + with pytest.raises(ValueError, match="is not a canonical channel ID"): + range_check_issues([("A", channel, _narrow())]) + + +def test_range_check_issues_name_the_channel_id(): + (issue,) = range_check_issues([("A", "PAX_488", _narrow())]) + assert issue.channel == "PAX_488" and "channel PAX_488 " in issue.message diff --git a/tests/test_rgb.py b/tests/test_rgb.py index 558c0af..078722b 100644 --- a/tests/test_rgb.py +++ b/tests/test_rgb.py @@ -24,6 +24,8 @@ from morphobricks_normalization.rgb import EPSILON, apply_rgb, check_rgb from simulation import cached_rgb_design, make_rgb_records, simulate_rgb_design +RGB = ("R", "G", "B") +RGB_SORTED = ("B", "G", "R") RGB_MESSAGE = ( "RGB normalization needs images with exactly three 8- or 16-bit colour channels, but these " "images have {} channels of type {}. Check the axis mapping (the samples axis must be the " @@ -45,7 +47,7 @@ def direct(pixels, low=1.0, high=99.8): def assert_rgb(parameters, offset, scale): - assert parameters.channels == (0, 1, 2) + assert parameters.channels == ("B", "G", "R") assert parameters.offset == (offset,) * 3 assert parameters.scale == (scale,) * 3 @@ -187,7 +189,7 @@ def test_a_constant_image_gives_zeros_not_an_error(): def test_fluorescence_still_refuses_zero_spread(): pixels = np.full((5, 6, 3), 200, np.uint8) with pytest.raises(ValueError, match="intensity spread is zero"): - fit(pixels, NormalizationSettings(scope="patch"), channel_axis=-1) + fit(pixels, NormalizationSettings(scope="patch"), channel_axis=-1, channels=("A", "B", "C")) def test_columns_hold_the_shared_value_and_round_trip_exactly(): @@ -195,11 +197,11 @@ def test_columns_hold_the_shared_value_and_round_trip_exactly(): settings = rgb() parameters = fit(records[:2], settings, group="control/2024_05_01/img_01.tif") columns = parameters.to_columns() - for c in (0, 1, 2): + for c in ("R", "G", "B"): assert columns[offset_column(c)] == repr(parameters.offset[0]) assert columns[scale_column(c)] == repr(parameters.scale[0]) assert columns["normalization_method"] == "rgb percentile 1-99.8" - assert offset_column(3) not in columns + assert offset_column("C0") not in columns again = Parameters.from_columns(columns, settings) assert again == parameters assert (again.offset, again.scale, again.channels) == ( @@ -210,14 +212,18 @@ def test_columns_hold_the_shared_value_and_round_trip_exactly(): def test_from_columns_refuses_incomplete_or_different_rgb_values(): - parameters = Parameters("rgb percentile 1-99.8", "image", "g", True, (2.0,) * 3, (5.0,) * 3) + parameters = Parameters( + "rgb percentile 1-99.8", "image", "g", True, (2.0,) * 3, (5.0,) * 3, RGB_SORTED + ) columns = parameters.to_columns() - with pytest.raises(ValueError, match="Normalization_Offset_C2, Normalization_Scale_C2"): - Parameters.from_columns({**columns, offset_column(2): "", scale_column(2): ""}, rgb()) + with pytest.raises(ValueError, match="Normalization_Offset_B, Normalization_Scale_B"): + Parameters.from_columns({**columns, offset_column("B"): "", scale_column("B"): ""}, rgb()) with pytest.raises(ValueError, match="also holds channel"): - Parameters.from_columns({**columns, offset_column(3): "1.0", scale_column(3): "2.0"}, rgb()) + Parameters.from_columns( + {**columns, offset_column("C3"): "1.0", scale_column("C3"): "2.0"}, rgb() + ) with pytest.raises(ValueError, match="the same offset and scale"): - Parameters.from_columns({**columns, offset_column(1): "2.5"}, rgb()) + Parameters.from_columns({**columns, offset_column("G"): "2.5"}, rgb()) with pytest.raises(ValueError, match="the settings use 'percentile 1-99.8'"): Parameters.from_columns(columns, NormalizationSettings()) @@ -230,7 +236,15 @@ def test_wrong_channel_counts_are_refused(shape): with pytest.raises(ValueError, match="have 1 channels of type uint16"): fit(np.zeros((6, 6), np.uint16), rgb(scope="patch")) histogram = compute_plane_histogram(np.arange(10, dtype=np.uint8)) - row = PlaneRecord("c/r/a.tif", "c", "r", "single", (histogram,) * shape[-1], position_id="p") + row = PlaneRecord( + "c/r/a.tif", + "c", + "r", + "single", + (histogram,) * shape[-1], + position_id="p", + channels=("R", "G", "B", "PHASE")[: shape[-1]], + ) with pytest.raises(ValueError) as error: fit([row], rgb()) assert str(error.value) == RGB_MESSAGE.format(shape[-1], "uint8") @@ -271,7 +285,7 @@ def test_approximate_histograms_stay_within_their_bound(): def test_a_group_without_pixels_names_the_three_channels(): empty = compute_plane_histogram(np.zeros((0, 0), np.uint8)) - row = PlaneRecord("c/r/a.tif", "c", "r", "single", (empty,) * 3, position_id="p") + row = PlaneRecord("c/r/a.tif", "c", "r", "single", (empty,) * 3, position_id="p", channels=RGB) with pytest.raises(NoValidPixelsError, match="Group 'g', channels R, G and B"): fit([row], rgb(), group="g") @@ -283,7 +297,7 @@ def test_apply_returns_a_new_array_and_never_changes_the_input(): normalized = parameters.apply(pixels, channel_axis=-1) np.testing.assert_array_equal(pixels, before) assert not np.shares_memory(normalized, pixels) - with pytest.raises(ValueError, match="give channel_axis"): + with pytest.raises(ValueError, match="give channels"): parameters.apply(pixels) float_pixels = pixels.astype(np.float32) again = parameters.apply(float_pixels, channel_axis=-1) @@ -371,28 +385,58 @@ def test_rgb_images_with_different_pixel_types_are_refused_before_pooling(monkey def test_rgb_images_with_one_pixel_type_and_several_times_are_pooled(kind): images, records = cached_rgb_design(seed=51, n_times=2, **DTYPES[kind]) parameters = fit(records[:4], rgb(scope="biological_repeat")) - assert parameters.channels == (0, 1, 2) + assert parameters.channels == RGB_SORTED -@pytest.mark.parametrize("subset", [[2], [0, 1], [1, 2], [0, 2]]) +@pytest.mark.parametrize("subset", [["B"], ["R", "G"], ["G", "B"], ["R", "B"]]) def test_rgb_parameters_refuse_a_channel_subset(subset): """Issue #50: a subset could not be read back by from_columns.""" - p = Parameters("rgb percentile 1-99.8", "image", "g", True, (1.0,) * 3, (4.0,) * 3) + p = Parameters("rgb percentile 1-99.8", "image", "g", True, (1.0,) * 3, (4.0,) * 3, RGB_SORTED) with pytest.raises(ValueError, match="three colour channels together"): p.select(subset) def test_rgb_parameters_select_all_channels_unchanged_and_fluorescence_subsets_work(): - p = Parameters("rgb percentile 1-99.8", "image", "g", True, (1.0,) * 3, (4.0,) * 3) - assert p.select([2, 1, 0]) == p + p = Parameters("rgb percentile 1-99.8", "image", "g", True, (1.0,) * 3, (4.0,) * 3, RGB_SORTED) + assert p.select(["B", "G", "R"]) == p settings = rgb() - assert Parameters.from_columns(p.select([0, 1, 2]).to_columns(), settings) == p + assert Parameters.from_columns(p.select(["R", "G", "B"]).to_columns(), settings) == p pixels = np.arange(27, dtype=np.uint8).reshape(3, 3, 3) assert np.array_equal( - p.select([0, 1, 2]).apply(pixels, channel_axis=0), p.apply(pixels, channel_axis=0) + p.select(RGB).apply(pixels, channel_axis=0), p.apply(pixels, channel_axis=0) + ) + assert np.array_equal( + p.apply(pixels, channel_axis=0, channels=RGB), p.apply(pixels, channel_axis=0) + ) + with pytest.raises(ValueError, match="channels R, G and B of an RGB imaging unit"): + p.apply(pixels, channel_axis=0, channels=["B", "G", "R"]) + fluorescence = Parameters( + "percentile 1-99.8", "image", "g", True, (1.0, 2.0, 3.0), (4.0,) * 3, ("A", "B", "C") + ) + assert fluorescence.select(["C"]).channels == ("C",) + + +def test_rgb_rows_need_the_rgb_channel_ids_in_order(): + _, records = cached_rgb_design(seed=52) + row = records[0] + swapped = PlaneRecord( + row.source_relative_path, + row.condition, + row.biological_repeat, + row.technical_repeat, + row.histograms, + position_id="p", + channels=("B", "G", "R"), ) - fluorescence = Parameters("percentile 1-99.8", "image", "g", True, (1.0, 2.0, 3.0), (4.0,) * 3) - assert fluorescence.select([2]).channels == (2,) + with pytest.raises(ValueError, match="R, G and B of an RGB imaging unit, in this order"): + fit([swapped], rgb()) + with pytest.raises(ValueError, match="R, G and B of an RGB imaging unit, in this order"): + fit( + np.zeros((4, 4, 3), np.uint8), + rgb(scope="patch"), + channel_axis=-1, + channels=("B", "G", "R"), + ) @pytest.mark.parametrize("dtype", [np.uint8, np.uint16, np.float32, np.float64]) diff --git a/tests/test_robust_range.py b/tests/test_robust_range.py index e7f6168..42029fe 100644 --- a/tests/test_robust_range.py +++ b/tests/test_robust_range.py @@ -7,6 +7,7 @@ """ import csv +from fractions import Fraction import numpy as np import pytest @@ -25,7 +26,9 @@ percentile_range, range_check_issues, read_plane_histograms, - write_plane_histograms, +) +from morphobricks_normalization import ( + write_plane_histograms as write_table, ) from morphobricks_normalization.csv_io import CsvFormatError from morphobricks_normalization.histogram import estimate_histogram_memory @@ -34,6 +37,13 @@ ROBUST = HistogramSettings(histogram_range="robust") ZERO_SPREAD = "intensity spread is zero" +C0 = ("C0",) # a channel named C0: a legal channel ID (ADR 0003), so the columns keep their names + + +def write_plane_histograms(records, path): + ids = sorted({c for r in records for c in r.channels}) + return write_table(records, path, channel_ids=ids) + def whole(plane): return lambda: (plane,) @@ -75,7 +85,7 @@ def issue_planes(seed=0): return clean, hot -def assert_same_histogram(a, b, sums_exact=True): +def assert_same_histogram(a, b): assert a.dtype == b.dtype and a.bin_count == b.bin_count assert (a.binning_min, a.binning_max) == (b.binning_min, b.binning_max) assert (a.pixel_min, a.pixel_max) == (b.pixel_min, b.pixel_max) @@ -83,12 +93,8 @@ def assert_same_histogram(a, b, sums_exact=True): np.testing.assert_array_equal(a.counts, b.counts) for ta, tb in ((a.underflow, b.underflow), (a.overflow, b.overflow)): assert ta.count == tb.count - if sums_exact: - assert (ta.sum, ta.sum_of_squares) == (tb.sum, tb.sum_of_squares) - else: - np.testing.assert_allclose( - [ta.sum, ta.sum_of_squares], [tb.sum, tb.sum_of_squares], rtol=1e-12 - ) + # Mean and std are rounded once from exact moments: equal whole or in blocks. + assert (ta.mean, ta.std) == (tb.mean, tb.std) # --- The reproduction of issue #12 ------------------------------------------------------------- @@ -104,7 +110,7 @@ def test_minmax_keeps_todays_result_and_the_check_flags_it(): # ...and the same zero-spread error (fit gives no new warning, N57). with pytest.raises(ValueError, match=ZERO_SPREAD): percentile_range(PooledHistogram([histogram]), 1, 99.8) - record = PlaneRecord("c/img.tif", "c", "r", "t", histogram, position_id="p") + record = PlaneRecord("c/img.tif", "c", "r", "t", histogram, position_id="p", channels=C0) with pytest.raises(ValueError, match=ZERO_SPREAD): fit([record], NormalizationSettings()) # The check sees the collapsed central range and what the robust range would give. @@ -113,8 +119,8 @@ def test_minmax_keeps_todays_result_and_the_check_flags_it(): assert check.robust_limits == (quantile(hot, 0.01), quantile(hot, 99.99)) assert check.outside_robust_count == 52 # 26 pixels below and 26 above assert check.robust_central_bins > 5000 - (issue,) = range_check_issues([(record.plane_id, 0, check)]) - assert issue.code == "narrow_histogram_range" and issue.channel == 0 + (issue,) = range_check_issues([(record.plane_id, "C0", check)]) + assert issue.code == "narrow_histogram_range" and issue.channel == "C0" assert issue.items == ("c/img.tif::z=0,t=0",) assert "fewer than 100 of the histogram's bins (as few as 1)" in issue.message assert "robust histogram range" in issue.action @@ -141,7 +147,7 @@ def test_robust_range_gives_percentiles_close_to_the_clean_planes(): assert parameters.offset == pytest.approx(exact[0], abs=0.01) # Pixels in the tails are represented by their mean, so the bound covers them. assert pooled.max_representative_error > 1e5 - record = PlaneRecord("c/img.tif", "c", "r", "t", histogram, position_id="p") + record = PlaneRecord("c/img.tif", "c", "r", "t", histogram, position_id="p", channels=C0) parameters = fit([record], NormalizationSettings()) assert not parameters.exact assert [i.code for i in parameters.issues] == ["approximate_parameters"] @@ -231,11 +237,11 @@ def test_blockwise_equals_whole_in_minmax_mode(dtype): @pytest.mark.parametrize("dtype", DTYPES) def test_blockwise_gives_the_same_robust_limits_and_counts(dtype): - # Only the tails' float sums may differ in the last bits (summation order). + # The tails too: their moments are exact, so block order cannot change a bit. plane = mixed_plane(dtype, np.random.default_rng(6)) expected, check = histogram_with_range_check(whole(plane), dtype, settings=ROBUST) histogram, block_check = histogram_with_range_check(in_blocks(plane), dtype, settings=ROBUST) - assert_same_histogram(histogram, expected, sums_exact=False) + assert_same_histogram(histogram, expected) assert block_check == check @@ -278,7 +284,8 @@ def test_small_integers_with_fewer_bins_than_values_use_the_robust_range(): quantile(plane, 99.99), ) assert histogram.overflow.count >= 6 and histogram.pixel_max == 65535 - assert histogram.overflow.sum == float(plane[plane > histogram.binning_max].sum()) + above = plane[plane > histogram.binning_max].astype(np.int64) + assert histogram.overflow.mean == float(Fraction(int(above.sum()), above.size)) minmax, minmax_check = histogram_with_range_check(in_blocks(plane), plane.dtype, 256) assert_same_histogram(minmax, compute_plane_histogram(plane, 256)) assert minmax_check.robust_limits == check.robust_limits @@ -313,7 +320,7 @@ def test_nan_and_infinite_pixels_are_left_out(): histogram = compute_plane_histogram(plane, settings=ROBUST) assert (histogram.valid_pixel_count, histogram.non_finite_pixel_count) == (2487, 13) assert histogram.binning_min == quantile(plane, 0.01) - assert np.isfinite([histogram.overflow.sum, histogram.underflow.sum]).all() + assert np.isfinite([histogram.overflow.mean, histogram.underflow.mean]).all() def test_coinciding_robust_limits(): @@ -323,7 +330,8 @@ def test_coinciding_robust_limits(): histogram = compute_plane_histogram(plane, settings=ROBUST) assert (histogram.binning_min, histogram.binning_max) == (0.0, 0.0) assert histogram.counts[0] == 39998 and histogram.counts[1:].sum() == 0 - assert (histogram.underflow.sum, histogram.overflow.sum) == (-5.0, 7.0) + assert (histogram.underflow.mean, histogram.overflow.mean) == (-5.0, 7.0) + assert histogram.underflow.std == histogram.overflow.std == 0.0 pooled = PooledHistogram([histogram]) assert pooled.percentile(0.001) == -5.0 and pooled.percentile(100) == 7.0 assert pooled.median == 0.0 @@ -395,6 +403,7 @@ def test_zscore_from_a_robust_table_uses_the_exact_sums(): "t", compute_plane_histogram(p, 1000, settings=ROBUST), position_id="p", + channels=C0, ) for i, p in enumerate(planes) ] @@ -405,7 +414,7 @@ def test_zscore_from_a_robust_table_uses_the_exact_sums(): assert not parameters.exact -# --- Statistics file, schema version 4 --------------------------------------------------------- +# --- Statistics file, schema version 5 --------------------------------------------------------- def robust_records(): @@ -421,6 +430,7 @@ def robust_records(): [compute_plane_histogram(p, 500, settings=ROBUST), compute_plane_histogram(p, 500)], t=i, position_id="p", + channels=("C0", "C1"), ) for i, p in enumerate(float_planes) ], [ @@ -431,12 +441,13 @@ def robust_records(): "t", compute_plane_histogram(p, settings=ROBUST), position_id="p", + channels=C0, ) for i, p in enumerate(int_planes) ] -def test_schema_4_round_trip(tmp_path): +def test_schema_5_round_trip_keeps_the_tails(tmp_path): for k, records in enumerate(robust_records()): path = write_plane_histograms(records, tmp_path / f"t{k}.csv") back = read_plane_histograms(path) @@ -448,65 +459,10 @@ def test_schema_4_round_trip(tmp_path): assert ha.max_representative_error == hb.max_representative_error with open(path, newline="", encoding="utf-8") as fh: row = next(csv.DictReader(fh)) - assert row["schema_version"] == "4" + assert row["schema_version"] == "5" assert int(row["Histogram_UnderflowCount_C0"]) == records[0].histograms[0].underflow.count - assert float(row["Histogram_OverflowSum_C0"]) == records[0].histograms[0].overflow.sum - - -def as_version_3(path): - """Rewrite a version-4 table as version 3: no tail columns, schema_version 3.""" - with open(path, newline="", encoding="utf-8") as fh: - header, *rows = list(csv.reader(fh)) - keep = [i for i, name in enumerate(header) if "flow" not in name] - version = header.index("schema_version") - for row in rows: - row[version] = "3" - with open(path, "w", newline="", encoding="utf-8") as fh: - csv.writer(fh).writerows([[r[i] for i in keep] for r in [header, *rows]]) - return path - - -def test_version_3_is_refused(tmp_path): - records = [ - PlaneRecord( - "c/a.tif", "c", "r", "t", compute_plane_histogram(np.arange(6)), position_id="p" - ) - ] - path = as_version_3(write_plane_histograms(records, tmp_path / "plane_histograms.csv")) - with pytest.raises(CsvFormatError) as error: - read_plane_histograms(path) - assert str(error.value) == ( - f"{path}: This plane_histograms.csv was written with schema version 3, without the " - "underflow and overflow columns. Run the preprocessing brick again on this experiment: " - "it rewrites the table with these columns (schema version 4)." - ) - - -def test_a_version_3_header_without_rows_is_refused_by_its_columns(tmp_path): - # No row, so no schema_version value to read: the missing tail columns refuse it. - records = [ - PlaneRecord( - "c/a.tif", "c", "r", "t", compute_plane_histogram(np.arange(6)), position_id="p" - ) - ] - path = as_version_3(write_plane_histograms(records, tmp_path / "plane_histograms.csv")) - lines = path.read_text(encoding="utf-8").splitlines(keepends=True) - path.write_text(lines[0], encoding="utf-8") - with pytest.raises(CsvFormatError, match="unexpected per-channel column names"): - read_plane_histograms(path) - - -def test_a_row_of_version_3_in_a_version_4_table_is_refused(tmp_path): - records = [ - PlaneRecord( - "c/a.tif", "c", "r", "t", compute_plane_histogram(np.arange(6)), position_id="p" - ) - ] - path = write_plane_histograms(records, tmp_path / "t.csv") - text = path.read_text(encoding="utf-8").replace(",4,", ",3,", 1) - path.write_text(text, encoding="utf-8") - with pytest.raises(CsvFormatError, match="schema version 3, without the underflow"): - read_plane_histograms(path) + assert float(row["Histogram_OverflowMean_C0"]) == records[0].histograms[0].overflow.mean + assert float(row["Histogram_OverflowStd_C0"]) == records[0].histograms[0].overflow.std @pytest.mark.parametrize( @@ -514,7 +470,9 @@ def test_a_row_of_version_3_in_a_version_4_table_is_refused(tmp_path): [ ("Histogram_UnderflowCount_C0", "0"), # counts no longer add up ("Histogram_BinningMin_C0", None), # limit at the extremum, yet a tail below it - ("Histogram_OverflowSum_C1", "3.0"), # sums of an empty tail + ("Histogram_OverflowMean_C1", "3.0"), # moments of an empty tail + ("Histogram_OverflowStd_C0", "-1.0"), # a negative spread + ("Histogram_OverflowStd_C0", "1e300"), # more than half the tail's range ], ) def test_inconsistent_tails_are_rejected(tmp_path, column, value): @@ -559,20 +517,20 @@ def check(bins, outside=3): issues = range_check_issues( [ - ("a::z=0,t=0", 1, check(5)), - ("a::z=0,t=0", 0, check(500)), # wide enough - ("b::z=0,t=0", 1, check(20, outside=4)), - ("b::z=0,t=0", 0, None), # exact histogram - ("c::z=0,t=0", 0, check(99)), + ("a::z=0,t=0", "DNA", check(5)), + ("a::z=0,t=0", "ACTIN", check(500)), # wide enough + ("b::z=0,t=0", "DNA", check(20, outside=4)), + ("b::z=0,t=0", "ACTIN", None), # exact histogram + ("c::z=0,t=0", "ACTIN", check(99)), ] ) assert [(i.channel, i.items) for i in issues] == [ - (0, ("c::z=0,t=0",)), - (1, ("a::z=0,t=0", "b::z=0,t=0")), + ("ACTIN", ("c::z=0,t=0",)), + ("DNA", ("a::z=0,t=0", "b::z=0,t=0")), ] assert "In 2 plane(s)" in issues[1].message and "as few as 5" in issues[1].message assert "7 pixel(s)" in issues[1].message and "at least 900 bins" in issues[1].message - assert range_check_issues([("a", 0, check(100))]) == [] + assert range_check_issues([("a", "DNA", check(100))]) == [] def test_saturated_fraction_is_blank_when_the_maximum_is_in_the_overflow(): @@ -582,7 +540,7 @@ def test_saturated_fraction_is_blank_when_the_maximum_is_in_the_overflow(): histogram = compute_plane_histogram(plane, 1000, settings=ROBUST) assert histogram.overflow.count >= 3 (summary,) = image_summaries( - [PlaneRecord("c/a.tif", "c", "r", "t", histogram, position_id="p")] + [PlaneRecord("c/a.tif", "c", "r", "t", histogram, position_id="p", channels=C0)] ) assert summary.saturated_fraction == (None,) assert summary.exact == (False,) diff --git a/tests/test_schema_versions.py b/tests/test_schema_versions.py new file mode 100644 index 0000000..99c8e05 --- /dev/null +++ b/tests/test_schema_versions.py @@ -0,0 +1,107 @@ +"""Tables of earlier schema versions are refused by the version-5 reader (ADR 0003 HV4). + +A table is recomputed, never converted: the reader names version 5 and the fix. +""" + +import csv + +import numpy as np +import pytest + +from morphobricks_normalization import ( + PlaneRecord, + compute_plane_histogram, + read_plane_histograms, + write_plane_histograms, +) +from morphobricks_normalization.csv_io import CsvFormatError + +C0 = ("C0",) + + +def as_version_3(path): + """Rewrite a version-5 table as version 3: no tail columns, schema_version 3.""" + with open(path, newline="", encoding="utf-8") as fh: + header, *rows = list(csv.reader(fh)) + keep = [i for i, name in enumerate(header) if "flow" not in name] + version = header.index("schema_version") + for row in rows: + row[version] = "3" + with open(path, "w", newline="", encoding="utf-8") as fh: + csv.writer(fh).writerows([[r[i] for i in keep] for r in [header, *rows]]) + return path + + +def test_version_3_is_refused(tmp_path): + records = [ + PlaneRecord( + "c/a.tif", + "c", + "r", + "t", + compute_plane_histogram(np.arange(6)), + position_id="p", + channels=C0, + ) + ] + path = as_version_3( + write_plane_histograms(records, tmp_path / "plane_histograms.csv", channel_ids=C0) + ) + with pytest.raises(CsvFormatError) as error: + read_plane_histograms(path) + assert str(error.value) == ( + f"{path}: This plane_histograms.csv was written with schema version 3, its channels " + "named by index and without the underflow and overflow columns. This version reads " + "schema version 5 only, with every channel named by its canonical channel ID from the " + "experiment description; old tables are not converted. Run the preprocessing brick " + "again on this experiment: it recomputes the table (schema version 5)." + ) + + +def test_a_version_3_header_without_rows_is_refused_by_its_columns(tmp_path): + # No row, so no schema_version value to read: the missing tail columns refuse it. + records = [ + PlaneRecord( + "c/a.tif", + "c", + "r", + "t", + compute_plane_histogram(np.arange(6)), + position_id="p", + channels=C0, + ) + ] + path = as_version_3( + write_plane_histograms(records, tmp_path / "plane_histograms.csv", channel_ids=C0) + ) + lines = path.read_text(encoding="utf-8").splitlines(keepends=True) + path.write_text(lines[0], encoding="utf-8") + with pytest.raises(CsvFormatError, match="does not have the per-channel columns of schema"): + read_plane_histograms(path) + + +@pytest.mark.parametrize( + "version, match", + [("3", "schema version 3, its channels named by index"), ("4", "schema version 4, its")], +) +def test_a_row_of_an_earlier_version_in_a_version_5_table_is_refused(tmp_path, version, match): + records = [ + PlaneRecord( + f"c/{name}.tif", + "c", + "r", + "t", + compute_plane_histogram(np.arange(6)), + position_id="p", + channels=C0, + ) + for name in ("a", "b") + ] + path = write_plane_histograms(records, tmp_path / "t.csv", channel_ids=C0) + with open(path, newline="", encoding="utf-8") as fh: + header, *rows = list(csv.reader(fh)) + rows[1][header.index("schema_version")] = version # the second row, not the first + with open(path, "w", newline="", encoding="utf-8") as fh: + csv.writer(fh).writerows([header, *rows]) + with pytest.raises(CsvFormatError, match=match): + read_plane_histograms(path) diff --git a/tests/test_scopes.py b/tests/test_scopes.py index fa76728..c29ed85 100644 --- a/tests/test_scopes.py +++ b/tests/test_scopes.py @@ -22,7 +22,14 @@ def record(condition, repeat, technical=SINGLE, image="img.tif", t=0): path = f"{condition}/{repeat}/{technical}/{image}" histogram = compute_plane_histogram(np.arange(4, dtype=np.uint8)) return PlaneRecord( - path, condition, repeat, technical, (histogram, histogram), t=t, position_id="p" + path, + condition, + repeat, + technical, + (histogram, histogram), + t=t, + position_id="p", + channels=("DNA", "ACTIN"), ) @@ -62,6 +69,7 @@ def test_pairing_differently_named_repeats_changes_only_the_grouping(): r.histograms, t=r.t, position_id="p", + channels=r.channels, ) for r in matching ] diff --git a/tests/test_settings.py b/tests/test_settings.py index 378cf63..8165954 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -47,6 +47,7 @@ def all_settings(): per_time_point=per_time_point, ) yield NormalizationSettings(image_type="rgb") + yield NormalizationSettings(scope="biological_repeat", partial_channels="planes_with_channel") def test_defaults_are_those_of_the_contract(): @@ -61,6 +62,7 @@ def test_defaults_are_those_of_the_contract(): } assert settings.consistency == NORMAL_CONSISTENCY == 1.482602218505602 assert settings.pooling == "pixels" + assert settings.partial_channels == "refuse" # A_SCOPE_INCOMPLETE unless chosen @pytest.mark.parametrize("settings", list(all_settings()), ids=repr) @@ -85,6 +87,21 @@ def test_keys_that_do_not_apply_are_omitted_and_their_options_reset(): assert NormalizationSettings(percentile=[2, 98]).percentile == (2.0, 98.0) +def test_the_choice_of_planes_with_a_channel_is_recorded_and_refusal_is_the_default(): + chosen = NormalizationSettings( + scope="biological_repeat", partial_channels="planes_with_channel" + ) + assert chosen.to_dict()["partial_channels"] == "planes_with_channel" + assert "partial_channels" not in NormalizationSettings().to_dict() # the default: refuse + assert NormalizationSettings.from_dict({"partial_channels": "refuse"}).partial_channels == ( + "refuse" + ) + rgb = NormalizationSettings(image_type="rgb", partial_channels="planes_with_channel") + assert rgb.partial_channels == "refuse" # RGB rows always hold R, G and B + with pytest.raises(ValueError, match="partial_channels 'pool' does not exist"): + NormalizationSettings(partial_channels="pool") + + @pytest.mark.parametrize("scope", ["condition", "technical_repeat", "experiment", "batch", ""]) def test_unknown_and_removed_scopes_are_refused(scope): with pytest.raises(ValueError) as error: diff --git a/tests/test_spec_pin.py b/tests/test_spec_pin.py new file mode 100644 index 0000000..e192e29 --- /dev/null +++ b/tests/test_spec_pin.py @@ -0,0 +1,236 @@ +"""Pin integrity of the vendored MorphoBricks-Spec files (offline, no git, no network). + +TEMPLATE: copy this file to ``tests/test_spec_pin.py`` of a brick. It is maintained in +MorphoBricks-Spec (``tools/brick_templates/``); change it there. Docs: ``docs/vendoring.md``. + +The vendored folder (``tests/spec_fixtures`` by default, made by ``tools/vendor.py``) must hash +to its ``PIN.toml``: ``fixtures/`` to the git tree-object ID ``git rev-parse :fixtures`` +and every other pinned file to its git blob ID. One changed, added or missing byte fails a +test, so a local edit of a fixture cannot make a reader pass. The hashing is git's own (the +scheme of the preprocessing brick's ``tests/spec_pin.py``), computed here with ``hashlib``. + +Edit only ``VENDORED`` below if the copy lives elsewhere. Needs Python 3.11 or newer +(``tomllib``) and pytest. These tests never skip. ``MORPHOBRICKS_SPEC_ROOT`` points them at +another folder laid out like the vendored one. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import shutil +import tomllib +from pathlib import Path + +import pytest + +VENDORED = Path(__file__).parent / "spec_fixtures" +ROOT_ENV = "MORPHOBRICKS_SPEC_ROOT" +#: Files of the folder that belong to the copy but are not Spec content. +OWN_FILES = frozenset({"PIN.toml", "README.md", ".gitattributes"}) + + +class SpecPinError(Exception): + """Raised when the vendored Spec files are missing, unreadable or malformed.""" + + +def spec_root() -> Path: + """Return the folder holding ``fixtures/``, ``docs/`` and ``PIN.toml``.""" + override = os.environ.get(ROOT_ENV) + return Path(override) if override else VENDORED + + +def read_pin(root: Path | None = None) -> dict: + """Return the parsed ``PIN.toml``, or raise ``SpecPinError`` when it is absent.""" + path = (root or spec_root()) / "PIN.toml" + if not path.is_file(): + raise SpecPinError(f"{path} is missing: the Spec files are not vendored here") + return tomllib.loads(path.read_text(encoding="utf-8")) + + +def git_blob_id(data: bytes) -> str: + """Return the git blob-object ID of ``data`` (``git hash-object``).""" + return hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest() + + +def git_tree_id(folder: Path) -> str: + """Return the git tree-object ID of a folder, as ``git write-tree`` would give it. + + Every file counts as mode 100644 (the vendored copy has no executable files, and a + file system without mode bits could not tell). A symbolic link, a missing folder or + anything that is neither a file nor a folder raises ``SpecPinError``. + """ + if not folder.is_dir() or folder.is_symlink(): + raise SpecPinError(f"{folder} is not a folder") + entries = [] + for entry in os.scandir(folder): + name = entry.name.encode("utf-8") + if entry.is_symlink(): + raise SpecPinError(f"{entry.path} is a symbolic link") + if entry.is_dir(): + entries.append((name + b"/", b"40000", name, bytes.fromhex(git_tree_id(Path(entry))))) + elif entry.is_file(): + blob = git_blob_id(Path(entry).read_bytes()) + entries.append((name, b"100644", name, bytes.fromhex(blob))) + else: + raise SpecPinError(f"{entry.path} is neither a file nor a folder") + body = b"".join(mode + b" " + name + b"\0" + sha for _, mode, name, sha in sorted(entries)) + return hashlib.sha1(b"tree %d\0" % len(body) + body).hexdigest() + + +def pin_problems(root: Path | None = None) -> list[str]: + """Return what differs between the vendored files and the pin (empty when they match).""" + root = root or spec_root() + pin = read_pin(root) + problems = [] + if not re.fullmatch(r"[0-9a-f]{40}", str(pin.get("commit", ""))): + problems.append("the pin is not a full 40-digit commit SHA") + tree = git_tree_id(root / "fixtures") + if tree != pin.get("fixtures_tree"): + problems.append(f"fixtures/ hashes to {tree}, the pin says {pin.get('fixtures_tree')}") + for relative, blob in pin.get("files", {}).items(): + path = root / relative + if not path.is_file(): + problems.append(f"{relative} is missing") + elif git_blob_id(path.read_bytes()) != blob: + problems.append(f"{relative} does not hash to its pin {blob}") + allowed = OWN_FILES | set(pin.get("files", {})) + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + if "__pycache__" in path.parts or not path.is_file() or relative.startswith("fixtures/"): + continue + if relative not in allowed: + problems.append(f"{relative} is not a pinned file") + return problems + + +def test_vendored_files_hash_to_the_pin(): + assert pin_problems() == [] + + +def test_the_pin_is_a_full_commit_sha_never_a_branch(): + pin = read_pin() + assert re.fullmatch(r"[0-9a-f]{40}", pin["commit"]) + assert re.fullmatch(r"[0-9a-f]{40}", pin["fixtures_tree"]) + assert pin["files"], "the pin names no file" + assert all(re.fullmatch(r"[0-9a-f]{40}", blob) for blob in pin["files"].values()) + + +def test_the_pinned_tree_is_the_folder_that_is_vendored(): + # The expected value is the pin's own: this fails when fixtures/ is not what it names. + assert git_tree_id(spec_root() / "fixtures") == read_pin()["fixtures_tree"] + + +def test_git_blob_and_tree_ids_equal_git_s(): + # Known answers computed with git itself (git hash-object, git write-tree). + assert git_blob_id(b"hello\n") == "ce013625030ba8dba906f756967f9e9ca394464a" + + +def test_git_tree_id_orders_a_folder_as_its_name_and_a_slash(tmp_path): + # `a.b` sorts before the folder `a` (compared as `a/`), `a-b` before both: the order that + # makes a hand-rolled tree hash differ from git's when it sorts by plain name. + (tmp_path / "b").mkdir() + (tmp_path / "a").mkdir() + (tmp_path / "a.txt").write_bytes(b"hello\n") + (tmp_path / "b" / "c").write_bytes(b"x") + (tmp_path / "a.b").write_bytes(b"y") + (tmp_path / "a-b").write_bytes(b"z") + (tmp_path / "a" / "f").write_bytes(b"w") + assert git_tree_id(tmp_path) == "fec6a54403e1c2a94724c1aab9f127c686f6dbb0" + + +def _copy(tmp_path: Path) -> Path: + root = tmp_path / "spec" + shutil.copytree(spec_root(), root) + return root + + +def _first(root: Path, *parts: str) -> Path: + files = sorted(p for p in (root.joinpath(*parts)).rglob("*") if p.is_file()) + assert files, f"nothing under {'/'.join(parts)}" + return files[0] + + +def _flip_byte(path: Path) -> None: + data = bytearray(path.read_bytes()) + data[len(data) // 2] ^= 0x01 + path.write_bytes(bytes(data)) + + +def _flip_fixture(root: Path) -> None: + _flip_byte(_first(root, "fixtures", "valid")) + + +def _add_fixture(root: Path) -> None: + (root / "fixtures" / "valid" / "v99_added.toml").write_bytes(b"# added\n") + + +def _remove_fixture(root: Path) -> None: + _first(root, "fixtures", "invalid").unlink() + + +def _rename_fixture(root: Path) -> None: + path = _first(root, "fixtures", "valid") + path.rename(path.with_name("renamed_" + path.name)) + + +def _crlf_fixture(root: Path) -> None: + path = _first(root, "fixtures", "valid") + path.write_bytes(path.read_bytes().replace(b"\n", b"\r\n")) + + +def _flip_pinned_file(root: Path) -> None: + _flip_byte(root / sorted(read_pin(root)["files"])[0]) + + +def _remove_pinned_file(root: Path) -> None: + (root / sorted(read_pin(root)["files"])[0]).unlink() + + +def _add_stray_file(root: Path) -> None: + (root / "docs").mkdir(exist_ok=True) + (root / "docs" / "stray.md").write_bytes(b"not pinned\n") + + +MUTATIONS = [ + _flip_fixture, + _add_fixture, + _remove_fixture, + _rename_fixture, + _crlf_fixture, + _flip_pinned_file, + _remove_pinned_file, + _add_stray_file, +] + + +@pytest.mark.parametrize("mutate", MUTATIONS, ids=lambda f: f.__name__.lstrip("_")) +def test_one_changed_added_or_missing_byte_fails_the_pin(tmp_path, mutate): + root = _copy(tmp_path) + assert pin_problems(root) == [] + mutate(root) + assert pin_problems(root) != [] + + +def test_a_symbolic_link_in_the_fixtures_is_refused(tmp_path): + root = _copy(tmp_path) + link = root / "fixtures" / "valid" / "v98_link.toml" + try: + link.symlink_to(_first(root, "fixtures", "valid")) + except OSError as error: # a file system without links: report it, never skip + pytest.fail(f"cannot create a link to test the refusal: {error}") + with pytest.raises(SpecPinError, match="symbolic link"): + pin_problems(root) + + +def test_a_pin_that_is_not_a_full_sha_is_reported(tmp_path): + root = _copy(tmp_path) + pin = root / "PIN.toml" + pin.write_text(pin.read_text(encoding="utf-8").replace(read_pin()["commit"], "main")) + assert any("40-digit" in problem for problem in pin_problems(root)) + + +def test_a_missing_pin_file_is_an_error_not_a_pass(tmp_path): + with pytest.raises(SpecPinError, match="PIN.toml is missing"): + pin_problems(tmp_path) diff --git a/tests/test_summaries.py b/tests/test_summaries.py index 2edcd69..043638b 100644 --- a/tests/test_summaries.py +++ b/tests/test_summaries.py @@ -18,15 +18,20 @@ summary_columns, write_plane_histograms, ) -from simulation import cached_design, simulate_plane +from simulation import cached_design, channel_ids, simulate_plane -CHANNEL_COLUMN = re.compile(r"^[A-Z][A-Za-z]*_[A-Z][A-Za-z]*_C(0|[1-9][0-9]*)$") +CHANNEL_COLUMN = re.compile( + r"^[A-Z][A-Za-z]*_[A-Z][A-Za-z]*_[A-Za-z][A-Za-z0-9]*(?:_[A-Za-z0-9]+)*$" +) -def record(path, planes, z=0, t=0, bin_count=None, labels=("control", "rep1", SINGLE)): - """One XYC plane of ``path``; ``planes`` holds one array per channel.""" +def record(path, planes, z=0, t=0, bin_count=None, labels=("control", "rep1", SINGLE), ids=None): + """One XYC plane of ``path``; ``planes`` holds one array per channel (``ch0``, ``ch1``...).""" histograms = [compute_plane_histogram(p, bin_count) for p in planes] - return PlaneRecord(path, *labels, histograms=histograms, z=z, t=t, position_id="p") + ids = channel_ids(len(planes)) if ids is None else ids + return PlaneRecord( + path, *labels, histograms=histograms, z=z, t=t, position_id="p", channels=ids + ) def direct(pixels, low=1.0, high=99.8): @@ -171,51 +176,78 @@ def test_a_channel_without_finite_pixels_is_blank(): assert (summary.median[1], summary.p_low[1], summary.p_high[1]) == (None, None, None) assert summary.saturated_fraction == (None, None) row = summary.to_row() - assert row["Intensity_MedianIntensity_C1"] is None + assert row["Intensity_MedianIntensity_ch1"] is None -def test_different_channel_counts_are_refused(): +def test_images_with_different_channels_are_summarized_by_channel_id(): + # ADR 0003 replaces N43: each summary holds its image's channels, by ID. rng = np.random.default_rng(7) plane = simulate_plane(rng, np.uint16, (8, 8)) - records = [record("a.tif", [plane]), record("b.tif", [plane, plane])] - with pytest.raises(ValueError, match="different numbers of channels.*a.tif.*b.tif"): + other = simulate_plane(rng, np.uint16, (8, 8)) + records = [ + record("a.tif", [plane], ids=("DNA",)), + record("b.tif", [other, plane], ids=("PHASE", "DNA")), + ] + a, b = image_summaries(records) + assert (a.channels, b.channels) == (("DNA",), ("DNA", "PHASE")) + assert a.median == (b.median[0],) # DNA is the same pixels in both, whatever its index + assert b.median[1] == image_summaries([record("c.tif", [other], ids=("PHASE",))])[0].median[0] + assert "Intensity_MedianIntensity_PHASE" not in a.to_row() # absent: no value, not zero + columns = summary_columns(["PHASE", "DNA"]) + assert [c for c in columns if c.startswith("Intensity_MedianIntensity")] == [ + "Intensity_MedianIntensity_DNA", + "Intensity_MedianIntensity_PHASE", + ] + + +def test_planes_of_one_image_with_different_channels_are_refused(): + plane = simulate_plane(np.random.default_rng(8), np.uint16, (8, 8)) + records = [ + record("a.tif", [plane], ids=("DNA",)), + record("a.tif", [plane], z=1, ids=("PHASE",)), + ] + with pytest.raises(ValueError, match="planes list different channels"): image_summaries(records) + with pytest.raises(ValueError, match="planes list different channels"): + list(iter_image_summaries(records)) def test_rows_have_the_contract_columns(): _, records = cached_design(seed=10, n_times=2) summaries = image_summaries(records) - columns = summary_columns(2) + columns = summary_columns(channel_ids(2)) assert columns == [ "source_relative_path", "condition", "biological_repeat", "technical_repeat", "image_file", - "Intensity_MedianIntensity_C0", - "Intensity_LowPercentile_C0", - "Intensity_HighPercentile_C0", - "Intensity_SaturatedFraction_C0", - "Histogram_Exact_C0", - "Intensity_MedianIntensity_C1", - "Intensity_LowPercentile_C1", - "Intensity_HighPercentile_C1", - "Intensity_SaturatedFraction_C1", - "Histogram_Exact_C1", + "Intensity_MedianIntensity_ch0", + "Intensity_LowPercentile_ch0", + "Intensity_HighPercentile_ch0", + "Intensity_SaturatedFraction_ch0", + "Histogram_Exact_ch0", + "Intensity_MedianIntensity_ch1", + "Intensity_LowPercentile_ch1", + "Intensity_HighPercentile_ch1", + "Intensity_SaturatedFraction_ch1", + "Histogram_Exact_ch1", ] assert all(CHANNEL_COLUMN.match(name) for name in columns[5:]) summary = summaries[0] row = summary.to_row() assert list(row) == columns assert row["image_file"] == summary.image_file - assert row["Intensity_MedianIntensity_C1"] == summary.median[1] - assert row["Intensity_HighPercentile_C0"] == summary.p_high[0] - assert row["Histogram_Exact_C1"] is True + assert row["Intensity_MedianIntensity_ch1"] == summary.median[1] + assert row["Intensity_HighPercentile_ch0"] == summary.p_high[0] + assert row["Histogram_Exact_ch1"] is True def test_summaries_from_the_written_table_equal_those_from_the_records(tmp_path): _, records = cached_design(seed=10, n_times=2) - path = write_plane_histograms(records, tmp_path / "plane_histograms.csv") + path = write_plane_histograms( + records, tmp_path / "plane_histograms.csv", channel_ids=channel_ids(2) + ) assert image_summaries(read_plane_histograms(path)) == image_summaries(records) @@ -311,7 +343,7 @@ def test_streaming_refuses_a_file_under_two_label_sets(): list(iter_image_summaries(rows)) -def test_streaming_refuses_repeated_planes_and_different_channel_counts(): +def test_streaming_refuses_repeated_planes_and_accepts_different_channels_per_image(): rows = grouped_records(n_images=1) with pytest.raises(ValueError, match="more than once"): list(iter_image_summaries([*rows, rows[0]])) @@ -319,8 +351,7 @@ def test_streaming_refuses_repeated_planes_and_different_channel_counts(): two = [ r for r in grouped_records(n_images=2, channels=2) if r.source_relative_path == "img1.tif" ] - with pytest.raises(ValueError, match="different numbers of channels"): - list(iter_image_summaries([*one, *two])) + assert list(iter_image_summaries([*one, *two])) == image_summaries([*one, *two]) def test_streaming_validates_percentiles_at_the_call(): diff --git a/tests/test_sums.py b/tests/test_sums.py index 38cf7cd..cb12af4 100644 --- a/tests/test_sums.py +++ b/tests/test_sums.py @@ -39,7 +39,9 @@ def _records(settings: HistogramSettings | None = None) -> list[PlaneRecord]: plane = rng.gamma(2.0, 300.0 + 7 * i, size=(64, 64)) hist = compute_plane_histogram(plane, 256, settings=settings) records.append( - PlaneRecord(f"c/r/img{i}.tif", "c", f"r{i % 2}", "t", [hist], position_id="p") + PlaneRecord( + f"c/r/img{i}.tif", "c", f"r{i % 2}", "t", [hist], position_id="p", channels=("DNA",) + ) ) return records @@ -133,7 +135,7 @@ def test_huge_planes_are_refused_by_name_not_by_fsum() -> None: for plane in (np.array([[big, -big], [big, -big]]), np.array([[big, big * 0.9], [1.0, 2.0]])): with np.errstate(over="ignore", invalid="ignore"): try: - fit(plane, NormalizationSettings(method="zscore")) + fit(plane, NormalizationSettings(method="zscore"), channels=["DNA"]) except ValueError as error: assert "z-score normalization cannot be used" in str(error) else: diff --git a/tests/test_tail_moments.py b/tests/test_tail_moments.py new file mode 100644 index 0000000..fd020d0 --- /dev/null +++ b/tests/test_tail_moments.py @@ -0,0 +1,165 @@ +"""Centred tail moments (schema version 5) against an exact Fraction oracle (#91 owner question). + +A tail stores its pixels' count, mean and population standard deviation, each the float64 +nearest its exact value. The oracle here takes every pixel at its exact rational value, so +nothing in it can cancel, overflow or round; the square root is taken in 60 digits and +rounded once. The raw sums of squares of the draft format are shown to cancel on the same +pixels. +""" + +from decimal import Decimal, localcontext +from fractions import Fraction + +import numpy as np +import pytest + +from morphobricks_normalization import ( + HistogramSettings, + PooledHistogram, + compute_block_histogram, + compute_plane_histogram, +) +from morphobricks_normalization.tail_moments import ( + TailMoments, + centred_moments, + counted_sums, + exact_sums, + sqrt_nearest, +) + +ROBUST = HistogramSettings(histogram_range="robust", robust_quantiles=(2.0, 98.0)) + + +def oracle(values): + """``(count, mean, std)`` of ``values``, exact and rounded once to float64.""" + exact = [Fraction(v) for v in np.asarray(values).ravel().tolist()] + n = len(exact) + mean = sum(exact) / n + variance = sum((v - mean) ** 2 for v in exact) / n + return n, float(mean), _sqrt(variance) + + +def _sqrt(value): + with localcontext() as context: + context.prec = 60 + return float((Decimal(value.numerator) / Decimal(value.denominator)).sqrt()) + + +def moments(values): + tail = TailMoments() + tail.add(np.asarray(values)) + return tail.moments() + + +@pytest.mark.parametrize( + "values", + [ + 1e12 + np.random.default_rng(1).uniform(0, 1e-3, 500), # large values, tiny spread + -3e15 + np.arange(400, dtype=np.float64) * 0.5, + np.random.default_rng(2).normal(0, 1, 300) * 1e200, # squares beyond float64 + np.array([5e-324, 1e-320, 2.5e-310]), # subnormals + np.array([-1.7e308, 1.7e308, 1.7e308]), + np.array([7.25] * 9), # a constant tail: std exactly 0 + np.random.default_rng(3).normal(100, 10, 1000).astype(np.float32), + ], +) +def test_float_tails_are_the_exact_moments_rounded_once(values): + assert moments(values) == oracle(values) + + +@pytest.mark.parametrize( + "values", + [ + np.array([2**62 + 1, 2**62 + 3, 2**62 + 4, 2**62 + 4], np.int64), # beyond float64 + np.array([2**64 - 1, 2**64 - 2, 2**63], np.uint64), + np.array([-(2**63), -(2**63) + 7], np.int64), + np.random.default_rng(4).integers(0, 65535, 2000).astype(np.uint16), + ], +) +def test_integer_tails_are_exact_even_where_float64_cannot_hold_the_values(values): + exact = [int(v) for v in values.tolist()] + n = len(exact) + mean = Fraction(sum(exact), n) + variance = sum((v - mean) ** 2 for v in exact) / n + assert moments(values) == (n, float(mean), _sqrt(variance)) + + +def test_large_values_with_a_small_spread_do_not_cancel(): + # Raw float64 sums lose the spread: the sum of squares is about 5e20, whose float64 step + # (65536) is far above the 500 * 8.3 the centred squares add up to. + values = 1e9 + np.random.default_rng(5).uniform(0, 10, 500) + n, mean, std = moments(values) + assert std == oracle(values)[2] and 2 < std < 4 + s, s2 = float(np.sum(values)), float(np.sum(values * values)) + raw = np.sqrt(max(s2 - s * (s / n), 0.0) / n) # the draft's formula + assert abs(raw - std) > std # wrong by more than the spread itself + + +def test_blocks_in_any_order_give_the_same_bits(): + values = np.random.default_rng(6).normal(5e9, 3.0, 3000) + whole = moments(values) + for pieces in (np.array_split(values, 7), np.array_split(values[::-1], 3)): + tail = TailMoments() + for piece in pieces: + tail.add(piece) + assert tail.moments() == whole + + +def test_exact_sums_and_counted_sums_agree_with_fractions(): + values = np.array([0.1, -2.5, 1e300, 3e-320]) + total, squares = exact_sums(values) + assert total == sum(Fraction(v) for v in values.tolist()) + assert squares == sum(Fraction(v) ** 2 for v in values.tolist()) + counts = np.array([0, 3, 0, 2, 5], np.int64) + total, squares = counted_sums(counts, 1000) + assert (total, squares) == ( + 3 * 1001 + 2 * 1003 + 5 * 1004, + 3 * 1001**2 + 2 * 1003**2 + 5 * 1004**2, + ) + + +@pytest.mark.parametrize( + "value", + [Fraction(2), Fraction(1, 3), Fraction(10**400 + 1, 7), Fraction(1, 10**600), Fraction(0)], +) +def test_square_roots_are_rounded_once(value): + assert sqrt_nearest(value) == (_sqrt(value) if value else 0.0) + with pytest.raises(ValueError, match="cannot be negative"): + sqrt_nearest(Fraction(-1)) + + +def test_no_values_give_zeros(): + assert centred_moments(0, Fraction(0), Fraction(0)) == (0, 0.0, 0.0) + + +def test_histogram_tails_whole_and_in_blocks_are_the_oracle_of_their_pixels(): + plane = 1e9 + np.random.default_rng(7).uniform(0, 10, (60, 70)) + whole = compute_plane_histogram(plane, 256, settings=ROBUST) + blocks = compute_block_histogram( + lambda: iter(np.array_split(plane, 9)), plane.dtype, 256, settings=ROBUST + ) + for name, pixels in ( + ("underflow", plane[plane < whole.binning_min]), + ("overflow", plane[plane > whole.binning_max]), + ): + tail = getattr(whole, name) + assert (tail.count, tail.mean, tail.std) == oracle(pixels) + assert getattr(blocks, name) == tail # bit for bit + + +def test_pooled_tails_are_exact_given_the_stored_moments_and_close_to_the_pixels(): + rng = np.random.default_rng(8) + planes = [1e9 + rng.uniform(0, 10, (50, 50)) + k * 2.0 for k in range(3)] + histograms = [compute_plane_histogram(p, 64, settings=ROBUST) for p in planes] + pooled = PooledHistogram(histograms) + # Exact pooling of what is stored: bins at their representatives, tails by moments. + stored = list( + zip(pooled.bin_counts.tolist(), map(Fraction, pooled.bin_values.tolist()), strict=True) + ) + n = pooled.pixel_count + mean = (sum(c * v for c, v in stored) + sum(c * Fraction(m) for c, m, _ in pooled.tails)) / n + squares = sum(c * (v - mean) ** 2 for c, v in stored) + squares += sum(c * Fraction(s) ** 2 + c * (Fraction(m) - mean) ** 2 for c, m, s in pooled.tails) + assert pooled.std == pytest.approx(_sqrt(squares / n), rel=1e-9) + # Against the pixels themselves: the bins' half-width is the only approximation. + assert pooled.std == pytest.approx(oracle(np.concatenate(planes))[2], rel=1e-2) diff --git a/tools/check.py b/tools/check.py index b12b08e..5d4c6d3 100644 --- a/tools/check.py +++ b/tools/check.py @@ -66,9 +66,13 @@ def config() -> dict: def _python_files() -> list[Path]: - """Return the repository's checked Python files (guard fixtures are deliberately bad).""" + """Return the repository's checked Python files. + + The guard fixtures are deliberately bad and ``spec_fixtures`` holds byte-exact copies of + MorphoBricks-Spec files (``tests/spec_fixtures/README.md``), so neither is checked. + """ files = [path for folder in LINTED for path in sorted((ROOT / folder).rglob("*.py"))] - return [path for path in files if "guard_fixtures" not in path.parts] + return [path for path in files if not {"guard_fixtures", "spec_fixtures"} & set(path.parts)] def _module_name(path: Path, package_dir: Path) -> str: diff --git a/tools/code_rules_baseline.json b/tools/code_rules_baseline.json index 41c1cdf..41e8958 100644 --- a/tools/code_rules_baseline.json +++ b/tools/code_rules_baseline.json @@ -9,21 +9,9 @@ "src/morphobricks_normalization/__init__.py : # Ruff findings missing-required-import": 1, "src/morphobricks_normalization/_version.py : # Ruff findings missing-required-import": 1, "src/morphobricks_normalization/checks.py _image_outliers: private function of # lines without a docstring": 42, - "src/morphobricks_normalization/checks.py _no_spread: private function of # lines without a docstring": 22, "src/morphobricks_normalization/checks.py _number: # Ruff findings any-type": 1, - "src/morphobricks_normalization/checks.py _repeat_shifts: private function of # lines without a docstring": 42, "src/morphobricks_normalization/checks.py _saturated_images: private function of # lines without a docstring": 22, - "src/morphobricks_normalization/csv_io.py _channel_layout: private function of # lines without a docstring": 15, - "src/morphobricks_normalization/csv_io.py _earlier_version_message: # Ruff findings missing-type-function-argument": 1, - "src/morphobricks_normalization/csv_io.py _earlier_version_message: private function of # lines without a docstring": 12, - "src/morphobricks_normalization/csv_io.py _row: private function of # lines without a docstring": 32, - "src/morphobricks_normalization/csv_io.py _text: # Ruff findings missing-type-function-argument": 1, - "src/morphobricks_normalization/csv_io.py _value: # Ruff findings missing-return-type-private-function": 1, - "src/morphobricks_normalization/fitting.py _check_rgb_columns: private function of # lines without a docstring": 20, "src/morphobricks_normalization/fitting.py _fit_issues: private function of # lines without a docstring": 28, - "src/morphobricks_normalization/fitting.py _pool: private function of # lines without a docstring": 14, - "src/morphobricks_normalization/fitting.py _records: private function of # lines without a docstring": 17, - "src/morphobricks_normalization/fitting.py _text: # Ruff findings any-type": 1, "src/morphobricks_normalization/histogram.py _bin_pass: # Ruff findings too-many-arguments": 1, "src/morphobricks_normalization/histogram.py _bin_pass: # parameters (max #)": 9, "src/morphobricks_normalization/histogram.py _float_bins: # Ruff findings too-many-arguments": 1, @@ -32,9 +20,10 @@ "src/morphobricks_normalization/histogram.py _integer_bins: # Ruff findings too-many-arguments": 1, "src/morphobricks_normalization/histogram.py _integer_bins: # parameters (max #)": 9, "src/morphobricks_normalization/histogram.py _integer_bins: private function of # lines without a docstring": 25, - "src/morphobricks_normalization/histogram.py: # lines (max #)": 861, + "src/morphobricks_normalization/histogram.py: # lines (max #)": 835, + "src/morphobricks_normalization/parameters.py _check_rgb_columns: private function of # lines without a docstring": 20, + "src/morphobricks_normalization/parameters.py _text: # Ruff findings any-type": 1, "src/morphobricks_normalization/pooling.py __init__: # Ruff findings missing-return-type-special-method": 1, - "src/morphobricks_normalization/records.py _check_channels: private function of # lines without a docstring": 19, "src/morphobricks_normalization/settings.py _percentile_pair: # Ruff findings any-type": 1, "src/morphobricks_normalization/settings.py _positive_number: # Ruff findings any-type": 1, "src/morphobricks_normalization/summaries.py _saturated_fraction: # Ruff findings missing-type-function-argument": 1, diff --git a/tools/dump_outputs.py b/tools/dump_outputs.py index 155d311..a2fa926 100644 --- a/tools/dump_outputs.py +++ b/tools/dump_outputs.py @@ -61,6 +61,11 @@ SMALLEST_NORMAL = float(np.finfo(np.float64).tiny) FLOAT64_MAX = float(np.finfo(np.float64).max) METHODS = ("minmax", "percentile", "zscore", "robust_zscore") +# The one channel of the dumped records. Since plane_histograms.csv schema version 5 a channel +# is named by its canonical ID (ADR 0003); "C0" is a legal ID, so the parameter and summary +# columns keep the names of earlier dumps and a diff against a version-4 base shows only what +# the format changed (the table's digest and Issue.channel). +DUMP_CHANNELS = ("C0",) def text(value: object) -> str: @@ -110,8 +115,8 @@ def histogram_text(hist: PlaneHistogram) -> str: hist.valid_pixel_count, hist.pixel_min, hist.pixel_max, - (hist.underflow.count, hist.underflow.sum, hist.underflow.sum_of_squares), - (hist.overflow.count, hist.overflow.sum, hist.overflow.sum_of_squares), + tail_fields(hist.underflow), + tail_fields(hist.overflow), hist.representatives, hist.max_representative_error, hist.is_exact, @@ -119,6 +124,17 @@ def histogram_text(hist: PlaneHistogram) -> str: return text(fields) +def tail_fields(tail: object) -> tuple: + """Return a tail's stored fields: count, mean and std (schema version 5). + + Commits before the centred tail moments stored count, sum and sum of squares; those + are dumped instead, so the corpus still runs on them (``PYTHONPATH=/src``). + """ + if hasattr(tail, "std"): + return tail.count, tail.mean, tail.std + return tail.count, tail.sum, tail.sum_of_squares + + def planes() -> Iterator[tuple[str, np.ndarray]]: """Yield named simulated planes of every supported pixel type. @@ -255,10 +271,11 @@ def experiment_lines(dtype: str, hists: list[PlaneHistogram], folder: Path) -> I [h], t=i % 2, position_id=f"img{i}", + channels=DUMP_CHANNELS, ) for i, h in enumerate(hists) ] - path = write_plane_histograms(records, folder / f"{dtype}.csv") + path = write_plane_histograms(records, folder / f"{dtype}.csv", channel_ids=DUMP_CHANNELS) yield f"csv {dtype} sha256 {hashlib.sha256(path.read_bytes()).hexdigest()}" back = read_plane_histograms(path) for record in back: