From a17609badad70549022933767d7504bea0a7b8c5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 7 Nov 2026 14:28:42 +0000 Subject: [PATCH] Reconcile the published description with the folder (Stage 1, PR 9) Units are unchanged, new or removed by exact folder labels; N35 reopening keeps confirmed_on; renamed folders resolve only by an explicit statement; removals come back as one grouped question per condition. Fixes the condition-folder layout in the transition date check (P33 b). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Scma6W3LC11s3qjXHZuGDm --- AGENTS.md | 5 + CHANGELOG.md | 20 + pyproject.toml | 3 + .../description_reconcile.py | 435 ++++++++++++++++++ .../description_removal.py | 265 +++++++++++ .../description_transition.py | 21 +- .../reconcile_observe.py | 31 ++ tests/test_condition_folder_reopens.py | 37 ++ tests/test_reconcile_observe.py | 28 ++ tests/test_spec_reconcile.py | 431 +++++++++++++++++ 10 files changed, 1271 insertions(+), 5 deletions(-) create mode 100644 src/morphobricks_preprocessing/description_reconcile.py create mode 100644 src/morphobricks_preprocessing/description_removal.py create mode 100644 src/morphobricks_preprocessing/reconcile_observe.py create mode 100644 tests/test_condition_folder_reopens.py create mode 100644 tests/test_reconcile_observe.py create mode 100644 tests/test_spec_reconcile.py diff --git a/AGENTS.md b/AGENTS.md index 518c5586..8150a8c7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,6 +107,11 @@ src/morphobricks_preprocessing/ description.py the model (Description, Condition, Channel, Repeat, Member, Unit, Mask) built from a reader-accepted document; description_from_bytes -> DescriptionError description_write.py the canonical writer (Spec experiment-v1 9.1) and section digests (9.3) + description_reconcile.py reconcile(published, observed): units unchanged/new/removed by exact folder + labels, N35 reopening (confirmed_on kept), stated renames (FolderRename), + confirm_repeat; description_removal.py candidate (the draft), removal_questions + (one grouped question per condition) and reconfirmations; reconcile_observe.py + observe_folders (image counts of a folder walk) description_transition.py the R_* comparison of a candidate with the published file (4.4a, 4.6, 4.9), Reconfirmation (the user's recorded acceptance of a changed ID) description_file.py read_description(path): symlink refusal (R_FILE_SYMLINK), bytes -> Description diff --git a/CHANGELOG.md b/CHANGELOG.md index a3747821..2ef5a1b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,26 @@ are in [docs/plan.md](docs/plan.md) ยง12. Upgrading from v0.3: see the ## Unreleased +- **Reconciliation of the published description with the folder (Stage 1, PR 9; conformance + P07, P08, P15, P33).** New modules `description_reconcile`, `description_removal` and + `reconcile_observe`. Each unit is `unchanged`, `new` or `removed`, matched only by its exact + source folder labels. A unit that is new beside a confirmed member, a removed unit, channel + evidence that no longer fits the declared list and a folder the user stated was renamed + (`FolderRename`) reopen the N35 confirmation of that repeat only (`confirmed = false` in the + draft, `confirmed_on` copied from the published file, a new date only from the user's + `confirm_repeat`). A change of `image_count` alone reopens nothing (it is reported as + `CountChange` and the member's count follows the folder). A renamed folder is a removed unit + plus a new pending one, never paired by name; a new folder is proposed only for an existing + repeat ID equal to its label. A removal needs the user's confirmation, asked once per + condition (`removal_questions`: every lost repeat, technical repeat and channel in one + question); a removed ID stays in `used_ids`, and a `reference` to a removed condition is never + cleared silently. Fix in the merged transition check: `confirmed_on` may now move with the + user's new confirmation when a condition ID holds another folder (conformance P33 b); the + layout it compares (`repeat_layout`) includes the condition's source folder. Not used by any + run, notebook or CLI command yet. Not done here: the pending new units are not in the draft + (they need channels and IDs, which are the draft model's), and image-count changes are + reported but not appended to `morphobricks.records.jsonl` (the file defines only + re-confirmation records). - **Content hashing of the sources is on by default (#159; owner decision of 7 November 2026).** A run records each image's content SHA-256 and reuses a row only for the same content, so an image edited in place with its size and modification time kept is read again (manifest diff --git a/pyproject.toml b/pyproject.toml index efd2eec7..5959ec64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -176,6 +176,9 @@ baseline = "tools/code_rules_baseline.json" "morphobricks_preprocessing.description" = "pure" "morphobricks_preprocessing.description_write" = "pure" "morphobricks_preprocessing.description_transition" = "pure" +"morphobricks_preprocessing.description_reconcile" = "pure" +"morphobricks_preprocessing.description_removal" = "pure" +"morphobricks_preprocessing.reconcile_observe" = "io" "morphobricks_preprocessing.description_file" = "io" "morphobricks_preprocessing.description_read" = "pure" "morphobricks_preprocessing.description_semantics" = "pure" diff --git a/src/morphobricks_preprocessing/description_reconcile.py b/src/morphobricks_preprocessing/description_reconcile.py new file mode 100644 index 00000000..029188d9 --- /dev/null +++ b/src/morphobricks_preprocessing/description_reconcile.py @@ -0,0 +1,435 @@ +"""Reconcile the published description with the folder as it is now (experiment-v1 section 10). + +Fresh discovery is compared with the published file: each imaging unit is **unchanged**, **new** +or **removed**, matched only by the exact source folder labels, never by an ID, a similar name +or a position. The result is a proposal for the user: it never rewrites confirmed semantics. + +- A unit that is new beside a confirmed member, a removed unit, a unit whose channel evidence + (count, header names) no longer fits its declared channel list, and a folder the user + explicitly said was renamed, reopen the N35 confirmation of that repeat only + (``confirmed = false`` in the draft). ``confirmed_on`` is copied from the published file, never + from the clock; only the user's new confirmation sets a new date (:func:`confirm_repeat`). +- A change of the number of images in a unit alone (new FOVs) does not reopen anything: it is + reported (:class:`CountChange`) and the member's ``image_count`` follows the folder. +- A renamed folder is a removed folder plus a new pending one, never paired automatically. Only a + :class:`FolderRename` statement of the user ties the new folder to the old entity, and only when + the old folder is gone and the new one is there. +- A new folder is pending: it joins no repeat, condition or technical repeat without the user, + and the only proposal is an existing repeat ID equal to the folder label (section 10). + +Pure: it compares two models and a plain description of the folders. +""" + +from __future__ import annotations + +import unicodedata +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, replace +from datetime import date +from enum import StrEnum + +from .description import Description, Unit +from .description_transition import DATE, repeat_layout + + +class Status(StrEnum): + """How a unit stands against the published file.""" + + UNCHANGED = "unchanged" + NEW = "new" + REMOVED = "removed" + + +NEW_UNIT = "new_unit" +REMOVED_UNIT = "removed_unit" +CHANNELS_CHANGED = "channels_changed" +RENAMED_FOLDER = "renamed_folder" +REOPENING_REASONS = (NEW_UNIT, REMOVED_UNIT, CHANNELS_CHANGED, RENAMED_FOLDER) + +CONDITION, MEMBER, TECHNICAL = "condition", "member", "technical" +RENAME_KINDS = (CONDITION, MEMBER, TECHNICAL) + + +@dataclass(frozen=True, order=True) +class UnitFolders: + """An imaging unit by its source folder labels (``single`` for a missing level).""" + + condition: str + source_repeat: str + technical: str + + +@dataclass(frozen=True) +class ObservedUnit: + """What the folder shows of one unit now. + + ``channel_count`` and ``header_names`` (raw names by C index, None where there is none) are + None when the headers were not read or do not state them: that is not evidence of a change. + """ + + image_count: int + channel_count: int | None = None + header_names: tuple[str | None, ...] | None = None + + def __post_init__(self) -> None: + if self.image_count < 1: + raise ValueError("an observed unit has at least one image") + + +@dataclass(frozen=True) +class FolderRename: + """The user's explicit statement that a folder was renamed, with the date they made it. + + ``kind`` is ``condition`` (``old``/``new`` are condition folders), ``member`` (source repeat + folders under the published condition folder ``condition``) or ``technical`` (technical-repeat + folders). ``confirmed_on`` is a user fact and has no default. + """ + + kind: str + old: str + new: str + confirmed_on: str + condition: str | None = None + + def __post_init__(self) -> None: + if self.kind not in RENAME_KINDS: + raise ValueError(f"a rename is of one of {RENAME_KINDS}, not {self.kind!r}") + if (self.kind == MEMBER) == (self.condition is None): + raise ValueError("a member rename names its condition folder, no other kind does") + if self.old == self.new: + raise ValueError(f"the folder {self.old!r} is not renamed") + if not DATE.fullmatch(self.confirmed_on): + raise ValueError(f"confirmed_on is YYYY-MM-DD, not {self.confirmed_on!r}") + date.fromisoformat(self.confirmed_on) + + +class RenameRefused(ValueError): + """Raised when a rename statement does not match the published file and the folder.""" + + +@dataclass(frozen=True) +class UnitChange: + """One unit: its folders now (published ones for a removed unit) and what is known of it. + + ``repeat_id``, ``condition_id`` and ``technical_id`` are the published IDs of an unchanged or + removed unit, None for a new one, which is pending. ``proposed_repeat`` is the only proposal + for a new unit: the published repeat ID equal to its source repeat label. + """ + + folders: UnitFolders + status: Status + repeat_id: str | None = None + condition_id: str | None = None + technical_id: str | None = None + proposed_repeat: str | None = None + reasons: tuple[str, ...] = () + image_count: int | None = None + + +@dataclass(frozen=True) +class CountChange: + """A member whose images in the folder differ from the published ``image_count``.""" + + repeat_id: str + condition_id: str + before: int + after: int + + +@dataclass(frozen=True) +class Reconciliation: + """The classification of every unit, the repeats to reopen and the reported count changes.""" + + units: tuple[UnitChange, ...] + reopened: Mapping[str, tuple[str, ...]] + counts: tuple[CountChange, ...] + renames: tuple[FolderRename, ...] = () + + def with_status(self, status: Status) -> tuple[UnitChange, ...]: + """Return the units with this status, in folder order.""" + return tuple(unit for unit in self.units if unit.status == status) + + @property + def unchanged(self) -> tuple[UnitChange, ...]: + """The units found as published.""" + return self.with_status(Status.UNCHANGED) + + @property + def new(self) -> tuple[UnitChange, ...]: + """The units in the folder that the file does not have: pending.""" + return self.with_status(Status.NEW) + + @property + def removed(self) -> tuple[UnitChange, ...]: + """The published units with no images in the folder: removed only on confirmation.""" + return self.with_status(Status.REMOVED) + + +@dataclass(frozen=True) +class PublishedUnit: + """A published unit with the IDs that hold it.""" + + folders: UnitFolders + repeat_id: str + condition_id: str + technical_id: str + unit: Unit + + +def published_units(description: Description) -> dict[UnitFolders, PublishedUnit]: + """Return the published units by source folder labels.""" + found = {} + for repeat_id, repeat in description.repeats.items(): + for condition_id, member in repeat.members.items(): + folder = description.conditions[condition_id].source_folder + for tid, unit in member.technical_repeats.items(): + key = UnitFolders(folder, member.source_repeat, unit.source_folder) + found[key] = PublishedUnit(key, repeat_id, condition_id, tid, unit) + return found + + +def renamed_key(key: UnitFolders, rename: FolderRename) -> UnitFolders: + """Return the unit's folders after one rename statement.""" + if rename.kind == CONDITION and key.condition == rename.old: + return replace(key, condition=rename.new) + if rename.kind == TECHNICAL and key.technical == rename.old: + return replace(key, technical=rename.new) + if rename.kind == MEMBER and (key.condition, key.source_repeat) == ( + rename.condition, + rename.old, + ): + return replace(key, source_repeat=rename.new) + return key + + +def _check_rename( + rename: FolderRename, published: Iterable[UnitFolders], observed: Iterable[UnitFolders] +) -> None: + """Refuse a rename whose old folder is not published or still exists, or new one is absent.""" + seen, held = list(observed), list(published) + moved = {key for key in held if renamed_key(key, rename) != key} + after = {renamed_key(key, rename) for key in moved} + if not moved: + raise RenameRefused(f"no published folder {rename.old!r} to rename ({rename.kind})") + if after & (set(held) - moved): + raise RenameRefused(f"{rename.new!r} is already a folder of the published file") + if any(renamed_key(key, rename) != key for key in seen): + raise RenameRefused(f"the folder {rename.old!r} is still in the experiment") + if not any(key in after for key in seen): + raise RenameRefused(f"the folder {rename.new!r} is not in the experiment") + + +def _channels_differ( + published: PublishedUnit, seen: ObservedUnit, description: Description +) -> bool: + """Return whether the observed channel evidence no longer fits the declared channel list.""" + unit = published.unit + if seen.channel_count is not None and seen.channel_count != len(unit.channels): + return True + if seen.header_names is None or unit.image_type == "rgb": + return False + for channel, name in zip(unit.channels, seen.header_names, strict=False): + known = {_nfc(entry) for entry in description.channels[channel].header_names or ()} + if name and _nfc(name) not in known: + return True + return False + + +def _nfc(text: str) -> str: + return unicodedata.normalize("NFC", text) + + +def _member_counts( + description: Description, + units: Mapping[UnitFolders, PublishedUnit], + now: Mapping[UnitFolders, UnitFolders], + observed: Mapping[UnitFolders, ObservedUnit], +) -> tuple[CountChange, ...]: + """Return the members whose units are all still in the folder and whose image count differs.""" + out = [] + for repeat_id, repeat in description.repeats.items(): + for condition_id, member in repeat.members.items(): + held = [ + u + for u in units.values() + if (u.repeat_id, u.condition_id) == (repeat_id, condition_id) + ] + seen = [observed.get(now[u.folders]) for u in held] + if None in seen: + continue # a unit is missing: the member's count is not known until it is settled + total = sum(item.image_count for item in seen if item is not None) + if total != member.image_count: + out.append(CountChange(repeat_id, condition_id, member.image_count, total)) + return tuple(out) + + +def reconcile( + published: Description, + observed: Mapping[UnitFolders, ObservedUnit], + renames: Iterable[FolderRename] = (), +) -> Reconciliation: + """Compare the published description with the folder and classify every unit. + + Parameters + ---------- + published + The confirmed description on disk. + observed + The units found in the folder now, by their exact source folder labels. + renames + The user's explicit statements that a folder was renamed; without one a renamed folder is + a removed unit plus a new one. + + Returns + ------- + Reconciliation + Every unit, the repeats whose N35 confirmation reopens and why, and the count changes. + + Raises + ------ + RenameRefused + When a statement names a folder that is not published, is still there, or is not found + under its new name. + """ + units = published_units(published) + statements = tuple(renames) + now = {key: key for key in units} + for rename in statements: + _check_rename(rename, now.values(), observed) + now = {key: renamed_key(current, rename) for key, current in now.items()} + changes, reasons = [], {key: set() for key in units} + renamed = {key for key in units if now[key] != key} + for key, unit in units.items(): + current = now[key] + seen = observed.get(current) + if seen is None: + changes.append(_change(unit, Status.REMOVED, (REMOVED_UNIT,))) + reasons[key].add(REMOVED_UNIT) + continue + found = [RENAMED_FOLDER] * (key in renamed) + found += [CHANNELS_CHANGED] * _channels_differ(unit, seen, published) + reasons[key].update(found) + changes.append(_change(unit, Status.UNCHANGED, tuple(found), current, seen)) + known = set(now.values()) + pending = [key for key in observed if key not in known] + changes += [_new(key, observed[key], published, now, reasons, units) for key in sorted(pending)] + reopened = _reopened(units, reasons) + counts = _member_counts(published, units, now, observed) + return Reconciliation( + tuple(sorted(changes, key=lambda c: c.folders)), reopened, counts, statements + ) + + +def _change( + unit: PublishedUnit, + status: Status, + reasons: tuple[str, ...], + folders: UnitFolders | None = None, + seen: ObservedUnit | None = None, +) -> UnitChange: + return UnitChange( + folders or unit.folders, + status, + unit.repeat_id, + unit.condition_id, + unit.technical_id, + None, + tuple(sorted(reasons)), + seen.image_count if seen else None, + ) + + +def _new( + key: UnitFolders, + seen: ObservedUnit, + published: Description, + now: Mapping[UnitFolders, UnitFolders], + reasons: dict[UnitFolders, set[str]], + units: Mapping[UnitFolders, PublishedUnit], +) -> UnitChange: + """Return a pending new unit; it reopens the repeat of an existing member it joins.""" + siblings = [ + units[old] + for old, current in now.items() + if (current.condition, current.source_repeat) == (key.condition, key.source_repeat) + ] + if siblings: # a new technical-repeat folder inside a confirmed member + for sibling in siblings: + reasons[sibling.folders].add(NEW_UNIT) + proposed = key.source_repeat if key.source_repeat in published.repeats else None + return UnitChange(key, Status.NEW, proposed_repeat=proposed, image_count=seen.image_count) + + +def _reopened( + units: Mapping[UnitFolders, PublishedUnit], reasons: Mapping[UnitFolders, set[str]] +) -> dict[str, tuple[str, ...]]: + """Return, per repeat ID, the sorted reasons of its units that reopen the confirmation.""" + out: dict[str, set[str]] = {} + for key, found in reasons.items(): + if found: + out.setdefault(units[key].repeat_id, set()).update(found) + return {repeat: tuple(sorted(found)) for repeat, found in sorted(out.items())} + + +def layout_changed(old: Description, new: Description) -> tuple[str, ...]: + """Return the repeat IDs in both files whose layout differs (section 10), sorted. + + A candidate that adds, removes or reassigns a member, unit, folder, channel list or image + type under an existing repeat reopens it, whether the change came from the folder or from + the user's assignment of a pending unit. The image count is not part of the layout. + """ + return tuple( + sorted( + key + for key in old.repeats.keys() & new.repeats.keys() + if repeat_layout(old, key) != repeat_layout(new, key) + ) + ) + + +def confirm_repeat(description: Description, repeat_id: str, confirmed_on: str) -> Description: + """Return the description with an open repeat confirmed by the user on ``confirmed_on``. + + The date is the calendar day of the user's action and has no default. + + Raises + ------ + ValueError + When the repeat is unknown, is not open (``confirmed`` is already true), or the date is + not a real ``YYYY-MM-DD`` day. + """ + if repeat_id not in description.repeats: + raise ValueError(f"no repeat {repeat_id!r}") + repeat = description.repeats[repeat_id] + if repeat.confirmed: + raise ValueError(f"repeat {repeat_id!r} is not open: it is confirmed already") + if not DATE.fullmatch(confirmed_on): + raise ValueError(f"confirmed_on is YYYY-MM-DD, not {confirmed_on!r}") + date.fromisoformat(confirmed_on) + repeats = { + **description.repeats, + repeat_id: replace(repeat, confirmed=True, confirmed_on=confirmed_on), + } + return replace(description, repeats=repeats) + + +__all__ = [ + "CHANNELS_CHANGED", + "NEW_UNIT", + "REMOVED_UNIT", + "RENAMED_FOLDER", + "CountChange", + "FolderRename", + "ObservedUnit", + "PublishedUnit", + "Reconciliation", + "RenameRefused", + "Status", + "UnitChange", + "UnitFolders", + "confirm_repeat", + "layout_changed", + "published_units", + "reconcile", + "renamed_key", +] diff --git a/src/morphobricks_preprocessing/description_removal.py b/src/morphobricks_preprocessing/description_removal.py new file mode 100644 index 00000000..bce98174 --- /dev/null +++ b/src/morphobricks_preprocessing/description_removal.py @@ -0,0 +1,265 @@ +"""The draft of a reconciliation and the removals the user must confirm (experiment-v1 4.6, 10). + +A removed folder is removed only on the user's confirmation, which is the recorded +re-confirmation of section 4.6. One folder going away can make several confirmed IDs "changed": +the repeat that lost a member, each technical repeat and each channel that lost a position, the +condition that lost its folder. They are all one decision of the user, so they come back +**grouped** (:class:`RemovalQuestion`, one per condition): the notebook shows one question and +one answer (:class:`Removal`, with its date) accepts every confirmation in it. + +Nothing is removed, renamed or defaulted without an answer, an ID that is removed stays in +``used_ids`` (section 4.9), and a ``reference`` to a removed condition is never cleared for the +user: the candidate refuses until the reference is dealt with. +""" + +from __future__ import annotations + +from collections.abc import Collection, Iterable +from dataclasses import dataclass, replace +from datetime import date + +from .description import Condition, Description, Repeat +from .description_reconcile import ( + FolderRename, + PublishedUnit, + Reconciliation, + Status, + UnitFolders, + published_units, + renamed_key, +) +from .description_rules import Refusal +from .description_transition import ( + CHANNELS, + CONDITIONS, + DATE, + REPEATS, + TECHNICAL, + Reconfirmation, + TransitionIssue, + transition_issues, +) + +Change = tuple[str, str, str | None] +"""A changed confirmed ID as ``Reconfirmation`` names it: (kind, old ID, new ID or None).""" + + +@dataclass(frozen=True) +class RemovalQuestion: + """One question for the notebook, to remove these folders of one condition. + + ``needs`` are every re-confirmation the removal makes necessary (a repeat, technical repeats + and channels, each by ID); ``last_member`` says the condition keeps no folder, so its ID is + retired; ``referenced_by`` are the conditions whose ``reference`` names this one and must be + changed first. + """ + + condition_id: str + condition_folder: str + units: tuple[UnitFolders, ...] + needs: tuple[Change, ...] + last_member: bool + referenced_by: tuple[str, ...] + + def answer(self, confirmed_on: str) -> Removal: + """Return the user's "Yes, remove" for this question, made on ``confirmed_on``.""" + return Removal(self.units, confirmed_on) + + +@dataclass(frozen=True) +class Removal: + """The user's confirmation that these published units are removed (a date, never defaulted).""" + + units: tuple[UnitFolders, ...] + confirmed_on: str + + def __post_init__(self) -> None: + if not DATE.fullmatch(self.confirmed_on): + raise ValueError(f"confirmed_on is YYYY-MM-DD, not {self.confirmed_on!r}") + date.fromisoformat(self.confirmed_on) + + +@dataclass(frozen=True) +class Reconfirmations: + """The records the candidate needs: those the user's answers cover, and those they do not.""" + + accepted: tuple[Reconfirmation, ...] + open: tuple[TransitionIssue, ...] + + +def _moves( + units: Iterable[UnitFolders], renames: Iterable[FolderRename] +) -> dict[UnitFolders, UnitFolders]: + """Return each published unit's folders after the renames, in the order stated.""" + now = {key: key for key in units} + for rename in renames: + now = {key: renamed_key(current, rename) for key, current in now.items()} + return now + + +def _build( + published: Description, rec: Reconciliation, removed: Collection[UnitFolders] +) -> Description: + """Return the published file reopened, renamed and with ``removed`` taken out (no checks).""" + now = _moves(published_units(published), rec.renames) + seen = {u.folders: u.image_count for u in rec.units if u.status == Status.UNCHANGED} + repeats, folders = {}, {} + for repeat_id, repeat in published.repeats.items(): + members = {} + for condition_id, member in repeat.members.items(): + kept, counts, after = {}, [], None + for tid, unit in member.technical_repeats.items(): + key = UnitFolders( + published.conditions[condition_id].source_folder, + member.source_repeat, + unit.source_folder, + ) + if key not in removed: + after = now[key] + kept[tid] = replace(unit, source_folder=after.technical) + counts.append(seen.get(after)) + if after is None: + continue + folders[condition_id] = after.condition + total = member.image_count if None in counts else sum(counts) + members[condition_id] = replace( + member, source_repeat=after.source_repeat, technical_repeats=kept, image_count=total + ) + if members: + confirmed = repeat.confirmed and repeat_id not in rec.reopened + repeats[repeat_id] = replace(repeat, members=members, confirmed=confirmed) + return replace(published, repeats=repeats, conditions=_conditions(published, repeats, folders)) + + +def _conditions( + published: Description, repeats: dict[str, Repeat], folders: dict[str, str] +) -> dict[str, Condition]: + """Return the conditions that keep a member, with their renamed folders; others are retired.""" + held = {cond for repeat in repeats.values() for cond in repeat.members} + had = {cond for repeat in published.repeats.values() for cond in repeat.members} + return { + key: replace(condition, source_folder=folders.get(key, condition.source_folder)) + for key, condition in published.conditions.items() + if key in held or key not in had + } + + +def _changes(published: Description, built: Description) -> tuple[TransitionIssue, ...]: + """Return the changed-entity refusals (the ones a re-confirmation accepts) of a candidate.""" + return tuple( + issue + for issue in transition_issues(published, built) + if issue.code == Refusal.R_CONFIRMED_ID_CHANGED + ) + + +def _affected(issue: TransitionIssue, units: dict[UnitFolders, PublishedUnit]) -> set[UnitFolders]: + """Return the published units that hold the changed ID.""" + matches = { + CONDITIONS: lambda u: u.condition_id == issue.old_id, + REPEATS: lambda u: u.repeat_id == issue.old_id, + TECHNICAL: lambda u: u.technical_id == issue.old_id, + CHANNELS: lambda u: issue.old_id in u.unit.channels, + }[issue.kind] + return {key for key, unit in units.items() if matches(unit)} + + +def removal_questions(published: Description, rec: Reconciliation) -> tuple[RemovalQuestion, ...]: + """Return one grouped question per condition that has removed units, in condition ID order. + + Each lists every re-confirmation its removal needs (each lost repeat, technical repeat and + channel by ID, and the condition's own when it lost its folder), so the notebook asks once. + A need that several conditions share belongs to the first of them. + """ + units = published_units(published) + removed = {u.folders for u in rec.removed} + built = _build(published, rec, removed) + issues = _changes(published, built) + by_condition: dict[str, set[UnitFolders]] = {} + for key in sorted(removed): + by_condition.setdefault(units[key].condition_id, set()).add(key) + assigned: set[Change] = set() + out = [] + for condition_id, held in sorted(by_condition.items()): + needs = [] + for issue in issues: + if issue.change not in assigned and _affected(issue, units) & held: + assigned.add(issue.change) + needs.append(issue.change) + everything = {k for k, u in units.items() if u.condition_id == condition_id} + refs = sorted(k for k, c in published.conditions.items() if c.reference == condition_id) + out.append( + RemovalQuestion( + condition_id, + published.conditions[condition_id].source_folder, + tuple(sorted(held)), + tuple(sorted(needs, key=lambda n: (n[0], n[1], n[2] or ""))), + everything <= held, + tuple(refs), + ) + ) + return tuple(out) + + +def candidate( + published: Description, rec: Reconciliation, removals: Iterable[Removal] = () +) -> Description: + """Return the draft: the published file as the folder and the user's answers leave it. + + Repeats the reconciliation reopened have ``confirmed = False`` and keep the published + ``confirmed_on``; the units the user confirmed removed (and a condition or repeat left + without members) are gone and their IDs stay in ``used_ids``; stated renames are applied; + pending new units are not in it. Nothing is published by this. + + Raises + ------ + ValueError + When a removal names a unit that is not a removed unit of the reconciliation, or removes + a condition that another condition still references. + """ + answers = tuple(removals) + gone = {u.folders for u in rec.removed} + asked = {key for removal in answers for key in removal.units} + if asked - gone: + names = ", ".join( + f"{k.condition}/{k.source_repeat}/{k.technical}" for k in sorted(asked - gone) + ) + raise ValueError(f"not a removed unit of this reconciliation: {names}") + built = _build(published, rec, asked) + dangling = sorted( + f"{key} refers to {c.reference}" + for key, c in built.conditions.items() + if c.reference is not None and c.reference not in built.conditions + ) + if dangling: + raise ValueError("a removed condition is still referenced: " + "; ".join(dangling)) + return built + + +def reconfirmations( + published: Description, + built: Description, + rec: Reconciliation, + removals: Iterable[Removal] = (), +) -> Reconfirmations: + """Return the re-confirmations of ``built`` that the user's answers cover, and the rest. + + Each changed confirmed ID takes the date of the first answer that touches one of its units; + an ID no answer touches is in ``open`` (the candidate would be refused for it). + """ + units = published_units(published) + answers = [(set(r.units), r.confirmed_on) for r in removals] + now = {key: key for key in units} + for rename in rec.renames: + after = {key: renamed_key(current, rename) for key, current in now.items()} + answers.append(({key for key in units if after[key] != now[key]}, rename.confirmed_on)) + now = after + accepted, left = set(), [] + for issue in _changes(published, built): + reached = _affected(issue, units) + dates = [day for held, day in answers if held & reached] + if dates: + accepted.add(Reconfirmation(issue.kind, issue.old_id, issue.new_id, dates[0])) + else: + left.append(issue) + return Reconfirmations(tuple(sorted(accepted)), tuple(left)) diff --git a/src/morphobricks_preprocessing/description_transition.py b/src/morphobricks_preprocessing/description_transition.py index e387ae11..b1c4ea84 100644 --- a/src/morphobricks_preprocessing/description_transition.py +++ b/src/morphobricks_preprocessing/description_transition.py @@ -13,7 +13,7 @@ from dataclasses import dataclass from datetime import date -from .description import USED_KINDS, Description, Repeat +from .description import USED_KINDS, Description from .description_rules import Refusal CONDITIONS, CHANNELS, REPEATS, TECHNICAL = USED_KINDS @@ -195,13 +195,24 @@ def _lost(old: Description, new: Description) -> list[TransitionIssue]: return out -def _layout(repeat: Repeat) -> frozenset: +def repeat_layout(description: Description, key: str) -> frozenset: """Return what reopens a repeat's confirmation (section 10): members, units, channels, type. - The image count is not part of it: new FOVs in an existing unit do not reopen it. + A member is its condition's source folder and ID, its source repeat folder, and each unit's + technical-repeat ID and folder, ordered channels and image type. The image count is not part + of it: new FOVs in an existing unit do not reopen it. """ + repeat = description.repeats[key] return frozenset( - (cond, member.source_repeat, tid, unit.source_folder, unit.channels, unit.image_type) + ( + cond, + description.conditions[cond].source_folder, + member.source_repeat, + tid, + unit.source_folder, + unit.channels, + unit.image_type, + ) for cond, member in repeat.members.items() for tid, unit in member.technical_repeats.items() ) @@ -213,7 +224,7 @@ def _dates(old: Description, new: Description) -> list[TransitionIssue]: TransitionIssue(Refusal.R_CONFIRMED_ON_CHANGED, REPEATS, key) for key in old.repeats.keys() & new.repeats.keys() if old.repeats[key].confirmed_on != new.repeats[key].confirmed_on - and _layout(old.repeats[key]) == _layout(new.repeats[key]) + and repeat_layout(old, key) == repeat_layout(new, key) ] diff --git a/src/morphobricks_preprocessing/reconcile_observe.py b/src/morphobricks_preprocessing/reconcile_observe.py new file mode 100644 index 00000000..f08b820a --- /dev/null +++ b/src/morphobricks_preprocessing/reconcile_observe.py @@ -0,0 +1,31 @@ +"""The units a folder walk found, as :func:`.description_reconcile.reconcile` takes them. + +Counts only: how many images each unit has now. Whether the headers still fit the declared +channels is the header evidence of the physical checks; when a caller has it (channel count, +raw header names) it builds the :class:`~.description_reconcile.ObservedUnit` itself. Nothing +is read from an image here, and no unit is renamed or matched by similarity. +""" + +from __future__ import annotations + +from collections.abc import Collection + +from .description_reconcile import ObservedUnit, UnitFolders +from .discovery import DiscoveryResult + + +def observe_folders( + discovery: DiscoveryResult, excluded: Collection[str] = () +) -> dict[UnitFolders, ObservedUnit]: + """Return each unit of the walk with its image count, leaving out the images ``excluded``. + + A folder whose images are all excluded is not a unit (it is a removed one if published). + The channel evidence of every unit is left unknown (None), never assumed. + """ + found: dict[UnitFolders, int] = {} + for leaf in discovery.leaves: + count = sum(1 for image in leaf.images if image not in excluded) + if count: + key = UnitFolders(*leaf.labels) + found[key] = found.get(key, 0) + count + return {key: ObservedUnit(count) for key, count in sorted(found.items())} diff --git a/tests/test_condition_folder_reopens.py b/tests/test_condition_folder_reopens.py new file mode 100644 index 00000000..3ea9563e --- /dev/null +++ b/tests/test_condition_folder_reopens.py @@ -0,0 +1,37 @@ +"""A condition folder renamed under its ID reopens the repeats of that condition (P33 b). + +Experiment-v1 section 10 reopens the N35 confirmation when the pairing of folders changes, and +a condition ID that now holds another folder is such a change. The published file's ``confirmed_on`` +may then move with the user's new confirmation, as it may for any other layout change. +""" + +from dataclasses import replace + +from morphobricks_preprocessing.description import description_from_bytes +from morphobricks_preprocessing.description_rules import Refusal +from morphobricks_preprocessing.description_transition import transition_issues +from spec_pin import fixtures + +V33 = description_from_bytes( + next(item for item in fixtures("valid") if item.name.startswith("v33")).data +) + + +def renamed_and_reconfirmed(day): + conditions = {**V33.conditions, "ctrl": replace(V33.conditions["ctrl"], source_folder="ctl")} + repeats = {key: replace(repeat, confirmed_on=day) for key, repeat in V33.repeats.items()} + return replace(V33, conditions=conditions, repeats=repeats) + + +def test_a_new_confirmation_date_is_accepted_after_a_condition_folder_changed(): + codes = {issue.code for issue in transition_issues(V33, renamed_and_reconfirmed("2026-11-09"))} + assert Refusal.R_CONFIRMED_ON_CHANGED not in codes + assert Refusal.R_CONFIRMED_ID_CHANGED in codes # the reassignment still needs its record + + +def test_the_date_still_may_not_move_when_nothing_was_reopened(): + unchanged = replace( + V33, repeats={k: replace(r, confirmed_on="2026-11-09") for k, r in V33.repeats.items()} + ) + codes = {issue.code for issue in transition_issues(V33, unchanged)} + assert codes == {Refusal.R_CONFIRMED_ON_CHANGED} diff --git a/tests/test_reconcile_observe.py b/tests/test_reconcile_observe.py new file mode 100644 index 00000000..9200152e --- /dev/null +++ b/tests/test_reconcile_observe.py @@ -0,0 +1,28 @@ +"""``observe_folders``: the image count of each unit of a folder walk, nothing else.""" + +from morphobricks_preprocessing.description_reconcile import ObservedUnit, UnitFolders +from morphobricks_preprocessing.discovery import discover_experiment +from morphobricks_preprocessing.reconcile_observe import observe_folders +from tiff_tree import write_tiff + + +def test_each_unit_has_its_image_count_and_no_channel_evidence(tmp_path): + for name in ("a.tif", "b.tif", "c.tif"): + write_tiff(tmp_path / "CTRL" / "rep1" / name) + write_tiff(tmp_path / "DrugA" / "rep1" / "acq1" / "a.tif") + seen = observe_folders(discover_experiment(tmp_path)) + assert seen == { + UnitFolders("CTRL", "rep1", "single"): ObservedUnit(3), + UnitFolders("DrugA", "rep1", "acq1"): ObservedUnit(1), + } + assert all(unit.channel_count is None and unit.header_names is None for unit in seen.values()) + + +def test_excluded_images_are_not_counted_and_an_emptied_folder_is_not_a_unit(tmp_path): + for name in ("a.tif", "b.tif"): + write_tiff(tmp_path / "CTRL" / "rep1" / name) + write_tiff(tmp_path / "DrugA" / "rep1" / "a.tif") + seen = observe_folders( + discover_experiment(tmp_path), excluded={"CTRL/rep1/a.tif", "DrugA/rep1/a.tif"} + ) + assert seen == {UnitFolders("CTRL", "rep1", "single"): ObservedUnit(1)} diff --git a/tests/test_spec_reconcile.py b/tests/test_spec_reconcile.py new file mode 100644 index 00000000..d2b56a17 --- /dev/null +++ b/tests/test_spec_reconcile.py @@ -0,0 +1,431 @@ +"""Reconciliation of the published description with the folder: conformance P07, P08, P15, P33. + +Experiment-v1 section 10. The folder is a plain mapping of source folder labels to what was +seen there (:class:`ObservedUnit`); the fixtures are the Spec's, and publication is the real +``publish_description`` into a temporary output folder. +""" + +from dataclasses import replace + +import pytest + +from morphobricks_preprocessing.description import Unit, description_from_bytes +from morphobricks_preprocessing.description_file import RECORDS_FILENAME +from morphobricks_preprocessing.description_reconcile import ( + CHANNELS_CHANGED, + NEW_UNIT, + REMOVED_UNIT, + RENAMED_FOLDER, + FolderRename, + ObservedUnit, + RenameRefused, + Status, + UnitFolders, + confirm_repeat, + layout_changed, + published_units, + reconcile, +) +from morphobricks_preprocessing.description_removal import ( + Removal, + candidate, + reconfirmations, + removal_questions, +) +from morphobricks_preprocessing.description_rules import Refusal +from morphobricks_preprocessing.description_transition import transition_issues +from morphobricks_preprocessing.description_write import canonical_bytes +from morphobricks_preprocessing.run.description_file import ( + ABSENT, + PublicationRefused, + publish_description, + read_records, +) +from spec_pin import fixtures + +BY_NAME = {item.name.split("_")[0]: item for item in fixtures("valid")} +DAY = "2026-11-09" + + +def model(name): + return description_from_bytes(BY_NAME[name].data) + + +def folder(description, changes=None): + """The folder as the description says it is (every unit with its published count).""" + seen = {} + for repeat in description.repeats.values(): + for condition_id, member in repeat.members.items(): + for unit in member.technical_repeats.values(): + key = UnitFolders( + description.conditions[condition_id].source_folder, + member.source_repeat, + unit.source_folder, + ) + share = member.image_count // len(member.technical_repeats) + seen[key] = ObservedUnit(share) + seen.update(changes or {}) + return seen + + +def published_in(tmp_path, description): + """Publish the description as the first file and return the output folder and its base.""" + return tmp_path, publish_description(tmp_path, description, base_sha256=ABSENT).sha256 + + +def key(condition, repeat, technical="single"): + return UnitFolders(condition, repeat, technical) + + +# P15 ------------------------------------------------------------------------------------ + + +def test_p15_new_fovs_in_an_existing_unit_keep_the_confirmation_and_its_date(tmp_path): + published = model("v13") + unit = key("DrugA", "rep2") + rec = reconcile(published, folder(published, {unit: ObservedUnit(15)})) + assert rec.reopened == {} + assert [c.status for c in rec.units] == [Status.UNCHANGED] * 9 + ((change,),) = [rec.counts] + assert (change.repeat_id, change.condition_id, change.before, change.after) == ( + "rep2", + "DrugA", + 10, + 15, + ) + draft = candidate(published, rec) + assert draft.repeats["rep2"].members["DrugA"].image_count == 15 + assert all(r.confirmed for r in draft.repeats.values()) + assert {k: r.confirmed_on for k, r in draft.repeats.items()} == { + k: r.confirmed_on for k, r in published.repeats.items() + } + assert layout_changed(published, draft) == () + output, base = published_in(tmp_path, published) + result = publish_description(output, draft, base_sha256=base) + assert result.records == () # the change is reported, and needs no re-confirmation + assert b'confirmed_on = "2026-10-30"' in (output / "morphobricks.toml").read_bytes() + + +def test_a_change_of_image_count_alone_never_reopens_even_with_fewer_images(): + published = model("v13") + rec = reconcile(published, folder(published, {key("CTRL", "rep1"): ObservedUnit(3)})) + assert rec.reopened == {} + assert candidate(published, rec).repeats["rep1"].members["CTRL"].image_count == 3 + + +def test_p15_a_changed_channel_list_reopens_with_the_date_kept(): + published = model("v01") + changed = {key("CTRL", "rep1"): ObservedUnit(12, channel_count=3)} + rec = reconcile(published, folder(published, changed)) + assert rec.reopened == {"rep1": (CHANNELS_CHANGED,)} + draft = candidate(published, rec) + assert draft.repeats["rep1"].confirmed is False + assert draft.repeats["rep1"].confirmed_on == published.repeats["rep1"].confirmed_on + assert draft.repeats["rep2"].confirmed is True + + +def test_the_header_evidence_that_still_fits_does_not_reopen_and_unknown_is_not_a_change(): + published = model("v01") + fits = ObservedUnit(12, channel_count=2, header_names=("DAPI", "Phalloidin-488")) + unknown = ObservedUnit(10, channel_count=None, header_names=None) + unnamed = ObservedUnit(11, channel_count=2, header_names=(None, None)) + seen = folder(published, {key("CTRL", "rep1"): fits, key("CTRL", "rep2"): unknown}) + seen[key("DrugA", "rep1")] = unnamed + assert reconcile(published, seen).reopened == {} + + +@pytest.mark.parametrize("names", [("dapi", "Phalloidin-488"), ("DAPI", "Phalloidin")]) +def test_a_header_name_that_is_not_a_confirmed_name_reopens_without_case_folding(names): + published = model("v01") + seen = folder(published, {key("CTRL", "rep1"): ObservedUnit(12, 2, names)}) + assert reconcile(published, seen).reopened == {"rep1": (CHANNELS_CHANGED,)} + + +# P07 ------------------------------------------------------------------------------------ + + +def test_p07_a_new_technical_repeat_is_pending_and_reopens_only_its_repeat(tmp_path): + published = model("v13") + new = key("DrugA", "rep2", "acquisition_2") + rec = reconcile(published, folder(published, {new: ObservedUnit(4)})) + assert [c.folders for c in rec.new] == [new] + pending = rec.new[0] + assert (pending.repeat_id, pending.technical_id, pending.proposed_repeat) == ( + None, + None, + "rep2", + ) + assert rec.reopened == {"rep2": (NEW_UNIT,)} + draft = candidate(published, rec) + assert {k: r.confirmed for k, r in draft.repeats.items()} == { + "rep1": True, + "rep2": False, + "rep3": True, + } + for name in ("rep1", "rep3"): # the dates of repeats that are not reopened are byte-identical + assert draft.repeats[name] == published.repeats[name] + assert draft.repeats["rep2"].confirmed_on == published.repeats["rep2"].confirmed_on + output, base = published_in(tmp_path, published) + with pytest.raises(Exception, match="E_REPEAT_UNCONFIRMED|not a valid"): + publish_description(output, draft, base_sha256=base) + assert (output / "morphobricks.toml").read_bytes() == canonical_bytes(published) + + +def test_p07_after_the_user_assigns_the_unit_and_confirms_the_new_date_is_accepted(tmp_path): + published = model("v06") + member = published.repeats["R002"].members["CTRL"] + new = key("CTRL", member.source_repeat, "acquisition_3") + rec = reconcile(published, folder(published, {new: ObservedUnit(4)})) + assert rec.reopened == {"R002": (NEW_UNIT,)} and [c.folders for c in rec.new] == [new] + draft = candidate(published, rec) + added = {**member.technical_repeats, "acq3": Unit("acquisition_3", ("DNA", "ACTIN"))} + members = {**draft.repeats["R002"].members, "CTRL": replace(member, technical_repeats=added)} + repeats = {**draft.repeats, "R002": replace(draft.repeats["R002"], members=members)} + used = (*draft.used_ids["technical_repeats"], "acq3") + assigned = replace( + draft, + repeats=repeats, + used_ids={**draft.used_ids, "technical_repeats": tuple(sorted(used))}, + ) + assert layout_changed(published, assigned) == ("R002",) + output, base = published_in(tmp_path, published) + result = publish_description(output, confirm_repeat(assigned, "R002", DAY), base_sha256=base) + assert result.records == () # adding a unit is not a changed entity (section 4.6) + assert 'confirmed_on = "2026-11-09"' in (output / "morphobricks.toml").read_text() + + +def test_the_new_date_is_refused_without_a_reopening(tmp_path): + published = model("v13") + draft = candidate(published, reconcile(published, folder(published))) + moved = replace( + draft, repeats={**draft.repeats, "rep1": replace(draft.repeats["rep1"], confirmed_on=DAY)} + ) + output, base = published_in(tmp_path, published) + with pytest.raises(PublicationRefused) as refused: + publish_description(output, moved, base_sha256=base) + assert refused.value.codes == {Refusal.R_CONFIRMED_ON_CHANGED} + + +def test_confirm_repeat_needs_an_open_repeat_and_a_real_date(): + published = model("v01") + with pytest.raises(ValueError, match="confirmed already"): + confirm_repeat(published, "rep1", DAY) + rec = reconcile(published, folder(published, {key("CTRL", "rep1"): ObservedUnit(12, 3)})) + draft = candidate(published, rec) + for bad in ("2026-13-01", "9 Nov 2026", ""): + with pytest.raises(ValueError): + confirm_repeat(draft, "rep1", bad) + with pytest.raises(ValueError, match="no repeat"): + confirm_repeat(draft, "nope", DAY) + assert confirm_repeat(draft, "rep1", DAY).repeats["rep1"].confirmed_on == DAY + + +# P08 ------------------------------------------------------------------------------------ + + +def renamed_member_folder(published): + seen = folder(published) + seen[key("DrugA", "experiment_23b")] = seen.pop(key("DrugA", "experiment_23")) + return seen + + +def test_p08_a_renamed_folder_is_removed_plus_new_and_never_paired(): + published = model("v03") + rec = reconcile(published, renamed_member_folder(published)) + assert [c.folders for c in rec.removed] == [key("DrugA", "experiment_23")] + assert [c.folders for c in rec.new] == [key("DrugA", "experiment_23b")] + assert rec.new[0].repeat_id is None and rec.new[0].proposed_repeat is None + assert rec.reopened == {"Day1": (REMOVED_UNIT,)} + draft = candidate(published, rec) + assert draft.repeats["Day1"].members["DrugA"].source_repeat == "experiment_23" # kept as was + assert draft.repeats["Day1"].confirmed is False and "Day2" in draft.repeats + assert draft.repeats["Day2"].confirmed is True + + +def test_p08_the_removal_is_one_grouped_question_and_publishes_only_when_answered(tmp_path): + published = model("v03") + rec = reconcile(published, renamed_member_folder(published)) + (question,) = removal_questions(published, rec) + assert question.condition_id == "DrugA" and not question.last_member + assert question.units == (key("DrugA", "experiment_23"),) + assert question.needs == ( + ("channels", "ACTIN", None), + ("channels", "DNA", None), + ("conditions", "DrugA", None), + ("repeats", "Day1", None), + ("technical_repeats", "single", None), + ) + answer = question.answer(DAY) + draft = confirm_repeat(candidate(published, rec, [answer]), "Day1", DAY) + assert "DrugA" not in draft.repeats["Day1"].members and "DrugA" in draft.repeats["Day2"].members + output, base = published_in(tmp_path, published) + with pytest.raises(PublicationRefused) as refused: # without the user's record + publish_description(output, draft, base_sha256=base) + assert refused.value.codes == {Refusal.R_CONFIRMED_ID_CHANGED} + found = reconfirmations(published, draft, rec, [answer]) + assert found.open == () and {c.confirmed_on for c in found.accepted} == {DAY} + assert len(found.accepted) == 5 + publish_description(output, draft, base_sha256=base, confirmations=found.accepted) + assert len(read_records(output)) == 5 + assert (output / RECORDS_FILENAME).exists() + + +def test_removing_a_condition_asks_once_for_every_lost_channel_and_technical_repeat(tmp_path): + published = model("v01") + gone = {k: v for k, v in folder(published).items() if k.condition != "DrugA"} + rec = reconcile(published, gone) + (question,) = removal_questions(published, rec) + assert question.last_member and question.referenced_by == () + assert question.units == (key("DrugA", "rep1"), key("DrugA", "rep2")) + assert {kind for kind, *_ in question.needs} == {"channels", "repeats", "technical_repeats"} + assert len(question.needs) == 5 # DNA, ACTIN, rep1, rep2 and the technical repeat single + answer = question.answer(DAY) + draft = confirm_repeat( + confirm_repeat(candidate(published, rec, [answer]), "rep1", DAY), "rep2", DAY + ) + assert "DrugA" not in draft.conditions and draft.used_ids == published.used_ids + output, base = published_in(tmp_path, published) + found = reconfirmations(published, draft, rec, [answer]) + assert {(c.kind, c.old_id) for c in found.accepted} == {(k, i) for k, i, _ in question.needs} + publish_description(output, draft, base_sha256=base, confirmations=found.accepted) + again = replace(published, used_ids=draft.used_ids) + assert Refusal.R_USED_ID_REUSED in {i.code for i in transition_issues(draft, again)} + + +def test_a_condition_that_another_still_references_cannot_be_removed_silently(): + published = model("v33") + gone = {k: v for k, v in folder(published).items() if k.condition != "control"} + rec = reconcile(published, gone) + (question,) = removal_questions(published, rec) + assert question.last_member and question.referenced_by == ("DrugA",) + with pytest.raises(ValueError, match="still referenced"): + candidate(published, rec, [question.answer(DAY)]) + + +def test_a_removal_of_something_that_is_not_removed_is_refused(): + published = model("v01") + rec = reconcile(published, folder(published)) + with pytest.raises(ValueError, match="not a removed unit"): + candidate(published, rec, [Removal((key("CTRL", "rep1"),), DAY)]) + with pytest.raises(ValueError, match="YYYY-MM-DD"): + Removal((), "yesterday") + + +# P33 ------------------------------------------------------------------------------------ + + +def on_disk_as_ctl(published): + return { + replace(k, condition="ctl" if k.condition == "control" else k.condition): v + for k, v in folder(published).items() + } + + +def test_p33_a_a_renamed_condition_folder_is_not_attached_to_the_old_id(): + published = model("v33") + rec = reconcile(published, on_disk_as_ctl(published)) + assert {c.folders.condition for c in rec.removed} == {"control"} + assert {c.folders.condition for c in rec.new} == {"ctl"} + assert all(c.condition_id is None and c.proposed_repeat in ("rep1", "rep2") for c in rec.new) + draft = candidate(published, rec) + assert draft.conditions["ctrl"].source_folder == "control" # never moved without a statement + attached = replace( + published, + conditions={ + **published.conditions, + "ctrl": replace(published.conditions["ctrl"], source_folder="ctl"), + }, + ) + codes = {i.code for i in transition_issues(published, attached)} + assert Refusal.R_CONFIRMED_ID_CHANGED in codes # t12: a hand-made attachment is refused + + +def test_p33_b_the_statement_keeps_the_id_moves_the_folder_and_reopens_the_repeats(tmp_path): + published = model("v33") + statement = FolderRename("condition", "control", "ctl", DAY) + rec = reconcile(published, on_disk_as_ctl(published), [statement]) + assert rec.new == () and rec.removed == () + assert rec.reopened == {"rep1": (RENAMED_FOLDER,), "rep2": (RENAMED_FOLDER,)} + draft = candidate(published, rec) + assert draft.conditions["ctrl"].source_folder == "ctl" + assert not any(r.confirmed for r in draft.repeats.values()) + assert {r.confirmed_on for r in draft.repeats.values()} == {"2026-11-06"} + found = reconfirmations(published, draft, rec) + assert found.open == () + assert [(c.kind, c.old_id, c.new_id, c.confirmed_on) for c in found.accepted] == [ + ("conditions", "ctrl", None, DAY) + ] + confirmed = confirm_repeat(confirm_repeat(draft, "rep1", DAY), "rep2", DAY) + output, base = published_in(tmp_path, published) + with pytest.raises(PublicationRefused) as refused: + publish_description(output, confirmed, base_sha256=base) + assert refused.value.codes == {Refusal.R_CONFIRMED_ID_CHANGED} # without the record + publish_description(output, confirmed, base_sha256=base, confirmations=found.accepted) + assert [r["old_id"] for r in read_records(output)] == ["ctrl"] + assert 'source_folder = "ctl"' in (output / "morphobricks.toml").read_text() + + +def test_p33_c_the_old_folder_removal_needs_its_confirmation_and_the_id_stays_used(): + published = model("v33") + rec = reconcile(published, on_disk_as_ctl(published)) + (question,) = removal_questions(published, rec) + assert question.condition_id == "ctrl" and question.last_member + assert ("conditions", "ctrl", None) not in question.needs # the ID goes, it is not "changed" + assert "ctrl" in published.used_ids["conditions"] + + +# Statements are checked against the file and the folder ------------------------------- + + +@pytest.mark.parametrize( + ("statement", "seen", "message"), + [ + (FolderRename("condition", "nope", "ctl", DAY), "ctl", "no published folder"), + (FolderRename("condition", "control", "ctl", DAY), "both", "still in the experiment"), + (FolderRename("condition", "control", "other", DAY), "ctl", "is not in the experiment"), + (FolderRename("condition", "control", "DrugA", DAY), "ctl", "already a folder"), + ], +) +def test_a_rename_statement_that_does_not_match_is_refused(statement, seen, message): + published = model("v33") + disk = on_disk_as_ctl(published) + if seen == "both": + disk.update(folder(published)) + with pytest.raises(RenameRefused, match=message): + reconcile(published, disk, [statement]) + + +def test_a_member_folder_rename_is_stated_under_its_condition(): + published = model("v03") + statement = FolderRename("member", "experiment_23", "experiment_23b", DAY, condition="DrugA") + rec = reconcile(published, renamed_member_folder(published), [statement]) + assert rec.new == () and rec.removed == () + assert rec.reopened == {"Day1": (RENAMED_FOLDER,)} + draft = candidate(published, rec) + assert draft.repeats["Day1"].members["DrugA"].source_repeat == "experiment_23b" + found = reconfirmations(published, draft, rec) + assert found.open == () and {c.confirmed_on for c in found.accepted} == {DAY} + assert ("repeats", "Day1") in {(c.kind, c.old_id) for c in found.accepted} + + +def test_a_technical_folder_rename_is_stated_by_its_folder_label(): + published = model("v06") + units = published_units(published) + old = next(u.folders for u in units.values() if u.folders.technical != "single") + seen = folder(published) + seen[replace(old, technical="renamed_acq")] = seen.pop(old) + statement = FolderRename("technical", old.technical, "renamed_acq", DAY) + rec = reconcile(published, seen, [statement]) + assert rec.new == () and rec.removed == () + assert RENAMED_FOLDER in sum(rec.reopened.values(), ()) + + +def test_a_statement_needs_a_real_date_and_its_condition_only_for_a_member(): + with pytest.raises(ValueError, match="YYYY-MM-DD"): + FolderRename("condition", "a", "b", "") + with pytest.raises(ValueError, match="names its condition"): + FolderRename("member", "a", "b", DAY) + with pytest.raises(ValueError, match="names its condition"): + FolderRename("condition", "a", "b", DAY, condition="c") + with pytest.raises(ValueError, match="one of"): + FolderRename("repeat", "a", "b", DAY)