From 6a2d22e69c9c6dd63e496f599fff65790fddd69a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 06:40:48 +0000 Subject: [PATCH 1/4] =?UTF-8?q?Re-pin=20Spec=201bed024;=20read=5Frecords?= =?UTF-8?q?=20follows=20=C2=A74.6=20(R=5FRECORDS=5FFILE=5FINVALID)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- CHANGELOG.md | 14 + docs/conformance.md | 2 +- .../draft_format.py | 2 +- .../records_file.py | 275 +++++++++++++++ .../run/description_file.py | 81 ++--- tests/spec_fixtures/PIN.toml | 9 +- tests/spec_fixtures/README.md | 8 +- tests/spec_fixtures/docs/experiment-v1.md | 44 ++- tests/spec_fixtures/fixtures/README.md | 61 +++- .../records/ri01_missing_final_lf.jsonl | 2 + .../records/ri02_malformed_line.jsonl | 3 + .../records/ri03_line_not_an_object.jsonl | 2 + .../fixtures/records/ri04_blank_line.jsonl | 3 + .../fixtures/records/ri05_unknown_type.jsonl | 2 + .../fixtures/records/ri06_type_missing.jsonl | 2 + .../fixtures/records/ri07_key_missing.jsonl | 2 + .../records/ri08_key_of_another_type.jsonl | 1 + .../fixtures/records/ri09_key_order.jsonl | 2 + .../records/ri10_nested_key_order.jsonl | 2 + .../records/ri11_non_ascii_escaped.jsonl | 2 + .../fixtures/records/ri12_invalid_utf8.jsonl | 2 + .../records/ri13_byte_order_mark.jsonl | 1 + .../fixtures/records/ri14_duplicate_key.jsonl | 2 + .../ri15_removed_unit_with_null_name.jsonl | 2 + .../ri16_removed_repeat_names_extra_key.jsonl | 2 + .../records/ri17_removed_entity_unknown.jsonl | 2 + .../records/ri18_image_count_zero.jsonl | 2 + .../records/ri19_image_count_float.jsonl | 2 + .../records/ri20_date_impossible.jsonl | 2 + .../ri21_technical_repeats_unsorted.jsonl | 2 + .../records/ri22_id_kind_unknown.jsonl | 2 + ...ri23_bad_line_after_good_new_records.jsonl | 6 + .../records/ri24_sha256_not_hex.jsonl | 2 + .../records/ri25_compact_separators.jsonl | 2 + .../records/ri26_crlf_line_ends.jsonl | 1 + .../records/ri27_optional_ascii_escape.jsonl | 2 + .../ri28_technical_repeats_empty.jsonl | 2 + .../records/ri29_image_count_unchanged.jsonl | 2 + .../records/ri30_extra_whitespace.jsonl | 2 + .../ri31_sha256_trailing_newline.jsonl | 2 + .../rv01_reconfirmation_old_bytes.jsonl | 3 + .../records/rv02_image_count_changed.jsonl | 2 + .../records/rv03_removed_every_entity.jsonl | 5 + .../records/rv04_old_and_new_mixed.jsonl | 7 + .../records/rv05_non_ascii_unescaped.jsonl | 4 + .../fixtures/records/rv06_empty_file.jsonl | 0 tests/spec_fixtures/tools/audit/refrecords.py | 333 ++++++++++++++++++ tests/spec_fixtures/tools/audit/refval.py | 9 +- tests/test_spec_pin.py | 6 +- tests/test_spec_records_goldens.py | 100 ++++++ 50 files changed, 949 insertions(+), 81 deletions(-) create mode 100644 src/morphobricks_preprocessing/records_file.py create mode 100644 tests/spec_fixtures/fixtures/records/ri01_missing_final_lf.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri02_malformed_line.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri03_line_not_an_object.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri04_blank_line.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri05_unknown_type.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri06_type_missing.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri07_key_missing.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri08_key_of_another_type.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri09_key_order.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri10_nested_key_order.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri11_non_ascii_escaped.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri12_invalid_utf8.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri13_byte_order_mark.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri14_duplicate_key.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri15_removed_unit_with_null_name.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri16_removed_repeat_names_extra_key.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri17_removed_entity_unknown.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri18_image_count_zero.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri19_image_count_float.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri20_date_impossible.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri21_technical_repeats_unsorted.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri22_id_kind_unknown.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri23_bad_line_after_good_new_records.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri24_sha256_not_hex.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri25_compact_separators.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri26_crlf_line_ends.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri27_optional_ascii_escape.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri28_technical_repeats_empty.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri29_image_count_unchanged.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri30_extra_whitespace.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/ri31_sha256_trailing_newline.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/rv01_reconfirmation_old_bytes.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/rv02_image_count_changed.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/rv03_removed_every_entity.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/rv04_old_and_new_mixed.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/rv05_non_ascii_unescaped.jsonl create mode 100644 tests/spec_fixtures/fixtures/records/rv06_empty_file.jsonl create mode 100644 tests/spec_fixtures/tools/audit/refrecords.py create mode 100644 tests/test_spec_records_goldens.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a2c87940..d2a9ec7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,20 @@ are in [docs/plan.md](docs/plan.md) §12. Upgrading from v0.3: see the ## Unreleased +- **Re-pin the vendored Spec to 1bed024 (MorphoBricks-Spec PR 16, owner decisions RF1 to RF6); + `read_records` follows section 4.6.** The Spec now fixes the bytes and values of + `morphobricks.records.jsonl` and names one refusal, `R_RECORDS_FILE_INVALID`, with the reasons + `E_RECORDS_ENCODING`, `_NO_FINAL_LF`, `_NOT_JSON_OBJECT`, `_UNKNOWN_TYPE`, `_KEY_SET`, `_KEY_ORDER`, + `_NON_ASCII_ESCAPED`, `_VALUE` and `_FORM`. `read_records` checked only the key set; it now refuses + every file the Spec's reference validator refuses (new pure module `records_file`): key order at every + level, escaped non-ASCII, duplicate keys, impossible dates and counts, upper-case or newline-ended + SHA-256, an unchanged or empty count record, and any byte that is not the canonical encoding + (`json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n"`, what the writer appends, so every + file the writer produces is still accepted). `RecordsFileError` now carries `code` and `reasons`. + The vendored files add `fixtures/records/` (37 goldens) and the reference module `refrecords.py` + (imported by `refval.py`, also pinned). `tests/test_spec_records_goldens.py` runs every golden + through `read_records` against the catalogue and the reference validator. A draft saved under the + previous pin is refused at resume with `spec_commit_mismatch`, as before. - **Notebook: check and save, reopen, described runs (Stage 1 PR 16b).** `PreprocessingNotebook(describe=True)` (the shipped notebook) saves the experiment as `morphobricks.toml` before the run. *Check and save* restates it in sentences (what each condition is compared against, "compared against nothing" in diff --git a/docs/conformance.md b/docs/conformance.md index 3bcb3e7a..36dcaaba 100644 --- a/docs/conformance.md +++ b/docs/conformance.md @@ -8,7 +8,7 @@ real run; a skip is never counted as a pass. | | | |---|---| | Base commit | `68b2a4b` (`origin/main`, merge of PR 211); the run included the changes of PR 17 | -| Spec pin | the full SHA in `tests/spec_fixtures/PIN.toml` (vendored `experiment-v1.md` at `c8f6852`) | +| Spec pin | the full SHA in `tests/spec_fixtures/PIN.toml` (vendored `experiment-v1.md` at `1bed024`) | | Command | `python tools/check.py` (Ruff, Ruff format, then `pytest -q` over the whole `tests/` folder), then `python tools/check.py --rules` | | Whole suite | 4324 passed, 4 skipped, 0 failed (301 s) | | Spec modules (`tests/test_spec_*.py`) | 1535 passed, 0 skipped, 0 failed (the Spec gate fails a skip or xfail there) | diff --git a/src/morphobricks_preprocessing/draft_format.py b/src/morphobricks_preprocessing/draft_format.py index 2e942500..ee11dc5e 100644 --- a/src/morphobricks_preprocessing/draft_format.py +++ b/src/morphobricks_preprocessing/draft_format.py @@ -35,7 +35,7 @@ DRAFT_VERSION = 3 IMAGE_TYPES = ("rgb", "channels") """The image types a draft may hold: ``channels`` is the answer "ordinary channel image".""" -SPEC_COMMIT = "c8f68526c08633526395ca0180fd6bd1a182fc84" +SPEC_COMMIT = "1bed024e278d104ba4b8f63ae76c020fb2e3470d" """The MorphoBricks-Spec commit the rules of this brick are pinned to (``tests/spec_fixtures``).""" BASE_MISSING = "R_DRAFT_BASE_MISSING" SHA256 = re.compile(r"[0-9a-f]{64}") diff --git a/src/morphobricks_preprocessing/records_file.py b/src/morphobricks_preprocessing/records_file.py new file mode 100644 index 00000000..15ad874c --- /dev/null +++ b/src/morphobricks_preprocessing/records_file.py @@ -0,0 +1,275 @@ +"""The bytes and values of ``morphobricks.records.jsonl`` (experiment-v1 section 4.6, RF1-RF5). + +A pure checker: the bytes of a records file in, every refusal out, nothing read from disk. A +refused file is ``R_RECORDS_FILE_INVALID`` (section 8); the ``E_RECORDS_*`` names below are its +reasons, the Spec's reference validator's. An accepted line equals ``encode_record`` of the +record it holds, which is also what the writer appends, so one function decides both sides. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from datetime import date + +CODE = "R_RECORDS_FILE_INVALID" +"""The catalogue code of a refused records file (section 8, owner decision RF3).""" +ENCODING = "E_RECORDS_ENCODING" +NO_FINAL_LF = "E_RECORDS_NO_FINAL_LF" +NOT_JSON_OBJECT = "E_RECORDS_NOT_JSON_OBJECT" +UNKNOWN_TYPE = "E_RECORDS_UNKNOWN_TYPE" +KEY_SET = "E_RECORDS_KEY_SET" +KEY_ORDER = "E_RECORDS_KEY_ORDER" +NON_ASCII_ESCAPED = "E_RECORDS_NON_ASCII_ESCAPED" +VALUE = "E_RECORDS_VALUE" +FORM = "E_RECORDS_FORM" + +_BOM = b"\xef\xbb\xbf" +_DATE = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}") +_SHA256 = re.compile(r"[0-9a-f]{64}") # fullmatch: a `$` would accept a final LF +_ESCAPE = re.compile(r"\\(?:u([0-9a-fA-F]{4})|.)", re.DOTALL) +ID_KINDS = ("conditions", "channels", "repeats", "technical_repeats") +ENTITIES = ("unit", "technical_repeat", "repeat", "condition") +_REMOVED_NAMES = ( + "condition_id", + "condition_source_folder", + "repeat_id", + "source_repeat", + "technical_repeat_id", + "technical_repeat_source_folder", +) +RECORD_KEYS: dict[str, frozenset[str]] = { + "reconfirmation": frozenset( + {"type", "id_kind", "old_id", "new_id", "confirmed_on", "old_sha256", "new_sha256"} + ), + "image_count_changed": frozenset( + { + "type", + "condition_id", + "condition_source_folder", + "repeat_id", + "source_repeat", + "technical_repeats", + "old_image_count", + "new_image_count", + "recorded_on", + } + ), + "removed": frozenset({"type", "entity", "confirmed_on", *_REMOVED_NAMES}), +} +"""The key set of each record type (Spec experiment-v1 4.6): a line carries exactly these keys.""" +_NAMED = { + "unit": _REMOVED_NAMES, + "technical_repeat": ("technical_repeat_id", "technical_repeat_source_folder"), + "repeat": ("repeat_id",), + "condition": ("condition_id", "condition_source_folder"), +} +"""The keys a ``removed`` line names per entity; every other name key is null.""" + + +@dataclass(frozen=True) +class RecordsIssue: + """One refusal: the ``E_RECORDS_*`` reason, the 1-based line and a short detail.""" + + reason: str + line: int + detail: str = "" + + +class _Duplicate(ValueError): + """A key written twice in one object.""" + + +def encode_record(record: dict) -> bytes: + """Return the canonical bytes of one record: code point key order, unescaped non-ASCII. + + The separators are ``, `` and ``: `` and the line ends in one LF (section 4.6, RF1); this + is what ``_append_records`` writes. + """ + return (json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n").encode("utf-8") + + +def _pairs(items: list[tuple[str, object]]) -> dict: + out: dict = {} + for key, value in items: + if key in out: + raise _Duplicate(key) + out[key] = value + return out + + +def _constant(name: str) -> object: + raise ValueError(f"{name} is not JSON") + + +def _sorted_keys(value: object) -> bool: + """Return whether every object nested in ``value`` lists its keys in code point order.""" + if isinstance(value, dict): + keys = list(value) + return keys == sorted(keys) and all(_sorted_keys(v) for v in value.values()) + if isinstance(value, list): + return all(_sorted_keys(v) for v in value) + return True + + +def _escaped_non_ascii(text: str) -> bool: + return any(m.group(1) and int(m.group(1), 16) >= 0x80 for m in _ESCAPE.finditer(text)) + + +def _text(value: object) -> bool: + return type(value) is str and not any(0xD800 <= ord(c) <= 0xDFFF for c in value) + + +def _date(value: object) -> bool: + if type(value) is not str or not _DATE.fullmatch(value): + return False + try: + date.fromisoformat(value) + except ValueError: + return False + return True + + +def _sha(value: object) -> bool: + return type(value) is str and _SHA256.fullmatch(value) is not None + + +def _count(value: object) -> bool: + return type(value) is int and value >= 1 + + +def _bad_reconfirmation(rec: dict) -> list[str]: + checks = { + "id_kind": rec["id_kind"] in ID_KINDS, + "old_id": _text(rec["old_id"]), + "new_id": rec["new_id"] is None or _text(rec["new_id"]), + "confirmed_on": _date(rec["confirmed_on"]), + "old_sha256": rec["old_sha256"] == "absent" or _sha(rec["old_sha256"]), + "new_sha256": _sha(rec["new_sha256"]), + } + return [name for name, ok in checks.items() if not ok] + + +def _bad_removed(rec: dict) -> list[str]: + if rec["entity"] not in ENTITIES: + return ["entity"] + named = _NAMED[rec["entity"]] + bad = [ + key + for key in _REMOVED_NAMES + if (not _text(rec[key]) if key in named else rec[key] is not None) + ] + return bad + ([] if _date(rec["confirmed_on"]) else ["confirmed_on"]) + + +def _bad_units(units: object) -> bool: + """Return whether ``units`` is not a non-empty array of ``{id, source_folder}`` sorted by id.""" + if type(units) is not list or not units: + return True + for unit in units: + if type(unit) is not dict or set(unit) != {"id", "source_folder"}: + return True + if not (_text(unit["id"]) and _text(unit["source_folder"])): + return True + ids = [unit["id"] for unit in units] + return ids != sorted(ids) + + +def _bad_image_count(rec: dict) -> list[str]: + names = ("condition_id", "condition_source_folder", "repeat_id", "source_repeat") + bad = [key for key in names if not _text(rec[key])] + bad += ["technical_repeats"] if _bad_units(rec["technical_repeats"]) else [] + bad += [k for k in ("old_image_count", "new_image_count") if not _count(rec[k])] + if not bad and rec["old_image_count"] == rec["new_image_count"]: + bad.append("new_image_count") # the record states a change (RF5) + return bad + ([] if _date(rec["recorded_on"]) else ["recorded_on"]) + + +_VALUE_CHECKS = { + "reconfirmation": _bad_reconfirmation, + "removed": _bad_removed, + "image_count_changed": _bad_image_count, +} + + +def _parse(text: str, number: int) -> tuple[dict | None, RecordsIssue | None]: + try: + rec = json.loads(text, object_pairs_hook=_pairs, parse_constant=_constant) + except _Duplicate as error: + return None, RecordsIssue(NOT_JSON_OBJECT, number, f"key {error} twice") + except ValueError as error: + return None, RecordsIssue(NOT_JSON_OBJECT, number, str(error)) + if type(rec) is not dict: + return None, RecordsIssue(NOT_JSON_OBJECT, number, type(rec).__name__) + return rec, None + + +def _line_issues(raw: bytes, number: int) -> tuple[dict | None, list[RecordsIssue]]: + """Return the record of one line (None when it is refused) and every refusal of it.""" + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + return None, [RecordsIssue(ENCODING, number, "not UTF-8")] + rec, issue = _parse(text, number) + if rec is None: + return None, [issue] + kind = rec.get("type") + if type(kind) is not str or kind not in RECORD_KEYS: + return None, [RecordsIssue(UNKNOWN_TYPE, number, repr(kind))] + if set(rec) != RECORD_KEYS[kind]: + odd = sorted(set(rec) ^ RECORD_KEYS[kind]) + return None, [RecordsIssue(KEY_SET, number, f"{kind}: differs in {odd}")] + issues = [] + bad = _VALUE_CHECKS[kind](rec) + if bad: + issues.append(RecordsIssue(VALUE, number, f"{kind}: {bad}")) + if not _sorted_keys(rec): + issues.append(RecordsIssue(KEY_ORDER, number)) + if _escaped_non_ascii(text): + issues.append(RecordsIssue(NON_ASCII_ESCAPED, number)) + if issues: + return None, issues + if raw + b"\n" != encode_record(rec): + form = "not the canonical bytes (whitespace, CR, optional escape)" + return None, [RecordsIssue(FORM, number, form)] + return rec, [] + + +def check_records(data: bytes) -> tuple[tuple[dict, ...], tuple[RecordsIssue, ...]]: + """Return the records of a records file's bytes and every refusal (none when accepted). + + Every line is checked, so one file may give several reasons. A file of zero bytes holds no + record and is accepted (RF4). + + Parameters + ---------- + data : bytes + The exact bytes of ``morphobricks.records.jsonl``. + + Returns + ------- + tuple + The accepted lines' records in order, and the refusals; the file is accepted exactly + when the second is empty. + """ + if not data: + return (), () + issues: list[RecordsIssue] = [] + bom = data.startswith(_BOM) + if bom: + issues.append(RecordsIssue(ENCODING, 1, "byte-order mark")) + parts = data.split(b"\n") # LF only: U+2028, U+0085 and CR do not end a line + if parts[-1] != b"": + issues.append(RecordsIssue(NO_FINAL_LF, len(parts), "last line lacks its LF")) + else: + parts = parts[:-1] + records = [] + for number, raw in enumerate(parts, start=1): + if number == 1 and bom: + continue + record, found = _line_issues(raw, number) + issues += found + if record is not None: + records.append(record) + return tuple(records), tuple(issues) diff --git a/src/morphobricks_preprocessing/run/description_file.py b/src/morphobricks_preprocessing/run/description_file.py index f9158d66..47610bf0 100644 --- a/src/morphobricks_preprocessing/run/description_file.py +++ b/src/morphobricks_preprocessing/run/description_file.py @@ -27,7 +27,6 @@ from __future__ import annotations -import json import os import re import tempfile @@ -53,6 +52,8 @@ unwaived, ) from ..description_write import canonical_bytes +from ..records_file import CODE as RECORDS_FILE_INVALID +from ..records_file import RecordsIssue, check_records, encode_record from .description_records import image_count_records, removal_records from .lock import output_lock @@ -71,42 +72,19 @@ def __init__(self, issues: tuple[DescriptionIssue, ...]) -> None: super().__init__(f"the description was not published: {shown}") -RECORD_KEYS: dict[str, frozenset[str]] = { - "reconfirmation": frozenset( - {"type", "id_kind", "old_id", "new_id", "confirmed_on", "old_sha256", "new_sha256"} - ), - "image_count_changed": frozenset( - { - "type", - "condition_id", - "condition_source_folder", - "repeat_id", - "source_repeat", - "technical_repeats", - "old_image_count", - "new_image_count", - "recorded_on", - } - ), - "removed": frozenset( - { - "type", - "entity", - "condition_id", - "condition_source_folder", - "repeat_id", - "source_repeat", - "technical_repeat_id", - "technical_repeat_source_folder", - "confirmed_on", - } - ), -} -"""The key set of each record type (Spec experiment-v1 4.6): a line carries exactly these keys.""" +class RecordsFileError(OSError): + """Raised when ``morphobricks.records.jsonl`` is refused (``R_RECORDS_FILE_INVALID``). + ``code`` is the catalogue code, ``issues`` every refusal and ``reasons`` their distinct + ``E_RECORDS_*`` names. + """ -class RecordsFileError(OSError): - """Raised when ``morphobricks.records.jsonl`` cannot be appended to as it stands.""" + def __init__(self, issues: tuple[RecordsIssue, ...]) -> None: + self.code = RECORDS_FILE_INVALID + self.issues = issues + self.reasons = frozenset(issue.reason for issue in issues) + shown = "; ".join(f"{i.reason} at line {i.line}" for i in issues[:6]) + super().__init__(f"{RECORDS_FILENAME} is refused ({self.code}): {shown}") @dataclass(frozen=True) @@ -164,32 +142,14 @@ def read_records(output_folder: str | os.PathLike) -> tuple[dict, ...]: DescriptionError ``R_FILE_SYMLINK`` when the records file is a symbolic link. RecordsFileError - When a line is not a JSON object, is not one of the three Spec record types with exactly - that type's keys (4.6), or the last line is torn (no final newline). + ``R_RECORDS_FILE_INVALID`` (section 4.6) for every file the Spec's reference validator + refuses; ``reasons`` lists the ``E_RECORDS_*`` names, the message the first line of each. """ data = read_file_bytes(Path(output_folder) / RECORDS_FILENAME) - if not data: - return () - if not data.endswith(b"\n"): - raise RecordsFileError(f"{RECORDS_FILENAME} ends in the middle of a line") - try: - # Lines end at LF only (Spec 4.6): str.splitlines would also cut at U+2028, U+0085, ... - rows = tuple(json.loads(line) for line in data.decode("utf-8").split("\n")[:-1]) - except ValueError as error: - raise RecordsFileError(f"{RECORDS_FILENAME} holds a line that is not JSON") from error - if not all(isinstance(row, dict) for row in rows): - raise RecordsFileError(f"{RECORDS_FILENAME} holds a line that is not an object") - for number, row in enumerate(rows, start=1): - keys = RECORD_KEYS.get(row.get("type")) if isinstance(row.get("type"), str) else None - if keys is None: - raise RecordsFileError(f"{RECORDS_FILENAME} line {number}: no known record type") - if set(row) != keys: - odd = sorted(set(row) ^ keys) - raise RecordsFileError( - f"{RECORDS_FILENAME} line {number}: a {row['type']} record has exactly the keys " - f"{sorted(keys)}; these differ: {odd}" - ) - return rows + records, issues = check_records(data or b"") + if issues: + raise RecordsFileError(issues) + return records def _replace_file_durably(path: Path, data: bytes) -> None: @@ -218,8 +178,7 @@ def _append_records(folder: Path, records: tuple[dict, ...]) -> None: read_records(folder) # refuses a symbolic link and a torn or foreign file before writing path = folder / RECORDS_FILENAME current = read_file_bytes(path) or b"" - lines = "".join(json.dumps(r, ensure_ascii=False, sort_keys=True) + "\n" for r in records) - _replace_file_durably(path, current + lines.encode("utf-8")) + _replace_file_durably(path, current + b"".join(encode_record(r) for r in records)) def _compare( diff --git a/tests/spec_fixtures/PIN.toml b/tests/spec_fixtures/PIN.toml index 8ed39686..4deebf9e 100644 --- a/tests/spec_fixtures/PIN.toml +++ b/tests/spec_fixtures/PIN.toml @@ -1,13 +1,14 @@ # The MorphoBricks-Spec commit these files are copied from (conformance plan section 1). # Updating the pin is a reviewed change: see README.md in this folder. repository = "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/CellMigrationLab/MorphoBricks-Spec" -commit = "c8f68526c08633526395ca0180fd6bd1a182fc84" +commit = "1bed024e278d104ba4b8f63ae76c020fb2e3470d" # `git rev-parse :fixtures`, the git tree-object ID of the fixtures/ folder. The # vendored folder must hash to it, so one changed, added or missing byte fails a test. -fixtures_tree = "87d097408e5b794f31f902ec30f2a17dd941a1bd" +fixtures_tree = "f61b1c2ba96a36eea9b2c101b13f6d7a3ff95b8b" # `git rev-parse :`, the git blob ID of each other vendored file. [files] -"docs/experiment-v1.md" = "e3d311b63e7ec7d2128a307822339806a10785dd" -"tools/audit/refval.py" = "de5a2cdb53ecb5bda7493c5b8cab2908e6e32fe6" +"docs/experiment-v1.md" = "b12678cbac2c83e7a4d64cdc2883d4a8e327bb1f" +"tools/audit/refval.py" = "0ea83094cfa8c1bcb6ea596e17b3f0c948ad5dfb" +"tools/audit/refrecords.py" = "ef06a1ea59a93c695dbba21d2ee792667ad8e8eb" diff --git a/tests/spec_fixtures/README.md b/tests/spec_fixtures/README.md index 59c3a022..51e348d1 100644 --- a/tests/spec_fixtures/README.md +++ b/tests/spec_fixtures/README.md @@ -8,6 +8,7 @@ at the commit named in `PIN.toml`, laid out as in the Spec repository: | `fixtures/` | `fixtures/` | The conformance fixtures: every test of the experiment-description reader runs on them | | `docs/experiment-v1.md` | `docs/experiment-v1.md` | The normative text; tests compare the reader's tables of codes, names and fields with it | | `tools/audit/refval.py` | `tools/audit/refval.py` | The reference validator, used **only as a test oracle**; nothing in `src/` imports it | +| `tools/audit/refrecords.py` | `tools/audit/refrecords.py` | The reference validator of the records file (§4.6), imported by `refval.py`; also a test oracle only | Rules (conformance plan, sections 1 and 2): @@ -26,10 +27,11 @@ Rules (conformance plan, sections 1 and 2): 1. In a checkout of the Spec, check out the new commit and read `git rev-parse HEAD`, `git rev-parse HEAD:fixtures`, - `git rev-parse HEAD:docs/experiment-v1.md` and `git rev-parse HEAD:tools/audit/refval.py`. + `git rev-parse HEAD:docs/experiment-v1.md`, `git rev-parse HEAD:tools/audit/refval.py` and + `git rev-parse HEAD:tools/audit/refrecords.py`. 2. Replace the vendored files from that commit, for example `git archive HEAD fixtures | tar -x -C tests/spec_fixtures`, and `git show HEAD:` for - the two single files. -3. Put the four values in `PIN.toml`. + the three single files. +3. Put the five values in `PIN.toml`. 4. Run the whole suite. The reader, its tables and its tests change in the same pull request when the Spec changed text or fixtures (the Spec changes first). diff --git a/tests/spec_fixtures/docs/experiment-v1.md b/tests/spec_fixtures/docs/experiment-v1.md index e3d311b6..b12678cb 100644 --- a/tests/spec_fixtures/docs/experiment-v1.md +++ b/tests/spec_fixtures/docs/experiment-v1.md @@ -399,17 +399,39 @@ stays in the brick that owns it. `morphobricks.toml`, and copies the records unchanged into the manifest of the next run. The records file is never rewritten or truncated, and nothing about a re-confirmation is stored in `morphobricks.toml`. Products keyed on that ID are invalidated. -- **Encoding of the records file.** UTF-8, one JSON object per line, each line ending in LF, keys - in code point order, non-ASCII characters unescaped. The key `type` tells the record types +- **Encoding of the records file (owner decisions RF1, RF4).** UTF-8 without a byte-order mark, + one JSON object per line, each line ending in LF (a CR before the LF is not allowed), keys in + code point order at every level (the objects inside `technical_repeats` included), non-ASCII + characters unescaped, the separators `, ` between members and `: ` after a key, and no other + whitespace outside strings. A string uses only the escapes `\"`, `\\`, `\b`, `\f`, `\n`, `\r`, + `\t` and, for the other control characters U+0000 to U+001F, `\u00xx` with lower-case hex; no + other character is escaped. A file of zero bytes holds no record and is accepted. The key + `type` tells the record types apart: `"reconfirmation"`, `"image_count_changed"` and `"removed"`. Each type has its own fixed key set; a line of one type carries no key of another. The first type is the only one written before this addition, and its lines are unchanged. Keys of `reconfirmation`: `type`, `id_kind` (`conditions`, `channels`, `repeats` or `technical_repeats`), `old_id`, `new_id` (`null` when the same ID lost or was reassigned something), `confirmed_on` (`YYYY-MM-DD`, the date of the user's action), `old_sha256` (of the published file replaced, or `absent`) and - `new_sha256` (of the file written). The records are appended after the file is replaced. A records - file whose last line lacks its LF, that holds a line that is not a JSON object, or that holds an - object whose `type` is none of the three, is refused before anything is published. + `new_sha256` (of the file written). A SHA-256 is 64 lower-case hex digits. The records are + appended after the file is replaced. +- **Refusal of a records file (owner decisions RF2, RF3, RF4, RF5; `R_RECORDS_FILE_INVALID`).** A + reader refuses the whole file, before anything is published, when any of these holds (the reason + is given in parentheses, one file may have several): the bytes are not UTF-8 or start with a + byte-order mark (`E_RECORDS_ENCODING`); the last line lacks its LF (`E_RECORDS_NO_FINAL_LF`); a + line is not a JSON object, including an empty line and an object with a key twice + (`E_RECORDS_NOT_JSON_OBJECT`); `type` is missing or none of the three (`E_RECORDS_UNKNOWN_TYPE`); + the keys are not exactly the key set of the type (`E_RECORDS_KEY_SET`); keys are not in code point + order at every level (`E_RECORDS_KEY_ORDER`); a non-ASCII character is written as an escape + (`E_RECORDS_NON_ASCII_ESCAPED`); a value is invalid: an enumeration (`id_kind`, `entity`) outside + its values, a date that is not a real `YYYY-MM-DD`, a SHA-256 that is not 64 lower-case hex + digits (`old_sha256` may also be `absent`), a key that must be `null` (or must not be) in a + `removed` line, a count that is not an integer of at least 1, `old_image_count` equal to + `new_image_count`, `technical_repeats` empty or not sorted by `id` (`E_RECORDS_VALUE`); or the + line differs from the canonical bytes above in anything else, whitespace, a CR, an optional + escape (`E_RECORDS_FORM`). The reasons are named by the reference validator + (`tools/audit/refrecords.py`); the code is the Spec's. The record values that name IDs and + folder labels are text; the Spec states no further rule for them here. - **Record of a removal (`removed`, owner decision RR2).** Written for **every** removal of an imaging unit, technical repeat, repeat or condition that the user confirms (§10), including a removal whose ID is held nowhere else afterwards; it is written in addition to a @@ -431,8 +453,9 @@ stays in the brick that owns it. confirmation and reopens nothing: a count change alone is not a reopening, and `R_CONFIRMED_ON_CHANGED` (§4.4a) is unchanged. Keys: `type`, `condition_id`, `condition_source_folder`, `repeat_id`, `source_repeat`, `technical_repeats` (the member's - imaging units as an array of objects `{"id", "source_folder"}`, sorted by `id` in code point - order), `old_image_count`, `new_image_count` (integers, each at least 1) and `recorded_on` + imaging units as a non-empty array of objects `{"id", "source_folder"}`, sorted by `id` in code + point order), `old_image_count`, `new_image_count` (integers, each at least 1, and different: + the record states a change) and `recorded_on` (`YYYY-MM-DD`, the date the change was recorded at publication; it is not a confirmation and is never copied into `confirmed_on`). - **Order and content of the `removed` lines of one publication (RR2, clarified with RR3).** Each @@ -441,6 +464,12 @@ stays in the brick that owns it. `confirmed_on` date of the first removal answer that touches it. When a removal is combined with an ID correction in the same publication, the lines carry the IDs as published (before the correction). +- **Order of the lines of one publication (owner decision RF6).** Preprocessing writes them as one + append: the `reconfirmation` lines (by `id_kind`, `old_id`, `new_id`, `confirmed_on`), then the + `removed` lines (entity lines, then `unit` lines, above), then the `image_count_changed` lines + by `condition_id`, then `repeat_id`. The file has no publication boundary, so a reader of the + file alone cannot check this order or that each unit is written once; the writer tests of + scenarios P34 and P35 check it, and a file that breaks it is not refused for that. - **A case-only correction needs a new ID (AQ2).** `ACTIN` cannot be corrected to `Actin`: the retired spelling stays in `used_ids` and the new one collides with it (`E_ID_CASE_COLLISION`, §4.9). The correction is a new, non-colliding ID (for example @@ -708,6 +737,7 @@ Required preprocessing refusals (`R_*`): | `R_USED_ID_REUSED` | A candidate declares an ID that `used_ids` lists as retired (§4.9) | | `R_USED_ID_DROPPED` | A candidate omits an ID listed in `used_ids` of the file it replaces (§4.9) | | `R_CONFIRMED_ON_CHANGED` | `confirmed_on` of a repeat whose confirmation was not reopened differs from the published file (§4.4a) | +| `R_RECORDS_FILE_INVALID` | `morphobricks.records.jsonl` breaks the encoding, key set or value rules of §4.6; reasons `E_RECORDS_ENCODING`, `E_RECORDS_NO_FINAL_LF`, `E_RECORDS_NOT_JSON_OBJECT`, `E_RECORDS_UNKNOWN_TYPE`, `E_RECORDS_KEY_SET`, `E_RECORDS_KEY_ORDER`, `E_RECORDS_NON_ASCII_ESCAPED`, `E_RECORDS_VALUE`, `E_RECORDS_FORM` | | `R_PUBLISHED_FILE_CHANGED` | The SHA-256 of the published file at publication differs from the draft's recorded base (§10) | | `R_DRAFT_BASE_MISSING` | A draft has no recorded base SHA-256 (§10) | | `R_FOLDER_NOT_NFC` | A folder label or a discovered mask type name read from disk is not NFC (§3.3) | diff --git a/tests/spec_fixtures/fixtures/README.md b/tests/spec_fixtures/fixtures/README.md index f5bd34a0..050430ba 100644 --- a/tests/spec_fixtures/fixtures/README.md +++ b/tests/spec_fixtures/fixtures/README.md @@ -2,8 +2,9 @@ Normative test data for [experiment-v1.md](../docs/experiment-v1.md). Each file starts with a comment header (`Fixture`, `Expected`, `Source`, description). How these become executable tests -in preprocessing is described in the [conformance plan](../docs/conformance-plan.md); nothing here -is executed by this repository. +in preprocessing is described in the [conformance plan](../docs/conformance-plan.md); the audit tool +`tools/audit/refval.py` runs them here. The records files in `records/` have no header; see +their section. ## Rules @@ -218,6 +219,12 @@ This is a check of the fixtures against the text, not a conformance claim for an | RR1 | Record type `image_count_changed` (a confirmed member's image count changes; nothing reopens) | P34 | | RR3 | `image_count_changed` is written for every confirmed member whose count changed, also when the publication changes other things | P34 | | RR2 | Record type `removed` for every confirmed removal of a unit, technical repeat, repeat or condition | P35 | +| RF1 | Records file separators are `, ` and `: `; other whitespace refused | `ri25`, `ri27`, `ri30` | +| RF2 | Every records-file violation is a refusal (key set, order, escapes, duplicates, values) | `ri01`-`ri24` | +| RF3 | One code, `R_RECORDS_FILE_INVALID`, with `E_RECORDS_*` reasons | all `ri` records files | +| RF4 | Empty records file accepted; BOM and CR before LF refused; lower-case SHA hex | `rv06`, `ri13`, `ri24`, `ri26`, `ri31` | +| RF5 | `technical_repeats` not empty; `old_image_count` differs from `new_image_count` | `ri28`, `ri29` | +| RF6 | Line order within one publication stated; checked by writer tests, not by the file | P34, P35 | ## Valid fixtures (37) @@ -431,3 +438,53 @@ Valid alone; checked against the named published predecessor (refused with the l | [`i133_condition_single_not_alone`](invalid/i133_condition_single_not_alone.toml) | Owner decision SQ6, matrix audit MS-1 | `E_SINGLE_MISUSE` | 4 | The condition `single` (no condition folder level) next to the condition DrugA: it must be the only condition (experiment-v1 3.2). | | [`i134_duplicate_key_inline_table`](invalid/i134_duplicate_key_inline_table.toml) | Stage 1 design PR 0, experiment-v1 2.2 | `E_DUPLICATE_KEY` | 1 | `display_name` written twice inside the inline table of the channel DNA: a key declared twice in any spelling, inline tables included, is `E_DUPLICATE_KEY`, not `E_TOML_SYNTAX` (experiment-v1 2.2). | | [`i135_used_id_channel_reserved`](invalid/i135_used_id_channel_reserved.toml) | Owner decision UR1 (Spec issue 11) | `E_ID_RESERVED` | 4 | The retired channel ID `Nucleus` in `used_ids.channels`: a listed ID reserves the ID, so it follows the reserved list of 3.2 (case-folded) like a declared channel ID (experiment-v1 4.9). | + +## Records fixtures (37) + +Byte-exact test data for `morphobricks.records.jsonl` (experiment-v1 §4.6, owner decisions RF1-RF6 +of 9 November 2026). Unlike the TOML fixtures a JSONL file has no comment header, so each case is +listed here. Every file is checked by `tools/audit/refrecords.py` through `refval.py fixtures`: an +`accepted` file gives no error and is byte-identical to the canonical encoding of its records; a +`refused` file gives exactly the listed reason and its first error is on the listed line. A reader +MUST accept the `rv` files and refuse the `ri` files with `R_RECORDS_FILE_INVALID`; the +`E_RECORDS_*` names are its reasons (experiment-v1 §8). + +| Fixture | Expected | Case | +|---|---|---| +| [`rv01_reconfirmation_old_bytes`](records/rv01_reconfirmation_old_bytes.jsonl) | `accepted` | Three `reconfirmation` lines exactly as written before the `removed` and `image_count_changed` types existed (a `null` `new_id`, an `old_sha256` of `absent`); the bytes are unchanged and equal the canonical encoding. | +| [`rv02_image_count_changed`](records/rv02_image_count_changed.jsonl) | `accepted` | Two `image_count_changed` lines (RR1, RR3): one unit, and two units sorted by `id`, with a space inside a `source_folder`. | +| [`rv03_removed_every_entity`](records/rv03_removed_every_entity.jsonl) | `accepted` | `removed` lines of all four entities: the entity lines (condition, repeat, technical repeat) first, then each `unit` line (RR2); a key the entity does not name is `null`. | +| [`rv04_old_and_new_mixed`](records/rv04_old_and_new_mixed.jsonl) | `accepted` | One file mixing old and new records, as a records file grows: `reconfirmation`, `removed` and `image_count_changed` lines in append order. | +| [`rv05_non_ascii_unescaped`](records/rv05_non_ascii_unescaped.jsonl) | `accepted` | Non-ASCII text written unescaped (umlaut, accent, CJK, an astral character), a U+2028 inside a string that must not end the line, and an escaped quote, backslash and tab. | +| [`rv06_empty_file`](records/rv06_empty_file.jsonl) | `accepted` | Zero bytes: no record and no torn line (RF4). | +| [`ri01_missing_final_lf`](records/ri01_missing_final_lf.jsonl) | refused: `E_RECORDS_NO_FINAL_LF` (line 2) | The last line is a complete record without its LF (a torn append). | +| [`ri02_malformed_line`](records/ri02_malformed_line.jsonl) | refused: `E_RECORDS_NOT_JSON_OBJECT` (line 2) | A line cut in the middle of an object; the lines after it are checked too. | +| [`ri03_line_not_an_object`](records/ri03_line_not_an_object.jsonl) | refused: `E_RECORDS_NOT_JSON_OBJECT` (line 2) | A line that is a JSON array. | +| [`ri04_blank_line`](records/ri04_blank_line.jsonl) | refused: `E_RECORDS_NOT_JSON_OBJECT` (line 2) | An empty line between two records. | +| [`ri05_unknown_type`](records/ri05_unknown_type.jsonl) | refused: `E_RECORDS_UNKNOWN_TYPE` (line 2) | `type` is `renamed`, none of the three. | +| [`ri06_type_missing`](records/ri06_type_missing.jsonl) | refused: `E_RECORDS_UNKNOWN_TYPE` (line 2) | No `type` key. | +| [`ri07_key_missing`](records/ri07_key_missing.jsonl) | refused: `E_RECORDS_KEY_SET` (line 2) | An `image_count_changed` line without `recorded_on`. | +| [`ri08_key_of_another_type`](records/ri08_key_of_another_type.jsonl) | refused: `E_RECORDS_KEY_SET` (line 1) | A `reconfirmation` line that also carries `recorded_on`, a key of another type. | +| [`ri09_key_order`](records/ri09_key_order.jsonl) | refused: `E_RECORDS_KEY_ORDER` (line 2) | The keys of a `reconfirmation` in the order of the text, not in code point order. | +| [`ri10_nested_key_order`](records/ri10_nested_key_order.jsonl) | refused: `E_RECORDS_KEY_ORDER` (line 2) | Inside `technical_repeats`, `source_folder` before `id`. | +| [`ri11_non_ascii_escaped`](records/ri11_non_ascii_escaped.jsonl) | refused: `E_RECORDS_NON_ASCII_ESCAPED` (line 2) | `\u00e9` written for a non-ASCII character. | +| [`ri12_invalid_utf8`](records/ri12_invalid_utf8.jsonl) | refused: `E_RECORDS_ENCODING` (line 2) | The byte 0xE9 alone (Latin-1) in a line. | +| [`ri13_byte_order_mark`](records/ri13_byte_order_mark.jsonl) | refused: `E_RECORDS_ENCODING` (line 1) | The file starts with a byte-order mark (RF4). | +| [`ri14_duplicate_key`](records/ri14_duplicate_key.jsonl) | refused: `E_RECORDS_NOT_JSON_OBJECT` (line 2) | `confirmed_on` twice in one object; the second value would silently win. | +| [`ri15_removed_unit_with_null_name`](records/ri15_removed_unit_with_null_name.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | A `unit` line with a `null` where the unit names all six keys. | +| [`ri16_removed_repeat_names_extra_key`](records/ri16_removed_repeat_names_extra_key.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | A `repeat` line with a `condition_id` that is not `null`. | +| [`ri17_removed_entity_unknown`](records/ri17_removed_entity_unknown.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `entity` is `channel`. | +| [`ri18_image_count_zero`](records/ri18_image_count_zero.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `new_image_count` is 0 (at least 1). | +| [`ri19_image_count_float`](records/ri19_image_count_float.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `new_image_count` is `15.0`, not an integer. | +| [`ri20_date_impossible`](records/ri20_date_impossible.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `confirmed_on` is `2026-02-30`. | +| [`ri21_technical_repeats_unsorted`](records/ri21_technical_repeats_unsorted.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `technical_repeats` not sorted by `id`. | +| [`ri22_id_kind_unknown`](records/ri22_id_kind_unknown.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `id_kind` is `masks`. | +| [`ri23_bad_line_after_good_new_records`](records/ri23_bad_line_after_good_new_records.jsonl) | refused: `E_RECORDS_VALUE` (line 5) | Four valid lines of three types, then an `image_count_changed` line with `old_image_count` 0; the line after it is valid. | +| [`ri24_sha256_not_hex`](records/ri24_sha256_not_hex.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `new_sha256` is 64 upper-case `B` (lower-case hex, RF4). | +| [`ri25_compact_separators`](records/ri25_compact_separators.jsonl) | refused: `E_RECORDS_FORM` (line 2) | Key order and ASCII rules met, but no space after `,` and `:` (RF1). | +| [`ri26_crlf_line_ends`](records/ri26_crlf_line_ends.jsonl) | refused: `E_RECORDS_FORM` (line 1) | A CR before the LF (RF4). | +| [`ri27_optional_ascii_escape`](records/ri27_optional_ascii_escape.jsonl) | refused: `E_RECORDS_FORM` (line 2) | `\u0031` written for the ASCII character `1`: only the minimal escapes are allowed. | +| [`ri28_technical_repeats_empty`](records/ri28_technical_repeats_empty.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | An `image_count_changed` line with `technical_repeats` `[]` (RF5). | +| [`ri29_image_count_unchanged`](records/ri29_image_count_unchanged.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `old_image_count` equals `new_image_count`: the record states no change (RF5). | +| [`ri30_extra_whitespace`](records/ri30_extra_whitespace.jsonl) | refused: `E_RECORDS_FORM` (line 2) | A space after `{` and a trailing space before the LF (RF1). | +| [`ri31_sha256_trailing_newline`](records/ri31_sha256_trailing_newline.jsonl) | refused: `E_RECORDS_VALUE` (line 2) | `new_sha256` is 64 lower-case hex digits followed by an escaped LF (`\n`): the line is canonical bytes, but the value is not 64 hex digits (RF4). Regression for a `$` anchor that matched before a final LF. | diff --git a/tests/spec_fixtures/fixtures/records/ri01_missing_final_lf.jsonl b/tests/spec_fixtures/fixtures/records/ri01_missing_final_lf.jsonl new file mode 100644 index 00000000..922c9b0c --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri01_missing_final_lf.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} \ No newline at end of file diff --git a/tests/spec_fixtures/fixtures/records/ri02_malformed_line.jsonl b/tests/spec_fixtures/fixtures/records/ri02_malformed_line.jsonl new file mode 100644 index 00000000..4a3c1740 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri02_malformed_line.jsonl @@ -0,0 +1,3 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"type": "removed", "entity": +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri03_line_not_an_object.jsonl b/tests/spec_fixtures/fixtures/records/ri03_line_not_an_object.jsonl new file mode 100644 index 00000000..e243c719 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri03_line_not_an_object.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +["reconfirmation"] diff --git a/tests/spec_fixtures/fixtures/records/ri04_blank_line.jsonl b/tests/spec_fixtures/fixtures/records/ri04_blank_line.jsonl new file mode 100644 index 00000000..fae05b36 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri04_blank_line.jsonl @@ -0,0 +1,3 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} + +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri05_unknown_type.jsonl b/tests/spec_fixtures/fixtures/records/ri05_unknown_type.jsonl new file mode 100644 index 00000000..83f18122 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri05_unknown_type.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "renamed"} diff --git a/tests/spec_fixtures/fixtures/records/ri06_type_missing.jsonl b/tests/spec_fixtures/fixtures/records/ri06_type_missing.jsonl new file mode 100644 index 00000000..9c085921 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri06_type_missing.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7"} diff --git a/tests/spec_fixtures/fixtures/records/ri07_key_missing.jsonl b/tests/spec_fixtures/fixtures/records/ri07_key_missing.jsonl new file mode 100644 index 00000000..fa8868f9 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri07_key_missing.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri08_key_of_another_type.jsonl b/tests/spec_fixtures/fixtures/records/ri08_key_of_another_type.jsonl new file mode 100644 index 00000000..4859140c --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri08_key_of_another_type.jsonl @@ -0,0 +1 @@ +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "recorded_on": "2026-11-08", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri09_key_order.jsonl b/tests/spec_fixtures/fixtures/records/ri09_key_order.jsonl new file mode 100644 index 00000000..d63e47ed --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri09_key_order.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"type": "reconfirmation", "id_kind": "conditions", "old_id": "DrugA", "new_id": null, "confirmed_on": "2026-11-08", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf"} diff --git a/tests/spec_fixtures/fixtures/records/ri10_nested_key_order.jsonl b/tests/spec_fixtures/fixtures/records/ri10_nested_key_order.jsonl new file mode 100644 index 00000000..39ee817a --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri10_nested_key_order.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"source_folder": "single", "id": "single"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri11_non_ascii_escaped.jsonl b/tests/spec_fixtures/fixtures/records/ri11_non_ascii_escaped.jsonl new file mode 100644 index 00000000..8b2d681b --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri11_non_ascii_escaped.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Caf\u00e9", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri12_invalid_utf8.jsonl b/tests/spec_fixtures/fixtures/records/ri12_invalid_utf8.jsonl new file mode 100644 index 00000000..dc5649b8 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri12_invalid_utf8.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Caf�", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri13_byte_order_mark.jsonl b/tests/spec_fixtures/fixtures/records/ri13_byte_order_mark.jsonl new file mode 100644 index 00000000..df931439 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri13_byte_order_mark.jsonl @@ -0,0 +1 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri14_duplicate_key.jsonl b/tests/spec_fixtures/fixtures/records/ri14_duplicate_key.jsonl new file mode 100644 index 00000000..1cfd89cd --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri14_duplicate_key.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "confirmed_on": "2026-11-09", "confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri15_removed_unit_with_null_name.jsonl b/tests/spec_fixtures/fixtures/records/ri15_removed_unit_with_null_name.jsonl new file mode 100644 index 00000000..40b54164 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri15_removed_unit_with_null_name.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "unit", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeat_id": "single", "technical_repeat_source_folder": null, "type": "removed"} diff --git a/tests/spec_fixtures/fixtures/records/ri16_removed_repeat_names_extra_key.jsonl b/tests/spec_fixtures/fixtures/records/ri16_removed_repeat_names_extra_key.jsonl new file mode 100644 index 00000000..b0968a69 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri16_removed_repeat_names_extra_key.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": null, "confirmed_on": "2026-11-09", "entity": "repeat", "repeat_id": "rep1", "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} diff --git a/tests/spec_fixtures/fixtures/records/ri17_removed_entity_unknown.jsonl b/tests/spec_fixtures/fixtures/records/ri17_removed_entity_unknown.jsonl new file mode 100644 index 00000000..223f04e4 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri17_removed_entity_unknown.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": null, "condition_source_folder": null, "confirmed_on": "2026-11-09", "entity": "channel", "repeat_id": "rep1", "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} diff --git a/tests/spec_fixtures/fixtures/records/ri18_image_count_zero.jsonl b/tests/spec_fixtures/fixtures/records/ri18_image_count_zero.jsonl new file mode 100644 index 00000000..b5421dc8 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri18_image_count_zero.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 0, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri19_image_count_float.jsonl b/tests/spec_fixtures/fixtures/records/ri19_image_count_float.jsonl new file mode 100644 index 00000000..1f92bedd --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri19_image_count_float.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15.0, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri20_date_impossible.jsonl b/tests/spec_fixtures/fixtures/records/ri20_date_impossible.jsonl new file mode 100644 index 00000000..518c68ff --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri20_date_impossible.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-02-30", "entity": "unit", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeat_id": "single", "technical_repeat_source_folder": "single", "type": "removed"} diff --git a/tests/spec_fixtures/fixtures/records/ri21_technical_repeats_unsorted.jsonl b/tests/spec_fixtures/fixtures/records/ri21_technical_repeats_unsorted.jsonl new file mode 100644 index 00000000..a197bc33 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri21_technical_repeats_unsorted.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "new_image_count": 4, "old_image_count": 9, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "acquisition_2", "source_folder": "a"}, {"id": "acquisition_1", "source_folder": "b"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri22_id_kind_unknown.jsonl b/tests/spec_fixtures/fixtures/records/ri22_id_kind_unknown.jsonl new file mode 100644 index 00000000..177c67ec --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri22_id_kind_unknown.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "masks", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri23_bad_line_after_good_new_records.jsonl b/tests/spec_fixtures/fixtures/records/ri23_bad_line_after_good_new_records.jsonl new file mode 100644 index 00000000..f027c9c0 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri23_bad_line_after_good_new_records.jsonl @@ -0,0 +1,6 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "condition", "repeat_id": null, "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "unit", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeat_id": "single", "technical_repeat_source_folder": "single", "type": "removed"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "new_image_count": 4, "old_image_count": 0, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "acquisition_1", "source_folder": "acq 1"}, {"id": "acquisition_2", "source_folder": "acq 2"}], "type": "image_count_changed"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri24_sha256_not_hex.jsonl b/tests/spec_fixtures/fixtures/records/ri24_sha256_not_hex.jsonl new file mode 100644 index 00000000..1b3c6c01 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri24_sha256_not_hex.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri25_compact_separators.jsonl b/tests/spec_fixtures/fixtures/records/ri25_compact_separators.jsonl new file mode 100644 index 00000000..13f0b4f5 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri25_compact_separators.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on":"2026-11-08","id_kind":"conditions","new_id":null,"new_sha256":"9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf","old_id":"DrugA","old_sha256":"f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7","type":"reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri26_crlf_line_ends.jsonl b/tests/spec_fixtures/fixtures/records/ri26_crlf_line_ends.jsonl new file mode 100644 index 00000000..0c9d0e69 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri26_crlf_line_ends.jsonl @@ -0,0 +1 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri27_optional_ascii_escape.jsonl b/tests/spec_fixtures/fixtures/records/ri27_optional_ascii_escape.jsonl new file mode 100644 index 00000000..fbce527d --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri27_optional_ascii_escape.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day\u0031", "old_sha256": "absent", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/ri28_technical_repeats_empty.jsonl b/tests/spec_fixtures/fixtures/records/ri28_technical_repeats_empty.jsonl new file mode 100644 index 00000000..e7635f2a --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri28_technical_repeats_empty.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri29_image_count_unchanged.jsonl b/tests/spec_fixtures/fixtures/records/ri29_image_count_unchanged.jsonl new file mode 100644 index 00000000..02e4d901 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri29_image_count_unchanged.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 10, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri30_extra_whitespace.jsonl b/tests/spec_fixtures/fixtures/records/ri30_extra_whitespace.jsonl new file mode 100644 index 00000000..8492bf38 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri30_extra_whitespace.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{ "condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type" : "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/ri31_sha256_trailing_newline.jsonl b/tests/spec_fixtures/fixtures/records/ri31_sha256_trailing_newline.jsonl new file mode 100644 index 00000000..dabba5e0 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/ri31_sha256_trailing_newline.jsonl @@ -0,0 +1,2 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf\n", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/rv01_reconfirmation_old_bytes.jsonl b/tests/spec_fixtures/fixtures/records/rv01_reconfirmation_old_bytes.jsonl new file mode 100644 index 00000000..491df325 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/rv01_reconfirmation_old_bytes.jsonl @@ -0,0 +1,3 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-07", "id_kind": "channels", "new_id": "Nuclei", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DAPI", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} diff --git a/tests/spec_fixtures/fixtures/records/rv02_image_count_changed.jsonl b/tests/spec_fixtures/fixtures/records/rv02_image_count_changed.jsonl new file mode 100644 index 00000000..e033a312 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/rv02_image_count_changed.jsonl @@ -0,0 +1,2 @@ +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "new_image_count": 4, "old_image_count": 9, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "acquisition_1", "source_folder": "acq 1"}, {"id": "acquisition_2", "source_folder": "acq 2"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/rv03_removed_every_entity.jsonl b/tests/spec_fixtures/fixtures/records/rv03_removed_every_entity.jsonl new file mode 100644 index 00000000..cf8e5751 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/rv03_removed_every_entity.jsonl @@ -0,0 +1,5 @@ +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "condition", "repeat_id": null, "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} +{"condition_id": null, "condition_source_folder": null, "confirmed_on": "2026-11-09", "entity": "repeat", "repeat_id": "rep1", "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} +{"condition_id": null, "condition_source_folder": null, "confirmed_on": "2026-11-09", "entity": "technical_repeat", "repeat_id": null, "source_repeat": null, "technical_repeat_id": "acquisition_2", "technical_repeat_source_folder": "acquisition_2", "type": "removed"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "unit", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeat_id": "single", "technical_repeat_source_folder": "single", "type": "removed"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "unit", "repeat_id": "rep2", "source_repeat": "rep2", "technical_repeat_id": "single", "technical_repeat_source_folder": "single", "type": "removed"} diff --git a/tests/spec_fixtures/fixtures/records/rv04_old_and_new_mixed.jsonl b/tests/spec_fixtures/fixtures/records/rv04_old_and_new_mixed.jsonl new file mode 100644 index 00000000..3bba8093 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/rv04_old_and_new_mixed.jsonl @@ -0,0 +1,7 @@ +{"confirmed_on": "2026-11-06", "id_kind": "repeats", "new_id": "R001", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Day1", "old_sha256": "absent", "type": "reconfirmation"} +{"confirmed_on": "2026-11-07", "id_kind": "channels", "new_id": "Nuclei", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DAPI", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "condition", "repeat_id": null, "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "confirmed_on": "2026-11-09", "entity": "unit", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeat_id": "single", "technical_repeat_source_folder": "single", "type": "removed"} +{"condition_id": "CTRL", "condition_source_folder": "CTRL", "new_image_count": 15, "old_image_count": 10, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "single"}], "type": "image_count_changed"} +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": null, "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "DrugA", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} +{"condition_id": "DrugA", "condition_source_folder": "DrugA", "new_image_count": 4, "old_image_count": 9, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "acquisition_1", "source_folder": "acq 1"}, {"id": "acquisition_2", "source_folder": "acq 2"}], "type": "image_count_changed"} diff --git a/tests/spec_fixtures/fixtures/records/rv05_non_ascii_unescaped.jsonl b/tests/spec_fixtures/fixtures/records/rv05_non_ascii_unescaped.jsonl new file mode 100644 index 00000000..a0adebe6 --- /dev/null +++ b/tests/spec_fixtures/fixtures/records/rv05_non_ascii_unescaped.jsonl @@ -0,0 +1,4 @@ +{"confirmed_on": "2026-11-08", "id_kind": "conditions", "new_id": "Kontrolle", "new_sha256": "9f22bfc4999c1b45655fb5dc377b77782c844cb09b77a1b15d7befa6948172cf", "old_id": "Zellkörper-Ctrl", "old_sha256": "f97cd441688b56cef4e4b4f56aac7dd7cd04aeed7afe90b2c40017f464e30af7", "type": "reconfirmation"} +{"condition_id": "対照", "condition_source_folder": "対照/𝔸", "confirmed_on": "2026-11-09", "entity": "unit", "repeat_id": "日1", "source_repeat": "実験 1", "technical_repeat_id": "single", "technical_repeat_source_folder": "single", "type": "removed"} +{"condition_id": "Café", "condition_source_folder": "Café", "new_image_count": 4, "old_image_count": 3, "recorded_on": "2026-11-10", "repeat_id": "rep1", "source_repeat": "rep1", "technical_repeats": [{"id": "single", "source_folder": "ligne
deux"}], "type": "image_count_changed"} +{"condition_id": null, "condition_source_folder": null, "confirmed_on": "2026-11-09", "entity": "repeat", "repeat_id": "say \"hi\" \\ back\ttab", "source_repeat": null, "technical_repeat_id": null, "technical_repeat_source_folder": null, "type": "removed"} diff --git a/tests/spec_fixtures/fixtures/records/rv06_empty_file.jsonl b/tests/spec_fixtures/fixtures/records/rv06_empty_file.jsonl new file mode 100644 index 00000000..e69de29b diff --git a/tests/spec_fixtures/tools/audit/refrecords.py b/tests/spec_fixtures/tools/audit/refrecords.py new file mode 100644 index 00000000..ef06a1ea --- /dev/null +++ b/tests/spec_fixtures/tools/audit/refrecords.py @@ -0,0 +1,333 @@ +"""AUDIT TOOL ONLY: reference validator and canonical encoder of `morphobricks.records.jsonl`. + +This is not a runtime package and no brick may import it. It implements, and only, the records +rules of docs/experiment-v1.md section 4.6: the three typed record forms (`reconfirmation`, +`image_count_changed`, `removed`, with RR3) and the byte rules of the file. It sits beside +`refval.py` (which stays the validator of `morphobricks.toml`) and is run by `refval.py fixtures`. + +A refused file is `R_RECORDS_FILE_INVALID` (section 8, owner decision RF3); the `E_RECORDS_*` +names below are its reasons, named by this tool (owner decisions RF1-RF5 of 9 November 2026). + +Usage: + python -I refrecords.py check FILE [--json] +""" + +from __future__ import annotations + +import json +import re +import sys +from datetime import date +from pathlib import Path + +DATE = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}") # matched with fullmatch +SHA256 = re.compile(r"[0-9a-f]{64}") # lower-case hex (RF4); matched with fullmatch, not `$` +ID_KINDS = ("conditions", "channels", "repeats", "technical_repeats") +ENTITIES = ("unit", "technical_repeat", "repeat", "condition") +RECONFIRMATION = ("type", "id_kind", "old_id", "new_id", "confirmed_on", "old_sha256", "new_sha256") +IMAGE_COUNT = ( + "type", + "condition_id", + "condition_source_folder", + "repeat_id", + "source_repeat", + "technical_repeats", + "old_image_count", + "new_image_count", + "recorded_on", +) +REMOVED_NAMES = ( + "condition_id", + "condition_source_folder", + "repeat_id", + "source_repeat", + "technical_repeat_id", + "technical_repeat_source_folder", +) +REMOVED = ("type", "entity", *REMOVED_NAMES, "confirmed_on") +KEY_SETS = { + "reconfirmation": frozenset(RECONFIRMATION), + "image_count_changed": frozenset(IMAGE_COUNT), + "removed": frozenset(REMOVED), +} +# the keys a `removed` line names, per entity; every other name key is null (section 4.6) +NAMED = { + "unit": REMOVED_NAMES, + "technical_repeat": ("technical_repeat_id", "technical_repeat_source_folder"), + "repeat": ("repeat_id",), + "condition": ("condition_id", "condition_source_folder"), +} +UNIT_KEYS = ("id", "source_folder") +ESCAPE = re.compile(r'\\(?:u([0-9a-fA-F]{4})|.)', re.DOTALL) + + +class Report: + """Errors of one records file: (reason, line number, detail).""" + + def __init__(self) -> None: + self.errors: list[tuple[str, int, str]] = [] + self.records: list[dict] = [] + + def e(self, code: str, line: int, detail: str = "") -> None: + self.errors.append((code, line, detail)) + + @property + def codes(self) -> set[str]: + return {c for c, _, _ in self.errors} + + def as_dict(self) -> dict: + return { + "records": len(self.records), + "errors": [list(map(str, x)) for x in self.errors], + "error_codes": sorted(self.codes), + } + + +# canonical encoder -------------------------------------------------------------------------- + + +def encode_record(record: dict) -> bytes: + """Return the canonical bytes of one record (section 4.6, RF1): keys in code point order, + non-ASCII unescaped, the separators `, ` and `: `, the minimal string escapes, one LF. This is + Python's `json.dumps(ensure_ascii=False, sort_keys=True)`, which wrote every line so far. + """ + return (json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n").encode("utf-8") + + +def encode_records(records: list[dict]) -> bytes: + """Return the bytes of a records file holding exactly these records, in order.""" + return b"".join(encode_record(r) for r in records) + + +# parsing ------------------------------------------------------------------------------------ + + +class _Duplicate(ValueError): + pass + + +def _pairs(items: list[tuple[str, object]]) -> dict: + out: dict = {} + for key, value in items: + if key in out: + raise _Duplicate(key) + out[key] = value + return out + + +def _constant(name: str) -> object: + raise ValueError(f"{name} is not JSON") + + +def _sorted_keys(value: object) -> bool: + """True when every object nested in ``value`` lists its keys in code point order.""" + if isinstance(value, dict): + keys = list(value) + return keys == sorted(keys) and all(_sorted_keys(v) for v in value.values()) + if isinstance(value, list): + return all(_sorted_keys(v) for v in value) + return True + + +def _escaped_non_ascii(line: str) -> bool: + """True when the line writes a non-ASCII character as a `\\uXXXX` escape.""" + return any(m.group(1) and int(m.group(1), 16) >= 0x80 for m in ESCAPE.finditer(line)) + + +# values ------------------------------------------------------------------------------------- + + +def _text(v: object) -> bool: + return type(v) is str and not any(0xD800 <= ord(c) <= 0xDFFF for c in v) + + +def _date(v: object) -> bool: + if type(v) is not str or not DATE.fullmatch(v): + return False + try: + date.fromisoformat(v) + except ValueError: + return False + return True + + +def _count(v: object) -> bool: + return type(v) is int and v >= 1 + + +def _bad_reconfirmation(rec: dict) -> list[str]: + bad = [] + if rec["id_kind"] not in ID_KINDS: + bad.append("id_kind") + if not _text(rec["old_id"]): + bad.append("old_id") + if rec["new_id"] is not None and not _text(rec["new_id"]): + bad.append("new_id") + if not _date(rec["confirmed_on"]): + bad.append("confirmed_on") + old = rec["old_sha256"] + if not (old == "absent" or (type(old) is str and SHA256.fullmatch(old))): + bad.append("old_sha256") + if not (type(rec["new_sha256"]) is str and SHA256.fullmatch(rec["new_sha256"])): + bad.append("new_sha256") + return bad + + +def _bad_removed(rec: dict) -> list[str]: + if rec["entity"] not in ENTITIES: + return ["entity"] + bad = [] + named = NAMED[rec["entity"]] + for key in REMOVED_NAMES: + if key in named and not _text(rec[key]) or key not in named and rec[key] is not None: + bad.append(key) + if not _date(rec["confirmed_on"]): + bad.append("confirmed_on") + return bad + + +def _bad_units(units: object) -> bool: + if type(units) is not list: + return True + ids = [] + for unit in units: + if type(unit) is not dict or set(unit) != set(UNIT_KEYS): + return True + if not (_text(unit["id"]) and _text(unit["source_folder"])): + return True + ids.append(unit["id"]) + return ids != sorted(ids) # sorted by id in code point order (ties cannot occur in a member) + + +def _bad_image_count(rec: dict) -> list[str]: + names = ("condition_id", "condition_source_folder", "repeat_id", "source_repeat") + bad = [k for k in names if not _text(rec[k])] + if _bad_units(rec["technical_repeats"]) or not rec["technical_repeats"]: + bad.append("technical_repeats") # an array of {id, source_folder}, sorted, not empty + bad += [k for k in ("old_image_count", "new_image_count") if not _count(rec[k])] + if not bad and rec["old_image_count"] == rec["new_image_count"]: + bad.append("new_image_count") # the record states a change (RF5) + if not _date(rec["recorded_on"]): + bad.append("recorded_on") + return bad + + +CHECKS = { + "reconfirmation": _bad_reconfirmation, + "removed": _bad_removed, + "image_count_changed": _bad_image_count, +} + + +def _check_line(raw: bytes, number: int, r: Report) -> None: + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + r.e("E_RECORDS_ENCODING", number, "not UTF-8") + return + try: + rec = json.loads(text, object_pairs_hook=_pairs, parse_constant=_constant) + except _Duplicate as err: + r.e("E_RECORDS_NOT_JSON_OBJECT", number, f"key {err} twice") + return + except ValueError as err: + r.e("E_RECORDS_NOT_JSON_OBJECT", number, str(err)) + return + if type(rec) is not dict: + r.e("E_RECORDS_NOT_JSON_OBJECT", number, type(rec).__name__) + return + kind = rec.get("type") + if type(kind) is not str or kind not in KEY_SETS: + r.e("E_RECORDS_UNKNOWN_TYPE", number, repr(kind)) + return + if set(rec) != KEY_SETS[kind]: + odd = sorted(set(rec) ^ KEY_SETS[kind]) + r.e("E_RECORDS_KEY_SET", number, f"{kind}: differs in {odd}") + return + before = len(r.errors) + bad = CHECKS[kind](rec) + if bad: + r.e("E_RECORDS_VALUE", number, f"{kind}: {bad}") + if not _sorted_keys(rec): + r.e("E_RECORDS_KEY_ORDER", number) + if _escaped_non_ascii(text): + r.e("E_RECORDS_NON_ASCII_ESCAPED", number) + if len(r.errors) > before: + return + r.records.append(rec) + if raw + b"\n" != encode_record(rec): + r.e("E_RECORDS_FORM", number, "not the canonical bytes (whitespace, CR, optional escape)") + return + + +def validate_records(data: bytes) -> Report: + """Validate the bytes of a `morphobricks.records.jsonl`; every line is checked.""" + r = Report() + if not data: + return r # an empty file holds no record and is accepted (RF4) + if data.startswith(b"\xef\xbb\xbf"): + r.e("E_RECORDS_ENCODING", 1, "byte-order mark") # RF4 + parts = data.split(b"\n") # LF only: U+2028, U+0085 and CR do not end a line + if parts[-1] != b"": + r.e("E_RECORDS_NO_FINAL_LF", len(parts), "last line lacks its LF") + else: + parts = parts[:-1] + for number, raw in enumerate(parts, start=1): + if number == 1 and data.startswith(b"\xef\xbb\xbf"): + continue + _check_line(raw, number, r) + return r + + +CODE = re.compile(r"\b(E_RECORDS_[A-Z_]+)\b") +ROW = re.compile(r"\| \[`(r[vi]\d+[^`]*)`\]\([^)]+\) \|([^|]*)\|") + + +def run_fixtures(root: Path) -> tuple[int, int]: + """Check every file of fixtures/records against its row of fixtures/README.md. + + Returns (files checked, mismatches). A file with no row, or a row with no file, is a mismatch. + """ + fx = root / "fixtures" + readme = (fx / "README.md").read_text(encoding="utf-8") + rows = {m.group(1): m.group(2).strip() for m in map(ROW.match, readme.splitlines()) if m} + files = {f.stem: f for f in sorted((fx / "records").glob("*.jsonl"))} + bad = 0 + for name in sorted(set(rows) ^ set(files)): + bad += 1 + print(f"NO-CATALOGUE-ROW-OR-FILE records/{name}") + for name in sorted(set(rows) & set(files)): + data = files[name].read_bytes() + r = validate_records(data) + cell = rows[name] + codes = set(CODE.findall(cell)) + if name.startswith("rv"): + # an accepted file is the canonical encoding of the records it holds + ok = not r.errors and encode_records(r.records) == data + want = "accepted" + else: + line = re.search(r"\(line (\d+)\)", cell) + ok = r.codes == codes and bool(line) and r.errors[0][1] == int(line.group(1)) + want = f"refused {sorted(codes)} first at line {line and line.group(1)}" + if not ok: + bad += 1 + print(f"MISMATCH records/{name}: got errors={r.errors[:3]}; expected {want}") + return len(files), bad + + +def main(argv: list[str]) -> int: + if len(argv) >= 2 and argv[0] == "check": + r = validate_records(Path(argv[1]).read_bytes()) + if "--json" in argv: + print(json.dumps(r.as_dict(), ensure_ascii=False)) + else: + for e in r.errors: + print("ERROR", *e) + print(len(r.records), "records", "OK" if not r.errors else "REFUSED") + return 1 if r.errors else 0 + print(__doc__) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/spec_fixtures/tools/audit/refval.py b/tests/spec_fixtures/tools/audit/refval.py index de5a2cdb..0ea83094 100644 --- a/tests/spec_fixtures/tools/audit/refval.py +++ b/tests/spec_fixtures/tools/audit/refval.py @@ -3,7 +3,8 @@ This is not a runtime package and no brick may import it. It implements the normative text of docs/experiment-v1.md (phases 1-5 of section 6 and the warnings), the canonical writer of section 9.1, the section digests of section 9.3 and the predecessor checks of sections 4.4a, -4.6 and 4.9 that two TOML files can express. Where the text is ambiguous the choice made here is +4.6 and 4.9 that two TOML files can express. The records file of section 4.6 is `refrecords.py`; +`fixtures` runs its goldens too. Where the text is ambiguous the choice made here is marked "AMBIGUITY" and reported in docs/audits/matrix-audit-2026-11-07.md. Usage: @@ -25,6 +26,9 @@ from datetime import date from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) # -I drops the script folder +import refrecords # noqa: E402 (records of section 4.6; a sibling audit module) + SENTINELS = {"", "na", "nan", "none", "null", "n/a", ""} TOKEN = re.compile(r"^[A-Za-z][A-Za-z0-9]*(?:_[A-Za-z0-9]+)*$") DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") @@ -905,6 +909,9 @@ def run_fixtures(root: Path) -> int: if section_digests(docs[k]) != section_digests(docs["v01"]): bad += 1 print(f"MISMATCH {k} digests != v01") + n_records, bad_records = refrecords.run_fixtures(root) # the records goldens (section 4.6) + bad += bad_records + counts["records"] = n_records print(f"checked {counts}; round trip on {sum(1 for k in docs if k[0] in 'vt')} files; " f"mismatches: {bad}") return bad diff --git a/tests/test_spec_pin.py b/tests/test_spec_pin.py index ffa31fa1..3af45be6 100644 --- a/tests/test_spec_pin.py +++ b/tests/test_spec_pin.py @@ -32,7 +32,11 @@ def test_the_pin_is_a_full_commit_sha_never_a_branch(): pin = read_pin() assert re.fullmatch(r"[0-9a-f]{40}", pin["commit"]) assert re.fullmatch(r"[0-9a-f]{40}", pin["fixtures_tree"]) - assert set(pin["files"]) == {"docs/experiment-v1.md", "tools/audit/refval.py"} + assert set(pin["files"]) == { + "docs/experiment-v1.md", + "tools/audit/refval.py", + "tools/audit/refrecords.py", + } assert all(re.fullmatch(r"[0-9a-f]{40}", blob) for blob in pin["files"].values()) diff --git a/tests/test_spec_records_goldens.py b/tests/test_spec_records_goldens.py new file mode 100644 index 00000000..d3288fd6 --- /dev/null +++ b/tests/test_spec_records_goldens.py @@ -0,0 +1,100 @@ +"""Every records golden of the vendored Spec through ``read_records`` (experiment-v1 4.6, RF1-RF5). + +``fixtures/records/`` holds byte-exact files and ``fixtures/README.md`` lists each one as +``accepted`` or ``refused`` with its ``E_RECORDS_*`` reasons. The reader must accept the first +and refuse the second with ``R_RECORDS_FILE_INVALID`` and exactly the listed reasons, and it +must agree with the vendored reference validator on every file. +""" + +from __future__ import annotations + +import importlib.util +import re +from functools import cache +from pathlib import Path +from types import ModuleType + +import pytest +from spec_pin import spec_root + +from morphobricks_preprocessing.run.description_file import ( + RECORDS_FILENAME, + RecordsFileError, + read_records, +) + +ROW = re.compile(r"\| \[`(r[vi]\d+[^`]*)`\]\(records/[^)]+\) \|([^|]*)\|") +REASON = re.compile(r"\b(E_RECORDS_[A-Z_]+)\b") + + +def _catalogue() -> dict[str, frozenset[str]]: + """Return each golden's expected reasons from the README (empty set: accepted).""" + text = (spec_root() / "fixtures" / "README.md").read_text(encoding="utf-8") + rows = {m.group(1): m.group(2).strip() for m in map(ROW.match, text.splitlines()) if m} + return { + name: frozenset() if cell == "`accepted`" else frozenset(REASON.findall(cell)) + for name, cell in rows.items() + } + + +@cache +def _oracle() -> ModuleType: + path = spec_root() / "tools" / "audit" / "refrecords.py" + spec = importlib.util.spec_from_file_location("spec_refrecords", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +CATALOGUE = _catalogue() +FILES = {f.stem: f for f in sorted((spec_root() / "fixtures" / "records").glob("*.jsonl"))} +REFUSED_BY_TASK = { + "ri09": "E_RECORDS_KEY_ORDER", + "ri10": "E_RECORDS_KEY_ORDER", + "ri11": "E_RECORDS_NON_ASCII_ESCAPED", + "ri14": "E_RECORDS_NOT_JSON_OBJECT", + **{f"ri{n}": "E_RECORDS_VALUE" for n in (15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 28, 29, 31)}, + **{f"ri{n}": "E_RECORDS_FORM" for n in (25, 26, 27, 30)}, +} + + +def test_every_golden_has_a_catalogue_row_and_every_row_a_golden(): + assert set(CATALOGUE) == set(FILES) + assert len(FILES) == 37 + + +@pytest.mark.parametrize("name", sorted(FILES)) +def test_read_records_follows_the_catalogue(tmp_path, name): + (tmp_path / RECORDS_FILENAME).write_bytes(FILES[name].read_bytes()) + expected = CATALOGUE[name] + if not expected: + assert name.startswith("rv") + read_records(tmp_path) + return + with pytest.raises(RecordsFileError) as refusal: + read_records(tmp_path) + assert refusal.value.code == "R_RECORDS_FILE_INVALID" + assert refusal.value.reasons == expected + + +@pytest.mark.parametrize("name", sorted(FILES)) +def test_read_records_agrees_with_the_reference_validator(tmp_path, name): + data = FILES[name].read_bytes() + (tmp_path / RECORDS_FILENAME).write_bytes(data) + verdict = _oracle().validate_records(data) + try: + records = read_records(tmp_path) + except RecordsFileError as refusal: + assert refusal.reasons == verdict.codes + assert refusal.issues[0].line == verdict.errors[0][1] + else: + assert not verdict.errors + assert records == tuple(verdict.records) + + +def test_the_goldens_beyond_the_key_set_are_refused_with_their_reason(): + by_prefix = {name[:4]: reasons for name, reasons in CATALOGUE.items()} + for prefix, reason in REFUSED_BY_TASK.items(): + assert by_prefix[prefix] == {reason}, prefix + refused = {name[:4] for name, reasons in CATALOGUE.items() if reasons} + assert {f"ri{n:02d}" for n in range(1, 32)} == refused From 20e1734f8be00f6624cf9d35216474191dbd0010 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 06:49:35 +0000 Subject: [PATCH 2/4] Declare records_file as a pure module and gate its golden test Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- AGENTS.md | 3 +++ pyproject.toml | 1 + tests/spec_gate.py | 1 + 3 files changed, 5 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index fe7ff177..554b13f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -122,6 +122,9 @@ src/morphobricks_preprocessing/ description_file.py read_description(path): symlink refusal (R_FILE_SYMLINK), bytes -> Description run/description_file.py publish_description: lock, base SHA-256, strict re-read, transitions, atomic write, append-only morphobricks.records.jsonl (Spec experiment-v1 4.6, 10) + records_file.py pure checker of morphobricks.records.jsonl (Spec experiment-v1 4.6): check_records + (bytes -> records + E_RECORDS_* issues), encode_record, RECORD_KEYS; read_records + in run/description_file.py refuses with R_RECORDS_FILE_INVALID draft_ids.py proposed IDs: the folder label or raw header name only when the reader would accept it as written, else none (never repaired); draft.py the draft (IdSlot proposed/confirmed, open questions, unit_rows with condition_id, diff --git a/pyproject.toml b/pyproject.toml index 98d50ebc..8541b510 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -240,6 +240,7 @@ baseline = "tools/code_rules_baseline.json" "morphobricks_preprocessing.description_walk" = "pure" "morphobricks_preprocessing.description_warnings" = "pure" "morphobricks_preprocessing.description_report" = "pure" +"morphobricks_preprocessing.records_file" = "pure" "morphobricks_preprocessing.description_rules" = "pure" "morphobricks_preprocessing.description_structure" = "pure" "morphobricks_preprocessing.design_edit" = "pure" diff --git a/tests/spec_gate.py b/tests/spec_gate.py index 7462412c..2583d8de 100644 --- a/tests/spec_gate.py +++ b/tests/spec_gate.py @@ -40,6 +40,7 @@ "test_spec_reader_refusal_paths.py", "test_spec_reconcile.py", "test_spec_records.py", + "test_spec_records_goldens.py", "test_spec_refval_crosscheck.py", "test_spec_rules_table.py", "test_spec_run_aliases.py", From 36ac981f52770dc4d634ca2dd13bfd8bfaf71d35 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 06:57:35 +0000 Subject: [PATCH 3/4] Sort the golden test's imports Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- tests/test_spec_records_goldens.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_spec_records_goldens.py b/tests/test_spec_records_goldens.py index d3288fd6..868239bf 100644 --- a/tests/test_spec_records_goldens.py +++ b/tests/test_spec_records_goldens.py @@ -11,17 +11,16 @@ import importlib.util import re from functools import cache -from pathlib import Path from types import ModuleType import pytest -from spec_pin import spec_root from morphobricks_preprocessing.run.description_file import ( RECORDS_FILENAME, RecordsFileError, read_records, ) +from spec_pin import spec_root ROW = re.compile(r"\| \[`(r[vi]\d+[^`]*)`\]\(records/[^)]+\) \|([^|]*)\|") REASON = re.compile(r"\b(E_RECORDS_[A-Z_]+)\b") From 98bbdc206c69a89bf38c4d752f1272640b263382 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 07:25:42 +0000 Subject: [PATCH 4/4] Write the seeded records file as bytes: write_text makes CRLF on Windows Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01G3qp9AxjpGPw9xfWAUJNqc --- tests/test_run_description.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_run_description.py b/tests/test_run_description.py index 3bccc9cd..2f458b1a 100644 --- a/tests/test_run_description.py +++ b/tests/test_run_description.py @@ -51,7 +51,7 @@ def test_the_manifest_records_the_file_its_digests_and_the_records_unchanged(tmp root, out = _folder(tmp_path) (out / "morphobricks.toml").write_bytes(V01.data) line = json.dumps(RECORD, sort_keys=True) + "\n" - (out / "morphobricks.records.jsonl").write_text(line, encoding="utf-8") + (out / "morphobricks.records.jsonl").write_bytes(line.encode("utf-8")) # no CRLF (4.6) assert run_experiment(root, MAPPING, output_folder=out).written entry = _manifest(out)["experiment_description"] assert entry["sha256"] == sha256_hex(V01.data)