From a56a9a574fd22317d964dccdd9b641aac0008d94 Mon Sep 17 00:00:00 2001 From: Eric McDonald Date: Thu, 13 Nov 2025 19:41:19 -0800 Subject: [PATCH 01/12] [WIP] Refactor 'decode' logic to improve accuracy of character set selection. --- sources/detextive/__/imports.py | 1 + sources/detextive/charsets.py | 45 ++++++-- sources/detextive/core.py | 3 + sources/detextive/decoders.py | 193 ++++++++++++++++++++++++++------ sources/detextive/detectors.py | 2 +- sources/detextive/inference.py | 21 ++-- 6 files changed, 213 insertions(+), 52 deletions(-) diff --git a/sources/detextive/__/imports.py b/sources/detextive/__/imports.py index ff4cf81..5dd8e79 100644 --- a/sources/detextive/__/imports.py +++ b/sources/detextive/__/imports.py @@ -24,6 +24,7 @@ import collections.abc as cabc import codecs +import contextlib as ctxl import dataclasses as dcls import enum import locale diff --git a/sources/detextive/charsets.py b/sources/detextive/charsets.py index 79352df..84bb8e7 100644 --- a/sources/detextive/charsets.py +++ b/sources/detextive/charsets.py @@ -35,7 +35,10 @@ ) -def attempt_decodes( +_charsets_permissive: dict[ str, bool ] = { } # TODO: Accretive dictionary. + + +def attempt_decodes( # noqa: PLR0915 content: _nomina.Content, /, *, behaviors: _Behaviors = _BEHAVIORS_DEFAULT, inference: __.Absential[ str ] = __.absent, @@ -50,7 +53,7 @@ def attempt_decodes( confidence = _core.confidence_from_bytes_quantity( content, behaviors = behaviors ) on_decode_error = behaviors.on_decode_error - trials: list[ str ] = [ ] + trials: set[ str ] = set( ) for codec in behaviors.trial_codecs: match codec: case _CodecSpecifiers.FromInference: @@ -65,14 +68,16 @@ def attempt_decodes( charset = supplement case str( ): charset = codec case _: continue + charset = normalize_charset( + charset, bom_cognizant = behaviors.remove_bom ) + if charset in trials: continue try: text = content.decode( charset, errors = on_decode_error ) - except UnicodeDecodeError: - trials.append( charset ) - continue + except UnicodeDecodeError: continue + finally: trials.add( charset ) result = _CharsetResult( charset = charset, confidence = confidence ) return text, result raise _exceptions.ContentDecodeFailure( - charset = trials, location = location ) + charset = tuple( trials ), location = location ) def discover_os_charset_default( ) -> str: @@ -82,9 +87,33 @@ def discover_os_charset_default( ) -> str: return normalize_charset( discoverer( ) ) -def normalize_charset( charset: str ) -> str: +def is_permissive_charset( charset: str ) -> bool: + ''' Checks if charset accepts all byte sequences (8-bit encoding). + + Returns ``True`` for CP1252, ISO-8859-*, etc.... + Returns ``False`` for ASCII, UTF-8, SHIFT-JIS, etc.... + ''' + charset_ = normalize_charset( charset ) + if charset_ in _charsets_permissive: + return _charsets_permissive[ charset_ ] + try: + texta = bytes( range( 256 ) ).decode( + charset_, errors = 'strict' ) + textd = bytes( range( 255, -1, -1 ) ).decode( + charset_, errors = 'strict' ) + except ( UnicodeDecodeError, LookupError ): + _charsets_permissive[ charset_ ] = False + return False + permissivity = ( len( texta ) == len( textd ) == 256 ) # noqa: PLR2004 + _charsets_permissive[ charset_ ] = permissivity + return permissivity + + +def normalize_charset( charset: str, bom_cognizant: bool = False ) -> str: ''' Normalizes character set encoding names. ''' - return __.codecs.lookup( charset ).name + charset_ = __.codecs.lookup( charset ).name + if bom_cognizant and charset_ == 'utf-8': return 'utf-8-sig' + return charset_ def trial_decode_as_confident( # noqa: PLR0913 diff --git a/sources/detextive/core.py b/sources/detextive/core.py index a64f79c..7691eea 100644 --- a/sources/detextive/core.py +++ b/sources/detextive/core.py @@ -114,6 +114,9 @@ class Behaviors( __.immut.DataclassObject ): 'codecs' module. ''' ), ] = 'strict' + remove_bom: __.typx.Annotated[ + bool, __.ddoc.Doc( ''' Remove byte-ordering mark? ''' ) + ] = True text_validate: __.typx.Annotated[ BehaviorTristate, __.ddoc.Doc( ''' When to validate text. ''' ), diff --git a/sources/detextive/decoders.py b/sources/detextive/decoders.py index 5685e23..eb3f9aa 100644 --- a/sources/detextive/decoders.py +++ b/sources/detextive/decoders.py @@ -23,10 +23,9 @@ from . import __ from . import charsets as _charsets -from . import core as _core +from . import detectors as _detectors from . import exceptions as _exceptions from . import inference as _inference -from . import mimetypes as _mimetypes from . import nomina as _nomina from . import validation as _validation @@ -37,6 +36,7 @@ BehaviorTristate as _BehaviorTristate, BehaviorsArgument as _BehaviorsArgument, CharsetResult as _CharsetResult, + CodecSpecifiers as _CodecSpecifiers, ) @@ -52,49 +52,172 @@ def decode( # noqa: PLR0913 mimetype_supplement: _nomina.MimetypeSupplementArgument = __.absent, ) -> str: ''' Decodes bytes array to Unicode text. ''' + # TODO: Deprecation warnings for 'mimetype_*' arguments. if content == b'': return '' - behaviors_ = __.dcls.replace( - behaviors, trial_decode = _BehaviorTristate.Never ) - try: - mimetype_result, charset_result = ( - _inference.infer_mimetype_charset_confidence( + result: __.Absential[ _CharsetResult ] = __.absent + text: __.Absential[ str ] = __.absent + if not __.is_absent( http_content_type ): + text = _attempt_decode_http_content_type( + content, http_content_type, + behaviors = behaviors, profile = profile, location = location ) + if not __.is_absent( text ): return text + if __.is_absent( result ): + behaviors_ = __.dcls.replace( + behaviors, trial_decode = _BehaviorTristate.Never ) + with __.ctxl.suppress( _exceptions.CharsetDetectFailure ): + result = _detectors.detect_charset_confidence( content, behaviors = behaviors_, - charset_default = charset_default, - mimetype_default = mimetype_default, - http_content_type = http_content_type, - charset_supplement = charset_supplement, - mimetype_supplement = mimetype_supplement, - location = location ) ) - except _exceptions.Omnierror: - charset = ( - 'utf-8-sig' if __.is_absent( charset_supplement ) - else charset_supplement ) - confidence = _core.confidence_from_bytes_quantity( content, behaviors ) - charset_result = _CharsetResult( - charset = charset, confidence = confidence ) - else: - if ( not _mimetypes.is_textual_mimetype( mimetype_result.mimetype ) - and charset_result.charset is None - ): raise _exceptions.ContentDecodeImpossibility( location = location ) - # When any reasonable doubt exists, we attempt decodes. - # Trial decodes and text validation is the only way to be certain. - text, result = _charsets.attempt_decodes( - content, + default = charset_default, + supplement = charset_supplement, + location = location ) + return _attempt_decodes( + content, result, behaviors = behaviors, - inference = ( - 'utf-8-sig' if charset_result.charset is None - else charset_result.charset ), + profile = profile, supplement = charset_supplement, location = location ) + + +def _attempt_decode_http_content_type( + content: _nomina.Content, + http_content_type: str, /, *, + behaviors: _BehaviorsArgument, + profile: _validation.ProfileArgument, + location: _nomina.LocationArgument, +) -> __.Absential[ str ]: + charset: __.Absential[ __.typx.Optional[ str ] ] = __.absent + result: __.Absential[ _CharsetResult ] = __.absent + error = _exceptions.ContentDecodeImpossibility( location = location ) + _, charset = _inference.parse_http_content_type( http_content_type ) + if charset is None: raise error + if __.is_absent( charset ): return __.absent + behaviors_ = __.dcls.replace( + behaviors, trial_codecs = ( _CodecSpecifiers.FromInference, ) ) + try: + text, result = _charsets.attempt_decodes( + content, + behaviors = behaviors_, inference = charset, location = location ) + except _exceptions.ContentDecodeFailure: return __.absent + # Allow other errors propagate. + if not __.is_absent( text ) and not __.is_absent( result ): + return _validate_text( + text, result.confidence, + behaviors = behaviors, profile = profile, location = location ) + return __.absent + + +def _append_charset( + permissives: list[ str ], + restrictives: list[ str ], + charset: str, + bom_cognizant: bool, +) -> None: + charset_ = _charsets.normalize_charset( + charset, bom_cognizant = bom_cognizant ) + if _charsets.is_permissive_charset( charset_ ): + if charset_ in permissives: return + permissives.append( charset_ ) + else: + if charset_ in restrictives: return + restrictives.append( charset_ ) + + +def _attempt_decodes( # noqa: PLR0913 + content: _nomina.Content, + detection: __.Absential[ _CharsetResult ], /, *, + behaviors: _BehaviorsArgument, + profile: _validation.ProfileArgument, + supplement: __.Absential[ str ], + location: _nomina.LocationArgument, +) -> str: + error = _exceptions.ContentDecodeImpossibility( location = location ) + permissives, restrictives = _prepare_charsets( + detection, behaviors = behaviors, supplement = supplement ) + on_decode_error = behaviors.on_decode_error + # Try restrictive charsets before permissive charsets, since: + # (1) Restrictive charsets can have decoding errors from invalid byte + # sequences. + # (2) Restrictive charsets can produce shorter strings, if the are + # multi-byte encodings. Permissive charsets decoding the same byte + # sequences will likly result in mojibake. + for charset in restrictives: + try: text = content.decode( charset, errors = on_decode_error ) + except UnicodeDecodeError: continue + try: + return _validate_text( + text, 0.0, + behaviors = behaviors, profile = profile, location = location ) + except _exceptions.TextInvalidity: continue + for charset in permissives: + try: text = content.decode( charset, errors = on_decode_error ) + except UnicodeDecodeError: continue + try: + return _validate_text( + text, 0.0, + behaviors = behaviors, profile = profile, location = location ) + except _exceptions.TextInvalidity: continue + raise error + + +def _prepare_charsets( + detection: __.Absential[ _CharsetResult ], /, *, + behaviors: _BehaviorsArgument, + supplement: __.Absential[ str ], +) -> tuple[ tuple[ str, ... ], tuple[ str, ... ] ]: + permissives: list[ str ] = [ ] + restrictives: list[ str ] = [ ] + os_charset = _charsets.discover_os_charset_default( ) + _append_charset( + permissives, restrictives, os_charset, behaviors.remove_bom ) + python_charset = __.locale.getpreferredencoding( ) + _append_charset( + permissives, restrictives, python_charset, behaviors.remove_bom ) + if not __.is_absent( supplement ): + _prepend_charset( + permissives, restrictives, supplement, behaviors.remove_bom ) + if not __.is_absent( detection ) and detection.charset is not None: + # Suspicious charset detections go at end. + if detection.confidence < behaviors.trial_decode_confidence: + _append_charset( + permissives, restrictives, detection.charset, + behaviors.remove_bom ) + else: + _prepend_charset( + permissives, restrictives, detection.charset, + behaviors.remove_bom ) + return tuple( permissives ), tuple( restrictives ) + + +def _prepend_charset( + permissives: list[ str ], + restrictives: list[ str ], + charset: str, + bom_cognizant: bool, +) -> None: + charset_ = _charsets.normalize_charset( + charset, bom_cognizant = bom_cognizant ) + if _charsets.is_permissive_charset( charset_ ): + if charset_ in permissives: return + permissives.insert( 0, charset_ ) + else: + if charset_ in restrictives: return + restrictives.insert( 0, charset_ ) + + +def _validate_text( + text: str, confidence: float, /, *, + behaviors: _BehaviorsArgument, + profile: _validation.ProfileArgument, + location: _nomina.LocationArgument, +) -> str: + error = _exceptions.TextInvalidity( location = location ) should_validate = False match behaviors.text_validate: case _BehaviorTristate.Always: should_validate = True case _BehaviorTristate.AsNeeded: - should_validate = ( - result.confidence < behaviors.text_validate_confidence ) + should_validate = confidence < behaviors.text_validate_confidence case _BehaviorTristate.Never: pass - if should_validate and not profile( text ): - raise _exceptions.TextInvalidity( location = location ) + if should_validate and not profile( text ): raise error return text diff --git a/sources/detextive/detectors.py b/sources/detextive/detectors.py index fff8000..ef89984 100644 --- a/sources/detextive/detectors.py +++ b/sources/detextive/detectors.py @@ -116,7 +116,7 @@ def detect_charset_confidence( # noqa: PLR0913 ) -> _CharsetResult: ''' Detects character set candidates with confidence scores. ''' if b'' == content: - return _CharsetResult( charset = 'utf-8', confidence = 1.0 ) + return _CharsetResult( charset = default, confidence = 1.0 ) for name in behaviors.charset_detectors_order: detector = charset_detectors.get( name ) if detector is None: continue diff --git a/sources/detextive/inference.py b/sources/detextive/inference.py index d77b32b..7d1cff4 100644 --- a/sources/detextive/inference.py +++ b/sources/detextive/inference.py @@ -200,6 +200,17 @@ def parse_http_content_type( return __.absent, __.absent +def validate_httpct_charset( + content: _nomina.Content, + charset: str, /, *, + behaviors: _Behaviors = _BEHAVIORS_DEFAULT, +) -> __.Absential[ _CharsetResult ]: + behaviors_ = __.dcls.replace( + behaviors, trial_codecs = ( _CodecSpecifiers.FromInference, ) ) + return _charsets.trial_decode_as_confident( + content, behaviors = behaviors_, inference = charset ) + + def _determine_parse_detect( detect_tristate: _BehaviorTristate, should_parse = False ) -> tuple[ bool, bool ]: @@ -229,14 +240,8 @@ def _validate_http_content_type( elif charset is None: charset_result = _CharsetResult( charset = None, confidence = 0.9 ) else: - # HTTP header provides explicit charset - only try that, not OS default - behaviors_http = __.dcls.replace( - behaviors, trial_codecs = ( _CodecSpecifiers.FromInference, ) ) - charset_result = _charsets.trial_decode_as_confident( - content, - behaviors = behaviors_http, - inference = charset, - supplement = charset_supplement ) + charset_result = validate_httpct_charset( + content, charset, behaviors = behaviors ) if __.is_absent( mimetype ): mimetype_result = __.absent else: mimetype_result = _MimetypeResult( From 3b2ba0943fc113bcfefa23043677b40c20b65f00 Mon Sep 17 00:00:00 2001 From: Eric McDonald Date: Thu, 13 Nov 2025 19:46:09 -0800 Subject: [PATCH 02/12] Update notes. (Coauthor: Anthropic Claude Sonnet 4.5) --- .auxiliary/notes/decode-refactor.md | 434 +++++++++------------------- 1 file changed, 130 insertions(+), 304 deletions(-) diff --git a/.auxiliary/notes/decode-refactor.md b/.auxiliary/notes/decode-refactor.md index 4cbb8e8..e06ab8b 100644 --- a/.auxiliary/notes/decode-refactor.md +++ b/.auxiliary/notes/decode-refactor.md @@ -4,359 +4,185 @@ The current `decode()` implementation has become overly complex with multiple special cases, three different `trial_codecs` usage patterns, and platform-specific encoding issues. The Windows Python 3.11+ doctest failures revealed fundamental issues with how we handle charset detection and validation. -## Core Insight: 8-bit Charsets Are Uninformative +## Core Insight: Charset Detection is Fundamentally Hard -**Key realization**: 8-bit character sets (cp1252, iso-8859-*, etc.) accept any byte sequence because they have one-to-one correspondence between byte values and code points. Trial decodes with these charsets tell us nothing about correctness. +**Key realization**: Without context, charset detection is heuristics all the way down. No amount of algorithmic complexity can solve the fundamental ambiguity problem. -Only **7-bit** (ASCII) and **multi-byte** (UTF-8, Shift-JIS, etc.) charsets provide informative feedback through decode success/failure. +**Examples of inherent ambiguity:** +- UTF-8 Turkish text decoded as ISO-8859-9 produces valid-looking mojibake +- ISO-8859-9 Turkish text decoded as UTF-8 also produces mojibake +- Both are "valid" decodings with different results +- Without external context (user knowledge, file source, HTTP headers), detection is guessing -## Design Principles +## Design Philosophy: Simplicity + User Control -1. **Ignore MIME type in `decode()`** - Focus solely on getting correct text -2. **Consider confidence for non-8-bit detections** - Even multi-byte charsets can be misdetected; 7-bit (ASCII) especially unreliable -3. **Distrust 8-bit detections** - They always succeed but may produce mojibake -4. **Respect configurable validation behavior** - Honor existing `text_validate` settings -5. **Shortest string wins for multi-byte** - Mojibake produces longer strings -6. **User supplement gets priority among 8-bit** - Respect user knowledge +After extensive analysis of multi-tier categorization schemes (permissive vs restrictive, multi-byte vs single-byte, etc.), we conclude: -## New Architecture +**Better to be simple and honest about limitations than complex and pretending to solve the unsolvable.** -### Helper Function: `is_permissive_charset()` +### What We Learned -```python -# Module-level cache (always on) -_PERMISSIVE_CHARSET_CACHE: dict[str, bool] = {} +1. **CP1252 is not fully permissive**: Has 5 undefined bytes (0x81, 0x8d, 0x8f, 0x90, 0x9d) +2. **ISO-8859-* variants are fully permissive**: All 256 bytes decode (many variants exist for different languages) +3. **ASCII compatibility is universal**: All major 8-bit encodings preserve ASCII in bytes 0x00-0x7F +4. **UTF-8 vs CP1252 length heuristic works**: UTF-8 multi-byte always produces shorter strings than 8-bit misinterpretation +5. **But length heuristic fails for other encodings**: Turkish ISO-8859-9 vs UTF-8 can produce same-length mojibake + +### Implementation Findings +`is_permissive_charset()` successfully implemented: +```python def is_permissive_charset(charset: str) -> bool: - """Check if charset accepts all byte sequences (8-bit encoding). - - Returns True for: cp1252, iso-8859-*, koi8-r, etc. - Returns False for: utf-8, ascii, shift-jis, etc. - - Tests both ascending and descending byte sequences to detect - multi-byte sequence introducers, and checks decoded length - to ensure 1:1 byte-to-character mapping. - """ - # Normalize and check cache - charset_normalized = normalize_charset(charset) - if charset_normalized in _PERMISSIVE_CHARSET_CACHE: - return _PERMISSIVE_CHARSET_CACHE[charset_normalized] - - try: - # Test ascending sequence - ascending = bytes(range(256)) - text_asc = ascending.decode(charset, errors='strict') - - # Test descending sequence (catches multi-byte introducers) - descending = bytes(range(255, -1, -1)) - text_desc = descending.decode(charset, errors='strict') - - # Check lengths: must be exactly 256 chars (1:1 mapping) - is_permissive = (len(text_asc) == 256 and len(text_desc) == 256) - - _PERMISSIVE_CHARSET_CACHE[charset_normalized] = is_permissive - return is_permissive - - except (UnicodeDecodeError, LookupError): - # Some bytes failed → informative charset - _PERMISSIVE_CHARSET_CACHE[charset_normalized] = False - return False + """Check if charset accepts all 256 byte values.""" + # Test ascending and descending sequences + # Check length == 256 (1:1 byte-to-char mapping) + # Cache results ``` -**Implementation notes:** -- Cache always enabled (minimal memory footprint) -- Tests both ascending and descending byte sequences -- Checks decoded length to detect multi-byte encodings -- Handles unknown/future charsets automatically +Results: +- ✅ ISO-8859-1: `True` (fully permissive) +- ✅ CP1252: `False` (5 undefined bytes) +- ✅ ASCII: `False` (only 128 values) +- ✅ UTF-8: `False` (multi-byte sequences) -### New Function: `detect_charset_reliable()` +But this revealed new complexity: need to subcategorize "restrictive" into multi-byte vs single-byte to avoid CP1252 mojibake before UTF-8 attempts. -Wrapper around `detect_charset_confidence()` that validates suspicious detections via trial decode: +**This led to a design rabbit hole that misses the forest for the trees.** -```python -def detect_charset_reliable(content, ...): - """Detect charset with validation of suspicious results. - - Part of public API. Applications can use this for more reliable - detection than raw detect_charset(). - """ - result = detect_charset_confidence(content, ...) - detected, confidence = result.charset, result.confidence - - # Consider confidence, especially for 7-bit and multi-byte - # Even non-8-bit charsets can be misdetected - if not is_permissive_charset(detected): - # If confidence is high enough, trust it - # Reuse existing threshold from behaviors DTO - if confidence >= behaviors.charset_confidence_threshold: - return result - # Otherwise, try defaults as well - - # Detected is 8-bit or low-confidence, try defaults - python_default = sys.getdefaultencoding() # utf-8 - os_default = discover_os_charset_default() # varies - - for default in [python_default, os_default]: - if not is_permissive_charset(default): - try: - content.decode(default) - # Return with appropriate confidence - return CharsetResult(charset=default, confidence=...) - except UnicodeDecodeError: - continue - - # All informative charsets failed, return original detection - return result -``` +## Simplified Design (Current Direction) -**Note**: Also add `detect_charset_confidence_reliable()` variant that returns full result object. +### Principles -### Helper Function: `_decode_with_http_content_type()` +1. **Put user in control**: Provide supplement as `str` or codec specifier +2. **Use sensible defaults**: OS charset for local files, Python charset (usually UTF-8) for general use +3. **Trust high-confidence detection**: But allow it to be overridden by user/context +4. **Keep it simple**: Fewer tiers, clearer behavior, easier to reason about -Extract HTTP Content-Type handling into helper: +### Trial Order Strategy ```python -def _decode_with_http_content_type( - content, http_content_type, behaviors, profile, location -): - """Attempt decode with charset from HTTP Content-Type header. - - Returns decoded text if successful, None if should fall back to detection. - Always falls back (never raises) on failure. - """ - charset = parse_charset_from_content_type(http_content_type) - if not charset or is_absent(charset): - return None - - # Use existing trial decode helpers - try: - text, result = attempt_decodes( - content, - behaviors=behaviors, - inference=charset, - location=location - ) - # Validate if configured - if should_validate_text(behaviors, result.confidence): - if not profile(text): - return None # Fall back - return text - except ContentDecodeFailure: - return None # Fall back +trial_order = [ + UserSupplement, # User knows their data (highest priority) + OsDefault, # Sensible for local filesystem content + PythonDefault, # Usually UTF-8, can be set via PYTHONIOENCODING +] + +# Insert detected charset based on confidence: +if detection.confidence >= behaviors.trial_decode_confidence: + trial_order.insert(1, FromInference) # After user, before OS +else: + trial_order.append(FromInference) # At end (suspicious) ``` -### Refactored `decode()` Flow +### User Supplement Enhancement +Allow `charset_supplement` to be either: +- **`str`**: Specific charset name (e.g., `'utf-8'`, `'iso-8859-9'`) +- **Codec specifier**: `OsDefault`, `PythonDefault`, etc. + +**Use cases:** ```python -def decode(content, http_content_type=None, charset_supplement=None, - behaviors=..., profile=..., location=...): - """Decode bytes to text with intelligent charset selection.""" - - if content == b'': - return '' - - # 1. Try authoritative charset from HTTP Content-Type - if http_content_type: - text = _decode_with_http_content_type( - content, http_content_type, behaviors, profile, location) - if text is not None: - return text - # Fall back to detection - - # 2. Detect charset with validation - result = detect_charset_confidence_reliable( - content, behaviors=behaviors, supplement=charset_supplement) - detected = result.charset - - # 3. Build candidate lists - reuse existing trial decode helpers - # Use attempt_decodes() and related functions rather than - # reinventing the wheel - - trial_candidates = [] # Non-8-bit charsets - actual_candidates = [] # 8-bit charsets - - # Add detected - if not is_permissive_charset(detected): - trial_candidates.append(detected) - else: - actual_candidates.append(detected) - - # Add defaults if different from detected and non-8-bit - python_default = sys.getdefaultencoding() # utf-8 - os_default = discover_os_charset_default() # varies - - for default in [python_default, os_default]: - if (default not in trial_candidates - and default not in actual_candidates - and not is_permissive_charset(default)): - trial_candidates.append(default) - - # Add supplement - if not is_absent(charset_supplement): - if is_permissive_charset(charset_supplement): - actual_candidates.insert(0, charset_supplement) - else: - trial_candidates.append(charset_supplement) - - # 4. Try candidates using existing helpers - # Validation timing respects behaviors.text_validate configuration - text = _try_decode_candidates( - content, trial_candidates, actual_candidates, - behaviors, profile, location) - - if text is not None: - return text - - # 5. No valid decode found - raise ContentDecodeFailure(location=location) -``` +# Internet/web content - prefer UTF-8 +decode(content, charset_supplement='utf-8') -**Implementation notes:** -- Reuse existing `attempt_decodes()` and codec trial functions -- Respect `behaviors.text_validate` configuration (Never/AsNeeded/Always) -- Extract helpers to avoid monolithic decode function +# Local filesystem - use OS charset +decode(content, charset_supplement=OsDefault) -### Decision Priority +# Known legacy encoding +decode(content, charset_supplement='iso-8859-9') +``` -When multiple decodes succeed: +### Optional: Use `is_permissive_charset()` for Filtering -1. **Shortest string always wins** (less mojibake) -2. **Tie-breaker**: User supplement over other charsets (user knowledge) -3. **Secondary tie-breaker**: Non-8-bit over 8-bit (more informative) +One lightweight use of the permissive check: -**Implementation**: ```python -def _try_decode_candidates(...): - results = [] - - # Try all candidates and collect successful decodes - for charset in all_candidates: - try: - text = content.decode(charset) - if should_validate and not profile(text): - continue - results.append(( - len(text), # Primary: shortest - charset != charset_supplement, # Tie-break: supplement wins - is_permissive_charset(charset), # Secondary: non-8-bit wins - charset, - text - )) - except UnicodeDecodeError: - continue - - if results: - # Sort by tuple: shortest, then supplement, then non-8-bit - results.sort() - return results[0][4] # Return text - - return None +# Skip truly permissive charsets if non-permissive options exist +candidates = build_candidate_list() +non_permissive = [c for c in candidates if not is_permissive_charset(c)] +if non_permissive: + candidates = non_permissive # Prefer informative attempts ``` -### Validation Timing +This prevents trying ISO-8859-1 when UTF-8 is available, without complex multi-tier logic. -Text validation timing is **configurable** via `behaviors.text_validate`: -- **Never**: Skip validation entirely -- **AsNeeded**: Validate based on confidence threshold -- **Always**: Always validate +## Current Implementation Status -The existing behavior configuration is preserved. Validation can happen during candidate selection or after - the difference is minimal in practice since validation is already configurable. +### Implemented ✅ -## OS Default vs Python Default +1. **`is_permissive_charset()`** - Working perfectly with caching +2. **HTTP Content-Type handling** - Extracts and validates charset, falls back gracefully +3. **Separate permissive/restrictive lists** - In `_attempt_decodes()` +4. **BOM handling** - `remove_bom` behavior parameter +5. **Charset deduplication** - Normalized before adding to trial list +6. **Empty content uses default** - Not hardcoded to UTF-8 -- **Python default**: `sys.getdefaultencoding()` → always UTF-8 in Python 3 - - Can be overridden via `PYTHONIOENCODING` or CLI flag -- **OS default**: `locale.getencoding()` (3.11+) or `sys.getfilesystemencoding()` - - cp1252 on Windows, UTF-8 on modern Linux/Mac +### Issues Discovered 🔍 -**Strategy**: Try both when they differ, preferring Python default first. +1. **Complexity creep**: Permissive vs restrictive revealed need for multi-byte vs single-byte subcategorization +2. **CP1252 vs UTF-8 ordering**: CP1252 is "restrictive" but still produces mojibake before UTF-8 +3. **Turkish/Finnish ambiguity**: Historical encodings have legitimate sequences that look like UTF-8 mojibake +4. **No magic bullet**: Algorithmic complexity doesn't solve fundamental ambiguity -**Special case**: Don't trial decode with cp1252 even if it's OS default (8-bit uninformative). +### Next Steps 🎯 -## Impact on Existing APIs +**Decision point**: Continue with complex categorization OR simplify to user-centric approach? -### `detect_charset()` -- **No change** - Returns raw detector output -- Used when applications just want to know what chardet/charset-normalizer says +**Recommendation**: Simplify +- Remove complex permissive/restrictive/multi-byte categorization +- Use simple context-based trial order (User → OS → Python → Detection) +- Keep `is_permissive_charset()` only for optional filtering +- Document limitations honestly +- Empower users with supplement options -### `detect_charset_reliable()` (new) -- Validates suspicious (8-bit) or low-confidence detections -- **Part of public API** along with `detect_charset_confidence_reliable()` -- Used internally by `decode()` +## Charset Evaluation Results -### `decode()` -- **Major refactor** - New candidate selection logic -- Ignores MIME type entirely -- Uses helper functions to avoid monolithic implementation -- Reuses existing trial decode functions -- HTTP Content-Type: always falls back to detection on failure (not configurable) +Comprehensive testing of `chardet` vs `charset-normalizer`: -### `infer_*()` functions -- Minor updates may be needed later (defer for now) -- HTTP Content-Type with charset: trial decode only with specified charset +**Key findings:** +- charset-normalizer: 92% accurate on UTF-8, 17% on Latin-1/CP1252 +- chardet: 58% accurate on UTF-8, 83% on Latin-1/CP1252 +- Overall: Both tied at 65% accuracy +- charset-normalizer is slower but better for UTF-8 +- chardet is faster and better for legacy 8-bit encodings -### `trial_codecs` behavior parameter -- **Deprecated** - Document as ignored -- Keep in API for compatibility but don't use -- New situational logic replaces fixed codec lists +**Decision**: Stick with chardet for now, provides good balance. -## Charset-Normalizer Investigation +See: `.auxiliary/notes/charset-detector-evaluation-results.md` -Before implementing, test `charset-normalizer` vs `chardet`: +## Related Files -1. Compare on wide variety of byte patterns -2. Verify it "normalizes" to useful/standard encodings -3. Measure performance characteristics -4. Document findings +- Implementation: `sources/detextive/decoders.py`, `sources/detextive/charsets.py` +- Evaluations: `.auxiliary/evaluations/compare-charset-detectors.py` (and related) +- Results: `.auxiliary/notes/charset-detector-evaluation-results.md` -`charset-normalizer` is already in dev environment. +## Open Questions -## Related Issues +1. Should we simplify back to context-based trial order? +2. Keep or remove permissive/restrictive categorization? +3. How much complexity is justified for marginal accuracy gains? +4. What's the right balance between "smart" and "simple"? -### Windows Python 3.11+ Doctest Failure +## The Honest Documentation Approach -Current failure: -``` -Expected: 'Café ★' -Got: 'Café ★' +```python +""" +decode() attempts decoding in context-aware order: +1. User supplement (you know your data best) +2. OS default (sensible for local files) +3. Python default (usually UTF-8) +4. Detected charset (if confidence is high) + +Charset detection is heuristic and cannot solve fundamental +ambiguities without context. For best results: +- Provide charset_supplement when encoding is known +- Use http_content_type for web content +- Validate results with is_valid_text() +- Consider confidence scores from detect_charset_confidence() + +There is no magic bullet for charset detection. We provide +sensible defaults and give you control over the process. +""" ``` -Our code is producing UTF-8-as-cp1252 mojibake on Windows. The refactor should fix this by: -1. Detecting UTF-8 via `detect_charset_reliable()` -2. Trying UTF-8 (non-8-bit informative charset) -3. Successfully decoding and validating - -### Three Trial Codecs Usage Patterns - -Previously documented patterns become: -1. **Opportunistic Decoding** → New `decode()` logic -2. **Authoritative Validation** → HTTP Content-Type handling -3. **Detection Confirmation** → `detect_charset_reliable()` - -The fixed lists are replaced by situational logic based on charset properties. - -## Implementation Plan - -1. Implement and test `is_permissive_charset()` with caching -2. Implement `detect_charset_reliable()` -3. Refactor `decode()` with new candidate selection -4. Update documentation to deprecate `trial_codecs` -5. Test charset-normalizer vs chardet -6. Verify Windows Python 3.11+ doctests pass -7. Update architecture documentation - -## Resolved Design Questions - -1. **Authoritative charset failure**: Always fall back to detection (not configurable). Users who want exceptions can parse the header themselves and call `.decode()` directly. -2. **`detect_charset_reliable()` public API**: Yes, add both `detect_charset_reliable()` and `detect_charset_confidence_reliable()` to public API. -3. **`infer_*()` functions refactoring**: Defer for later; minor updates may be needed but not part of this refactor. -4. **Validation timing**: Respect existing `behaviors.text_validate` configuration; difference between during/after selection is minimal. -5. **Trust non-8-bit detections**: No, must consider confidence levels. Even multi-byte charsets can be misdetected; 7-bit (ASCII) is especially unreliable. -6. **Reuse existing functions**: Yes, use `attempt_decodes()` and existing trial decode helpers rather than reimplementing. - -## All Design Questions Resolved - -1. **Confidence threshold**: Use existing `behaviors.charset_confidence_threshold` from DTO -2. **Permissive charset caching**: Always enabled (no flag needed, minimal memory) -3. **Candidate prioritization**: Shortest always wins, user supplement is tie-breaker -4. **Multi-byte detection**: Test both ascending and descending byte sequences, check decoded length == 256 +**Complexity should serve users, not obscure limitations.** From e067736acf8f0645bef07184a9c2aa88faa3f2a5 Mon Sep 17 00:00:00 2001 From: Eric McDonald Date: Sat, 15 Nov 2025 17:22:09 -0800 Subject: [PATCH 03/12] Document confidence scoring and text validation strategies. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add comprehensive documentation for confidence scoring approach: - Size-based scaling rationale and formula - Detector-specific strategies (intrinsic vs constant confidence) - Base confidence values for magic (0.95/0.75) and charset-normalizer (0.85) - Examples and interaction with behavior thresholds Add analysis of text validation and confidence threshold: - text_validate_confidence is effectively unused (always 0.0 in main path) - Validation checks textuality, not detection confidence (orthogonal concerns) - Recommend removing confidence threshold, keeping tristate control Fix docstring in is_permissive_charset() to correctly reflect that CP1252 is not permissive (has 5 undefined bytes). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .auxiliary/notes/confidence.md | 307 ++++++++++++++++++++++++++++ .auxiliary/notes/text-validation.md | 200 ++++++++++++++++++ sources/detextive/charsets.py | 4 +- 3 files changed, 509 insertions(+), 2 deletions(-) create mode 100644 .auxiliary/notes/confidence.md create mode 100644 .auxiliary/notes/text-validation.md diff --git a/.auxiliary/notes/confidence.md b/.auxiliary/notes/confidence.md new file mode 100644 index 0000000..9cde9a2 --- /dev/null +++ b/.auxiliary/notes/confidence.md @@ -0,0 +1,307 @@ +# Confidence Scoring Strategy + +## Overview + +This document describes the confidence scoring strategy for detection results in detextive. The core principle is that confidence should reflect **both detection quality AND sample size adequacy**. + +## Design Philosophy + +### Why Scale All Confidence by Content Size? + +1. **Small samples are inherently less reliable**: A charset detection on 10 bytes is fundamentally less trustworthy than the same detection on 1000 bytes, regardless of what the detector reports. + +2. **Empirical justification**: `chardet` is known to be overconfident on small samples, sometimes reporting high confidence on minimal data that could be interpreted multiple ways. + +3. **Cost-benefit alignment**: Trial decoding and validation are **cheaper** for small content. Being more conservative (lower confidence → more validation) when it matters least (small files) is a win-win. + +4. **Smooth, predictable behavior**: Linear scaling avoids arbitrary threshold discontinuities. A step function would create sudden behavior changes at threshold boundaries, while linear scaling provides gradual, intuitive confidence progression. + +5. **Philosophical consistency**: "Honest about limitations" means acknowledging that charset/MIME detection is fundamentally harder with less data. Our confidence scores should reflect this reality. + +## Size Scaling Formula + +```python +def confidence_from_bytes_quantity( + content: Content, behaviors: Behaviors = BEHAVIORS_DEFAULT +) -> float: + return min(1.0, len(content) / behaviors.bytes_quantity_confidence_divisor) +``` + +**Default divisor**: 1024 bytes + +This means: +- 512 bytes → 0.5 scaling factor +- 1024 bytes → 1.0 scaling factor (full confidence) +- 2048 bytes → 1.0 (capped at maximum) + +## Detector-Specific Strategies + +### Detectors With Intrinsic Confidence + +These detectors provide their own confidence scores based on detection quality. We multiply by the size scaling factor. + +#### chardet (Charset Detection) + +```python +def _detect_via_chardet( + content: Content, behaviors: Behaviors +) -> CharsetResult | types.NotImplementedType: + try: import chardet + except ImportError: return NotImplemented + result_ = chardet.detect(content) + charset, confidence = result_['encoding'], result_['confidence'] + + # Scale confidence by content size + size_factor = confidence_from_bytes_quantity(content, behaviors=behaviors) + confidence = confidence * size_factor + + return CharsetResult(charset=charset, confidence=confidence) +``` + +**Rationale**: `chardet` reports confidence based on statistical analysis, but doesn't account for sample size adequacy. A 95% confidence on 10 bytes should be treated much more skeptically than 95% on 1000 bytes. + +#### puremagic (MIME Type Detection) + +```python +def _detect_via_puremagic( + content: Content, behaviors: Behaviors +) -> MimetypeResult | types.NotImplementedType: + try: import puremagic + except ImportError: return NotImplemented + try: + matches = puremagic.magic_string(content) + if not matches: return NotImplemented + match = matches[0] # Best match + + # Use puremagic's intrinsic confidence, scaled by size + size_factor = confidence_from_bytes_quantity(content, behaviors=behaviors) + confidence = match.confidence * size_factor + + return MimetypeResult(mimetype=match.mime_type, confidence=confidence) + except (puremagic.PureError, ValueError): + return NotImplemented +``` + +**Rationale**: `puremagic` provides confidence scores (typically 0.4-0.8) based on signature match quality. Longer, more specific signatures get higher confidence. Similar to `chardet`, these scores benefit from size scaling. + +**Note**: The current implementation uses `puremagic.from_string(content, mime=True)` which returns a simple string. To access confidence, we need to use `puremagic.magic_string(content)` instead, which returns `PureMagicWithConfidence` objects. + +### Detectors Without Intrinsic Confidence + +These detectors only return a detection result without quality assessment. We assign a base confidence constant, then scale by size. + +#### magic/libmagic (MIME Type Detection) + +```python +def _detect_via_magic( + content: Content, behaviors: Behaviors +) -> MimetypeResult | types.NotImplementedType: + try: import magic + except ImportError: return NotImplemented + try: mimetype = magic.from_buffer(content, mime=True) + except Exception: return NotImplemented + + # Use different base confidence for textual vs binary formats + if is_textual_mimetype(mimetype): + BASE_CONFIDENCE = 0.75 # Lower for text (heuristic-based) + else: + BASE_CONFIDENCE = 0.95 # Higher for binary (magic bytes) + + confidence = BASE_CONFIDENCE * confidence_from_bytes_quantity( + content, behaviors=behaviors) + return MimetypeResult(mimetype=mimetype, confidence=confidence) +``` + +**Rationale**: +- **Binary formats (0.95)**: libmagic excels at detecting structured binary formats with magic bytes (PNG: `\x89PNG`, PDF: `%PDF`, etc.). These are unambiguous byte patterns with decades of curated signatures. +- **Textual formats (0.75)**: Text detection is often heuristic-based. `text/plain` is frequently a fallback/guess. `text/html`, `text/xml`, and even `application/json` (which may be detected as `text/plain` on some platforms) are more ambiguous and context-dependent. + +**Platform note**: `magic` behavior varies across platforms and versions. On Windows, JSON content may return `text/plain` instead of `application/json`. The textual/non-textual distinction handles this gracefully. + +#### charset-normalizer (Charset Detection) + +```python +def _detect_via_charset_normalizer( + content: Content, behaviors: Behaviors +) -> CharsetResult | types.NotImplementedType: + try: import charset_normalizer + except ImportError: return NotImplemented + result_ = charset_normalizer.from_bytes(content).best() + charset = None if result_ is None else result_.encoding + + # charset-normalizer doesn't provide usable confidence + # Use base constant scaled by size + BASE_CONFIDENCE = 0.85 + confidence = BASE_CONFIDENCE * confidence_from_bytes_quantity( + content, behaviors=behaviors) + + return CharsetResult(charset=charset, confidence=confidence) +``` + +**Rationale**: +- charset-normalizer has `coherence` and related attributes, but these are not reliable confidence metrics (often 0.0) +- Evaluation results showed: 92% accurate on UTF-8, but only 17% on Latin-1/CP1252 +- Base confidence of 0.85 reflects that it's good but not as reliable as `chardet` (which provides its own confidence) +- Still higher than textual MIME detection (0.75) since charset detection is more targeted + +## Confidence Constants Summary + +| Detector | Type | Strategy | Base Confidence | Notes | +|----------|------|----------|-----------------|-------| +| `chardet` | Charset | Intrinsic × size | N/A (uses reported) | Statistical analysis | +| `puremagic` | MIME | Intrinsic × size | N/A (uses reported) | Signature match quality (0.4-0.8) | +| `magic` (binary) | MIME | Constant × size | 0.95 | Magic bytes, very reliable | +| `magic` (textual) | MIME | Constant × size | 0.75 | Heuristic-based, less reliable | +| `charset-normalizer` | Charset | Constant × size | 0.85 | Good for UTF-8, weaker for legacy | + +## Example Confidence Calculations + +### Small File (100 bytes) +Size factor: `100 / 1024 = 0.0977` (~0.1) + +- **chardet** (0.95 raw): `0.95 × 0.1 = 0.095` +- **magic** binary (0.95 base): `0.95 × 0.1 = 0.095` +- **magic** textual (0.75 base): `0.75 × 0.1 = 0.075` +- **charset-normalizer** (0.85 base): `0.85 × 0.1 = 0.085` +- **puremagic** (0.8 raw): `0.8 × 0.1 = 0.08` + +All appropriately conservative. With `trial_decode_confidence = 0.80`, all trigger validation. + +### Medium File (512 bytes) +Size factor: `512 / 1024 = 0.5` + +- **chardet** (0.95 raw): `0.95 × 0.5 = 0.475` +- **magic** binary (0.95 base): `0.95 × 0.5 = 0.475` +- **magic** textual (0.75 base): `0.75 × 0.5 = 0.375` +- **charset-normalizer** (0.85 base): `0.85 × 0.5 = 0.425` +- **puremagic** (0.8 raw): `0.8 × 0.5 = 0.4` + +Still below 0.80 threshold, but closer. More validation occurs. + +### Full Confidence (1024+ bytes) +Size factor: `1024 / 1024 = 1.0` (or higher, capped at 1.0) + +- **chardet** (0.95 raw): `0.95 × 1.0 = 0.95` +- **magic** binary (0.95 base): `0.95 × 1.0 = 0.95` +- **magic** textual (0.75 base): `0.75 × 1.0 = 0.75` +- **charset-normalizer** (0.85 base): `0.85 × 1.0 = 0.85` +- **puremagic** (0.8 raw): `0.8 × 1.0 = 0.8` + +Nice spread. Binary detections and high-confidence chardet skip validation. Textual MIME and charset-normalizer still trigger validation unless detection is very confident or sample is larger. + +## Interaction with Behavior Thresholds + +### `trial_decode_confidence` (default: 0.80) + +Minimum confidence to skip trial decoding during charset detection. + +With size scaling: +- Small files almost always trigger trial decode (good: cheap to validate) +- Medium files trigger if detector isn't confident +- Large files only skip if detector is confident + +### `text_validate_confidence` (default: 0.80) + +Minimum confidence to skip text validation. + +Similar behavior: more validation on small samples, less on large confident detections. + +## Special Cases + +### Empty Content + +Empty content (`b''`) always returns: +- Charset: default charset with 1.0 confidence +- MIME: `text/plain` with 1.0 confidence + +No detection is needed, so confidence is absolute. + +### Content with BOM + +BOMs (Byte Order Marks) provide near-certainty for UTF-8/UTF-16 detection regardless of size. However: +- This is already handled in `_normalize_charset_detection()` which checks for BOM and adjusts charset accordingly +- No special confidence handling needed; chardet will report high confidence, which is appropriate + +### Pure ASCII + +Small pure ASCII samples (like `b"Hello"`) get scaled down confidence, but: +- ASCII is promoted to UTF-8 via `charset_promotions` +- Small ASCII content is cheap to validate +- Erring on the side of validation is fine + +## Implementation Notes + +### Current State (Before Changes) + +- ✅ `chardet`: Returns raw confidence (no scaling) +- ✅ `charset-normalizer`: Uses `confidence_from_bytes_quantity()` +- ✅ `magic`: Uses `confidence_from_bytes_quantity()` +- ✅ `puremagic`: Uses `confidence_from_bytes_quantity()` + +### Needed Changes + +1. **Scale chardet confidence**: Multiply by size factor +2. **Add base constants**: Define base confidence for `magic` and `charset-normalizer` +3. **Textual/binary distinction for magic**: Use `is_textual_mimetype()` to select base confidence +4. **Extract puremagic confidence**: Switch from `from_string()` to `magic_string()` to access confidence scores + +## Documentation for Users + +Users should understand that confidence scores in detextive are composite: + +> **Confidence scores reflect both detection quality and sample adequacy.** +> +> A confidence of 0.95 from detextive means both: +> - The detector is highly confident in its result +> - There is sufficient content for reliable detection +> +> For small samples (< 1024 bytes), confidence is proportionally reduced to encourage validation. This is intentional: charset and MIME type detection are fundamentally less reliable with less data. + +## Rationale: Why Not Step Functions? + +An alternative approach would be minimum size thresholds: + +```python +if len(content) < 1024: + confidence = min(confidence, 0.79) # Force below threshold +``` + +**Problems with this approach**: +1. **Discontinuous behavior**: 1023 bytes → untrusted, 1024 bytes → fully trusted +2. **Arbitrary boundary**: Why 1024? Why not 512 or 2048? +3. **Loss of information**: A 1000-byte detection is more reliable than a 100-byte detection, but both get capped + +Linear scaling is more principled, predictable, and preserves relative quality differences across sample sizes. + +## Future Considerations + +### Tunable Parameters + +If users want different size/confidence tradeoffs, they can adjust: + +```python +behaviors = Behaviors( + bytes_quantity_confidence_divisor=512, # Smaller threshold + trial_decode_confidence=0.70, # Lower bar for skipping validation +) +``` + +### Alternative Scaling Functions + +Linear scaling is simple and effective, but alternatives could be considered: + +```python +# Logarithmic (slower growth, more conservative) +confidence = math.log(len(content) + 1) / math.log(1025) + +# Sigmoid (smooth S-curve with inflection point) +confidence = 1 / (1 + math.exp(-k * (len(content) - midpoint))) +``` + +For now, linear scaling aligns with the design philosophy: simple, honest, and predictable. + +## Related Documents + +- `.auxiliary/notes/decode-refactor.md` - Context-based trial order and design philosophy +- `.auxiliary/notes/charset-detector-evaluation-results.md` - Empirical detector performance data +- `documentation/architecture/designs/001-python-api.rst` - API design including confidence scoring diff --git a/.auxiliary/notes/text-validation.md b/.auxiliary/notes/text-validation.md new file mode 100644 index 0000000..7db5c6d --- /dev/null +++ b/.auxiliary/notes/text-validation.md @@ -0,0 +1,200 @@ +# Text Validation and the Irrelevance of Confidence Thresholds + +## Summary + +The `text_validate_confidence` parameter should be removed. Text validation checks whether decoded content looks like real text (not binary data that happened to decode successfully). This is orthogonal to charset detection confidence and doesn't benefit from a confidence threshold. + +## Current Behavior + +### `text_validate_confidence` Usage + +Currently used in `_validate_text()` to decide whether to validate: + +```python +match behaviors.text_validate: + case BehaviorTristate.AsNeeded: + should_validate = confidence < behaviors.text_validate_confidence +``` + +Default threshold: 0.80 + +### Where Validation Is Called + +1. **From `_attempt_decode_http_content_type()`**: + - Passes `result.confidence` from `attempt_decodes()` + - This is size-based confidence: `confidence_from_bytes_quantity()` + +2. **From `_attempt_decodes()` (main decode path)**: + - Passes `0.0` confidence (hardcoded!) + - **Always validates** regardless of threshold + +## What Validation Actually Checks + +`PROFILE_TEXTUAL` (the default validation profile) checks: + +- **Control characters**: Only allows `\t`, `\n`, `\r`, plus bidi/zero-width/formatting characters +- **Rejectable categories**: Rejects Unicode categories: + - `Cc` (control characters) + - `Cf` (format characters) + - `Co` (private use) + - `Cs` (surrogate) +- **Printables ratio**: Requires ≥85% printable characters +- **Explicit rejects**: DELETE character (0x7F) + +### What Validation Catches + +Validation catches **binary/non-textual data that successfully decoded**: + +- Binary PNG data decoded as CP1252 → fails printables ratio +- UTF-16 data decoded as UTF-8 → produces garbage with control characters +- Random binary content decoded as ISO-8859-1 → fails printables ratio +- Mojibake from wrong charset → may contain unprintables + +**Key insight**: Any charset can decode binary data without raising `UnicodeDecodeError`. Validation is the only way to catch these false positives. + +## Why Confidence Threshold Doesn't Help + +### 1. Always 0.0 in Main Decode Path + +In `_attempt_decodes()`, confidence is hardcoded to `0.0`: + +```python +return _validate_text( + text, 0.0, # ← Always 0.0 + behaviors=behaviors, profile=profile, location=location) +``` + +This means: +- Validation **always runs** in the main decode path +- The `text_validate_confidence` threshold is never actually checked +- The parameter is effectively dead code for normal decoding + +### 2. Validation Is Not About Detection Quality + +Confidence reflects: "How sure are we this is the right charset?" + +Validation checks: "Does this look like real text?" + +These are **orthogonal concerns**: +- High-confidence UTF-8 detection can still produce mojibake if the actual charset was CP1252 +- Low-confidence detection on small sample might be correct and produce valid text +- Wrong charset with high confidence → valid-looking text that happens to be garbage + +### 3. Sample Size Doesn't Reduce Need for Validation + +The argument for confidence threshold might be: +> "Large files with high-confidence detection don't need validation" + +But this is wrong because: +- Large binary files (images, executables) can still decode as text +- HTTP headers can lie about charset +- Validation is cheap (character category checks) +- Better to validate anyway + +### 4. Any Charset Can Encode Binary Data + +All charsets can represent control characters: +- UTF-8: `\x00`, `\x01`, `\x02`, etc. +- CP1252: Control chars in 0x00-0x1F range +- ISO-8859-1: Fully permissive, decodes everything + +There's no charset-based reason to skip validation. + +## Proposed Changes + +### Remove Confidence Threshold + +Change validation logic from: + +```python +match behaviors.text_validate: + case BehaviorTristate.AsNeeded: + should_validate = confidence < behaviors.text_validate_confidence +``` + +To: + +```python +match behaviors.text_validate: + case BehaviorTristate.AsNeeded: + should_validate = True # Always validate when AsNeeded +``` + +Or simplify the tristate entirely: +- `Always`: Validate +- `Never`: Don't validate +- `AsNeeded`: **Remove** (was equivalent to Always in practice) + +### Simplify to Boolean + +Even simpler option: + +```python +class Behaviors: + text_validate: bool = True # Just True/False +``` + +But keeping the tristate maintains API compatibility and clarity: +- `Always`: Validate (explicit) +- `AsNeeded`: Validate (matches current behavior) +- `Never`: Don't validate (opt-out for performance) + +### Remove Parameter + +Delete from `Behaviors`: + +```python +text_validate_confidence: float = 0.80 # ← Remove this +``` + +### Update Signature + +`_validate_text()` can keep the confidence parameter for now (for backward compatibility in internal calls), but ignore it: + +```python +def _validate_text( + text: str, confidence: float, /, *, # confidence unused + behaviors: BehaviorsArgument, + profile: ProfileArgument, + location: LocationArgument, +) -> str: + # Don't check confidence, just validate based on tristate + ... +``` + +Or remove it entirely and update all call sites. + +## Why Validation Is Important + +Validation is **critical** for detextive's reliability: + +1. **Catches wrong charsets**: ISO-8859-1 can decode UTF-8 as mojibake +2. **Catches binary data**: Images, executables, etc. that decode without errors +3. **Provides meaningful errors**: Better to fail with "TextInvalidity" than return garbage +4. **Aligns with design philosophy**: "Honest about limitations" → validate results + +## Performance Considerations + +**Validation is cheap**: +- Samples only first 8192 characters by default (`profile.sample_quantity`) +- Character category lookup is O(1) with Unicode data +- Ratio calculations are simple arithmetic +- Negligible compared to charset detection + +**No need to skip validation for performance.** + +## Recommendation + +1. **Remove `text_validate_confidence` parameter** from `Behaviors` +2. **Keep `text_validate` tristate** for user control +3. **Always validate when `AsNeeded`** (remove confidence check) +4. **Update documentation** to clarify that validation is about textuality, not confidence +5. **Update vulturefood.py** to remove `text_validate_confidence` entry + +This simplifies the API, removes dead code, and aligns behavior with actual needs. + +## Related Documents + +- `.auxiliary/notes/confidence.md` - Confidence scoring strategy +- `.auxiliary/notes/decode-refactor.md` - Design philosophy and simplification +- `sources/detextive/validation.py` - Validation profiles and logic diff --git a/sources/detextive/charsets.py b/sources/detextive/charsets.py index 84bb8e7..090fe6c 100644 --- a/sources/detextive/charsets.py +++ b/sources/detextive/charsets.py @@ -90,8 +90,8 @@ def discover_os_charset_default( ) -> str: def is_permissive_charset( charset: str ) -> bool: ''' Checks if charset accepts all byte sequences (8-bit encoding). - Returns ``True`` for CP1252, ISO-8859-*, etc.... - Returns ``False`` for ASCII, UTF-8, SHIFT-JIS, etc.... + Returns ``True`` for ISO-8859-*, etc.... + Returns ``False`` for ASCII, CP1252, UTF-8, SHIFT-JIS, etc.... ''' charset_ = normalize_charset( charset ) if charset_ in _charsets_permissive: From 452af933e1a9a29e3f18bf7974c3ab2afd6eec55 Mon Sep 17 00:00:00 2001 From: Eric McDonald Date: Thu, 12 Feb 2026 20:38:03 -0800 Subject: [PATCH 04/12] Simplify decode trial flow and realign tests. Refactor decoders to use charset trial decoding with validator hooks. Update default trial codec order to prefer UTF-8 before OS defaults and keep inference confidence gating. Adjust docs and tests for BOM-aware charset normalization and decode behavior. Co-Authored-By: GPT-5 Codex --- .../examples/advanced-configuration.rst | 12 +- sources/detextive/__/imports.py | 1 + sources/detextive/charsets.py | 33 +---- sources/detextive/core.py | 4 +- sources/detextive/decoders.py | 131 ++++-------------- tests/test_000_detextive/test_220_charsets.py | 6 +- .../test_000_detextive/test_400_inference.py | 10 +- tests/test_000_detextive/test_500_decoders.py | 12 +- 8 files changed, 58 insertions(+), 151 deletions(-) diff --git a/documentation/examples/advanced-configuration.rst b/documentation/examples/advanced-configuration.rst index 4cba43e..813f318 100644 --- a/documentation/examples/advanced-configuration.rst +++ b/documentation/examples/advanced-configuration.rst @@ -133,7 +133,7 @@ Let HTTP header inform detection: >>> mimetype 'application/json' >>> charset - 'utf-8' + 'utf-8-sig' Location-Based Inference =============================================================================== @@ -212,7 +212,9 @@ Apply validation profiles during high-level decoding: >>> text 'Text for terminal display' -Validation failures raise appropriate exceptions. Note that we provide ``http_content_type`` here to bypass MIME type detection, which would reject this content as binary before text validation runs: +Validation filtering can exhaust all decode attempts and raise a decode error. +Note that we provide ``http_content_type`` here to bypass MIME type detection, +which would reject this content as binary before decoding runs: .. doctest:: AdvancedConfiguration @@ -223,9 +225,9 @@ Validation failures raise appropriate exceptions. Note that we provide ``http_co ... problematic, ... profile = detextive.PROFILE_TERMINAL_SAFE, ... http_content_type = 'text/plain' ) - ... except detextive.exceptions.TextInvalidity as exception: - ... print( "Text validation failed" ) - Text validation failed + ... except detextive.exceptions.ContentDecodeFailure as exception: + ... print( "Decode failed after validation filtering" ) + Decode failed after validation filtering Error Handling =============================================================================== diff --git a/sources/detextive/__/imports.py b/sources/detextive/__/imports.py index 5dd8e79..ae8296f 100644 --- a/sources/detextive/__/imports.py +++ b/sources/detextive/__/imports.py @@ -27,6 +27,7 @@ import contextlib as ctxl import dataclasses as dcls import enum +import functools as funct import locale import mimetypes import os diff --git a/sources/detextive/charsets.py b/sources/detextive/charsets.py index 090fe6c..d820a6d 100644 --- a/sources/detextive/charsets.py +++ b/sources/detextive/charsets.py @@ -35,15 +35,15 @@ ) -_charsets_permissive: dict[ str, bool ] = { } # TODO: Accretive dictionary. - - -def attempt_decodes( # noqa: PLR0915 +def attempt_decodes( # noqa: C901,PLR0912,PLR0913,PLR0915 content: _nomina.Content, /, *, behaviors: _Behaviors = _BEHAVIORS_DEFAULT, inference: __.Absential[ str ] = __.absent, supplement: __.Absential[ str ] = __.absent, location: __.Absential[ _nomina.Location ] = __.absent, + validator: __.Absential[ + __.cabc.Callable[ [ str, _CharsetResult ], None ] + ] = __.absent, ) -> tuple[ str, _CharsetResult ]: ''' Attempts to decode content with various character sets. @@ -75,6 +75,9 @@ def attempt_decodes( # noqa: PLR0915 except UnicodeDecodeError: continue finally: trials.add( charset ) result = _CharsetResult( charset = charset, confidence = confidence ) + if not __.is_absent( validator ): + try: validator( text, result ) + except _exceptions.TextInvalidity: continue return text, result raise _exceptions.ContentDecodeFailure( charset = tuple( trials ), location = location ) @@ -87,28 +90,6 @@ def discover_os_charset_default( ) -> str: return normalize_charset( discoverer( ) ) -def is_permissive_charset( charset: str ) -> bool: - ''' Checks if charset accepts all byte sequences (8-bit encoding). - - Returns ``True`` for ISO-8859-*, etc.... - Returns ``False`` for ASCII, CP1252, UTF-8, SHIFT-JIS, etc.... - ''' - charset_ = normalize_charset( charset ) - if charset_ in _charsets_permissive: - return _charsets_permissive[ charset_ ] - try: - texta = bytes( range( 256 ) ).decode( - charset_, errors = 'strict' ) - textd = bytes( range( 255, -1, -1 ) ).decode( - charset_, errors = 'strict' ) - except ( UnicodeDecodeError, LookupError ): - _charsets_permissive[ charset_ ] = False - return False - permissivity = ( len( texta ) == len( textd ) == 256 ) # noqa: PLR2004 - _charsets_permissive[ charset_ ] = permissivity - return permissivity - - def normalize_charset( charset: str, bom_cognizant: bool = False ) -> str: ''' Normalizes character set encoding names. ''' charset_ = __.codecs.lookup( charset ).name diff --git a/sources/detextive/core.py b/sources/detextive/core.py index 7691eea..7079607 100644 --- a/sources/detextive/core.py +++ b/sources/detextive/core.py @@ -129,9 +129,11 @@ class Behaviors( __.immut.DataclassObject ): __.cabc.Sequence[ str | CodecSpecifiers ], __.ddoc.Doc( ''' Sequence of codec names or specifiers. ''' ), ] = ( - CodecSpecifiers.OsDefault, CodecSpecifiers.UserSupplement, + 'utf-8', CodecSpecifiers.FromInference, + CodecSpecifiers.OsDefault, + CodecSpecifiers.PythonDefault, ) trial_decode: __.typx.Annotated[ BehaviorTristate, diff --git a/sources/detextive/decoders.py b/sources/detextive/decoders.py index eb3f9aa..8d379c0 100644 --- a/sources/detextive/decoders.py +++ b/sources/detextive/decoders.py @@ -32,7 +32,6 @@ from .core import ( # isort: skip BEHAVIORS_DEFAULT as _BEHAVIORS_DEFAULT, CHARSET_DEFAULT as _CHARSET_DEFAULT, - MIMETYPE_DEFAULT as _MIMETYPE_DEFAULT, BehaviorTristate as _BehaviorTristate, BehaviorsArgument as _BehaviorsArgument, CharsetResult as _CharsetResult, @@ -45,15 +44,13 @@ def decode( # noqa: PLR0913 behaviors: _BehaviorsArgument = _BEHAVIORS_DEFAULT, profile: _validation.ProfileArgument = _validation.PROFILE_TEXTUAL, charset_default: _nomina.CharsetDefaultArgument = _CHARSET_DEFAULT, - mimetype_default: _nomina.MimetypeDefaultArgument = _MIMETYPE_DEFAULT, http_content_type: _nomina.HttpContentTypeArgument = __.absent, location: _nomina.LocationArgument = __.absent, charset_supplement: _nomina.CharsetSupplementArgument = __.absent, - mimetype_supplement: _nomina.MimetypeSupplementArgument = __.absent, ) -> str: ''' Decodes bytes array to Unicode text. ''' - # TODO: Deprecation warnings for 'mimetype_*' arguments. if content == b'': return '' + charset: __.Absential[ str ] = __.absent result: __.Absential[ _CharsetResult ] = __.absent text: __.Absential[ str ] = __.absent if not __.is_absent( http_content_type ): @@ -71,12 +68,21 @@ def decode( # noqa: PLR0913 default = charset_default, supplement = charset_supplement, location = location ) - return _attempt_decodes( - content, result, + if ( result.charset + and result.confidence >= behaviors.trial_decode_confidence + ): charset = result.charset + validator = __.funct.partial( + _validate_text_in_decode_attempt, behaviors = behaviors, profile = profile, - supplement = charset_supplement, location = location ) + return _charsets.attempt_decodes( + content, + behaviors = behaviors, + inference = charset, + supplement = charset_supplement, + location = location, + validator = validator )[ 0 ] def _attempt_decode_http_content_type( @@ -107,104 +113,6 @@ def _attempt_decode_http_content_type( return __.absent -def _append_charset( - permissives: list[ str ], - restrictives: list[ str ], - charset: str, - bom_cognizant: bool, -) -> None: - charset_ = _charsets.normalize_charset( - charset, bom_cognizant = bom_cognizant ) - if _charsets.is_permissive_charset( charset_ ): - if charset_ in permissives: return - permissives.append( charset_ ) - else: - if charset_ in restrictives: return - restrictives.append( charset_ ) - - -def _attempt_decodes( # noqa: PLR0913 - content: _nomina.Content, - detection: __.Absential[ _CharsetResult ], /, *, - behaviors: _BehaviorsArgument, - profile: _validation.ProfileArgument, - supplement: __.Absential[ str ], - location: _nomina.LocationArgument, -) -> str: - error = _exceptions.ContentDecodeImpossibility( location = location ) - permissives, restrictives = _prepare_charsets( - detection, behaviors = behaviors, supplement = supplement ) - on_decode_error = behaviors.on_decode_error - # Try restrictive charsets before permissive charsets, since: - # (1) Restrictive charsets can have decoding errors from invalid byte - # sequences. - # (2) Restrictive charsets can produce shorter strings, if the are - # multi-byte encodings. Permissive charsets decoding the same byte - # sequences will likly result in mojibake. - for charset in restrictives: - try: text = content.decode( charset, errors = on_decode_error ) - except UnicodeDecodeError: continue - try: - return _validate_text( - text, 0.0, - behaviors = behaviors, profile = profile, location = location ) - except _exceptions.TextInvalidity: continue - for charset in permissives: - try: text = content.decode( charset, errors = on_decode_error ) - except UnicodeDecodeError: continue - try: - return _validate_text( - text, 0.0, - behaviors = behaviors, profile = profile, location = location ) - except _exceptions.TextInvalidity: continue - raise error - - -def _prepare_charsets( - detection: __.Absential[ _CharsetResult ], /, *, - behaviors: _BehaviorsArgument, - supplement: __.Absential[ str ], -) -> tuple[ tuple[ str, ... ], tuple[ str, ... ] ]: - permissives: list[ str ] = [ ] - restrictives: list[ str ] = [ ] - os_charset = _charsets.discover_os_charset_default( ) - _append_charset( - permissives, restrictives, os_charset, behaviors.remove_bom ) - python_charset = __.locale.getpreferredencoding( ) - _append_charset( - permissives, restrictives, python_charset, behaviors.remove_bom ) - if not __.is_absent( supplement ): - _prepend_charset( - permissives, restrictives, supplement, behaviors.remove_bom ) - if not __.is_absent( detection ) and detection.charset is not None: - # Suspicious charset detections go at end. - if detection.confidence < behaviors.trial_decode_confidence: - _append_charset( - permissives, restrictives, detection.charset, - behaviors.remove_bom ) - else: - _prepend_charset( - permissives, restrictives, detection.charset, - behaviors.remove_bom ) - return tuple( permissives ), tuple( restrictives ) - - -def _prepend_charset( - permissives: list[ str ], - restrictives: list[ str ], - charset: str, - bom_cognizant: bool, -) -> None: - charset_ = _charsets.normalize_charset( - charset, bom_cognizant = bom_cognizant ) - if _charsets.is_permissive_charset( charset_ ): - if charset_ in permissives: return - permissives.insert( 0, charset_ ) - else: - if charset_ in restrictives: return - restrictives.insert( 0, charset_ ) - - def _validate_text( text: str, confidence: float, /, *, behaviors: _BehaviorsArgument, @@ -221,3 +129,16 @@ def _validate_text( case _BehaviorTristate.Never: pass if should_validate and not profile( text ): raise error return text + + +def _validate_text_in_decode_attempt( + text: str, result: _CharsetResult, /, *, + behaviors: _BehaviorsArgument, + profile: _validation.ProfileArgument, + location: _nomina.LocationArgument, +) -> None: + _validate_text( + text, 0.0, + behaviors = behaviors, + profile = profile, + location = location ) diff --git a/tests/test_000_detextive/test_220_charsets.py b/tests/test_000_detextive/test_220_charsets.py index 91b6964..b72c66e 100644 --- a/tests/test_000_detextive/test_220_charsets.py +++ b/tests/test_000_detextive/test_220_charsets.py @@ -100,7 +100,7 @@ def test_220_codec_specifiers_user_supplement( ): text, result = _charsets.attempt_decodes( _patterns.UTF8_BASIC, behaviors = behaviors, supplement = 'utf-8' ) assert text == 'Hello, world!' - assert result.charset == 'utf-8' + assert result.charset == 'utf-8-sig' def test_230_codec_specifiers_string_codec( ): @@ -121,7 +121,7 @@ def test_240_invalid_codec_type_handling( ): text, result = _charsets.attempt_decodes( content, behaviors = behaviors ) assert text == 'test content' - assert result.charset == 'utf-8' + assert result.charset == 'utf-8-sig' #============================================================================# @@ -148,4 +148,4 @@ def test_310_from_inference_codec_skipped_when_absent( ): ) ) text, result = _charsets.attempt_decodes( content, behaviors = behaviors ) assert text == 'Hello, world!' - assert result.charset is not None \ No newline at end of file + assert result.charset is not None diff --git a/tests/test_000_detextive/test_400_inference.py b/tests/test_000_detextive/test_400_inference.py index 5cfd6ac..086712e 100644 --- a/tests/test_000_detextive/test_400_inference.py +++ b/tests/test_000_detextive/test_400_inference.py @@ -62,7 +62,7 @@ def test_120_infer_charset_confidence_http_content_type_parsing( ): http_content_type = 'text/plain; charset=iso-8859-1' result = _inference.infer_charset_confidence( content, http_content_type = http_content_type ) - assert result.charset == 'iso-8859-1' + assert result.charset == 'iso8859-1' def test_130_infer_charset_confidence_detection_fallback( ): @@ -131,7 +131,7 @@ def test_200_http_content_type_parsing_success( ): utf8_content, behaviors = behaviors, http_content_type = 'text/plain; charset=utf-8' ) ) assert mimetype_result.mimetype == 'text/plain' - assert charset_result.charset == 'utf-8' + assert charset_result.charset == 'utf-8-sig' def test_210_location_based_mimetype_inference( ): @@ -305,7 +305,7 @@ def test_340_http_validation_mimetype_present( ): content, http_content_type = 'application/json; charset=utf-8' ) ) assert mimetype_result.mimetype == 'application/json' - assert charset_result.charset == 'utf-8' + assert charset_result.charset == 'utf-8-sig' def test_350_http_validation_mimetype_not_absent( ): @@ -317,6 +317,4 @@ def test_350_http_validation_mimetype_not_absent( ): http_content_type = 'application/json; charset=utf-8' ) ) assert mimetype_result.mimetype == 'application/json' assert mimetype_result.confidence == 0.9 - assert charset_result.charset == 'utf-8' - - + assert charset_result.charset == 'utf-8-sig' diff --git a/tests/test_000_detextive/test_500_decoders.py b/tests/test_000_detextive/test_500_decoders.py index b98e598..c769524 100644 --- a/tests/test_000_detextive/test_500_decoders.py +++ b/tests/test_000_detextive/test_500_decoders.py @@ -104,8 +104,11 @@ def test_420_validation_failure_handling( ): http_content_type = 'text/plain; charset=iso-8859-1' ) -def test_430_content_decode_impossibility( ): - ''' ContentDecodeImpossibility with charset=None and non-textual type. ''' +def test_430_decode_ignores_mimetype_context( ): + ''' Decode path remains charset-driven. + + Even with non-textual MIME signal. + ''' # Use a custom detector that returns charset=None def charset_none_detector( content, behaviors ): return detextive.core.CharsetResult( charset = None, confidence = 0.8 ) @@ -122,6 +125,5 @@ def mimetype_png_detector( content, behaviors ): behaviors = detextive.Behaviors( charset_detectors_order = ( 'test-decode-charset-none', ), mimetype_detectors_order = ( 'test-decode-mimetype-png', ) ) - # This should trigger ContentDecodeImpossibility - with pytest.raises( detextive.exceptions.ContentDecodeImpossibility ): - _decoders.decode( content, behaviors = behaviors ) \ No newline at end of file + text = _decoders.decode( content, behaviors = behaviors ) + assert text == 'some binary data' From 4b699038a9224f94e3a7b305db0a1f09090bf7a3 Mon Sep 17 00:00:00 2001 From: Eric McDonald Date: Thu, 12 Feb 2026 21:09:09 -0800 Subject: [PATCH 05/12] Prune stale notes and consolidate refactor context. Remove resolved Windows encoding investigation notes and keep active research notes focused on current v3 decisions. Update ideas scope to post-v3.0+ and retain CP1252 historical finding in decode refactor notes. Co-Authored-By: GPT-5 Codex --- .../charset-detector-evaluation-results.md | 58 +---- .auxiliary/notes/decode-refactor.md | 241 ++++-------------- .auxiliary/notes/ideas.md | 8 +- .auxiliary/notes/text-validation.md | 2 +- .auxiliary/notes/windows-encoding.md | 61 ----- 5 files changed, 73 insertions(+), 297 deletions(-) delete mode 100644 .auxiliary/notes/windows-encoding.md diff --git a/.auxiliary/notes/charset-detector-evaluation-results.md b/.auxiliary/notes/charset-detector-evaluation-results.md index 679d3a1..3eb7da0 100644 --- a/.auxiliary/notes/charset-detector-evaluation-results.md +++ b/.auxiliary/notes/charset-detector-evaluation-results.md @@ -11,10 +11,9 @@ Both detectors have strengths and weaknesses: - **Overall accuracy**: Tied at 65% on ground-truth tests - **Performance**: chardet is generally faster (19 vs 4 wins in speed tests) -**Recommendation**: Consider using **both** detectors with fallback logic: -1. Try charset-normalizer first for UTF-8 preference -2. Fall back to chardet if low confidence or decode fails -3. Apply `is_permissive_charset()` filtering to both +**Recommendation**: Treat this as detector-behavior reference data. For decode +selection, prefer deterministic trial ordering plus textual validation rather +than complex detector arbitration. ## Detailed Findings @@ -131,55 +130,20 @@ Both detectors have strengths and weaknesses: ## Recommendation for Detextive -### Proposed Strategy +### Practical Strategy -Use a **hybrid approach** with situational logic: +1. Keep detector results as hints, not authoritative truth. +2. Use straightforward trial order in decode. +3. Validate decoded text to reject non-textual output. +4. Use HTTP charset when explicitly provided and decodable. -```python -def detect_charset_reliable(content, behaviors): - """Reliable charset detection using hybrid approach.""" +### Detector Choice Guidance - # 1. Try charset-normalizer first (UTF-8 preference) - norm_result = detect_via_charset_normalizer(content) - - # 2. If normalizer detected UTF-8 or other multi-byte, trust it - if norm_result.charset and not is_permissive_charset(norm_result.charset): - return norm_result - - # 3. For 8-bit or uncertain, try chardet - chardet_result = detect_via_chardet(content) - - # 4. Apply logic: - # - If chardet detected multi-byte non-8-bit, prefer it - # - If chardet detected 8-bit, verify with trial decode - # - If both detected 8-bit, treat as uncertain - - if chardet_result.charset and not is_permissive_charset(chardet_result.charset): - # chardet found informative charset - if chardet_result.confidence >= behaviors.charset_confidence_threshold: - return chardet_result - - # 5. Fall back to defaults with trial decode - return try_defaults(content, behaviors) -``` - -### Why This Works - -1. **UTF-8 preference**: normalizer catches modern UTF-8 content that chardet misses -2. **8-bit accuracy**: chardet catches Latin-1/Win1252 that normalizer mangles -3. **Safety net**: `is_permissive_charset()` prevents accepting uninformative 8-bit -4. **Confidence gating**: Only trust chardet when confidence is high - -### Alternative: Just Use chardet - -If hybrid is too complex, **stick with chardet**: +If a single detector is preferred, **chardet remains a reasonable default**: - More consistent behavior across encoding types - Better confidence scores - Faster performance -- We can compensate for UTF-8 issues with: - - Always trying UTF-8 first in trial decode - - Using shortest-wins heuristic - - Text validation +- Compensate for UTF-8 misses with trial order and text validation. ## Test Scripts diff --git a/.auxiliary/notes/decode-refactor.md b/.auxiliary/notes/decode-refactor.md index e06ab8b..f128aab 100644 --- a/.auxiliary/notes/decode-refactor.md +++ b/.auxiliary/notes/decode-refactor.md @@ -1,188 +1,57 @@ # Decode Function Refactor -## Problem Statement - -The current `decode()` implementation has become overly complex with multiple special cases, three different `trial_codecs` usage patterns, and platform-specific encoding issues. The Windows Python 3.11+ doctest failures revealed fundamental issues with how we handle charset detection and validation. - -## Core Insight: Charset Detection is Fundamentally Hard - -**Key realization**: Without context, charset detection is heuristics all the way down. No amount of algorithmic complexity can solve the fundamental ambiguity problem. - -**Examples of inherent ambiguity:** -- UTF-8 Turkish text decoded as ISO-8859-9 produces valid-looking mojibake -- ISO-8859-9 Turkish text decoded as UTF-8 also produces mojibake -- Both are "valid" decodings with different results -- Without external context (user knowledge, file source, HTTP headers), detection is guessing - -## Design Philosophy: Simplicity + User Control - -After extensive analysis of multi-tier categorization schemes (permissive vs restrictive, multi-byte vs single-byte, etc.), we conclude: - -**Better to be simple and honest about limitations than complex and pretending to solve the unsolvable.** - -### What We Learned - -1. **CP1252 is not fully permissive**: Has 5 undefined bytes (0x81, 0x8d, 0x8f, 0x90, 0x9d) -2. **ISO-8859-* variants are fully permissive**: All 256 bytes decode (many variants exist for different languages) -3. **ASCII compatibility is universal**: All major 8-bit encodings preserve ASCII in bytes 0x00-0x7F -4. **UTF-8 vs CP1252 length heuristic works**: UTF-8 multi-byte always produces shorter strings than 8-bit misinterpretation -5. **But length heuristic fails for other encodings**: Turkish ISO-8859-9 vs UTF-8 can produce same-length mojibake - -### Implementation Findings - -`is_permissive_charset()` successfully implemented: -```python -def is_permissive_charset(charset: str) -> bool: - """Check if charset accepts all 256 byte values.""" - # Test ascending and descending sequences - # Check length == 256 (1:1 byte-to-char mapping) - # Cache results -``` - -Results: -- ✅ ISO-8859-1: `True` (fully permissive) -- ✅ CP1252: `False` (5 undefined bytes) -- ✅ ASCII: `False` (only 128 values) -- ✅ UTF-8: `False` (multi-byte sequences) - -But this revealed new complexity: need to subcategorize "restrictive" into multi-byte vs single-byte to avoid CP1252 mojibake before UTF-8 attempts. - -**This led to a design rabbit hole that misses the forest for the trees.** - -## Simplified Design (Current Direction) - -### Principles - -1. **Put user in control**: Provide supplement as `str` or codec specifier -2. **Use sensible defaults**: OS charset for local files, Python charset (usually UTF-8) for general use -3. **Trust high-confidence detection**: But allow it to be overridden by user/context -4. **Keep it simple**: Fewer tiers, clearer behavior, easier to reason about - -### Trial Order Strategy - -```python -trial_order = [ - UserSupplement, # User knows their data (highest priority) - OsDefault, # Sensible for local filesystem content - PythonDefault, # Usually UTF-8, can be set via PYTHONIOENCODING -] - -# Insert detected charset based on confidence: -if detection.confidence >= behaviors.trial_decode_confidence: - trial_order.insert(1, FromInference) # After user, before OS -else: - trial_order.append(FromInference) # At end (suspicious) -``` - -### User Supplement Enhancement - -Allow `charset_supplement` to be either: -- **`str`**: Specific charset name (e.g., `'utf-8'`, `'iso-8859-9'`) -- **Codec specifier**: `OsDefault`, `PythonDefault`, etc. - -**Use cases:** -```python -# Internet/web content - prefer UTF-8 -decode(content, charset_supplement='utf-8') - -# Local filesystem - use OS charset -decode(content, charset_supplement=OsDefault) - -# Known legacy encoding -decode(content, charset_supplement='iso-8859-9') -``` - -### Optional: Use `is_permissive_charset()` for Filtering - -One lightweight use of the permissive check: - -```python -# Skip truly permissive charsets if non-permissive options exist -candidates = build_candidate_list() -non_permissive = [c for c in candidates if not is_permissive_charset(c)] -if non_permissive: - candidates = non_permissive # Prefer informative attempts -``` - -This prevents trying ISO-8859-1 when UTF-8 is available, without complex multi-tier logic. - -## Current Implementation Status - -### Implemented ✅ - -1. **`is_permissive_charset()`** - Working perfectly with caching -2. **HTTP Content-Type handling** - Extracts and validates charset, falls back gracefully -3. **Separate permissive/restrictive lists** - In `_attempt_decodes()` -4. **BOM handling** - `remove_bom` behavior parameter -5. **Charset deduplication** - Normalized before adding to trial list -6. **Empty content uses default** - Not hardcoded to UTF-8 - -### Issues Discovered 🔍 - -1. **Complexity creep**: Permissive vs restrictive revealed need for multi-byte vs single-byte subcategorization -2. **CP1252 vs UTF-8 ordering**: CP1252 is "restrictive" but still produces mojibake before UTF-8 -3. **Turkish/Finnish ambiguity**: Historical encodings have legitimate sequences that look like UTF-8 mojibake -4. **No magic bullet**: Algorithmic complexity doesn't solve fundamental ambiguity - -### Next Steps 🎯 - -**Decision point**: Continue with complex categorization OR simplify to user-centric approach? - -**Recommendation**: Simplify -- Remove complex permissive/restrictive/multi-byte categorization -- Use simple context-based trial order (User → OS → Python → Detection) -- Keep `is_permissive_charset()` only for optional filtering -- Document limitations honestly -- Empower users with supplement options - -## Charset Evaluation Results - -Comprehensive testing of `chardet` vs `charset-normalizer`: - -**Key findings:** -- charset-normalizer: 92% accurate on UTF-8, 17% on Latin-1/CP1252 -- chardet: 58% accurate on UTF-8, 83% on Latin-1/CP1252 -- Overall: Both tied at 65% accuracy -- charset-normalizer is slower but better for UTF-8 -- chardet is faster and better for legacy 8-bit encodings - -**Decision**: Stick with chardet for now, provides good balance. - -See: `.auxiliary/notes/charset-detector-evaluation-results.md` - -## Related Files - -- Implementation: `sources/detextive/decoders.py`, `sources/detextive/charsets.py` -- Evaluations: `.auxiliary/evaluations/compare-charset-detectors.py` (and related) -- Results: `.auxiliary/notes/charset-detector-evaluation-results.md` - -## Open Questions - -1. Should we simplify back to context-based trial order? -2. Keep or remove permissive/restrictive categorization? -3. How much complexity is justified for marginal accuracy gains? -4. What's the right balance between "smart" and "simple"? - -## The Honest Documentation Approach - -```python -""" -decode() attempts decoding in context-aware order: -1. User supplement (you know your data best) -2. OS default (sensible for local files) -3. Python default (usually UTF-8) -4. Detected charset (if confidence is high) - -Charset detection is heuristic and cannot solve fundamental -ambiguities without context. For best results: -- Provide charset_supplement when encoding is known -- Use http_content_type for web content -- Validate results with is_valid_text() -- Consider confidence scores from detect_charset_confidence() - -There is no magic bullet for charset detection. We provide -sensible defaults and give you control over the process. -""" -``` - -**Complexity should serve users, not obscure limitations.** +## Current State (v3 branch) + +This note captures the historical rationale that led to the current decode +simplification and trial-order model now implemented on `decode-refactor`. + +## Core Insight + +Charset detection remains heuristic and context dependent. In ambiguous cases, +there is no universally reliable algorithmic shortcut. Simpler, deterministic +behavior with explicit user controls is easier to reason about and maintain. + +## Practical Lessons + +1. UTF and 8-bit cross-decoding can both produce plausible-looking mojibake. +2. Header and detector signals can help, but neither is perfectly reliable. +3. Platform defaults differ (notably Windows shell contexts), so trial ordering + must avoid over-trusting local defaults. +4. Validation remains essential because successful decode does not imply + textual validity. +5. Historical observation from this refactor cycle: CP1252 has undefined byte + mappings (e.g., 0x81, 0x8d, 0x8f, 0x90, 0x9d), so it is not a total + byte-to-codepoint mapping. + +## Implemented Direction + +1. Simplified `decode()` internals to use `charsets.attempt_decodes(...)` + directly with a validator hook. +2. Removed bespoke decode helper pipeline in `decoders.py`. +3. Kept explicit HTTP header handling path first when `http_content_type` is + supplied. +4. Updated default trial order to prioritize user/context and UTF-8 ahead of + OS defaults: + - `UserSupplement` + - `'utf-8'` + - `FromInference` + - `OsDefault` + - `PythonDefault` +5. Preserved behavior that `decode()` is not gated by MIME inference in its + normal path. + +## Test/Doc Alignment Completed + +1. Test expectations updated for BOM-aware normalized charset names where + applicable (`utf-8-sig`). +2. Inference tests aligned to normalized codec naming (`iso8859-1` form). +3. Decoder behavior tests updated to reflect charset-driven decode semantics. +4. Doctests updated for the same behavior shifts. + +## Deferred Follow-ups + +1. Evaluate whether `charset_promotions` should remain. +2. Continue API-shape work for single-call metadata-rich decode + (`decode_inform` direction). +3. Keep detector-confidence refinements scoped separately from core decode + simplification. diff --git a/.auxiliary/notes/ideas.md b/.auxiliary/notes/ideas.md index 89e9e8e..e176061 100644 --- a/.auxiliary/notes/ideas.md +++ b/.auxiliary/notes/ideas.md @@ -1,6 +1,10 @@ -# Future Ideas for Detextive +# Future Ideas for Detextive (Post v3.0+) -## Postprocessors for v2.1+ +## Scope + +These are intentionally deferred until after v3.0 stabilization. + +## Post-v3.0+ Postprocessors Text postprocessing features to enhance decoded content: diff --git a/.auxiliary/notes/text-validation.md b/.auxiliary/notes/text-validation.md index 7db5c6d..9df87a7 100644 --- a/.auxiliary/notes/text-validation.md +++ b/.auxiliary/notes/text-validation.md @@ -96,7 +96,7 @@ But this is wrong because: All charsets can represent control characters: - UTF-8: `\x00`, `\x01`, `\x02`, etc. - CP1252: Control chars in 0x00-0x1F range -- ISO-8859-1: Fully permissive, decodes everything +- ISO-8859-1: Decodes a very broad range of byte values There's no charset-based reason to skip validation. diff --git a/.auxiliary/notes/windows-encoding.md b/.auxiliary/notes/windows-encoding.md deleted file mode 100644 index baf83a1..0000000 --- a/.auxiliary/notes/windows-encoding.md +++ /dev/null @@ -1,61 +0,0 @@ -# Windows Doctest Encoding Issue - -## Current Status - -Python 3.11 on Windows doctest failure: -``` -File "examples\basic-usage.rst", line 178, in BasicUsage -Failed example: - text -Expected: - 'Caf� \u2605' -Got: - 'Café ★' -``` - -## Analysis - -### Critical Clue -This test **previously passed** on Windows Python 3.10 and 3.11 before our charset validation fixes (commits 1aa0565, 2d98cec). - -### What Changed - -**Before our fixes:** -- Python 3.10 on Windows: `discover_os_charset_default()` used `sys.getfilesystemencoding()` → cp1252 -- Python 3.11 on Windows: `discover_os_charset_default()` used `locale.getencoding()` → cp1252 -- Charset detection confirmation tried OsDefault (cp1252) first -- Content `b'Caf\xc3\xa9 \xe2\x98\x85'` decoded with cp1252 → mojibake `'Caf� ★'` -- Mojibake matched doctest expectation → test passed (wrong result) - -**After our fixes (commit 2d98cec):** -- Charset detection confirmation excludes OsDefault -- Tries only UserSupplement and FromInference -- chardet correctly detects content as utf-8 -- Content decodes correctly as `'Café ★'` -- Doesn't match garbled expectation → test fails (correct result!) - -### Why Python 3.10 Still Passes - -Our fix in `_confirm_charset_detection()` works the same on both Python versions. Need to investigate why Python 3.10 still passes - possibly chardet behaves differently between versions? - -### Question - -**Should we fix the doctest expectation to match the correct output?** - -This seems straightforward, but: -1. Why did the broken output match the doctest in the first place? -2. Is the doctest file encoding declaration being respected on Windows? -3. Could this be a Sphinx/doctest encoding configuration issue? - -## Next Steps - -1. Check if file has correct encoding declaration (has `.. -*- coding: utf-8 -*-`) -2. Verify what Python 3.10 on Windows actually produces now -3. Consider if we need Windows-specific doctest handling -4. Update doctest expectation if appropriate - -## Related Files - -- `documentation/examples/basic-usage.rst` line 178 -- `sources/detextive/detectors.py` `_confirm_charset_detection()` -- Commits: 1aa0565 (MIME validation fix), 2d98cec (charset validation fix) From f2fe5ddddf9252b2fe1bf30365bd3fc83b9ce6e6 Mon Sep 17 00:00:00 2001 From: Eric McDonald Date: Thu, 12 Feb 2026 21:41:08 -0800 Subject: [PATCH 06/12] Remove charset promotions behavior and preserve detection confidence. Co-Authored-By: GPT-5 Codex --- .auxiliary/notes/confidence.md | 2 +- .auxiliary/notes/decode-refactor.md | 5 ++--- .../005-validation-behavior-configuration.rst | 4 +--- sources/detextive/core.py | 16 ---------------- sources/detextive/detectors.py | 4 ++-- 5 files changed, 6 insertions(+), 25 deletions(-) diff --git a/.auxiliary/notes/confidence.md b/.auxiliary/notes/confidence.md index 9cde9a2..2b3c3ac 100644 --- a/.auxiliary/notes/confidence.md +++ b/.auxiliary/notes/confidence.md @@ -225,7 +225,7 @@ BOMs (Byte Order Marks) provide near-certainty for UTF-8/UTF-16 detection regard ### Pure ASCII Small pure ASCII samples (like `b"Hello"`) get scaled down confidence, but: -- ASCII is promoted to UTF-8 via `charset_promotions` +- UTF-8 trial ordering still keeps decode behavior deterministic - Small ASCII content is cheap to validate - Erring on the side of validation is fine diff --git a/.auxiliary/notes/decode-refactor.md b/.auxiliary/notes/decode-refactor.md index f128aab..4cf2dd4 100644 --- a/.auxiliary/notes/decode-refactor.md +++ b/.auxiliary/notes/decode-refactor.md @@ -50,8 +50,7 @@ behavior with explicit user controls is easier to reason about and maintain. ## Deferred Follow-ups -1. Evaluate whether `charset_promotions` should remain. -2. Continue API-shape work for single-call metadata-rich decode +1. Continue API-shape work for single-call metadata-rich decode (`decode_inform` direction). -3. Keep detector-confidence refinements scoped separately from core decode +2. Keep detector-confidence refinements scoped separately from core decode simplification. diff --git a/documentation/architecture/decisions/005-validation-behavior-configuration.rst b/documentation/architecture/decisions/005-validation-behavior-configuration.rst index c522580..bb45299 100644 --- a/documentation/architecture/decisions/005-validation-behavior-configuration.rst +++ b/documentation/architecture/decisions/005-validation-behavior-configuration.rst @@ -81,7 +81,6 @@ configuration object. mimetype_detect: BehaviorTristate = BehaviorTristate.AsNeeded # Charset handling sophistication - charset_promotions: Mapping[str, str] = {'ascii': 'utf-8'} charset_trial_codecs: Sequence[str | CodecSpecifiers] = ( CodecSpecifiers.Inference, CodecSpecifiers.UserDefault) charset_trial_decode: BehaviorTristate = BehaviorTristate.AsNeeded @@ -94,7 +93,6 @@ configuration object. **Advanced Charset Handling:** -* **charset_promotions**: Mapping for upgrading detected charsets (e.g., ASCII→UTF-8) * **charset_trial_codecs**: Sequence of codecs to try during trial decoding * **CodecSpecifiers**: Enum for dynamic codec resolution (Inference, OsDefault, UserDefault) @@ -231,4 +229,4 @@ to provide complete control over validation execution and error handling: This decision provides the foundation for performance-aware and context-sensitive validation that addresses the rigid validation limitations of the v1.x functional -approach while maintaining backward compatibility through sensible defaults. \ No newline at end of file +approach while maintaining backward compatibility through sensible defaults. diff --git a/sources/detextive/core.py b/sources/detextive/core.py index 7079607..cf7cda6 100644 --- a/sources/detextive/core.py +++ b/sources/detextive/core.py @@ -25,12 +25,6 @@ from . import nomina as _nomina -_STANDARD_CHARSET_PROMOTIONS = ( - ( 'ascii', 'utf-8-sig' ), - ( 'utf-8', 'utf-8-sig' ), -) - - CHARSET_DEFAULT = 'utf-8' MIMETYPE_DEFAULT = 'application/octet-stream' @@ -80,16 +74,6 @@ class Behaviors( __.immut.DataclassObject ): DetectFailureActions, __.ddoc.Doc( ''' Action to take on charset detection failure. ''' ), ] = DetectFailureActions.Default - charset_promotions: __.typx.Annotated[ - __.cabc.Mapping[ str, str ], - __.ddoc.Doc( - ''' Which detected charsets to promote to other charsets. - - E.g., 7-bit ASCII to UTF-8. - ''' ), - ] = __.dcls.field( - default_factory = ( - lambda: __.immut.Dictionary( _STANDARD_CHARSET_PROMOTIONS ) ) ) mimetype_detect: __.typx.Annotated[ BehaviorTristate, __.ddoc.Doc( ''' When to detect MIME type from content. ''' ), diff --git a/sources/detextive/detectors.py b/sources/detextive/detectors.py index ef89984..592860b 100644 --- a/sources/detextive/detectors.py +++ b/sources/detextive/detectors.py @@ -207,7 +207,6 @@ def _confirm_charset_detection( # noqa: PLR0911 result = _normalize_charset_detection( content, behaviors, result ) if result.charset is None: return result # pragma: no cover charset, confidence = result.charset, result.confidence - charset = behaviors.charset_promotions.get( charset, charset ) if charset.startswith( 'utf-' ): behaviors_no_fallback = __.dcls.replace( behaviors, @@ -222,7 +221,6 @@ def _confirm_charset_detection( # noqa: PLR0911 confidence = confidence, location = location ) return _normalize_charset_detection( content, behaviors, result ) - result = _CharsetResult( charset = charset, confidence = confidence ) match behaviors.trial_decode: case _BehaviorTristate.Never: return result case _: # Shake out false positives, like 'MacRoman'. @@ -244,6 +242,8 @@ def _confirm_charset_detection( # noqa: PLR0911 location = location ) except _exceptions.ContentDecodeFailure: return result if charset == result_.charset: return result # pragma: no cover + result_ = _CharsetResult( + charset = result_.charset, confidence = confidence ) return _normalize_charset_detection( content, behaviors, result_ ) From 871297878b764d8818473249796ff702fa6370db Mon Sep 17 00:00:00 2001 From: Eric McDonald Date: Fri, 13 Feb 2026 08:15:58 -0800 Subject: [PATCH 07/12] Clarify default versus supplement semantics for decoding and inference. Co-Authored-By: GPT-5 Codex --- .../architecture/openspec/specs/api/design.md | 6 +--- .../examples/advanced-configuration.rst | 32 ++++++++++++++++--- sources/detextive/decoders.py | 10 +++--- sources/detextive/inference.py | 28 +++++++++++++--- sources/detextive/nomina.py | 11 ++++--- tests/test_000_detextive/test_500_decoders.py | 10 ++++++ 6 files changed, 76 insertions(+), 21 deletions(-) diff --git a/documentation/architecture/openspec/specs/api/design.md b/documentation/architecture/openspec/specs/api/design.md index 1e9dae5..ee4134a 100644 --- a/documentation/architecture/openspec/specs/api/design.md +++ b/documentation/architecture/openspec/specs/api/design.md @@ -265,18 +265,14 @@ def decode( content: Content, /, *, behaviors: Behaviors = BEHAVIORS_DEFAULT, profile: TextValidationProfile = PROFILE_TEXTUAL, - charset_default: str = CHARSET_DEFAULT, - mimetype_default: str = MIMETYPE_DEFAULT, http_content_type: __.Absential[ str ] = __.absent, location: __.Absential[ Location ] = __.absent, charset_supplement: __.Absential[ str ] = __.absent, - mimetype_supplement: __.Absential[ str ] = __.absent, ) -> str: ''' High-level bytes-to-text decoding with validation. Performs comprehensive detection, decoding, and validation - for robust text extraction from byte content. Supports - configurable default values for graceful degradation. + for robust text extraction from byte content. ''' ``` diff --git a/documentation/examples/advanced-configuration.rst b/documentation/examples/advanced-configuration.rst index 813f318..22f10b9 100644 --- a/documentation/examples/advanced-configuration.rst +++ b/documentation/examples/advanced-configuration.rst @@ -157,10 +157,16 @@ Use Path objects for precise location context: >>> mimetype in ('application/json', 'text/plain') # text/plain on Windows with python-magic-bin True -Default Value Handling +Default and Supplement Semantics ------------------------------------------------------------------------------- -Specify fallback values when detection confidence is insufficient: +Use the parameters with distinct intent: + +* ``*_supplement``: user-supplied hint to guide inference. +* ``*_default``: fallback value returned when inference cannot determine a + better result. + +Supplements guide inference but are not fallback return values: .. code-block:: python @@ -171,8 +177,26 @@ Specify fallback values when detection confidence is insufficient: mimetype_supplement = 'text/plain', charset_supplement = 'utf-8' ) - print( f"Result (with defaults): {mimetype}, {charset}" ) - # Output: Result (with defaults): text/plain, utf-8 + print( f"Result (with supplements): {mimetype}, {charset}" ) + +Defaults provide fallback return values when detectors do not produce a result: + +.. code-block:: python + + behaviors = detextive.Behaviors( + charset_detectors_order = ( 'nonexistent-detector', ), + mimetype_detectors_order = ( 'nonexistent-detector', ), + charset_on_detect_failure = detextive.DetectFailureActions.Default, + mimetype_on_detect_failure = detextive.DetectFailureActions.Default ) + + mimetype, charset = detextive.infer_mimetype_charset( + b'\x80\x81\x82', + behaviors = behaviors, + mimetype_default = 'text/plain', + charset_default = 'latin-1' ) + + print( f"Fallback result: {mimetype}, {charset}" ) + # Output: Fallback result: text/plain, latin-1 Text Validation Profiles =============================================================================== diff --git a/sources/detextive/decoders.py b/sources/detextive/decoders.py index 8d379c0..67716d0 100644 --- a/sources/detextive/decoders.py +++ b/sources/detextive/decoders.py @@ -31,7 +31,6 @@ from .core import ( # isort: skip BEHAVIORS_DEFAULT as _BEHAVIORS_DEFAULT, - CHARSET_DEFAULT as _CHARSET_DEFAULT, BehaviorTristate as _BehaviorTristate, BehaviorsArgument as _BehaviorsArgument, CharsetResult as _CharsetResult, @@ -43,12 +42,16 @@ def decode( # noqa: PLR0913 content: _nomina.Content, /, *, behaviors: _BehaviorsArgument = _BEHAVIORS_DEFAULT, profile: _validation.ProfileArgument = _validation.PROFILE_TEXTUAL, - charset_default: _nomina.CharsetDefaultArgument = _CHARSET_DEFAULT, http_content_type: _nomina.HttpContentTypeArgument = __.absent, location: _nomina.LocationArgument = __.absent, charset_supplement: _nomina.CharsetSupplementArgument = __.absent, ) -> str: - ''' Decodes bytes array to Unicode text. ''' + ''' Decodes bytes array to Unicode text. + + Uses trial decoding and validation; does not provide default-return + semantics. The ``charset_supplement`` parameter is a trial hint and + not a fallback return value. + ''' if content == b'': return '' charset: __.Absential[ str ] = __.absent result: __.Absential[ _CharsetResult ] = __.absent @@ -65,7 +68,6 @@ def decode( # noqa: PLR0913 result = _detectors.detect_charset_confidence( content, behaviors = behaviors_, - default = charset_default, supplement = charset_supplement, location = location ) if ( result.charset diff --git a/sources/detextive/inference.py b/sources/detextive/inference.py index 7d1cff4..2ae6dbf 100644 --- a/sources/detextive/inference.py +++ b/sources/detextive/inference.py @@ -50,7 +50,12 @@ def infer_charset( # noqa: PLR0913 mimetype_supplement: _nomina.MimetypeSupplementArgument = __.absent, location: _nomina.LocationArgument = __.absent, ) -> __.typx.Optional[ str ]: - ''' Infers charset through various means. ''' + ''' Infers charset through various means. + + ``charset_default`` is the returned fallback when inference cannot + determine another charset. ``charset_supplement`` is a user-supplied + hint used during inference/validation. + ''' result = infer_charset_confidence( content, behaviors = behaviors, @@ -71,7 +76,12 @@ def infer_charset_confidence( # noqa: PLR0913 mimetype_supplement: _nomina.MimetypeSupplementArgument = __.absent, location: _nomina.LocationArgument = __.absent, ) -> _CharsetResult: - ''' Infers charset with confidence level through various means. ''' + ''' Infers charset with confidence level through various means. + + ``charset_default`` is the returned fallback when inference cannot + determine another charset. ``charset_supplement`` is a user-supplied + hint used during inference/validation. + ''' if content == b'': return _CharsetResult( charset = 'utf-8', confidence = 1.0 ) should_parse, should_detect = ( @@ -107,7 +117,12 @@ def infer_mimetype_charset( # noqa: PLR0913 charset_supplement: _nomina.CharsetSupplementArgument = __.absent, mimetype_supplement: _nomina.MimetypeSupplementArgument = __.absent, ) -> tuple[ str, __.typx.Optional[ str ] ]: - ''' Infers MIME type and charset through various means. ''' + ''' Infers MIME type and charset through various means. + + ``*_default`` values are returned fallbacks on inference failure. + ``*_supplement`` values are user-supplied hints used to guide + inference before fallback behavior is applied. + ''' mimetype_result, charset_result = ( infer_mimetype_charset_confidence( content, @@ -131,7 +146,12 @@ def infer_mimetype_charset_confidence( # noqa: PLR0913 charset_supplement: _nomina.CharsetSupplementArgument = __.absent, mimetype_supplement: _nomina.MimetypeSupplementArgument = __.absent, ) -> tuple[ _MimetypeResult, _CharsetResult ]: - ''' Infers MIME type and charset through various means. ''' + ''' Infers MIME type and charset through various means. + + ``*_default`` values are returned fallbacks on inference failure. + ``*_supplement`` values are user-supplied hints used to guide + inference before fallback behavior is applied. + ''' should_parse, should_detect_charset = ( _determine_parse_detect( behaviors.charset_detect ) ) should_parse, should_detect_mimetype = ( diff --git a/sources/detextive/nomina.py b/sources/detextive/nomina.py index d47aaee..9db03f1 100644 --- a/sources/detextive/nomina.py +++ b/sources/detextive/nomina.py @@ -40,12 +40,14 @@ ] CharsetDefaultArgument: __.typx.TypeAlias = __.typx.Annotated[ str, - __.ddoc.Doc( ''' Default character set to use when detection fails. ''' ), + __.ddoc.Doc( + ''' Fallback character set returned on inference/detection + failure. ''' ), ] CharsetSupplementArgument: __.typx.TypeAlias = __.typx.Annotated[ __.Absential[ str ], __.ddoc.Doc( - ''' Supplemental character set to use for trial decodes. ''' ), + ''' User-supplied character set hint for trial decode attempts. ''' ), ] HttpContentTypeArgument: __.typx.TypeAlias = __.typx.Annotated[ __.Absential[ str ], @@ -62,9 +64,10 @@ ] MimetypeDefaultArgument: __.typx.TypeAlias = __.typx.Annotated[ str, - __.ddoc.Doc( ''' Default MIME type to use when detection fails. ''' ), + __.ddoc.Doc( + ''' Fallback MIME type returned on inference/detection failure. ''' ), ] MimetypeSupplementArgument: __.typx.TypeAlias = __.typx.Annotated[ __.Absential[ str ], - __.ddoc.Doc( ''' Supplemental MIME type to use for inference. ''' ), + __.ddoc.Doc( ''' User-supplied MIME type hint for inference. ''' ), ] diff --git a/tests/test_000_detextive/test_500_decoders.py b/tests/test_000_detextive/test_500_decoders.py index c769524..1595c05 100644 --- a/tests/test_000_detextive/test_500_decoders.py +++ b/tests/test_000_detextive/test_500_decoders.py @@ -89,6 +89,16 @@ def test_200_decode_empty_content_returns_empty_string( ): assert result == '' +def test_210_decode_no_default_fallback_on_detection_failure( ): + ''' Decode does not use inference-style default charset fallbacks. ''' + behaviors = detextive.Behaviors( + charset_detectors_order = ( 'nonexistent-detector', ), + charset_on_detect_failure = detextive.DetectFailureActions.Default, + trial_codecs = ( 'utf-8', ) ) + with pytest.raises( detextive.exceptions.ContentDecodeFailure ): + _decoders.decode( b'\xa0', behaviors = behaviors ) + + # Error Handling Tests (400-499): Exception scenarios and recovery def test_420_validation_failure_handling( ): From f2ae3833e57e9746569255018a94b1cb13ad8ccc Mon Sep 17 00:00:00 2001 From: Eric McDonald Date: Fri, 13 Feb 2026 09:47:56 -0800 Subject: [PATCH 08/12] Add decode_inform API with textual MIME metadata. Co-Authored-By: GPT-5 Codex --- .auxiliary/configuration/vulturefood.py | 2 + sources/detextive/decoders.py | 176 +++++++++++++++--- tests/test_000_detextive/test_120_core.py | 2 +- tests/test_000_detextive/test_500_decoders.py | 49 +++++ 4 files changed, 197 insertions(+), 32 deletions(-) diff --git a/.auxiliary/configuration/vulturefood.py b/.auxiliary/configuration/vulturefood.py index 97e516c..324e7e9 100644 --- a/.auxiliary/configuration/vulturefood.py +++ b/.auxiliary/configuration/vulturefood.py @@ -10,9 +10,11 @@ # Refactor 2.0 - public API functions not yet exposed in __init__.py detect_charset # public API function detect_mimetype # public API function +decode_inform # public API function infer_charset # public API function infer_mimetype_charset # public API function is_valid_text # public API function +DecodeInformResult # public API result type # Exception classes for public API TextualMimetypeInvalidity # exception class for public API diff --git a/sources/detextive/decoders.py b/sources/detextive/decoders.py index 67716d0..dbc8ca0 100644 --- a/sources/detextive/decoders.py +++ b/sources/detextive/decoders.py @@ -26,6 +26,8 @@ from . import detectors as _detectors from . import exceptions as _exceptions from . import inference as _inference +from . import lineseparators as _lineseparators +from . import mimetypes as _mimetypes from . import nomina as _nomina from . import validation as _validation @@ -35,9 +37,31 @@ BehaviorsArgument as _BehaviorsArgument, CharsetResult as _CharsetResult, CodecSpecifiers as _CodecSpecifiers, + MimetypeResult as _MimetypeResult, ) +_MIMETYPE_DEFAULT_TEXTUAL = 'text/plain' + + +class DecodeInformResult( __.immut.DataclassObject ): + ''' Decoded text with supplemental inference metadata. ''' + + text: __.typx.Annotated[ + str, __.ddoc.Doc( ''' Decoded text content. ''' ) + ] + charset: __.typx.Annotated[ + _CharsetResult, __.ddoc.Doc( ''' Charset used for decoding. ''' ) + ] + mimetype: __.typx.Annotated[ + _MimetypeResult, __.ddoc.Doc( ''' Inferred MIME type metadata. ''' ) + ] + linesep: __.typx.Annotated[ + __.typx.Optional[ _lineseparators.LineSeparators ], + __.ddoc.Doc( ''' Detected line separator from content sample. ''' ), + ] + + def decode( # noqa: PLR0913 content: _nomina.Content, /, *, behaviors: _BehaviorsArgument = _BEHAVIORS_DEFAULT, @@ -52,15 +76,92 @@ def decode( # noqa: PLR0913 semantics. The ``charset_supplement`` parameter is a trial hint and not a fallback return value. ''' - if content == b'': return '' - charset: __.Absential[ str ] = __.absent + _, httpct_charset = _parse_http_content_type( http_content_type ) + return _decode_content_charset_result( + content, behaviors, profile, + httpct_charset = httpct_charset, + location = location, + charset_supplement = charset_supplement )[ 0 ] + + +def decode_inform( # noqa: PLR0913 + content: _nomina.Content, /, *, + behaviors: _BehaviorsArgument = _BEHAVIORS_DEFAULT, + profile: _validation.ProfileArgument = _validation.PROFILE_TEXTUAL, + mimetype_default: _nomina.MimetypeDefaultArgument = ( + _MIMETYPE_DEFAULT_TEXTUAL ), + http_content_type: _nomina.HttpContentTypeArgument = __.absent, + location: _nomina.LocationArgument = __.absent, + charset_supplement: _nomina.CharsetSupplementArgument = __.absent, +) -> DecodeInformResult: + ''' Decodes bytes and returns supplemental inference metadata. ''' + httpct_mimetype, httpct_charset = ( + _parse_http_content_type( http_content_type ) ) + text, charset_result = _decode_content_charset_result( + content, behaviors, profile, + httpct_charset = httpct_charset, + location = location, + charset_supplement = charset_supplement ) + mimetype_result = _infer_mimetype( + content, behaviors, + mimetype_default = mimetype_default, + httpct_mimetype = httpct_mimetype, + location = location, + charset = charset_result.charset ) + linesep = _lineseparators.LineSeparators.detect_bytes( content ) + return DecodeInformResult( + text = text, + charset = charset_result, + mimetype = mimetype_result, + linesep = linesep ) + + +def _attempt_decode_http_content_type( + content: _nomina.Content, + behaviors: _BehaviorsArgument, + profile: _validation.ProfileArgument, /, *, + httpct_charset: __.Absential[ __.typx.Optional[ str ] ], + location: _nomina.LocationArgument, +) -> __.Absential[ tuple[ str, _CharsetResult ] ]: result: __.Absential[ _CharsetResult ] = __.absent - text: __.Absential[ str ] = __.absent - if not __.is_absent( http_content_type ): - text = _attempt_decode_http_content_type( - content, http_content_type, + error = _exceptions.ContentDecodeImpossibility( location = location ) + if httpct_charset is None: raise error + if __.is_absent( httpct_charset ): return __.absent + behaviors_ = __.dcls.replace( + behaviors, trial_codecs = ( _CodecSpecifiers.FromInference, ) ) + try: + text, result = _charsets.attempt_decodes( + content, + behaviors = behaviors_, + inference = httpct_charset, + location = location ) + except _exceptions.ContentDecodeFailure: return __.absent + # Allow other errors propagate. + if not __.is_absent( text ) and not __.is_absent( result ): + text = _validate_text( + text, result.confidence, behaviors = behaviors, profile = profile, location = location ) - if not __.is_absent( text ): return text + return text, result + return __.absent + + +def _decode_content_charset_result( # noqa: PLR0913 + content: _nomina.Content, + behaviors: _BehaviorsArgument, + profile: _validation.ProfileArgument, /, *, + httpct_charset: __.Absential[ __.typx.Optional[ str ] ], + location: _nomina.LocationArgument, + charset_supplement: _nomina.CharsetSupplementArgument, +) -> tuple[ str, _CharsetResult ]: + if content == b'': + return '', _CharsetResult( charset = 'utf-8', confidence = 1.0 ) + charset: __.Absential[ str ] = __.absent + result: __.Absential[ _CharsetResult ] = __.absent + httpct_result: __.Absential[ tuple[ str, _CharsetResult ] ] = __.absent + httpct_result = _attempt_decode_http_content_type( + content, behaviors, profile, + httpct_charset = httpct_charset, location = location ) + if not __.is_absent( httpct_result ): return httpct_result if __.is_absent( result ): behaviors_ = __.dcls.replace( behaviors, trial_decode = _BehaviorTristate.Never ) @@ -84,35 +185,48 @@ def decode( # noqa: PLR0913 inference = charset, supplement = charset_supplement, location = location, - validator = validator )[ 0 ] + validator = validator ) -def _attempt_decode_http_content_type( +def _infer_mimetype( # noqa: PLR0913 content: _nomina.Content, - http_content_type: str, /, *, - behaviors: _BehaviorsArgument, - profile: _validation.ProfileArgument, + behaviors: _BehaviorsArgument, /, *, + mimetype_default: _nomina.MimetypeDefaultArgument, + httpct_mimetype: __.Absential[ str ], location: _nomina.LocationArgument, -) -> __.Absential[ str ]: - charset: __.Absential[ __.typx.Optional[ str ] ] = __.absent - result: __.Absential[ _CharsetResult ] = __.absent - error = _exceptions.ContentDecodeImpossibility( location = location ) - _, charset = _inference.parse_http_content_type( http_content_type ) - if charset is None: raise error - if __.is_absent( charset ): return __.absent - behaviors_ = __.dcls.replace( - behaviors, trial_codecs = ( _CodecSpecifiers.FromInference, ) ) - try: - text, result = _charsets.attempt_decodes( + charset: __.typx.Optional[ str ], +) -> _MimetypeResult: + charset_ = __.absent if charset is None else charset + if ( not __.is_absent( httpct_mimetype ) + and _mimetypes.is_textual_mimetype( httpct_mimetype ) + ): + return _MimetypeResult( mimetype = httpct_mimetype, confidence = 0.9 ) + result: __.Absential[ _MimetypeResult ] = __.absent + if not __.is_absent( location ): + mimetype = _mimetypes.mimetype_from_location( location ) + if ( not __.is_absent( mimetype ) + and _mimetypes.is_textual_mimetype( mimetype ) + ): + return _MimetypeResult( mimetype = mimetype, confidence = 0.9 ) + if behaviors.mimetype_detect is not _BehaviorTristate.Never: + result = _detectors.detect_mimetype_confidence( content, - behaviors = behaviors_, inference = charset, location = location ) - except _exceptions.ContentDecodeFailure: return __.absent - # Allow other errors propagate. - if not __.is_absent( text ) and not __.is_absent( result ): - return _validate_text( - text, result.confidence, - behaviors = behaviors, profile = profile, location = location ) - return __.absent + behaviors = behaviors, + default = mimetype_default, + charset = charset_, + location = location ) + if __.is_absent( result ): + return _MimetypeResult( mimetype = mimetype_default, confidence = 1.0 ) + if _mimetypes.is_textual_mimetype( result.mimetype ): return result + return _MimetypeResult( mimetype = mimetype_default, confidence = 1.0 ) + + +def _parse_http_content_type( + http_content_type: _nomina.HttpContentTypeArgument +) -> tuple[ __.Absential[ str ], __.Absential[ __.typx.Optional[ str ] ] ]: + if __.is_absent( http_content_type ): + return __.absent, __.absent + return _inference.parse_http_content_type( http_content_type ) def _validate_text( diff --git a/tests/test_000_detextive/test_120_core.py b/tests/test_000_detextive/test_120_core.py index d8d54cc..d06c33b 100644 --- a/tests/test_000_detextive/test_120_core.py +++ b/tests/test_000_detextive/test_120_core.py @@ -31,4 +31,4 @@ def test_000_imports( ): assert hasattr( _core, 'Behaviors' ) assert hasattr( _core, 'BehaviorTristate' ) assert hasattr( _core, 'CodecSpecifiers' ) - assert hasattr( _core, 'DetectFailureActions' ) \ No newline at end of file + assert hasattr( _core, 'DetectFailureActions' ) diff --git a/tests/test_000_detextive/test_500_decoders.py b/tests/test_000_detextive/test_500_decoders.py index 1595c05..5cf923c 100644 --- a/tests/test_000_detextive/test_500_decoders.py +++ b/tests/test_000_detextive/test_500_decoders.py @@ -36,6 +36,8 @@ def test_000_imports( ): ''' Decode function is accessible from main module. ''' assert hasattr( detextive, 'decode' ) + assert hasattr( detextive, 'decode_inform' ) + assert hasattr( _decoders, 'DecodeInformResult' ) # High-Level Decode Tests (100-199): decode function with various parameters @@ -67,6 +69,53 @@ def test_110_decode_inference_failure_fallback_to_supplement( ): assert result == 'Hello, world!' +def test_120_decode_inform_reports_decode_and_metadata( ): + ''' decode_inform returns text, charset, mimetype, and linesep. ''' + content = b'Hello,\nworld!\n' + result = _decoders.decode_inform( content, location = 'test.txt' ) + assert result.text == 'Hello,\nworld!\n' + assert result.charset.charset is not None + assert result.mimetype.mimetype == 'text/plain' + assert result.linesep == detextive.LineSeparators.LF + + +def test_130_decode_inform_honors_http_content_type( ): + ''' decode_inform prefers HTTP Content-Type metadata when available. ''' + content = b'{"message": "hello"}' + result = _decoders.decode_inform( + content, + http_content_type = 'application/json; charset=utf-8' ) + assert result.text == '{"message": "hello"}' + assert result.charset.charset == 'utf-8-sig' + assert result.mimetype.mimetype == 'application/json' + + +def test_140_decode_inform_empty_content( ): + ''' decode_inform returns deterministic metadata for empty content. ''' + result = _decoders.decode_inform( b'' ) + assert result.text == '' + assert result.charset.charset == 'utf-8' + assert result.charset.confidence == 1.0 + assert result.mimetype.mimetype == 'text/plain' + assert result.linesep is None + + +def test_150_decode_inform_mimetype_inference_fallback( ): + ''' decode_inform falls back to text/plain on inference absence. ''' + behaviors = detextive.Behaviors( + mimetype_detect = detextive.BehaviorTristate.Never ) + result = _decoders.decode_inform( b'hello', behaviors = behaviors ) + assert result.mimetype.mimetype == 'text/plain' + + +def test_160_decode_inform_non_textual_mimetype_coerced( ): + ''' decode_inform coerces non-textual mimetype to text/plain. ''' + result = _decoders.decode_inform( + b'hello', + location = 'artifact.png' ) + assert result.mimetype.mimetype == 'text/plain' + + def test_190_decode_validation_profile_parameters( ): ''' Validation profile parameters are applied correctly. ''' content = b'\x00\x01\x02\xff' # Binary content that fails text validation From 2fbad9168da72cafaeb5ae25e0c06b8eae00896a Mon Sep 17 00:00:00 2001 From: Eric McDonald Date: Fri, 13 Feb 2026 16:52:18 -0800 Subject: [PATCH 09/12] Improve v3 coverage plan and decoding test coverage. Co-Authored-By: GPT-5 Codex --- .../architecture/testplans/index.rst | 1 + .../testplans/v3-coverage-strategy.rst | 79 +++++++++++++++++++ documentation/examples/basic-usage.rst | 28 +++++++ sources/detextive/decoders.py | 38 ++++----- .../test_000_detextive/test_310_detectors.py | 15 +++- tests/test_000_detextive/test_500_decoders.py | 34 +++++++- 6 files changed, 170 insertions(+), 25 deletions(-) create mode 100644 documentation/architecture/testplans/v3-coverage-strategy.rst diff --git a/documentation/architecture/testplans/index.rst b/documentation/architecture/testplans/index.rst index 57b89b5..c1029e4 100644 --- a/documentation/architecture/testplans/index.rst +++ b/documentation/architecture/testplans/index.rst @@ -25,5 +25,6 @@ Test Plans :maxdepth: 2 summary + v3-coverage-strategy v2-test-suite content-patterns diff --git a/documentation/architecture/testplans/v3-coverage-strategy.rst b/documentation/architecture/testplans/v3-coverage-strategy.rst new file mode 100644 index 0000000..49c30ff --- /dev/null +++ b/documentation/architecture/testplans/v3-coverage-strategy.rst @@ -0,0 +1,79 @@ +.. vim: set fileencoding=utf-8: +.. -*- coding: utf-8 -*- +.. +--------------------------------------------------------------------------+ + | | + | Licensed under the Apache License, Version 2.0 (the "License"); | + | you may not use this file except in compliance with the License. | + | You may obtain a copy of the License at | + | | + | http://www.apache.org/licenses/LICENSE-2.0 | + | | + | Unless required by applicable law or agreed to in writing, software | + | distributed under the License is distributed on an "AS IS" BASIS, | + | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | + | See the License for the specific language governing permissions and | + | limitations under the License. | + | | + +--------------------------------------------------------------------------+ + + +******************************************************************************* +V3 Coverage Strategy +******************************************************************************* + +Goal +=============================================================================== + +Maintain full line/branch coverage for v3 behavior while keeping +tests readable and representative of user workflows. + +Testing split +=============================================================================== + +Common usage paths (user-facing) +------------------------------------------------------------------------------- + +- Prefer doctests in ``documentation/examples``. +- Cover stable API entry points and common data flows. +- Keep examples readable and copy/paste friendly. + +Edge and failure paths (engineering-facing) +------------------------------------------------------------------------------- + +- Prefer pytest in ``tests/test_000_detextive``. +- Cover boundary and error behaviors. +- Use dependency injection surfaces (detector registries, behavior arguments) + instead of monkey-patching internals. + +Component focus areas +=============================================================================== + +Detection and Inference +------------------------------------------------------------------------------- + +- Detector ordering and fallback behavior. +- Header/location context handling. +- Default-vs-error policy behavior with confidence expectations. + +Decoding +------------------------------------------------------------------------------- + +- Trial decode sequencing and validator interactions. +- HTTP Content-Type charset behavior, including non-textual rejection. +- ``decode_inform`` textual MIME guarantees and metadata shape. + +Validation +------------------------------------------------------------------------------- + +- Profile behavior for reasonable vs unreasonable text. +- Confidence-threshold behavior and validation gating. + +Coverage workflow +=============================================================================== + +1. Run ``hatch --env develop run testers`` for the combined coverage view. +2. Use ``hatch --env develop run coverage report --show-missing`` to identify + exact uncovered lines/branches. +3. Add/adjust doctests for common user-facing paths. +4. Add/adjust pytest cases for edge/error branches. +5. Re-run ``testers`` and verify coverage remains at target. diff --git a/documentation/examples/basic-usage.rst b/documentation/examples/basic-usage.rst index b726eeb..f9e3aed 100644 --- a/documentation/examples/basic-usage.rst +++ b/documentation/examples/basic-usage.rst @@ -189,6 +189,34 @@ Location context improves decoding decisions: >>> text 'Sample content for analysis' +Combined Decode and Metadata +------------------------------------------------------------------------------- + +The ``decode_inform`` function returns decoded text with charset/MIME metadata: + +.. doctest:: BasicUsage + + >>> import detextive + >>> result = detextive.decode_inform( b'Hello, world!\n', location = 'notes.txt' ) + >>> result.text + 'Hello, world!\n' + >>> result.mimetype.mimetype + 'text/plain' + >>> result.charset.charset + 'utf-8-sig' + >>> result.linesep + + +HTTP header context is honored when textual: + +.. doctest:: BasicUsage + + >>> result = detextive.decode_inform( + ... b'{"message":"ok"}', + ... http_content_type = 'application/json; charset=utf-8' ) + >>> result.mimetype.mimetype + 'application/json' + Content Validation =============================================================================== diff --git a/sources/detextive/decoders.py b/sources/detextive/decoders.py index dbc8ca0..fdcbcac 100644 --- a/sources/detextive/decoders.py +++ b/sources/detextive/decoders.py @@ -123,7 +123,6 @@ def _attempt_decode_http_content_type( httpct_charset: __.Absential[ __.typx.Optional[ str ] ], location: _nomina.LocationArgument, ) -> __.Absential[ tuple[ str, _CharsetResult ] ]: - result: __.Absential[ _CharsetResult ] = __.absent error = _exceptions.ContentDecodeImpossibility( location = location ) if httpct_charset is None: raise error if __.is_absent( httpct_charset ): return __.absent @@ -136,13 +135,10 @@ def _attempt_decode_http_content_type( inference = httpct_charset, location = location ) except _exceptions.ContentDecodeFailure: return __.absent - # Allow other errors propagate. - if not __.is_absent( text ) and not __.is_absent( result ): - text = _validate_text( - text, result.confidence, - behaviors = behaviors, profile = profile, location = location ) - return text, result - return __.absent + _validate_text( + text, result.confidence, + behaviors = behaviors, profile = profile, location = location ) + return text, result def _decode_content_charset_result( # noqa: PLR0913 @@ -162,18 +158,17 @@ def _decode_content_charset_result( # noqa: PLR0913 content, behaviors, profile, httpct_charset = httpct_charset, location = location ) if not __.is_absent( httpct_result ): return httpct_result - if __.is_absent( result ): - behaviors_ = __.dcls.replace( - behaviors, trial_decode = _BehaviorTristate.Never ) - with __.ctxl.suppress( _exceptions.CharsetDetectFailure ): - result = _detectors.detect_charset_confidence( - content, - behaviors = behaviors_, - supplement = charset_supplement, - location = location ) - if ( result.charset - and result.confidence >= behaviors.trial_decode_confidence - ): charset = result.charset + behaviors_ = __.dcls.replace( + behaviors, trial_decode = _BehaviorTristate.Never ) + with __.ctxl.suppress( _exceptions.CharsetDetectFailure ): + result = _detectors.detect_charset_confidence( + content, + behaviors = behaviors_, + supplement = charset_supplement, + location = location ) + if ( result.charset + and result.confidence >= behaviors.trial_decode_confidence + ): charset = result.charset validator = __.funct.partial( _validate_text_in_decode_attempt, behaviors = behaviors, @@ -234,7 +229,7 @@ def _validate_text( behaviors: _BehaviorsArgument, profile: _validation.ProfileArgument, location: _nomina.LocationArgument, -) -> str: +) -> None: error = _exceptions.TextInvalidity( location = location ) should_validate = False match behaviors.text_validate: @@ -244,7 +239,6 @@ def _validate_text( should_validate = confidence < behaviors.text_validate_confidence case _BehaviorTristate.Never: pass if should_validate and not profile( text ): raise error - return text def _validate_text_in_decode_attempt( diff --git a/tests/test_000_detextive/test_310_detectors.py b/tests/test_000_detextive/test_310_detectors.py index 1990469..5f13de1 100644 --- a/tests/test_000_detextive/test_310_detectors.py +++ b/tests/test_000_detextive/test_310_detectors.py @@ -295,6 +295,19 @@ def test_330_detect_mimetype_decode_failure_error_behavior( ): behaviors = behaviors, charset = 'utf-8' ) +def test_335_detect_mimetype_trial_decode_never_error_behavior( ): + ''' MIME type detection raises when trial decode is disabled. ''' + behaviors = detextive.Behaviors( + mimetype_detectors_order = ( 'nonexistent-detector', ), + trial_decode = detextive.BehaviorTristate.Never, + mimetype_on_detect_failure = detextive.DetectFailureActions.Error ) + with pytest.raises( detextive.exceptions.MimetypeDetectFailure ): + detextive.detect_mimetype_confidence( + b'test content', + behaviors = behaviors, + charset = 'utf-8' ) + + def test_340_detect_mimetype_text_validation_never( ): ''' MIME type detection respects text validation disabled setting. ''' behaviors = detextive.Behaviors( @@ -402,4 +415,4 @@ def test_600_python_magic_vs_python_magic_bin( ): assert result_puremagic is not None assert result_magic is not None assert result_puremagic.confidence >= 0.0 - assert result_magic.confidence >= 0.0 \ No newline at end of file + assert result_magic.confidence >= 0.0 diff --git a/tests/test_000_detextive/test_500_decoders.py b/tests/test_000_detextive/test_500_decoders.py index 5cf923c..4cc6e71 100644 --- a/tests/test_000_detextive/test_500_decoders.py +++ b/tests/test_000_detextive/test_500_decoders.py @@ -25,6 +25,7 @@ import detextive import detextive.decoders as _decoders +import detextive.detectors as _detectors from .patterns import ( EMPTY_CONTENT, @@ -101,7 +102,7 @@ def test_140_decode_inform_empty_content( ): def test_150_decode_inform_mimetype_inference_fallback( ): - ''' decode_inform falls back to text/plain on inference absence. ''' + ''' Falls back to text/plain when MIME inference is unavailable. ''' behaviors = detextive.Behaviors( mimetype_detect = detextive.BehaviorTristate.Never ) result = _decoders.decode_inform( b'hello', behaviors = behaviors ) @@ -109,13 +110,42 @@ def test_150_decode_inform_mimetype_inference_fallback( ): def test_160_decode_inform_non_textual_mimetype_coerced( ): - ''' decode_inform coerces non-textual mimetype to text/plain. ''' + ''' Coerces non-textual location MIME to text/plain. ''' result = _decoders.decode_inform( b'hello', location = 'artifact.png' ) assert result.mimetype.mimetype == 'text/plain' +def test_170_decode_inform_non_textual_http_header_rejected( ): + ''' Rejects non-textual HTTP Content-Type values with charset. ''' + with pytest.raises( detextive.exceptions.ContentDecodeImpossibility ): + _decoders.decode_inform( + b'hello', + http_content_type = 'image/png; charset=utf-8' ) + + +def test_180_decode_inform_header_charset_fallback_to_trials( ): + ''' Falls back to standard decode trials when HTTP charset decode fails.''' + result = _decoders.decode_inform( + b'Caf\xc3\xa9', + http_content_type = 'text/plain; charset=ascii' ) + assert result.text == 'Café' + + +def test_185_decode_inform_detector_non_textual_coerced_to_default( ): + ''' Coerces non-textual detector MIME result to textual default. ''' + detector_name = 'test-decode-inform-image-png' + def mimetype_png_detector( content, behaviors ): + return detextive.core.MimetypeResult( + mimetype = 'image/png', confidence = 0.9 ) + _detectors.mimetype_detectors[ detector_name ] = mimetype_png_detector + behaviors = detextive.Behaviors( + mimetype_detectors_order = ( detector_name, ) ) + result = _decoders.decode_inform( b'hello', behaviors = behaviors ) + assert result.mimetype.mimetype == 'text/plain' + + def test_190_decode_validation_profile_parameters( ): ''' Validation profile parameters are applied correctly. ''' content = b'\x00\x01\x02\xff' # Binary content that fails text validation From 4636cb9caf031038abdb89f8a2223ae3d18c8129 Mon Sep 17 00:00:00 2001 From: Eric McDonald Date: Fri, 13 Feb 2026 18:46:24 -0800 Subject: [PATCH 10/12] Honor decode-attempt validation confidence threshold. Propagate decode-attempt confidence into text validation gating. Add tests for both above-threshold skip and below-threshold validation behavior. Co-Authored-By: Codex --- sources/detextive/decoders.py | 2 +- tests/test_000_detextive/test_500_decoders.py | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/sources/detextive/decoders.py b/sources/detextive/decoders.py index fdcbcac..661be48 100644 --- a/sources/detextive/decoders.py +++ b/sources/detextive/decoders.py @@ -248,7 +248,7 @@ def _validate_text_in_decode_attempt( location: _nomina.LocationArgument, ) -> None: _validate_text( - text, 0.0, + text, result.confidence, behaviors = behaviors, profile = profile, location = location ) diff --git a/tests/test_000_detextive/test_500_decoders.py b/tests/test_000_detextive/test_500_decoders.py index 4cc6e71..88d1f2e 100644 --- a/tests/test_000_detextive/test_500_decoders.py +++ b/tests/test_000_detextive/test_500_decoders.py @@ -160,6 +160,32 @@ def test_190_decode_validation_profile_parameters( ): assert text is not None # Should succeed when validation is disabled +def test_195_decode_attempts_skip_validation_above_confidence_threshold( ): + ''' As-needed validation skips high-confidence decode attempts. ''' + content = b'Text with\x00null bytes' + behaviors = detextive.Behaviors( + charset_detectors_order = ( 'nonexistent-detector', ), + charset_on_detect_failure = detextive.DetectFailureActions.Error, + bytes_quantity_confidence_divisor = 1, + text_validate = detextive.BehaviorTristate.AsNeeded, + text_validate_confidence = 0.8 ) + text = _decoders.decode( content, behaviors = behaviors ) + assert text == 'Text with\x00null bytes' + + +def test_196_decode_attempts_validate_below_confidence_threshold( ): + ''' As-needed validation runs for low-confidence decode attempts. ''' + content = b'Text with\x00null bytes' + behaviors = detextive.Behaviors( + charset_detectors_order = ( 'nonexistent-detector', ), + charset_on_detect_failure = detextive.DetectFailureActions.Error, + bytes_quantity_confidence_divisor = 10_000, + text_validate = detextive.BehaviorTristate.AsNeeded, + text_validate_confidence = 0.8 ) + with pytest.raises( detextive.exceptions.ContentDecodeFailure ): + _decoders.decode( content, behaviors = behaviors ) + + # Default Parameter Tests (200-299): Custom default values and behaviors def test_200_decode_empty_content_returns_empty_string( ): From b57fe985ef132abc8a266ae7ae3507a0a5418a64 Mon Sep 17 00:00:00 2001 From: Eric McDonald Date: Fri, 13 Feb 2026 18:59:49 -0800 Subject: [PATCH 11/12] Refresh architecture docs and restore conservative decode validation. Update architecture summary and validation decision documentation for current v3 behavior. Restore conservative decode-attempt text validation confidence handling and remove threshold-gating tests. Co-Authored-By: Codex --- .../005-validation-behavior-configuration.rst | 68 ++--- documentation/architecture/filesystem.rst | 2 +- documentation/architecture/summary.rst | 232 +++++++----------- sources/detextive/decoders.py | 2 +- tests/test_000_detextive/test_500_decoders.py | 26 -- 5 files changed, 133 insertions(+), 197 deletions(-) diff --git a/documentation/architecture/decisions/005-validation-behavior-configuration.rst b/documentation/architecture/decisions/005-validation-behavior-configuration.rst index bb45299..b4d051a 100644 --- a/documentation/architecture/decisions/005-validation-behavior-configuration.rst +++ b/documentation/architecture/decisions/005-validation-behavior-configuration.rst @@ -72,18 +72,26 @@ configuration object. class BehaviorTristate(enum.Enum): Never = enum.auto() - AsNeeded = enum.auto() + AsNeeded = enum.auto() Always = enum.auto() class Behaviors(immut.Dataclass): # Core detection controls charset_detect: BehaviorTristate = BehaviorTristate.AsNeeded mimetype_detect: BehaviorTristate = BehaviorTristate.AsNeeded - - # Charset handling sophistication - charset_trial_codecs: Sequence[str | CodecSpecifiers] = ( - CodecSpecifiers.Inference, CodecSpecifiers.UserDefault) - charset_trial_decode: BehaviorTristate = BehaviorTristate.AsNeeded + + # Trial decoding and validation controls + trial_decode: BehaviorTristate = BehaviorTristate.AsNeeded + trial_decode_confidence: float = 0.80 + text_validate: BehaviorTristate = BehaviorTristate.AsNeeded + text_validate_confidence: float = 0.80 + trial_codecs: Sequence[str | CodecSpecifiers] = ( + CodecSpecifiers.UserSupplement, + 'utf-8', + CodecSpecifiers.FromInference, + CodecSpecifiers.OsDefault, + CodecSpecifiers.PythonDefault, + ) **BehaviorTristate Control:** @@ -93,35 +101,37 @@ configuration object. **Advanced Charset Handling:** -* **charset_trial_codecs**: Sequence of codecs to try during trial decoding -* **CodecSpecifiers**: Enum for dynamic codec resolution (Inference, OsDefault, UserDefault) +* **trial_codecs**: Sequence of codecs to try during trial decoding +* **CodecSpecifiers**: Enum for dynamic codec resolution + (FromInference, OsDefault, PythonDefault, UserSupplement) **Sophisticated Detection Control:** * **charset_detect**: Controls when charset detection from content occurs * **mimetype_detect**: Controls when MIME type detection from content occurs -* **charset_trial_decode**: Controls when trial decoding validation occurs +* **trial_decode**: Controls when trial decoding runs +* **text_validate**: Controls when decoded text is validated **Integration Pattern:** .. code-block:: python - def detect_mimetype_charset( + def infer_mimetype_charset( content: Content, location: Absential[Location] = absent, *, - behaviors: Absential[Behaviors] = absent, + behaviors: Behaviors = BEHAVIORS_DEFAULT, # ... other parameters - ) -> tuple[Absential[str], Absential[str]]: + ) -> tuple[str, Optional[str]]: **Default Behavior Design:** .. code-block:: python BEHAVIORS_DEFAULT = Behaviors( - trial_decode='as-needed', - validate_printable='as-needed', - printable_threshold=0.0, - assume_utf8_superset=True, + trial_decode=BehaviorTristate.AsNeeded, + trial_decode_confidence=0.80, + text_validate=BehaviorTristate.AsNeeded, + text_validate_confidence=0.80, ) Alternatives @@ -183,8 +193,8 @@ Consequences # Quick charset detection for decoding fast_behaviors = Behaviors( - trial_decode='never', - validate_printable='never', + trial_decode=BehaviorTristate.Never, + text_validate=BehaviorTristate.Never, ) **Security-Focused Configuration:** @@ -193,9 +203,8 @@ Consequences # Comprehensive validation for untrusted content secure_behaviors = Behaviors( - trial_decode='always', - validate_printable='always', - printable_threshold=0.05, # Allow 5% non-printable + trial_decode=BehaviorTristate.Always, + text_validate=BehaviorTristate.Always, ) **Content-Specific Configuration:** @@ -204,8 +213,8 @@ Consequences # Relaxed validation for code/data content code_behaviors = Behaviors( - printable_threshold=0.15, # Allow more control characters - validate_printable='as-needed', + text_validate=BehaviorTristate.AsNeeded, + text_validate_confidence=0.40, ) **Conditional Logic Implementation:** @@ -214,17 +223,18 @@ Internal implementation will evaluate behavior configuration to determine which validation steps to execute, maintaining performance characteristics appropriate for each configuration profile. -**Integration with Error Class Provider:** +**Integration with Failure Policies:** -Behaviors configuration works in conjunction with error class provider pattern -to provide complete control over validation execution and error handling: +Behaviors configuration works in conjunction with detect-failure actions +to provide control over validation execution and fallback behavior: .. code-block:: python - result = detect_mimetype_charset( + result = infer_mimetype_charset( content, location, - behaviors=secure_behaviors, - error_class_provider=security_error_mapper, + behaviors = secure_behaviors, + charset_default = 'utf-8', + mimetype_default = 'text/plain', ) This decision provides the foundation for performance-aware and context-sensitive diff --git a/documentation/architecture/filesystem.rst b/documentation/architecture/filesystem.rst index a2e67ac..c9b052b 100644 --- a/documentation/architecture/filesystem.rst +++ b/documentation/architecture/filesystem.rst @@ -67,7 +67,7 @@ The main Python package follows the standard ``sources/`` directory pattern: │ ├── py.typed # Type checking marker │ ├── core.py # Core types: Behaviors, Result, CodecSpecifiers │ ├── charsets.py # Charset decoding and trial decode logic - │ ├── decoders.py # High-level decode() function + │ ├── decoders.py # High-level decode/decode_inform APIs │ ├── detectors.py # Core detection functions with confidence │ ├── exceptions.py # Package exception hierarchy │ ├── inference.py # Charset and mimetype inference orchestration diff --git a/documentation/architecture/summary.rst b/documentation/architecture/summary.rst index f679bc4..ca54664 100644 --- a/documentation/architecture/summary.rst +++ b/documentation/architecture/summary.rst @@ -21,155 +21,107 @@ System Overview ******************************************************************************* -The **detextive** library implements a faithful functional reproduction to -consolidate text detection capabilities from multiple packages. The first -iteration prioritizes behavioral fidelity and minimal migration effort over -architectural sophistication. +The **detextive** library consolidates MIME detection, charset inference, +text decoding, and line-separator utilities behind a unified functional API. Major Components =============================================================================== -Core Detection Functions +Public API ------------------------------------------------------------------------------- -**Public Functional API** - Core detection and inference functions with confidence-aware behavior: - - * ``detect_charset(content, *, behaviors=BEHAVIORS_DEFAULT, ...)`` - Character encoding detection - * ``detect_charset_confidence(content, *, behaviors=BEHAVIORS_DEFAULT, ...)`` - Charset detection with confidence scoring - * ``detect_mimetype(content, *, behaviors=BEHAVIORS_DEFAULT, ...)`` - MIME type detection - * ``detect_mimetype_confidence(content, *, behaviors=BEHAVIORS_DEFAULT, ...)`` - MIME type detection with confidence scoring - * ``infer_charset(content, *, behaviors=BEHAVIORS_DEFAULT, ...)`` - Charset inference with validation - * ``infer_charset_confidence(content, *, behaviors=BEHAVIORS_DEFAULT, ...)`` - Charset inference with confidence scoring - * ``infer_mimetype_charset(content, *, behaviors=BEHAVIORS_DEFAULT, ...)`` - Combined MIME type and charset inference - * ``infer_mimetype_charset_confidence(content, *, behaviors=BEHAVIORS_DEFAULT, ...)`` - Combined detection with confidence scoring - * ``decode(content, *, behaviors=BEHAVIORS_DEFAULT, ...)`` - High-level bytes-to-text decoding with validation - * ``is_textual_mimetype(mimetype)`` - Textual MIME type validation - * ``is_valid_text(text, profile=PROFILE_TEXTUAL)`` - Unicode-aware text validation - -**Core Types and Configuration** - Shared data structures for confidence-aware behavior: - - * ``CharsetResult(charset, confidence)`` - Charset detection results with confidence scoring (0.0-1.0) - * ``MimetypeResult(mimetype, confidence)`` - MIME type detection results with confidence scoring (0.0-1.0) - * ``Behaviors`` - Configurable detection behavior with confidence thresholds and failure handling - * ``BehaviorTristate`` - When to apply behaviors (Never/AsNeeded/Always) - * ``CodecSpecifiers`` - Dynamic codec resolution (FromInference/OsDefault/UserSupplement/etc.) - * ``DetectFailureActions`` - Failure handling strategy (Default/Error) for graceful degradation - -**Text Validation System** - Unicode-aware text validation with configurable profiles: - - * ``TextValidationProfile`` - Validation rules and character acceptance policies - * ``PROFILE_TEXTUAL`` - General textuality validation (lenient) - * ``PROFILE_TERMINAL_SAFE`` - Terminal output safety (strict) - * ``PROFILE_PRINTER_SAFE`` - Printer output safety (form feed allowed) - -**Line Separator Processing** - Direct migration of proven enumeration and utilities: - - * ``LineSeparators`` enum - Detection, normalization, and nativization methods - -Component Relationships +The public API is composed of confidence-aware detection functions, +inference orchestration functions, and high-level decode functions: + +* ``detect_charset`` / ``detect_charset_confidence`` +* ``detect_mimetype`` / ``detect_mimetype_confidence`` +* ``infer_charset`` / ``infer_charset_confidence`` +* ``infer_mimetype_charset`` / ``infer_mimetype_charset_confidence`` +* ``decode`` +* ``decode_inform`` +* ``is_textual_mimetype`` +* ``is_valid_text`` +* ``LineSeparators`` utilities + +Core Types and Configuration +------------------------------------------------------------------------------- + +* ``Behaviors`` - policy object controlling parse/detect/trial/validation + behaviors and confidence thresholds. +* ``BehaviorTristate`` - execution mode for selected behavior paths + (Never/AsNeeded/Always). +* ``DetectFailureActions`` - fallback policy on detector failure + (Default/Error). +* ``CodecSpecifiers`` - dynamic trial codec slots + (FromInference/OsDefault/PythonDefault/UserSupplement). +* ``CharsetResult`` - charset with confidence score. +* ``MimetypeResult`` - MIME type with confidence score. +* ``DecodeInformResult`` - decoded text plus charset/mimetype/line-separator + metadata. + +Layered Runtime Architecture +=============================================================================== + +.. code-block:: text + + ┌──────────────────────────────────────────────────────┐ + │ Public API (__init__.py re-exports) │ + └──────────────────────────────────────────────────────┘ + │ + ┌──────────────────────────────────────────────────────┐ + │ Decoding Layer (decoders.py) │ + │ decode(), decode_inform() │ + │ - HTTP Content-Type parse + charset-first attempt │ + │ - detector-assisted trial decode + text validation │ + │ - optional MIME/line-separator metadata │ + └──────────────────────────────────────────────────────┘ + │ + ┌──────────────────────────────────────────────────────┐ + │ Inference Layer (inference.py) │ + │ infer_*() orchestration + header/location context │ + └──────────────────────────────────────────────────────┘ + │ + ┌──────────────────────────────────────────────────────┐ + │ Detection Layer (detectors.py) │ + │ detector registries + confidence results │ + └──────────────────────────────────────────────────────┘ + │ + ┌──────────────────────────────────────────────────────┐ + │ Support Layer │ + │ charsets.py, mimetypes.py, validation.py, │ + │ lineseparators.py │ + └──────────────────────────────────────────────────────┘ + +Decoder Flow (v3) =============================================================================== -**v2.0 Layered Architecture** - -.. code-block:: - - ┌─────────────────────────────────────────────────┐ - │ Public API Layer (decoders.py) │ - │ decode() - High-level bytes-to-text function │ - └─────────────────────────────────────────────────┘ - │ - ┌─────────────────────────────────────────────────┐ - │ Inference Layer (inference.py) │ - │ infer_charset_confidence() infer_mimetype() │ - │ Context-aware orchestration + HTTP parsing │ - └─────────────────────────────────────────────────┘ - │ - ┌─────────────────────────────────────────────────┐ - │ Detection Layer (detectors.py) │ - │ detect_charset_confidence() detect_mimetype() │ - │ Core detection with confidence scoring │ - └─────────────────────────────────────────────────┘ - │ - ┌─────────────────────────────────────────────────┐ - │ Support Modules (charsets.py, validation.py) │ - │ Trial decoding + Text validation + MIME utils │ - └─────────────────────────────────────────────────┘ - │ - ┌─────────────────────────────────────────────────┐ - │ External Dependencies │ - │ chardet charset-normalizer puremagic │ - │ python-magic mimetypes (stdlib) [optional] │ - └─────────────────────────────────────────────────┘ - -**v2.0 Data Flow** - -1. **Input Processing**: Functions receive byte content, behaviors configuration, optional default values, and HTTP/location context -2. **Registry-Based Detection**: Core detectors iterate through configured backends (chardet, charset-normalizer, puremagic, python-magic) returning CharsetResult/MimetypeResult objects with confidence scores -3. **Smart Decision Making**: Confidence thresholds drive AsNeeded behavior for trial decode and text validation -4. **Failure Handling**: DetectFailureActions configuration determines whether to return default values (graceful degradation) or raise exceptions -5. **Layered Inference**: Higher-level functions orchestrate detection, validation, and configurable error handling -6. **Validated Output**: Text validation ensures decoded content meets specified profiles for safety/quality - -Integration Patterns +``decode`` and ``decode_inform`` share the same decoding core: + +1. Parse ``http_content_type`` when provided. +2. If header MIME is non-textual, raise ``ContentDecodeImpossibility``. +3. If header charset is textual and decodable, decode with that charset first. +4. Otherwise, run detector-assisted trial decodes in configured codec order. +5. Apply text validation according to ``Behaviors.text_validate`` and + ``Behaviors.text_validate_confidence``. +6. Return text (``decode``) or structured metadata (``decode_inform``). + +Inference Flow =============================================================================== -**Drop-in Replacement Strategy** - Existing code can replace imports with minimal changes: - - .. code-block:: python - - # Before: from mimeogram.acquirers import _detect_charset - # After: from detextive import detect_charset - charset = detect_charset(content_bytes) - -**Behavioral Fidelity** - Preserves exact existing behavior: - - * UTF-8 bias with validation from mimeogram charset detection - * Extensible textual MIME type patterns from all implementations - * Fallback chains (puremagic → mimetypes) from mimeogram - * Complex parameter handling from ``detect_mimetype_and_charset`` - * Heuristic validation from ``is_reasonable_text_content`` - * Error handling and exception types maintained - -**Implementation Strategy** - * Direct consolidation of proven function logic - * Minimal abstraction to preserve performance characteristics - * Same dependencies and detection libraries as existing implementations - -Architectural Patterns +``infer_*`` functions use contextual hints and detection orchestration: + +1. Optionally parse ``http_content_type`` depending on behavior settings. +2. Consider ``location``-based MIME hints. +3. Run registered detectors for MIME and charset as configured. +4. Apply ``*_default`` values only for fallback return semantics. +5. Use ``*_supplement`` values as hints to guide detection/validation. + +Integration Notes =============================================================================== -**Faithful Functional Reproduction** - Direct consolidation of existing functional implementations without - architectural changes (see ADR-001). - -**Consolidation Pattern** - Multiple implementations merged into single functions: - - * **chardet**: Statistical charset detection with UTF-8 bias - * **puremagic**: Pure Python magic byte detection (primary) - * **mimetypes**: Standard library extension-based fallback - * **LineSeparators**: Byte-level line ending detection and normalization - -**v2.0 Evolution** - ADR-003 and ADR-006 document the context-aware detection architecture for v2.0 that - addresses real-world integration challenges: - - * Context-driven detection utilizing HTTP headers, location, and content analysis - * Confidence-based result types with specific CharsetResult/MimetypeResult objects - * Configurable validation behaviors for performance and security requirements - * Default return behavior pattern enabling graceful degradation for detection failures - * Enhanced function interfaces maintaining backward compatibility - -**Detector Registry Architecture** - ADR-002 documents the implemented pluggable backend system: - - * Dynamic detector registration with type aliases for CharsetDetector/MimetypeDetector functions - * Configurable detector precedence via Behaviors.charset_detectors_order and mimetype_detectors_order - * Graceful degradation with NotImplemented return pattern for missing optional dependencies - * Registry dictionaries (charset_detectors, mimetype_detectors) enabling runtime backend selection \ No newline at end of file +* ``decode`` is authoritative for byte-to-text conversion and raises on + irrecoverable decode failure. +* ``decode_inform`` is intended for callers that need text plus consistent + decode metadata in one call. +* Detector registries are pluggable and backend-optional by design. +* Trial codec ordering is behavior-driven and can be overridden by callers. diff --git a/sources/detextive/decoders.py b/sources/detextive/decoders.py index 661be48..fdcbcac 100644 --- a/sources/detextive/decoders.py +++ b/sources/detextive/decoders.py @@ -248,7 +248,7 @@ def _validate_text_in_decode_attempt( location: _nomina.LocationArgument, ) -> None: _validate_text( - text, result.confidence, + text, 0.0, behaviors = behaviors, profile = profile, location = location ) diff --git a/tests/test_000_detextive/test_500_decoders.py b/tests/test_000_detextive/test_500_decoders.py index 88d1f2e..4cc6e71 100644 --- a/tests/test_000_detextive/test_500_decoders.py +++ b/tests/test_000_detextive/test_500_decoders.py @@ -160,32 +160,6 @@ def test_190_decode_validation_profile_parameters( ): assert text is not None # Should succeed when validation is disabled -def test_195_decode_attempts_skip_validation_above_confidence_threshold( ): - ''' As-needed validation skips high-confidence decode attempts. ''' - content = b'Text with\x00null bytes' - behaviors = detextive.Behaviors( - charset_detectors_order = ( 'nonexistent-detector', ), - charset_on_detect_failure = detextive.DetectFailureActions.Error, - bytes_quantity_confidence_divisor = 1, - text_validate = detextive.BehaviorTristate.AsNeeded, - text_validate_confidence = 0.8 ) - text = _decoders.decode( content, behaviors = behaviors ) - assert text == 'Text with\x00null bytes' - - -def test_196_decode_attempts_validate_below_confidence_threshold( ): - ''' As-needed validation runs for low-confidence decode attempts. ''' - content = b'Text with\x00null bytes' - behaviors = detextive.Behaviors( - charset_detectors_order = ( 'nonexistent-detector', ), - charset_on_detect_failure = detextive.DetectFailureActions.Error, - bytes_quantity_confidence_divisor = 10_000, - text_validate = detextive.BehaviorTristate.AsNeeded, - text_validate_confidence = 0.8 ) - with pytest.raises( detextive.exceptions.ContentDecodeFailure ): - _decoders.decode( content, behaviors = behaviors ) - - # Default Parameter Tests (200-299): Custom default values and behaviors def test_200_decode_empty_content_returns_empty_string( ): From 862adc6f964b1f927a6168c440a6fc6a418f0aff Mon Sep 17 00:00:00 2001 From: Eric McDonald Date: Fri, 13 Feb 2026 19:39:25 -0800 Subject: [PATCH 12/12] Simplify detection toggles and enforce behavior flag validity. Treat supplied HTTP Content-Type as authoritative parse input in inference paths. Convert charset and MIME detection toggles to booleans and validate them via BehaviorsInvalidity. Update tests and architecture notes for the v3 behavior model. Co-Authored-By: Codex --- .../005-validation-behavior-configuration.rst | 8 +-- sources/detextive/core.py | 21 +++++-- sources/detextive/decoders.py | 2 +- sources/detextive/exceptions.py | 8 +++ sources/detextive/inference.py | 58 +++++------------ sources/detextive/nomina.py | 4 +- .../test_000_detextive/test_110_exceptions.py | 13 +++- tests/test_000_detextive/test_120_core.py | 19 ++++++ .../test_000_detextive/test_400_inference.py | 63 ++++++++++++++----- tests/test_000_detextive/test_500_decoders.py | 2 +- 10 files changed, 126 insertions(+), 72 deletions(-) diff --git a/documentation/architecture/decisions/005-validation-behavior-configuration.rst b/documentation/architecture/decisions/005-validation-behavior-configuration.rst index b4d051a..6a17e67 100644 --- a/documentation/architecture/decisions/005-validation-behavior-configuration.rst +++ b/documentation/architecture/decisions/005-validation-behavior-configuration.rst @@ -77,8 +77,8 @@ configuration object. class Behaviors(immut.Dataclass): # Core detection controls - charset_detect: BehaviorTristate = BehaviorTristate.AsNeeded - mimetype_detect: BehaviorTristate = BehaviorTristate.AsNeeded + charset_detect: bool = True + mimetype_detect: bool = True # Trial decoding and validation controls trial_decode: BehaviorTristate = BehaviorTristate.AsNeeded @@ -107,8 +107,8 @@ configuration object. **Sophisticated Detection Control:** -* **charset_detect**: Controls when charset detection from content occurs -* **mimetype_detect**: Controls when MIME type detection from content occurs +* **charset_detect**: Enables/disables charset detection from content +* **mimetype_detect**: Enables/disables MIME type detection from content * **trial_decode**: Controls when trial decoding runs * **text_validate**: Controls when decoded text is validated diff --git a/sources/detextive/core.py b/sources/detextive/core.py index cf7cda6..a6736fc 100644 --- a/sources/detextive/core.py +++ b/sources/detextive/core.py @@ -22,6 +22,7 @@ from . import __ +from . import exceptions as _exceptions from . import nomina as _nomina @@ -62,9 +63,9 @@ class Behaviors( __.immut.DataclassObject ): ''' Minimum number of bytes for full detection confidence. ''' ), ] = 1024 charset_detect: __.typx.Annotated[ - BehaviorTristate, - __.ddoc.Doc( ''' When to detect charset from content. ''' ), - ] = BehaviorTristate.AsNeeded + bool, + __.ddoc.Doc( ''' Whether to detect charset from content. ''' ), + ] = True charset_detectors_order: __.typx.Annotated[ __.cabc.Sequence[ str ], __.ddoc.Doc( @@ -75,9 +76,9 @@ class Behaviors( __.immut.DataclassObject ): __.ddoc.Doc( ''' Action to take on charset detection failure. ''' ), ] = DetectFailureActions.Default mimetype_detect: __.typx.Annotated[ - BehaviorTristate, - __.ddoc.Doc( ''' When to detect MIME type from content. ''' ), - ] = BehaviorTristate.AsNeeded + bool, + __.ddoc.Doc( ''' Whether to detect MIME type from content. ''' ), + ] = True mimetype_detectors_order: __.typx.Annotated[ __.cabc.Sequence[ str ], __.ddoc.Doc( @@ -128,6 +129,14 @@ class Behaviors( __.immut.DataclassObject ): float, __.ddoc.Doc( ''' Minimum confidence to skip trial decode. ''') ] = 0.80 + def __post_init__( self ) -> None: + if not isinstance( self.charset_detect, bool ): + raise _exceptions.BehaviorsInvalidity( + 'charset_detect', 'a boolean' ) + if not isinstance( self.mimetype_detect, bool ): + raise _exceptions.BehaviorsInvalidity( + 'mimetype_detect', 'a boolean' ) + BehaviorsArgument: __.typx.TypeAlias = __.typx.Annotated[ Behaviors, diff --git a/sources/detextive/decoders.py b/sources/detextive/decoders.py index fdcbcac..2f69cd7 100644 --- a/sources/detextive/decoders.py +++ b/sources/detextive/decoders.py @@ -203,7 +203,7 @@ def _infer_mimetype( # noqa: PLR0913 and _mimetypes.is_textual_mimetype( mimetype ) ): return _MimetypeResult( mimetype = mimetype, confidence = 0.9 ) - if behaviors.mimetype_detect is not _BehaviorTristate.Never: + if behaviors.mimetype_detect: result = _detectors.detect_mimetype_confidence( content, behaviors = behaviors, diff --git a/sources/detextive/exceptions.py b/sources/detextive/exceptions.py index 347e691..0d11293 100644 --- a/sources/detextive/exceptions.py +++ b/sources/detextive/exceptions.py @@ -33,6 +33,14 @@ class Omnierror( Omniexception, Exception ): ''' Base for error exceptions raised by package API. ''' +class BehaviorsInvalidity( Omnierror, TypeError, ValueError ): + + def __init__( self, attribute: str, expectation: str ) -> None: + message = ( + f"Behaviors attribute '{attribute}' must be {expectation}" ) + super( ).__init__( f"{message}." ) + + class CharsetDetectFailure( Omnierror, TypeError, ValueError ): def __init__( diff --git a/sources/detextive/inference.py b/sources/detextive/inference.py index 2ae6dbf..136197c 100644 --- a/sources/detextive/inference.py +++ b/sources/detextive/inference.py @@ -32,7 +32,6 @@ BEHAVIORS_DEFAULT as _BEHAVIORS_DEFAULT, CHARSET_DEFAULT as _CHARSET_DEFAULT, MIMETYPE_DEFAULT as _MIMETYPE_DEFAULT, - BehaviorTristate as _BehaviorTristate, Behaviors as _Behaviors, BehaviorsArgument as _BehaviorsArgument, CharsetResult as _CharsetResult, @@ -80,17 +79,17 @@ def infer_charset_confidence( # noqa: PLR0913 ``charset_default`` is the returned fallback when inference cannot determine another charset. ``charset_supplement`` is a user-supplied - hint used during inference/validation. + hint used during inference/validation. ``http_content_type`` is + parsed when supplied, independent of detector enablement behavior. ''' if content == b'': return _CharsetResult( charset = 'utf-8', confidence = 1.0 ) - should_parse, should_detect = ( - _determine_parse_detect( behaviors.charset_detect ) ) + should_detect = behaviors.charset_detect result = __.absent mimetype = mimetype_supplement http_content_type = ( '' if __.is_absent( http_content_type ) else http_content_type ) - if should_parse and http_content_type: + if http_content_type: mimetype_result, charset_result = _validate_http_content_type( content, behaviors, http_content_type, charset_supplement = charset_supplement, location = location ) @@ -146,31 +145,22 @@ def infer_mimetype_charset_confidence( # noqa: PLR0913 charset_supplement: _nomina.CharsetSupplementArgument = __.absent, mimetype_supplement: _nomina.MimetypeSupplementArgument = __.absent, ) -> tuple[ _MimetypeResult, _CharsetResult ]: - ''' Infers MIME type and charset through various means. - - ``*_default`` values are returned fallbacks on inference failure. - ``*_supplement`` values are user-supplied hints used to guide - inference before fallback behavior is applied. - ''' - should_parse, should_detect_charset = ( - _determine_parse_detect( behaviors.charset_detect ) ) - should_parse, should_detect_mimetype = ( - _determine_parse_detect( - behaviors.mimetype_detect, should_parse = should_parse ) ) + ''' Infers MIME type and charset through various means. ''' + should_detect_charset = behaviors.charset_detect + should_detect_mimetype = behaviors.mimetype_detect charset_result: __.Absential[ _CharsetResult ] = __.absent mimetype_result: __.Absential[ _MimetypeResult ] = __.absent http_content_type = ( '' if __.is_absent( http_content_type ) else http_content_type ) - if should_parse: - if http_content_type: - mimetype_result, charset_result = _validate_http_content_type( - content, behaviors, http_content_type, - charset_supplement = charset_supplement, location = location ) - if __.is_absent( mimetype_result ) and not __.is_absent( location ): - mimetype = _mimetypes.mimetype_from_location( location ) - if not __.is_absent( mimetype ): - mimetype_result = _MimetypeResult( - mimetype = mimetype, confidence = 0.9 ) + if http_content_type: + mimetype_result, charset_result = _validate_http_content_type( + content, behaviors, http_content_type, + charset_supplement = charset_supplement, location = location ) + if __.is_absent( mimetype_result ) and not __.is_absent( location ): + mimetype = _mimetypes.mimetype_from_location( location ) + if not __.is_absent( mimetype ): + mimetype_result = _MimetypeResult( + mimetype = mimetype, confidence = 0.9 ) if __.is_absent( mimetype_result ) and should_detect_mimetype: charset = ( charset_supplement @@ -231,22 +221,6 @@ def validate_httpct_charset( content, behaviors = behaviors_, inference = charset ) -def _determine_parse_detect( - detect_tristate: _BehaviorTristate, should_parse = False -) -> tuple[ bool, bool ]: - match detect_tristate: - case _BehaviorTristate.Always: - should_parse = should_parse or False - should_detect = True - case _BehaviorTristate.AsNeeded: - should_parse = should_parse or True - should_detect = True - case _BehaviorTristate.Never: # pragma: no branch - should_parse = should_parse or True - should_detect = False - return should_parse, should_detect - - def _validate_http_content_type( content: _nomina.Content, behaviors: _Behaviors, diff --git a/sources/detextive/nomina.py b/sources/detextive/nomina.py index 9db03f1..8c5fb58 100644 --- a/sources/detextive/nomina.py +++ b/sources/detextive/nomina.py @@ -41,8 +41,8 @@ CharsetDefaultArgument: __.typx.TypeAlias = __.typx.Annotated[ str, __.ddoc.Doc( - ''' Fallback character set returned on inference/detection - failure. ''' ), + ''' Fallback character set returned on inference/detection failure. + ''' ), ] CharsetSupplementArgument: __.typx.TypeAlias = __.typx.Annotated[ __.Absential[ str ], diff --git a/tests/test_000_detextive/test_110_exceptions.py b/tests/test_000_detextive/test_110_exceptions.py index 78b3c36..3c2053b 100644 --- a/tests/test_000_detextive/test_110_exceptions.py +++ b/tests/test_000_detextive/test_110_exceptions.py @@ -28,6 +28,7 @@ def test_000_imports( ): ''' Exception classes are accessible from main module. ''' + assert hasattr( _exceptions, 'BehaviorsInvalidity' ) assert hasattr( _exceptions, 'CharsetDetectFailure' ) assert hasattr( _exceptions, 'CharsetInferFailure' ) assert hasattr( _exceptions, 'MimetypeDetectFailure' ) @@ -186,6 +187,16 @@ def test_180_exception_hierarchy_inheritance( ): assert issubclass( _exceptions.Omnierror, Exception ) +def test_180_behaviors_invalidity_hierarchy_and_message( ): + ''' BehaviorsInvalidity uses package family and type semantics. ''' + exc = _exceptions.BehaviorsInvalidity( 'charset_detect', 'a boolean' ) + assert isinstance( exc, _exceptions.Omnierror ) + assert isinstance( exc, TypeError ) + assert ( + str( exc ) + == "Behaviors attribute 'charset_detect' must be a boolean." ) + + def test_181_mimetype_infer_failure_without_location( ): ''' MimetypeInferFailure constructs correctly without location. ''' exc = _exceptions.MimetypeInferFailure( ) @@ -246,4 +257,4 @@ def test_190_package_exception_catching( ): ] for exc in exceptions: assert isinstance( exc, _exceptions.Omnierror ) - assert isinstance( exc, _exceptions.Omniexception ) \ No newline at end of file + assert isinstance( exc, _exceptions.Omniexception ) diff --git a/tests/test_000_detextive/test_120_core.py b/tests/test_000_detextive/test_120_core.py index d06c33b..1ae7337 100644 --- a/tests/test_000_detextive/test_120_core.py +++ b/tests/test_000_detextive/test_120_core.py @@ -21,7 +21,10 @@ ''' Core types, enums, and behaviors. ''' +import pytest + import detextive.core as _core +import detextive.exceptions as _exceptions # Basic Tests (000-099): Module import verification, Constant value validation @@ -32,3 +35,19 @@ def test_000_imports( ): assert hasattr( _core, 'BehaviorTristate' ) assert hasattr( _core, 'CodecSpecifiers' ) assert hasattr( _core, 'DetectFailureActions' ) + + +def test_100_behaviors_detect_flags_require_boolean( ): + ''' Detect flags reject non-boolean values at construction time. ''' + with pytest.raises( _exceptions.BehaviorsInvalidity ): + _core.Behaviors( charset_detect = _core.BehaviorTristate.Never ) + with pytest.raises( _exceptions.BehaviorsInvalidity ): + _core.Behaviors( mimetype_detect = _core.BehaviorTristate.Never ) + + +def test_110_behaviors_detect_flags_accept_boolean( ): + ''' Detect flags accept explicit boolean values. ''' + behaviors = _core.Behaviors( + charset_detect = False, mimetype_detect = True ) + assert behaviors.charset_detect is False + assert behaviors.mimetype_detect is True diff --git a/tests/test_000_detextive/test_400_inference.py b/tests/test_000_detextive/test_400_inference.py index 086712e..72b9771 100644 --- a/tests/test_000_detextive/test_400_inference.py +++ b/tests/test_000_detextive/test_400_inference.py @@ -65,10 +65,24 @@ def test_120_infer_charset_confidence_http_content_type_parsing( ): assert result.charset == 'iso8859-1' +def test_125_infer_charset_httpct_honored_with_detect_enabled( ): + ''' Header charset is honored when charset detection is enabled. ''' + content = 'Café'.encode( 'iso-8859-1' ) + behaviors = detextive.Behaviors( + charset_detect = True, + charset_detectors_order = ( 'nonexistent-detector', ), + charset_on_detect_failure = detextive.DetectFailureActions.Error ) + result = _inference.infer_charset_confidence( + content, + behaviors = behaviors, + http_content_type = 'text/plain; charset=iso-8859-1' ) + assert result.charset == 'iso8859-1' + + def test_130_infer_charset_confidence_detection_fallback( ): ''' Falls back to detection when no other methods work. ''' behaviors = detextive.Behaviors( - charset_detect = detextive.BehaviorTristate.Always ) + charset_detect = True ) result = _inference.infer_charset_confidence( UTF8_BASIC, behaviors = behaviors ) assert result.charset is not None @@ -78,7 +92,7 @@ def test_130_infer_charset_confidence_detection_fallback( ): def test_140_infer_charset_confidence_failure_when_no_detection( ): ''' Raises CharsetInferFailure when no detection methods available. ''' behaviors = detextive.Behaviors( - charset_detect = detextive.BehaviorTristate.Never, + charset_detect = False, charset_detectors_order = ( 'nonexistent-detector', ), charset_on_detect_failure = detextive.DetectFailureActions.Error ) with pytest.raises( detextive.exceptions.CharsetInferFailure ): @@ -92,7 +106,7 @@ def test_150_charset_result_early_return( ): charset_result = _inference.infer_charset_confidence( content, behaviors = detextive.Behaviors( - charset_detect = detextive.BehaviorTristate.Always ), + charset_detect = True ), http_content_type = 'text/plain; charset=utf-8' ) assert hasattr( charset_result, 'charset' ) assert charset_result.charset is not None @@ -134,6 +148,25 @@ def test_200_http_content_type_parsing_success( ): assert charset_result.charset == 'utf-8-sig' +def test_205_httpct_honored_with_both_detect_enabled( ): + ''' Header parse is honored when both detect behaviors are enabled. ''' + content = UTF8_BASIC + behaviors = detextive.Behaviors( + charset_detect = True, + mimetype_detect = True, + charset_detectors_order = ( 'nonexistent-detector', ), + mimetype_detectors_order = ( 'nonexistent-detector', ), + charset_on_detect_failure = detextive.DetectFailureActions.Error, + mimetype_on_detect_failure = detextive.DetectFailureActions.Error ) + mimetype_result, charset_result = ( + _inference.infer_mimetype_charset_confidence( + content, + behaviors = behaviors, + http_content_type = 'text/plain; charset=utf-8' ) ) + assert mimetype_result.mimetype == 'text/plain' + assert charset_result.charset == 'utf-8-sig' + + def test_210_location_based_mimetype_inference( ): ''' Location-based mimetype inference when HTTP parsing absent. ''' utf8_content = 'Hello, world!'.encode( 'utf-8' ) @@ -163,11 +196,11 @@ def test_220_inference_failure_scenarios( ): content, behaviors = behaviors ) -def test_230_behavior_tristate_never( ): - ''' BehaviorTristate.Never disables detection. ''' +def test_230_mimetype_detection_disabled( ): + ''' Disabled MIME detection bypasses detector execution. ''' content = b'test content' behaviors = detextive.Behaviors( - mimetype_detect = detextive.BehaviorTristate.Never, + mimetype_detect = False, charset_on_detect_failure = detextive.DetectFailureActions.Default, mimetype_on_detect_failure = detextive.DetectFailureActions.Default ) mimetype_result, _ = _inference.infer_mimetype_charset_confidence( @@ -202,7 +235,7 @@ def test_260_charset_infer_failure_exception( ): ''' CharsetInferFailure raised when charset inference completely fails. ''' content = b'test content' behaviors = detextive.Behaviors( - charset_detect = detextive.BehaviorTristate.Never, + charset_detect = False, charset_on_detect_failure = detextive.DetectFailureActions.Error ) with pytest.raises( detextive.exceptions.CharsetInferFailure ): _inference.infer_mimetype_charset_confidence( @@ -215,7 +248,7 @@ def test_270_mimetype_infer_failure_exception( ): ''' MimetypeInferFailure raised when mimetype inference fails. ''' content = b'test content' behaviors = detextive.Behaviors( - mimetype_detect = detextive.BehaviorTristate.Never, + mimetype_detect = False, mimetype_on_detect_failure = detextive.DetectFailureActions.Error ) with pytest.raises( detextive.exceptions.MimetypeInferFailure ): _inference.infer_mimetype_charset_confidence( @@ -225,11 +258,11 @@ def test_270_mimetype_infer_failure_exception( ): def test_280_should_parse_false_branch( ): - ''' should_parse=False skips parsing and goes to detection. ''' + ''' Absent HTTP header uses regular detection paths. ''' content = b'test content' behaviors = detextive.Behaviors( - charset_detect = detextive.BehaviorTristate.Always, - mimetype_detect = detextive.BehaviorTristate.Always ) + charset_detect = True, + mimetype_detect = True ) result = _inference.infer_mimetype_charset_confidence( content, behaviors = behaviors, @@ -242,7 +275,7 @@ def test_290_location_mimetype_absent_branch( ): ''' Location-based mimetype inference when mimetype is absent. ''' content = b'test content' behaviors = detextive.Behaviors( - mimetype_detect = detextive.BehaviorTristate.AsNeeded ) + mimetype_detect = True ) result = _inference.infer_mimetype_charset_confidence( content, behaviors = behaviors, @@ -276,11 +309,11 @@ def test_310_http_validation_charset_absent( ): assert isinstance( charset_result.charset, str ) -def test_320_behavior_tristate_never_detection( ): - ''' BehaviorTristate.Never disables detection correctly. ''' +def test_320_mimetype_detection_disabled( ): + ''' Disabled MIME detection still honors parsed HTTP metadata. ''' content = b'test content' behaviors = detextive.Behaviors( - mimetype_detect = detextive.BehaviorTristate.Never ) + mimetype_detect = False ) result = _inference.infer_mimetype_charset_confidence( content, behaviors = behaviors, diff --git a/tests/test_000_detextive/test_500_decoders.py b/tests/test_000_detextive/test_500_decoders.py index 4cc6e71..8787129 100644 --- a/tests/test_000_detextive/test_500_decoders.py +++ b/tests/test_000_detextive/test_500_decoders.py @@ -104,7 +104,7 @@ def test_140_decode_inform_empty_content( ): def test_150_decode_inform_mimetype_inference_fallback( ): ''' Falls back to text/plain when MIME inference is unavailable. ''' behaviors = detextive.Behaviors( - mimetype_detect = detextive.BehaviorTristate.Never ) + mimetype_detect = False ) result = _decoders.decode_inform( b'hello', behaviors = behaviors ) assert result.mimetype.mimetype == 'text/plain'