From 03a8387a8e47628e9695da726a932f0166c8ef36 Mon Sep 17 00:00:00 2001 From: Pascal Tomecek Date: Mon, 21 Sep 2026 19:24:48 -0400 Subject: [PATCH] [bug] Ignore padding bytes when tokenizing structured numpy arrays Structured dtypes can carry padding between their fields, and those bytes are never written. normalize_token hashed the raw buffer, so two arrays holding identical field values could produce different cache tokens depending on whatever happened to occupy the padding. This is easiest to hit with np.empty followed by field assignment, which leaves the padding holding unrelated heap contents. Recurse per field for structured dtypes instead of hashing the buffer. Arrays with object fields are unaffected, since the earlier hasobject branch already handles them via tolist() and never sees padding. Tokens for structured arrays change as a result, so cache entries keyed on one are invalidated once. Signed-off-by: Pascal Tomecek --- ccflow/tests/utils/test_tokenize.py | 28 ++++++++++++++++++++++++++++ ccflow/utils/tokenize.py | 4 ++++ 2 files changed, 32 insertions(+) diff --git a/ccflow/tests/utils/test_tokenize.py b/ccflow/tests/utils/test_tokenize.py index f5d6bf23..98343a85 100644 --- a/ccflow/tests/utils/test_tokenize.py +++ b/ccflow/tests/utils/test_tokenize.py @@ -778,6 +778,34 @@ def test_numpy_structured_array(self): arr = np.array([(1, 2.0), (3, 4.0)], dtype=dt) assert normalize_token(arr)[0] == "ndarray" + def test_numpy_structured_array_ignores_padding(self): + dt = np.dtype([("a", np.uint8), ("b", np.uint64)], align=True) + a = np.frombuffer(bytearray(dt.itemsize), dtype=dt) + b = np.frombuffer(bytearray(b"\xff" * dt.itemsize), dtype=dt) + for arr in (a, b): + arr["a"] = 7 + arr["b"] = 12345 + assert a.tobytes() != b.tobytes() + assert normalize_token(a) == normalize_token(b) + + def test_numpy_structured_array_different_values(self): + dt = np.dtype([("a", np.uint8), ("b", np.uint64)], align=True) + a = np.zeros(1, dtype=dt) + b = np.zeros(1, dtype=dt) + a["b"] = 1 + b["b"] = 2 + assert normalize_token(a) != normalize_token(b) + + def test_numpy_structured_array_nested(self): + dt = np.dtype([("p", [("x", np.float32), ("y", np.float32)]), ("m", np.int16, (2, 2))], align=True) + a = np.zeros(2, dtype=dt) + b = np.zeros(2, dtype=dt) + a["p"]["x"] = 1.5 + b["p"]["x"] = 1.5 + assert normalize_token(a) == normalize_token(b) + b["m"][0, 0, 0] = 4 + assert normalize_token(a) != normalize_token(b) + def test_numpy_scalar(self): s = np.int64(42) assert normalize_token(s) == ("np_scalar", "int64", 42) diff --git a/ccflow/utils/tokenize.py b/ccflow/utils/tokenize.py index 5defde23..3381a7fb 100644 --- a/ccflow/utils/tokenize.py +++ b/ccflow/utils/tokenize.py @@ -362,6 +362,10 @@ def _normalize_ndarray(obj): # element-wise instead. Cycle detection is keyed on the array because tolist() returns a fresh list. if obj.dtype.hasobject: return _with_cycle_check(obj, lambda: ("ndarray", str(obj.dtype), obj.shape, normalize_token(obj.tolist()))) + # Structured dtypes may carry padding between fields. Those bytes are never written, so hashing the + # raw buffer gives arrays with identical field values different tokens. Recurse per field to skip them. + if obj.dtype.names is not None: + return ("ndarray", str(obj.dtype), obj.shape, tuple((name, normalize_token(obj[name])) for name in obj.dtype.names)) return ("ndarray", str(obj.dtype), obj.shape, hashlib.sha256(np.ascontiguousarray(obj).tobytes()).hexdigest()) @normalize_token.register(np.ma.MaskedArray)