From 6cdec365e65d03a45337ced83e86ac982beaf8dd Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 14 Jun 2026 09:46:27 +0000 Subject: [PATCH 1/2] buffers: complete the counter set + sensitive/clear enforcement (OWN024) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the small gaps from the stackalloc/logging design audit. Counters (§10): emit the four metrics that were specified but missing — ScratchTotalRequestedBytes / ScratchMaxRequestedBytes (via Requested()), ScratchPoolBytesReturned (via PoolReturned(), inside the rented-null guard), and ScratchForcedClears (via ForcedClear(), next to a clear-on-release). All scratch-scoped and under [Conditional("OWNSHARP_COUNTERS")]. Sensitive buffers (OWN024, §12): a `sensitive` option / policy key, and a check that a buffer marked sensitive but not cleared on release is rejected -- pooled/scratch arrays return to a shared ArrayPool, native memory goes back to the allocator, so leaving secret bytes unzeroed is the silent leak the flag exists to prevent. The report (json + text) now carries `sensitive`. README: document OWN024 + the full counter set; the unsafe-contract (UNS0xx, §6) and benchmark-matrix (§13) work stays flagged as roadmap. Tests: 5 OWN024 analysis cases (option + policy, cleared/uncleared, malformed) and 2 codegen-content cases for the new counter emission. Suite: analysis 100/100, codegen content 23/23, property fuzz clean (20k draws). --- README.md | 18 ++++++++++++------ ownlang/buffers.py | 20 ++++++++++++++++++-- ownlang/codegen.py | 42 +++++++++++++++++++++++++++++++++++++++++- ownlang/diagnostics.py | 1 + ownlang/report.py | 2 ++ tests/run_tests.py | 22 ++++++++++++++++++++++ tests/test_codegen.py | 11 +++++++++++ 7 files changed, 107 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 24fc11d6..f886dd4a 100644 --- a/README.md +++ b/README.md @@ -230,6 +230,7 @@ fn process(size: int) { | OWN019 | inline-ёмкость слишком велика для stack-backed политики (выше потолка стека) | | OWN021 | `stack`/`inline` динамического размера без статической границы (нет `max =`) | | OWN023 | `scratch` с `fallback = forbidden`, но размер может превысить inline-лимит | +| OWN024 | буфер помечен `sensitive`, но не зануляется на release (нет `clear = true`) | ### Неподдерживаемое / структурное / граница @@ -399,9 +400,10 @@ pool, а ты три часа смотришь на GC-график. Поэто Release вызовы вырезаются, логирование не становится новым bottleneck'ом. 3. **Runtime counters** — `OwnCounters` (`ScratchStackHits`, `ScratchPoolFallbacks`, - `ScratchPoolBytesRented`, `ScratchReleaseCount`) под `[Conditional("OWNSHARP_COUNTERS")]`. - Отвечают на главный вопрос: мы реально часто попадаем в стек, или inline-лимит - подобран мимо? + `ScratchPoolBytesRented`, `ScratchPoolBytesReturned`, `ScratchTotalRequestedBytes`, + `ScratchMaxRequestedBytes`, `ScratchReleaseCount`, `ScratchForcedClears`) под + `[Conditional("OWNSHARP_COUNTERS")]`. Отвечают на главный вопрос: мы реально часто + попадаем в стек, или inline-лимит подобран мимо? ### Политики @@ -456,9 +458,13 @@ faithful-inline (release ровно там, где написан; без `try/f **OWN021** (для динамики есть `stack`). Plain-локал, объявленный в теле буфера и использованный после release, не оборачивается в hoist'нутый `try` (иначе вышел бы из C#-scope) — такой буфер лоуэрится inline. -Булевы настройки (`clear_on_release`, `counters`) и `trace` валидируются: опечатка -вроде `clear_on_release = ture` — **OWN030**, а не тихое отключение clear на -sensitive-буфере. `native` хранит `byte*` (backing, освобождается на release), но наружу +Булевы настройки (`clear_on_release`, `counters`, `sensitive`) и `trace` +валидируются: опечатка вроде `clear_on_release = ture` — **OWN030**, а не тихое +отключение clear на sensitive-буфере. А `sensitive = true` без `clear = true` — +**OWN024**: пометил секретным — обязан занулить перед тем, как backing-память +(пул/аллокатор/кадр стека) переиспользуют. `counters` теперь и +`ScratchTotalRequestedBytes`/`ScratchMaxRequestedBytes` (распределение запросов), +`ScratchPoolBytesReturned` (баланс с `Rented`) и `ScratchForcedClears`. `native` хранит `byte*` (backing, освобождается на release), но наружу отдаёт `Span`-view — borrow/call видят тот же логический тип, что и pooled/stack/scratch. Borrow-параметр типа `Buffer` (и в `extern`, и в **локальной** `fn`) рендерится как `Span`/`ReadOnlySpan`, так что один diff --git a/ownlang/buffers.py b/ownlang/buffers.py index 04757192..46e82e47 100644 --- a/ownlang/buffers.py +++ b/ownlang/buffers.py @@ -60,11 +60,11 @@ class BufferMode(Enum): # heap, defeating the explicit storage guarantee. VALID_OPTIONS = frozenset({ "policy", "inline", "inline_bytes", "max", "max_bytes", - "fallback", "clear", "trace", "counters", + "fallback", "clear", "trace", "counters", "sensitive", }) VALID_POLICY_KEYS = frozenset({ "inline_bytes", "max_bytes", "fallback", "clear_on_release", - "trace", "counters", + "trace", "counters", "sensitive", }) # Modes whose backing storage may live on the stack. A stack-backed buffer must @@ -95,6 +95,7 @@ class BufferInfo: fallback_pool: bool # scratch: heap fallback allowed fallback_forbidden: bool # stack: heap fallback explicitly forbidden clear_on_release: bool # zero the bytes before returning/releasing + sensitive: bool # holds secret data: must be cleared on release trace: bool # emit OwnTrace hooks counters: bool # emit OwnCounters hooks policy_name: str | None @@ -358,10 +359,24 @@ def first_int(sources: list[tuple[bool, str]], label: str, # ---- flags from options / policy -------------------------------------- clear = _bool_flag(opts.get("clear"), base.get("clear_on_release"), False, "clear_on_release", diags, line) + sensitive = _bool_flag(opts.get("sensitive"), base.get("sensitive"), + False, "sensitive", diags, line) trace = _trace_flag(opts.get("trace"), base.get("trace"), True, diags, line) counters = _bool_flag(opts.get("counters"), base.get("counters"), True, "counters", diags, line) + # A buffer marked sensitive must be zeroed before its backing memory can be + # observed again — pooled/scratch arrays go back to a shared ArrayPool, native + # memory is handed back to the allocator, and even a stack frame is reused by + # the next call. Marking it sensitive without clearing is the silent leak the + # flag exists to prevent, so require an explicit `clear = true`. + if sensitive and not clear: + diags.append(Diagnostic( + "OWN024", + "buffer is marked sensitive but is not cleared on release; add " + "'clear = true' so its bytes are zeroed before the backing memory " + "is reused", line)) + info = BufferInfo( mode=mode, elem="byte", @@ -371,6 +386,7 @@ def first_int(sources: list[tuple[bool, str]], label: str, fallback_pool=fallback_pool, fallback_forbidden=fallback_forbidden, clear_on_release=clear, + sensitive=sensitive, trace=trace, counters=counters, policy_name=pol_name, diff --git a/ownlang/codegen.py b/ownlang/codegen.py index 729b80f2..19cc462e 100644 --- a/ownlang/codegen.py +++ b/ownlang/codegen.py @@ -252,6 +252,7 @@ def _buffer_lowering(self, name: str, intent: A.BufferIntent if info.trace: pre.append(f'OwnTrace.StackSelected("{fn}", "{name}", {size}, {L});') if sc: + pre.append(f"OwnCounters.Requested({size});") pre.append("OwnCounters.StackHit();") if info.size_const == L: pre.append(f"Span {name} = stackalloc byte[{L}];") @@ -266,6 +267,7 @@ def _buffer_lowering(self, name: str, intent: A.BufferIntent if info.trace: pre.append(f'OwnTrace.StackSelected("{fn}", "{name}", {size}, {L});') if sc: + pre.append(f"OwnCounters.Requested({size});") pre.append("OwnCounters.StackHit();") pre.append(f"Span {name}_backing = stackalloc byte[{L}];") pre.append(f"Span {name} = {name}_backing[..{size}];") @@ -273,6 +275,8 @@ def _buffer_lowering(self, name: str, intent: A.BufferIntent fin.append("OwnCounters.Release();") if info.clear_on_release: fin.append(f"{name}.Clear();") + if sc: + fin.append("OwnCounters.ForcedClear();") elif scratch_pool: if not info.size_is_const: @@ -284,6 +288,8 @@ def _buffer_lowering(self, name: str, intent: A.BufferIntent pre.append(f"byte[]? {name}_rented = null;") pre.append(f"Span {name}_backing = stackalloc byte[{L}];") pre.append(f"Span {name};") + if sc: + pre.append(f"OwnCounters.Requested({size});") pre.append(f"if ({size} <= {L})") pre.append("{") if info.trace: @@ -305,8 +311,16 @@ def _buffer_lowering(self, name: str, intent: A.BufferIntent fin.append("OwnCounters.Release();") if info.clear_on_release: fin.append(f"{name}.Clear();") + if sc: + fin.append("OwnCounters.ForcedClear();") fin.append(f"if ({name}_rented is not null)") - fin.append(f" ArrayPool.Shared.Return({name}_rented);") + if sc: + fin.append("{") + fin.append(f" ArrayPool.Shared.Return({name}_rented);") + fin.append(f" OwnCounters.PoolReturned({size});") + fin.append("}") + else: + fin.append(f" ArrayPool.Shared.Return({name}_rented);") elif info.mode == BufferMode.POOLED: # pooled is not scratch: trace it, but do not touch the Scratch.* @@ -684,7 +698,25 @@ def _usings(mod: A.Module) -> list[str]: public static long ScratchStackHits; public static long ScratchPoolFallbacks; public static long ScratchPoolBytesRented; + public static long ScratchPoolBytesReturned; + public static long ScratchTotalRequestedBytes; + public static long ScratchMaxRequestedBytes; public static long ScratchReleaseCount; + public static long ScratchForcedClears; + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void Requested(int bytes) + { + System.Threading.Interlocked.Add(ref ScratchTotalRequestedBytes, bytes); + long cur; + do + { + cur = System.Threading.Interlocked.Read(ref ScratchMaxRequestedBytes); + if (bytes <= cur) return; + } + while (System.Threading.Interlocked.CompareExchange( + ref ScratchMaxRequestedBytes, bytes, cur) != cur); + } [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] public static void StackHit() @@ -697,9 +729,17 @@ def _usings(mod: A.Module) -> list[str]: System.Threading.Interlocked.Add(ref ScratchPoolBytesRented, bytes); } + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void PoolReturned(int bytes) + => System.Threading.Interlocked.Add(ref ScratchPoolBytesReturned, bytes); + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] public static void Release() => System.Threading.Interlocked.Increment(ref ScratchReleaseCount); + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void ForcedClear() + => System.Threading.Interlocked.Increment(ref ScratchForcedClears); } ''' diff --git a/ownlang/diagnostics.py b/ownlang/diagnostics.py index 0a91355f..131ea41e 100644 --- a/ownlang/diagnostics.py +++ b/ownlang/diagnostics.py @@ -48,6 +48,7 @@ class Severity(Enum): "OWN019": "inline capacity too large for a stack-backed policy", "OWN021": "stack allocation requires a statically known bound", "OWN023": "scratch fallback forbidden but the size may exceed the inline limit", + "OWN024": "sensitive buffer is not cleared on release", # ---- unsupported ---- "OWN020": "unsupported construct (out of scope for the MVP)", # ---- name resolution & structural ---- diff --git a/ownlang/report.py b/ownlang/report.py index 60578787..1b0871c3 100644 --- a/ownlang/report.py +++ b/ownlang/report.py @@ -79,6 +79,7 @@ def build_report(mod: A.Module, diags: list[Diagnostic]) -> dict: else "forbidden")), "escapePolicy": info.escape_policy, "clearOnRelease": info.clear_on_release, + "sensitive": info.sensitive, "trace": info.trace, "counters": info.counters, "policy": info.policy_name, @@ -102,6 +103,7 @@ def render_report(report: dict) -> str: lines.append(f" Fallback: {e['fallback']}") lines.append(f" EscapePolicy: {e['escapePolicy']}") lines.append(f" ClearOnRelease: {str(e['clearOnRelease']).lower()}") + lines.append(f" Sensitive: {str(e['sensitive']).lower()}") if e["policy"]: lines.append(f" Policy: {e['policy']}") lines.append(" Generated branches:") diff --git a/tests/run_tests.py b/tests/run_tests.py index 027fa6d9..03cccf66 100644 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -280,8 +280,24 @@ def codes(src: str) -> list[str]: ("buf_policy_ok", "policy P { inline_bytes = 512; fallback = pool; } " "fn f(n: int){ let b = Buffer.scratch(n, policy = P); release b; }", []), + ("buf_sensitive_cleared_ok", + "fn f(n: int){ let b = Buffer.pooled(n, sensitive = true, clear = true); " + "release b; }", []), + ("buf_sensitive_policy_ok", + "policy Secret { sensitive = true; clear_on_release = true; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = Secret); release b; }", []), # ---- buffer storage policies: faults ---- + ("buf_sensitive_no_clear", + "fn f(n: int){ let b = Buffer.pooled(n, sensitive = true); release b; }", + ["OWN024"]), + ("buf_sensitive_policy_no_clear", + "policy Secret { sensitive = true; clear_on_release = false; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = Secret); release b; }", + ["OWN024"]), + ("buf_bad_sensitive", + "fn f(n: int){ let b = Buffer.scratch(n, sensitive = ture); release b; }", + ["OWN030"]), ("buf_stack_dyn_unbounded", "fn f(n: int){ let b = Buffer.stack(n); release b; }", ["OWN021"]), ("buf_stack_too_large", @@ -401,6 +417,12 @@ def buffer_smoke() -> list[str]: 'OwnTrace.ScratchSelected("parse", "tmp", size, 1024, "ArrayPool");', "OwnCounters.StackHit();", "OwnCounters.PoolFallback(size);", + "OwnCounters.Requested(size);", + "OwnCounters.PoolReturned(size);", + "public static long ScratchTotalRequestedBytes;", + "public static long ScratchMaxRequestedBytes;", + "public static long ScratchPoolBytesReturned;", + "public static long ScratchForcedClears;", "ArrayPool.Shared.Rent(size)", "ArrayPool.Shared.Return(tmp_rented)", "try", diff --git a/tests/test_codegen.py b/tests/test_codegen.py index e323a341..e139d069 100644 --- a/tests/test_codegen.py +++ b/tests/test_codegen.py @@ -274,6 +274,17 @@ def case(name: str, prelude: str, fn_src: str) -> Check: "fn c(){ let d = acquire Buffer(2); release d; }") \ .has("static void a(").has("static void c(") +# A scratch buffer with clear-on-release zeroes its bytes and bumps the +# ForcedClear counter (the one Scratch.* metric the golden smoke doesn't hit). +case("scratch_clear_forced_clear_counter", SCHEMATIC, + "fn f(n: int){ let b = Buffer.scratch(n, clear = true); release b; }") \ + .has(".Clear();").has("OwnCounters.ForcedClear();") + +# The requested/returned byte counters are emitted around the scratch arms. +case("scratch_byte_counters", SCHEMATIC, + "fn f(n: int){ let b = Buffer.scratch(n); release b; }") \ + .has("OwnCounters.Requested(n);").has("OwnCounters.PoolReturned(n);") + # --------------------------------------------------------------------------- # runner From 79956b110283d06c912f961db4f16fbcff39a0db Mon Sep 17 00:00:00 2001 From: PhysShell <45852143+PhysShell@users.noreply.github.com> Date: Sun, 14 Jun 2026 16:11:23 +0500 Subject: [PATCH 2/2] Update codegen.py for CAS early return comment --- ownlang/codegen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ownlang/codegen.py b/ownlang/codegen.py index 19cc462e..dc3c263e 100644 --- a/ownlang/codegen.py +++ b/ownlang/codegen.py @@ -708,6 +708,7 @@ def _usings(mod: A.Module) -> list[str]: public static void Requested(int bytes) { System.Threading.Interlocked.Add(ref ScratchTotalRequestedBytes, bytes); + // CAS retry loop: update max if bytes > current max; retry if concurrent update long cur; do {