diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..d3789e01 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +name: CI + +on: + push: + branches: ["**"] + pull_request: + workflow_dispatch: + +jobs: + tests: + name: tests (py${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # The PoC needs 3.11+ (see README). Run the floor and current releases. + python-version: ["3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + # Zero-dependency project: nothing to install. The suite runs the + # analyzer cases, the golden ArrayPool lowering, the codegen content + # assertions, and the property fuzzer (fixed seed) in one entrypoint. + - name: Run test suite + run: python tests/run_tests.py + + # A heavier, non-blocking fuzz pass so a flake-free regression that only + # shows up on other random draws still gets surfaced on every push. + fuzz-extended: + name: extended codegen fuzz + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Property fuzz (50k draws, rotating seed) + run: python tests/test_codegen_props.py 50000 ${{ github.run_number }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..ab4f781d --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +*.ownreport.json + +# .NET build output (golden_arraypool demo) +bin/ +obj/ diff --git a/README.md b/README.md index 49d5e53b..24fc11d6 100644 --- a/README.md +++ b/README.md @@ -45,12 +45,13 @@ C# (шаблоны emit_* → реальный .NET; try/finally на straight- ### Запуск ```bash -cd ownlang -python -m ownlang check examples/ok_extern_calls.own # проверка -python -m ownlang emit examples/golden_arraypool/buffer.own # проверка + печать C# -python -m ownlang cfg examples/bad_maybe_release.own # дамп CFG +# запускать из корня репозитория (там, где лежит пакет ownlang/ и examples/) +python -m ownlang check examples/ok_extern_calls.own # проверка +python -m ownlang emit examples/golden_arraypool/buffer.own # проверка + печать C# +python -m ownlang cfg examples/bad_maybe_release.own # дамп CFG +python -m ownlang report examples/buffer_scratch.own # buffer-отчёт + .ownreport.json -python tests/run_tests.py # 42 кейса + codegen + golden smoke +python tests/run_tests.py # кейсы + codegen + golden + buffer smoke ``` `check` возвращает ненулевой код при наличии ошибок — годится для CI. @@ -60,7 +61,10 @@ python tests/run_tests.py # 42 кейса + c ```bash cd examples/golden_arraypool -dotnet run # требует .NET SDK; в песочнице PoC его нет +# Здесь лежат buffer.own (источник) и Program.cs (сгенерённый process + host). +# Своего .csproj PoC не возит; чтобы запустить — заверни Program.cs в console-проект: +dotnet new console -o demo && cp Program.cs demo/ && cd demo && dotnet run +# (требует .NET SDK; в песочнице PoC его нет — проверено по построению, не запуском) ``` `buffer.own` объявляет ресурс `Buffer` с шаблонами `emit_*`, отображающими его на @@ -208,7 +212,7 @@ fn process(size: int) { | **OWN004** | borrow убегает из своей области (например, `return` borrow'а) | | **OWN005** | use/… после move (**definite**) | | **OWN006** | `borrow_mut` при живом shared borrow | -| **OWN007** | move/consume владельца под живым borrow'ом | +| **OWN007** | move/consume/return владельца под живым borrow'ом | | **OWN008** | release владельца под живым borrow'ом | | **OWN009** | операция над ресурсом, который **мог** быть освобождён на каком-то пути (**maybe**) | | **OWN010** | операция над ресурсом, который **мог** быть перемещён на каком-то пути (**maybe**) | @@ -216,6 +220,17 @@ fn process(size: int) { | **OWN012** | shared borrow при живом `borrow_mut` | | **OWN013** | прямое обращение к владельцу, пока он `borrow_mut` | +### Буферы: storage policies + +| Код | Что ловит | +|-----|-----------| +| OWN015 | stack-backed буфер (`stack`/`scratch`/`inline`) пытается убежать из функции (`return`) | +| OWN016 | stack-backed буфер отдан в `consume`-вызов (move в более долгоживущего владельца) | +| OWN017 | movable-буфер (`pooled`/`native`) escape'ит — модель это разрешает, но PoC-codegen пока не умеет честно лоуэрить escape (см. ниже) | +| OWN019 | inline-ёмкость слишком велика для stack-backed политики (выше потолка стека) | +| OWN021 | `stack`/`inline` динамического размера без статической границы (нет `max =`) | +| OWN023 | `scratch` с `fallback = forbidden`, но размер может превысить inline-лимит | + ### Неподдерживаемое / структурное / граница | Код | Что ловит | @@ -299,6 +314,179 @@ control-flow в `finally` — настоящая работа, она в roadmap --- +## Буферы: storage policies + логирование + +`stackalloc` — это не оптимизация сама по себе. Это **storage strategy с жёстким +lifetime-контрактом**. Поэтому буфер в OwnLang — это owned-ресурс (release ровно +один раз, escape-проверки, конфликты borrow'ов — всё как обычно), но с явной +**политикой хранения**. Модель: *пользователь задаёт intent → checker проверяет +lifetime/ownership → backend выбирает или строго соблюдает storage → codegen +генерит безопасный C# → логи показывают фактический выбор → benchmark доказывает +выигрыш*. Не «компилятор молча решил за тебя» — а «ты задал политику, компилятор +её соблюл, runtime показал, что реально выбралось». + +### Режимы + +``` +let a = Buffer.stack(256); // только stackalloc, fallback запрещён +let b = Buffer.stack(size, max = 1024); // динамика, но с забором (guard) +let c = Buffer.scratch(size, inline = 1024, fallback = pool); // стек, иначе ArrayPool +let d = Buffer.pooled(size); // только ArrayPool; movable, Return обязателен +let e = Buffer.native(size); // NativeMemory; unsafe, Free обязателен +let f = Buffer.inline(128); // фиксированный compile-time стековый буфер +``` + +Главное правило: **`stack` никогда не падает в heap**; **`scratch` может**, потому +что пользователь явно разрешил fallback. API, который врёт про память, — это не +абстракция. `stack`/`scratch`/`inline` — stack-backed → не могут escape (OWN015/016). + +Буфер можно `move` внутри функции — владение и storage-политика переходят на +нового владельца, и `release` нового имени освобождает исходный backing. Namespace +обязан быть `Buffer`: `Foo.stack(...)` (опечатка/чужой идентификатор) — это +**OWN030**, а не тихая аллокация. + +`pooled`/`native` в **ownership-модели** movable (теоретически их можно +`return`/`consume`). Но **deliverable здесь — checker, а codegen лишь доказывает, +что модель лоуэрится в настоящий .NET**, и не раздувается в самоцель. Честно +лоуэрить *escaping* буфер нечем: значение внутри функции — это `Span`, а +отдавать наружу надо handle (`byte[]`/`byte*`+длина), которым вызывающий сделает +`Return`/`Free`. Поэтому PoC **отвергает** escape movable-буфера (**OWN017**), а не +шипает C#, который течёт или не компилируется. Локально `pooled`/`native` работают +полноценно (rent→borrow→release с реальным `ArrayPool.Return`/`NativeMemory.Free`). +Полноценный movable-lowering (через `byte[]`-handle или обёртку +`IMemoryOwner`) — **roadmap**. + +### `scratch` лоуэрится так (это и есть golden buffer-пример) + +```csharp +byte[]? tmp_rented = null; +Span tmp_backing = stackalloc byte[1024]; +Span tmp; +if (size <= 1024) +{ + OwnTrace.ScratchSelected("parse", "tmp", size, 1024, "stackalloc"); + OwnCounters.StackHit(); + tmp = tmp_backing[..size]; +} +else +{ + OwnTrace.ScratchSelected("parse", "tmp", size, 1024, "ArrayPool"); + OwnCounters.PoolFallback(size); + tmp_rented = ArrayPool.Shared.Rent(size); + tmp = tmp_rented.AsSpan(0, size); +} +try { /* ... */ } +finally +{ + OwnCounters.Release(); + if (tmp_rented is not null) + ArrayPool.Shared.Return(tmp_rented); +} +``` + +### Логирование — обязательная часть, а не опция + +Без логов `scratch` стал бы той самой «умной» абстракцией, которая молча выбрала +pool, а ты три часа смотришь на GC-график. Поэтому логи — в трёх местах: + +1. **Compile-time report** (`python -m ownlang report file.own`): что checker/codegen + решил по каждому буферу — mode, inline-лимит, fallback, escape-policy, clear, + сгенерированные ветки и какие проверки прошли. Выводится текстом и пишется в + `file.ownreport.json` (удобно для ревью/CI). + +2. **Runtime trace** — хук `OwnTrace.*` в сгенерированном C#: какой backend реально + выбран при конкретном `size`. Под `[Conditional("OWNSHARP_TRACE")]` — в обычном + Release вызовы вырезаются, логирование не становится новым bottleneck'ом. + +3. **Runtime counters** — `OwnCounters` (`ScratchStackHits`, `ScratchPoolFallbacks`, + `ScratchPoolBytesRented`, `ScratchReleaseCount`) под `[Conditional("OWNSHARP_COUNTERS")]`. + Отвечают на главный вопрос: мы реально часто попадаем в стек, или inline-лимит + подобран мимо? + +### Политики + +`policy`-блок — это переиспользуемый набор дефолтов; буфер ссылается на него через +`policy =`, инлайновые опции перекрывают: + +``` +policy SensitiveScratch { + inline_bytes = 512; + fallback = pool; + counters = true; + clear_on_release = true; // занулить байты перед возвратом в пул +} + +fn handle(size: int) { + let secret = Buffer.scratch(size, policy = SensitiveScratch); + borrow_mut secret as m { Fill(m); } + release secret; // codegen: secret.Clear(); затем Return +} +``` + +### Runnable golden + +`buffer_scratch_program.cs.txt` — запускаемый пример: метод `parse` и классы +`OwnTrace`/`OwnCounters` вклеены **дословно** из `python -m ownlang emit +buffer_scratch.own`, а `Fill`/`Hash`/`Main` — host-код. Доказывает, что +buffer-модель лоуэрится в настоящий .NET с реальным `ArrayPool.Rent/Return`: + +```bash +dotnet run -p:DefineConstants="OWNSHARP_TRACE;OWNSHARP_COUNTERS" +# parse(64) -> stackalloc-ветка (heap не трогаем) +# parse(4096) -> ArrayPool-ветка (реальные Rent/Return), trace + counters в выводе +``` + +### Где это жульничает + +Элемент буфера зафиксирован как `byte` (как во всех примерах). В straight-line +функции (без `if`/`move`/owned-return) буферы и обычные ресурсы лоуэрятся в порядке +исходника, каждый в свой exception-safe `try/finally` со split'ом по точке +`release` — **но только если времена жизни laminar** (любая пара вложена или +раздельна) **и каждый `release` на верхнем уровне**: непересекающиеся остаются +раздельными (a возвращается до аренды b), вложенные — нестятся (LIFO). Частичное +пересечение (`let a; let b; release a; … release b;`), `release` во вложенном +`borrow`/`if`-блоке, или ресурс, consume'нутый вызовом, hoist'ить нельзя без +искажения lifetime'а / двойной очистки, поэтому такие функции лоуэрятся +faithful-inline (release ровно там, где написан; без `try/finally`). +`scratch`/`stack`/`native` динамического размера guard'ят некорректный (в т.ч. +отрицательный) запрос **до** любого trace/counter, чтобы битый ввод не портил +метрики. Размер буфера обязан быть целым числом — `Buffer.pooled(flag: bool)`, owned-ресурс +или plain неизвестного типа (например копия borrow'а) как размер это **OWN018**; +а `inline` требует compile-time литерала — `Buffer.inline(n, max = …)` это +**OWN021** (для динамики есть `stack`). Plain-локал, объявленный в теле буфера и +использованный после release, не оборачивается в hoist'нутый `try` (иначе вышел бы +из C#-scope) — такой буфер лоуэрится inline. +Булевы настройки (`clear_on_release`, `counters`) и `trace` валидируются: опечатка +вроде `clear_on_release = ture` — **OWN030**, а не тихое отключение clear на +sensitive-буфере. `native` хранит `byte*` (backing, освобождается на release), но наружу +отдаёт `Span`-view — borrow/call видят тот же логический тип, что и +pooled/stack/scratch. Borrow-параметр типа `Buffer` (и в `extern`, и в **локальной** +`fn`) рендерится как `Span`/`ReadOnlySpan`, так что один +`fn helper(x: &mut Buffer)` лоуэрится в одну C#-сигнатуру для всех storage-режимов, +а вызов `helper(b)` компилируется. Отчёт атрибутирует диагностики по идентичности +буфера (`name#line:col`, переносится через `move`-алиасы), а не по имени в тексте — +два одноимённых буфера в соседних скоупах не путаются. +В ветвистой функции (есть `if`/`move`/owned-return) используется inline-режим: +буфер с чистым вложением получает `try/finally`, а перекрывающиеся времена жизни, +ветвистый release и moved-алиасы — inline-release (реальный cleanup в местах +release'ов, без подъёма в `finally`; обычные ресурсы там тоже inline — подъём из +произвольного control-flow это roadmap). `native` динамического размера +guard'ит отрицательный запрос перед `NativeMemory.Alloc`. Escaping movable-буферы +отвергаются (OWN017), полноценный movable-lowering — roadmap. Неизвестные значения +**и имена** настроек (mode, namespace, policy, fallback, а также сами имена опций +буфера и ключей policy-блока) ловятся как **OWN030** — опечатка в +`fallback = forbidden`, `fallback = 0` или `fallbak = forbidden` не «протечёт» в +heap, а будет отвергнута. Повторённая опция/ключ (`fallback = forbidden, +fallback = pool`) — тоже **OWN030**: конфликтующее обещание не разрешается +правилом «последний выигрывает». Бенчмарк-матрица из дизайн-дока +(safe vs unsafe, stack vs pool на размерах 32 B … 1 MB) — **следующий слой**: +правило «unsafe-backend разрешён только при выигрыше ≥ 10-15 % с disassembly- +обоснованием» задаёт дисциплину, но прогон бенчей вне песочницы. Unsafe-контракты +(`UNS0xx`) пока не реализованы: `native` лоуэрится в `NativeMemory.Alloc/Free` в +`unsafe`-блоке, но pointer-escape проверки — roadmap. + +--- + ## Changelog: перенумерация кодов Коды переразложены в связную схему. Если ты смотришь вывод прошлой версии: @@ -379,16 +567,18 @@ control-flow в `finally` — настоящая работа, она в roadmap ownlang/ ownlang/ lexer.py # токенизатор; цикл/async лексятся как REJECTED; строки для emit_* - ast_nodes.py # dataclass-узлы AST (resource, extern, call, эффекты) + ast_nodes.py # dataclass-узлы AST (resource, extern, call, эффекты, buffer, policy) parser.py # recursive descent; грамматика в docstring + buffers.py # storage policies: режимы, резолв policy+intent, валидация cfg.py # resolver (Symbol/Kind) + collect_signatures + lowering, Invoke analysis.py # flow-sensitive dataflow: var-states + active loans + permissions diagnostics.py # коды OWN0xx в одном месте - codegen.py # C# codegen (emit_* шаблоны, try/finally hoist + inline) - __main__.py # CLI: check / emit / cfg + codegen.py # C# codegen (emit_* шаблоны, try/finally hoist + inline, буферы) + report.py # compile-time buffer report -> stdout + .ownreport.json + __main__.py # CLI: check / emit / cfg / report examples/ ok_*.own # проходят bad_*.own # падают с конкретным кодом - golden_arraypool/ # buffer.own + Program.cs + demo.csproj (dotnet run) + golden_arraypool/ # buffer.own + Program.cs (host-код; .csproj не входит) tests/run_tests.py # 42 кейса анализа + codegen smoke + golden smoke ``` diff --git a/codegen.py b/codegen.py deleted file mode 100644 index 591f385f..00000000 --- a/codegen.py +++ /dev/null @@ -1,258 +0,0 @@ -""" -C# code generator for OwnLang. - -Two modes, chosen automatically per function: - -* **try/finally hoist** — when a function has no branches, no `move`, and no - owned `return`, every owned resource is acquired and released exactly once. - We lower it to the textbook exception-safe pattern: acquire, `try { ... }`, - `finally { release }`, nested for multiple resources. The checker has already - proven release-exactly-once; the `finally` additionally makes it hold across - C# exceptions. Note we do NOT emit a runtime "released?" flag: because the - release is hoisted *out* of the `try` (it is not also in the body), it runs - exactly once with no guard needed. A runtime flag would only make sense if we - didn't trust the static result — and if we don't trust it, we shouldn't ship. - -* **faithful inline** — for functions with branches or ownership transfer, - releases are emitted inline exactly where the source put them. Hoisting - releases out of arbitrary control flow into `finally` is real work, flagged as - a next step in the README rather than faked. - -Resource lowering: if a `resource` declares `emit_type` / `emit_acquire` / -`emit_release` / `emit_borrow` templates, those produce REAL .NET (e.g. -`ArrayPool.Shared.Rent/Return`, `byte[]`, `.AsSpan()`). Otherwise the -schematic `Resource.method()` form is used, with method names taken from the -declaration. `extern fn` calls lower to `Name(args)`; a borrow binding renders -as its C# local (the span / ref), an owned argument as its variable. -""" - -from __future__ import annotations - -from . import ast_nodes as A - - -class CodegenError(Exception): - pass - - -def _csharp_type(t: A.TypeRef) -> str: - base = {"int": "int", "bool": "bool"}.get(t.name, t.name) - if t.borrowed: - return ("ref " if t.mutable else "ref readonly ") + base - return base - - -class _FnGen: - def __init__(self, mod: A.Module, fn: A.FnDecl): - self.mod = mod - self.fn = fn - self.res = {r.name: r for r in mod.resources} - self.owned_resource: dict[str, str] = {} - # borrow binding name -> owner resource type (so calls know the C# view) - self.binding_owner_res: dict[str, str] = {} - for p in fn.params: - if not p.type.borrowed and p.type.name in self.res: - self.owned_resource[p.name] = p.type.name - - # -- shape detection ---------------------------------------------------- - - def _is_simple(self) -> bool: - return not _contains_branch_or_transfer(self.fn.body) - - # -- emit --------------------------------------------------------------- - - def emit(self) -> str: - ret = _csharp_type(self.fn.ret) if self.fn.ret else "void" - params = ", ".join(f"{_csharp_type(p.type)} {p.name}" for p in self.fn.params) - head = f"public static {ret} {self.fn.name}({params})" - if self._is_simple(): - body = self._emit_simple(self.fn.body) - else: - body = self._emit_inline(self.fn.body, indent=1) - return f"{head}\n{{\n{body}}}\n" - - # -- simple (try/finally hoist) ---------------------------------------- - - def _emit_simple(self, stmts: list[A.Stmt]) -> str: - acquired: list[tuple[str, str, str]] = [] # (var, resource, args_csv) - other: list[A.Stmt] = [] - for st in stmts: - if isinstance(st, A.Let) and isinstance(st.rhs, A.Acquire): - rt = st.rhs.resource - self.owned_resource[st.name] = rt - args_csv = ", ".join(self._arg(x) for x in st.rhs.args) - acquired.append((st.name, rt, args_csv)) - elif isinstance(st, A.Release): - pass # consumed by the finally - else: - other.append(st) - - lines: list[str] = [] - ind = " " - - def open_resource(idx: int, base: str) -> None: - var, rt, args_csv = acquired[idx] - lines.append(f"{base}{self._local_type(rt)} {var} = {self._acquire_expr(rt, args_csv)};") - lines.append(f"{base}try") - lines.append(f"{base}{{") - inner = base + ind - if idx + 1 < len(acquired): - open_resource(idx + 1, inner) - else: - for st in other: - lines.extend(self._stmt_inline(st, inner)) - lines.append(f"{base}}}") - lines.append(f"{base}finally") - lines.append(f"{base}{{") - lines.append(f"{base}{ind}{self._release_stmt(var, rt)}") - lines.append(f"{base}}}") - - if acquired: - open_resource(0, ind) - else: - for st in other: - lines.extend(self._stmt_inline(st, ind)) - return "".join(l + "\n" for l in lines) - - # -- faithful inline ---------------------------------------------------- - - def _emit_inline(self, stmts: list[A.Stmt], indent: int) -> str: - ind = " " * indent - out: list[str] = [] - for st in stmts: - out.extend(self._stmt_inline(st, ind)) - return "".join(l + "\n" for l in out) - - def _stmt_inline(self, st: A.Stmt, ind: str) -> list[str]: - if isinstance(st, A.Let): - if isinstance(st.rhs, A.Acquire): - rt = st.rhs.resource - self.owned_resource[st.name] = rt - args_csv = ", ".join(self._arg(x) for x in st.rhs.args) - return [f"{ind}{self._local_type(rt)} {st.name} = {self._acquire_expr(rt, args_csv)};"] - if isinstance(st.rhs, A.Move): - self.owned_resource[st.name] = self.owned_resource.get(st.rhs.var, "") - return [f"{ind}var {st.name} = {st.rhs.var}; " - f"// ownership moved from {st.rhs.var}"] - if isinstance(st.rhs, A.IntLit): - return [f"{ind}var {st.name} = {st.rhs.value};"] - if isinstance(st.rhs, A.VarRef): - return [f"{ind}var {st.name} = {st.rhs.name};"] - if isinstance(st, A.Release): - rt = self.owned_resource.get(st.var, "") - return [f"{ind}{self._release_stmt(st.var, rt)}"] - if isinstance(st, A.Use): - return [f"{ind}Use({st.var});"] - if isinstance(st, A.Call): - return [f"{ind}{st.callee}({', '.join(self._arg(a) for a in st.args)});"] - if isinstance(st, A.BorrowBlock): - kind = "mutable" if st.kind == A.BorrowKind.MUT else "shared" - rt = self.owned_resource.get(st.owner, "") - self.binding_owner_res[st.binding] = rt - head = [f"{ind}{{ // {kind} borrow of {st.owner} as {st.binding}", - f"{ind} var {st.binding} = {self._borrow_expr(st.owner, rt)};"] - body: list[str] = [] - for inner in st.body: - body.extend(self._stmt_inline(inner, ind + " ")) - return head + body + [f"{ind}}}"] - if isinstance(st, A.If): - out = [f"{ind}if ({st.cond_text or 'cond'})", f"{ind}{{"] - for s in st.then_body: - out.extend(self._stmt_inline(s, ind + " ")) - out.append(f"{ind}}}") - if st.else_body: - out.append(f"{ind}else") - out.append(f"{ind}{{") - for s in st.else_body: - out.extend(self._stmt_inline(s, ind + " ")) - out.append(f"{ind}}}") - return out - if isinstance(st, A.Return): - return [f"{ind}return {st.var};" if st.var else f"{ind}return;"] - raise CodegenError(f"cannot codegen {st!r}") - - # -- template helpers --------------------------------------------------- - - def _local_type(self, resource: str) -> str: - r = self.res.get(resource) - if r and r.emit_type: - return r.emit_type - return "var" - - def _acquire_expr(self, resource: str, args_csv: str) -> str: - r = self.res.get(resource) - if r and r.emit_acquire: - return r.emit_acquire.replace("{args}", args_csv) - method = _member(r, "acquire") if r else "Create" - return f"{resource}.{method}({args_csv})" - - def _release_stmt(self, var: str, resource: str) -> str: - r = self.res.get(resource) - if r and r.emit_release: - return r.emit_release.replace("{0}", var) + ";" - method = _member(r, "release") if r else "Dispose" - return f"{var}.{method}();" - - def _borrow_expr(self, owner_var: str, resource: str) -> str: - r = self.res.get(resource) - if r and r.emit_borrow: - return r.emit_borrow.replace("{0}", owner_var) - return owner_var - - def _arg(self, e: A.Expr) -> str: - if isinstance(e, A.IntLit): - return str(e.value) - if isinstance(e, A.VarRef): - return e.name - return "/* expr */" - - -def _member(r: A.ResourceDecl, role: str) -> str: - for m in r.members: - if m.role == role: - return m.name - return "Create" if role == "acquire" else "Dispose" - - -def _contains_branch_or_transfer(stmts: list[A.Stmt]) -> bool: - for st in stmts: - if isinstance(st, A.If): - return True - if isinstance(st, A.Return) and st.var is not None: - return True - if isinstance(st, A.Let) and isinstance(st.rhs, A.Move): - return True - if isinstance(st, A.BorrowBlock): - if _contains_branch_or_transfer(st.body): - return True - return False - - -def _usings(mod: A.Module) -> list[str]: - out = ["using System;"] - blob = " ".join( - (r.emit_acquire or "") + (r.emit_release or "") + (r.emit_type or "") - for r in mod.resources - ) - if "ArrayPool" in blob: - out.append("using System.Buffers;") - return out - - -def generate(mod: A.Module) -> str: - parts = [ - "// by OwnLang PoC. Ownership was checked at the .own", - "// level; this C# is the lowering. Do not hand-edit.", - ] - parts.extend(_usings(mod)) - parts.append("") - parts.append(f"public static class {mod.name}") - parts.append("{") - bodies = [_FnGen(mod, fn).emit() for fn in mod.functions] - indented = [] - for b in bodies: - indented.append("\n".join(" " + line if line else line - for line in b.splitlines())) - parts.append("\n\n".join(indented)) - parts.append("}") - return "\n".join(parts) + "\n" diff --git a/bad_leak_branch.own.txt b/examples/bad_leak_branch.own similarity index 100% rename from bad_leak_branch.own.txt rename to examples/bad_leak_branch.own diff --git a/examples/bad_maybe_release.own b/examples/bad_maybe_release.own new file mode 100644 index 00000000..f14ce2ed --- /dev/null +++ b/examples/bad_maybe_release.own @@ -0,0 +1,18 @@ +module MaybeReleaseDemo + +resource Buffer { + acquire rent + release give +} + +// 'b' is released on the 'then' path only. At the join its state is +// inconsistent, so the following `use b` touches a maybe-released value +// (OWN009). The 'else' path never releases, so it also leaks (OWN001). +// Good for `python -m ownlang cfg examples/bad_maybe_release.own`. +fn use_after_maybe(flag: int) { + let b = acquire Buffer(flag); + if (flag) { + release b; + } + use b; +} diff --git a/examples/bad_scratch_escape.own b/examples/bad_scratch_escape.own new file mode 100644 index 00000000..3ea1f305 --- /dev/null +++ b/examples/bad_scratch_escape.own @@ -0,0 +1,10 @@ +module BadScratchEscape + +// OWN015: a scratch buffer may be stack-backed, so returning it would hand back +// a span into a frame that is about to pop. If you need to return a buffer, use +// Buffer.pooled(...) — a movable owned resource whose Return the checker enforces. + +fn leak(size: int) -> Buffer { + let tmp = Buffer.scratch(size, inline = 1024); + return tmp; +} diff --git a/examples/buffer_scratch.own b/examples/buffer_scratch.own new file mode 100644 index 00000000..8417de87 --- /dev/null +++ b/examples/buffer_scratch.own @@ -0,0 +1,32 @@ +module ScratchDemo + +// A scratch buffer: prefer the stack, fall back to ArrayPool when the request +// exceeds the inline limit. The user states the intent; the checker proves it +// stays local; codegen emits both branches; and OwnTrace/OwnCounters log which +// backend was actually chosen at runtime. +// +// python -m ownlang check buffer_scratch.own # ownership is clean +// python -m ownlang emit buffer_scratch.own # stack-first + pool fallback +// python -m ownlang report buffer_scratch.own # what the compiler decided + +policy DefaultScratch { + inline_bytes = 1024; + fallback = pool; + trace = debug; + counters = true; + clear_on_release = false; +} + +extern fn Fill(borrow_mut Buffer); +extern fn Hash(borrow Buffer); + +fn parse(size: int) { + let tmp = Buffer.scratch(size, inline = 1024, fallback = pool); + borrow_mut tmp as bytes { + Fill(bytes); + } + borrow tmp as view { + Hash(view); + } + release tmp; +} diff --git a/examples/buffer_stack.own b/examples/buffer_stack.own new file mode 100644 index 00000000..9b8388ae --- /dev/null +++ b/examples/buffer_stack.own @@ -0,0 +1,26 @@ +module StackDemo + +// Strict stack mode: stackalloc only, the heap fallback is FORBIDDEN, the buffer +// cannot escape. A constant bound lowers to `stackalloc byte[N]`; a dynamic size +// requires a `max =` guard so the real stack frame is bounded — "we put up a +// fence", not "the user promised". + +extern fn Fill(borrow_mut Buffer); + +fn fixed_buf() { + let line = Buffer.stack(256); // -> Span line = stackalloc byte[256]; + borrow_mut line as m { + Fill(m); + } + release line; +} + +fn bounded(size: int) +{ + // dynamic size, but bounded: codegen emits a guard then slices the backing. + let buf = Buffer.stack(size, max = 1024); + borrow_mut buf as m { + Fill(m); + } + release buf; +} diff --git a/Program.cs.txt b/examples/golden_arraypool/Program.cs similarity index 100% rename from Program.cs.txt rename to examples/golden_arraypool/Program.cs diff --git a/buffer.own.txt b/examples/golden_arraypool/buffer.own similarity index 100% rename from buffer.own.txt rename to examples/golden_arraypool/buffer.own diff --git a/examples/ok_extern_calls.own b/examples/ok_extern_calls.own new file mode 100644 index 00000000..671fba46 --- /dev/null +++ b/examples/ok_extern_calls.own @@ -0,0 +1,30 @@ +module ExternCalls + +resource Buffer { + acquire rent + release give +} + +// Host (C#) calls must declare what they do with ownership. The checker +// trusts these effect annotations and nothing else: an undeclared call that +// is handed an owned value is a hard OWN040. Borrow params are noescape, so +// the value cannot leak out through them. +extern fn Fill(borrow_mut Buffer); // writes through the buffer, hands it back +extern fn Hash(borrow Buffer); // reads only, hands it back +extern fn Store(consume Buffer); // takes ownership; caller must not reuse + +// Happy path: lend the buffer to the host mutably, then immutably, then hand +// it back ourselves. Both extern calls borrow, so 'b' is still owned after. +fn read_write(size: int) { + let b = acquire Buffer(size); + Fill(b); + Hash(b); + release b; +} + +// Happy path: give the buffer to the host for good. Store consumes it, so we +// neither release nor use 'b' afterwards — either of those would be OWN002. +fn hand_off(size: int) { + let b = acquire Buffer(size); + Store(b); +} diff --git a/ok_pool.own.txt b/examples/ok_pool.own similarity index 100% rename from ok_pool.own.txt rename to examples/ok_pool.own diff --git a/ownlang/__init__.py b/ownlang/__init__.py new file mode 100644 index 00000000..42bdc001 --- /dev/null +++ b/ownlang/__init__.py @@ -0,0 +1 @@ +"""OwnLang — a micro ownership/borrow checker PoC for .NET codegen.""" diff --git a/__main__.py b/ownlang/__main__.py similarity index 66% rename from __main__.py rename to ownlang/__main__.py index 901657f7..634aa8e6 100644 --- a/__main__.py +++ b/ownlang/__main__.py @@ -4,6 +4,7 @@ python -m ownlang check file.own # report ownership diagnostics python -m ownlang emit file.own # check, then print generated C# python -m ownlang cfg file.own # dump the control-flow graph + python -m ownlang report file.own # buffer storage report + .ownreport.json Exit code is non-zero if any error-level diagnostic was produced. """ @@ -14,9 +15,11 @@ from .parser import parse, ParseError from .lexer import LexError -from .cfg import build_cfg, collect_signatures, CFG +from .cfg import build_cfg, collect_signatures, collect_policies, CFG from .analysis import analyze from .codegen import generate +from .buffers import validate_policies +from .report import build_report, render_report from .diagnostics import Diagnostic, Severity @@ -28,9 +31,10 @@ def _collect(src: str) -> tuple[list[Diagnostic], object | None]: return [Diagnostic("OWN020", str(e).split(": ", 1)[-1], line)], None rnames = {r.name for r in mod.resources} sigs = collect_signatures(mod) - diags: list[Diagnostic] = [] + pols = collect_policies(mod) + diags: list[Diagnostic] = list(validate_policies(pols)) for fn in mod.functions: - cfg, d1 = build_cfg(fn, rnames, sigs) + cfg, d1 = build_cfg(fn, rnames, sigs, pols) d2 = analyze(cfg) diags.extend(d1) diags.extend(d2) @@ -73,12 +77,38 @@ def cmd_cfg(path: str) -> int: return 1 rnames = {r.name for r in mod.resources} sigs = collect_signatures(mod) + pols = collect_policies(mod) for fn in mod.functions: - cfg, _ = build_cfg(fn, rnames, sigs) + cfg, _ = build_cfg(fn, rnames, sigs, pols) _print_cfg(cfg) return 0 +def cmd_report(path: str) -> int: + """Emit the compile-time buffer report: what storage policy the checker and + codegen settled on for every buffer, and which checks passed. Prints a + human summary to stdout and writes the machine-readable .ownreport.json.""" + src = _read(path) + diags, mod = _collect(src) + if mod is None: + for d in diags: + print(d.render(path), file=sys.stderr) + return 1 + # surface diagnostics (e.g. a mistyped buffer mode) without crashing the + # report; the report still covers every well-formed buffer. + errors = [d for d in diags if d.severity == Severity.ERROR] + for d in diags: + print(d.render(path), file=sys.stderr) + report = build_report(mod, diags) # type: ignore[arg-type] + print(render_report(report)) + out_path = path.rsplit(".", 1)[0] + ".ownreport.json" + import json + with open(out_path, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2) + print(f"\nwrote {out_path}") + return 1 if errors else 0 + + def _print_cfg(cfg: CFG) -> None: print(f"fn {cfg.fn_name} (entry: B{cfg.entry}, " f"params: {[p.name for p in cfg.params]})") @@ -91,10 +121,15 @@ def _print_cfg(cfg: CFG) -> None: def _fmt_instr(ins) -> str: - from .cfg import (Acquire, MoveInto, Release, Use, Invoke, + from .cfg import (Acquire, AcquireBuffer, MoveInto, Release, Use, Invoke, BorrowStart, BorrowEnd, Return) if isinstance(ins, Acquire): return f"acquire {ins.sym.name} : {ins.resource}" + if isinstance(ins, AcquireBuffer): + i = ins.info + size = i.size_const if i.size_is_const else (i.size_var or "?") + return (f"buffer {ins.sym.name} : {i.mode.value}(size={size}, " + f"inline={i.inline_bytes}, fallback={'pool' if i.fallback_pool else 'none'})") if isinstance(ins, MoveInto): return f"move {ins.src.name} -> {ins.dst.name}" if isinstance(ins, Release): @@ -122,11 +157,12 @@ def _read(path: str) -> str: def main(argv: list[str]) -> int: - if len(argv) < 2 or argv[0] not in {"check", "emit", "cfg"}: + if len(argv) < 2 or argv[0] not in {"check", "emit", "cfg", "report"}: print(__doc__) return 2 cmd, path = argv[0], argv[1] - return {"check": cmd_check, "emit": cmd_emit, "cfg": cmd_cfg}[cmd](path) + return {"check": cmd_check, "emit": cmd_emit, "cfg": cmd_cfg, + "report": cmd_report}[cmd](path) if __name__ == "__main__": diff --git a/analysis.py b/ownlang/analysis.py similarity index 83% rename from analysis.py rename to ownlang/analysis.py index 2ab46e47..b681a75c 100644 --- a/analysis.py +++ b/ownlang/analysis.py @@ -38,7 +38,8 @@ from .cfg import ( CFG, Block, Symbol, Kind, - Acquire, MoveInto, Release, Use, Invoke, BorrowStart, BorrowEnd, Return, + Acquire, AcquireBuffer, MoveInto, Release, Use, Invoke, + BorrowStart, BorrowEnd, Return, ) from .ast_nodes import Effect from .diagnostics import Diagnostic @@ -103,8 +104,9 @@ def initial_state(self) -> State: s.var[id(p)] = {VarState.OWNED} return s - def err(self, code: str, msg: str, line: int) -> None: - self.diags.append(Diagnostic(code, msg, line)) + def err(self, code: str, msg: str, line: int, + subject: str | None = None) -> None: + self.diags.append(Diagnostic(code, msg, line, subject=subject)) # -- loan / permission helpers ----------------------------------------- @@ -129,24 +131,28 @@ def binding_live(self, st: State, sym: Symbol) -> bool: # gone / maybe-gone cases shared by use/move/release/borrow/consume. def _state_problem(self, st: State, sym: Symbol, verb: str, line: int) -> bool: S = st.var.get(id(sym), {VarState.OWNED}) + subj = sym.origin if VarState.OWNED not in S: if VarState.MOVED in S: - self.err("OWN005", f"{verb} '{sym.name}' after it was moved", line) + self.err("OWN005", f"{verb} '{sym.name}' after it was moved", + line, subject=subj) elif VarState.ESCAPED in S and VarState.RELEASED not in S: self.err("OWN002", - f"{verb} '{sym.name}' after it was consumed", line) + f"{verb} '{sym.name}' after it was consumed", line, + subject=subj) else: - self.err("OWN002", f"{verb} '{sym.name}' after it was released", line) + self.err("OWN002", f"{verb} '{sym.name}' after it was released", + line, subject=subj) return True if S & {VarState.RELEASED, VarState.ESCAPED}: self.err("OWN009", f"{verb} '{sym.name}', which may have been released on some " - f"path", line) + f"path", line, subject=subj) return True if VarState.MOVED in S: self.err("OWN010", f"{verb} '{sym.name}', which may have been moved on some " - f"path", line) + f"path", line, subject=subj) return True return False @@ -230,7 +236,8 @@ def leak_check(self, st: State, at_line: int, context: str, name = sym.name if sym else f"#{symid}" self.err("OWN001", f"'{name}' is owned but not released {context} " - f"(leaks on at least one path)", at_line) + f"(leaks on at least one path)", at_line, + subject=(sym.origin if sym else None)) def _sym_by_id(self, symid: int) -> Symbol | None: if not hasattr(self, "_symindex"): @@ -263,6 +270,10 @@ def step(self, ins, st: State) -> None: st.var[id(ins.sym)] = {VarState.OWNED} return + if isinstance(ins, AcquireBuffer): + st.var[id(ins.sym)] = {VarState.OWNED} + return + if isinstance(ins, MoveInto): self._consume_like(st, ins.src, "move", ins.line, code_borrowed="OWN007") st.var[id(ins.src)] = {VarState.MOVED} @@ -270,19 +281,21 @@ def step(self, ins, st: State) -> None: return if isinstance(ins, Release): + subj = ins.sym.origin S = st.var.get(id(ins.sym), {VarState.OWNED}) if S == {VarState.RELEASED}: - self.err("OWN003", f"'{ins.sym.name}' is released twice", ins.line) + self.err("OWN003", f"'{ins.sym.name}' is released twice", + ins.line, subject=subj) elif VarState.RELEASED in S: self.err("OWN003", f"'{ins.sym.name}' may already be released on some path " - f"before this release", ins.line) + f"before this release", ins.line, subject=subj) elif not self._state_problem(st, ins.sym, "release", ins.line): shared, mut = self.loans_on(st, ins.sym) if shared or mut: self.err("OWN008", f"cannot release '{ins.sym.name}' while it is borrowed", - ins.line) + ins.line, subject=subj) st.var[id(ins.sym)] = {VarState.RELEASED} return @@ -327,16 +340,37 @@ def step(self, ins, st: State) -> None: self.leak_check(st, at_line=ins.line, context="before return", exclude=ins.sym) if ins.sym is not None: + subj = ins.sym.origin S = st.var.get(id(ins.sym), {VarState.OWNED}) if VarState.OWNED not in S: if VarState.MOVED in S: self.err("OWN005", f"'{ins.sym.name}' returned after it was moved", - ins.line) + ins.line, subject=subj) else: self.err("OWN002", f"'{ins.sym.name}' returned after it was released", - ins.line) + ins.line, subject=subj) + else: + # returning an owner is an escape (consume): it needs Own + # permission, so a live loan on it is OWN007, just like move. + shared, mut = self.loans_on(st, ins.sym) + if shared or mut: + self.err("OWN007", + f"cannot return '{ins.sym.name}' while it is " + f"borrowed", ins.line, subject=subj) + elif ins.sym.buffer is not None and ins.sym.buffer.stack_backed: + self.err("OWN015", + f"'{ins.sym.name}' is a {ins.sym.buffer.mode.value} " + f"buffer and may be stack-backed; it cannot escape " + f"the current function", ins.line, subject=subj) + elif ins.sym.buffer is not None: + self.err("OWN017", + f"'{ins.sym.name}' is a {ins.sym.buffer.mode.value} " + f"buffer; the PoC code generator cannot lower an " + f"escaping buffer to faithful .NET (the caller gets " + f"no handle to Return/Free), so returning it is " + f"rejected", ins.line, subject=subj) st.var[id(ins.sym)] = {VarState.ESCAPED} return @@ -381,6 +415,18 @@ def _apply_effect(self, st: State, sym: Symbol, eff: Effect, if sym.kind == Kind.OWNED: if eff == Effect.CONSUME: self._consume_like(st, sym, "consume", line, code_borrowed="OWN007") + if sym.buffer is not None and sym.buffer.stack_backed: + self.err("OWN016", + f"'{sym.name}' is a {sym.buffer.mode.value} buffer " + f"and may be stack-backed; it cannot be moved to a " + f"longer-lived owner by consuming it in '{callee}'", + line, subject=sym.origin) + elif sym.buffer is not None: + self.err("OWN017", + f"'{sym.name}' is a {sym.buffer.mode.value} buffer; " + f"the PoC code generator cannot lower an escaping " + f"buffer, so consuming it in '{callee}' is rejected", + line, subject=sym.origin) st.var[id(sym)] = {VarState.ESCAPED} elif eff == Effect.BORROW_MUT: self._check_mut_borrowable(st, sym, line) diff --git a/ast_nodes.py b/ownlang/ast_nodes.py similarity index 77% rename from ast_nodes.py rename to ownlang/ast_nodes.py index db839df1..33110275 100644 --- a/ast_nodes.py +++ b/ownlang/ast_nodes.py @@ -60,7 +60,23 @@ class Move: line: int -Expr = IntLit | VarRef | Acquire | Move +@dataclass +class BufferIntent: + """Buffer.(size, name = value, ...) -> Owned with a storage + policy. `mode` is one of stack/scratch/pooled/native/inline. `size` is the + single positional argument (an IntLit or VarRef), or None. `options` maps a + named option (inline, max, fallback, clear, trace, counters, policy) to its + value expression. `ns` is the namespace as written (must be "Buffer").""" + mode: str + size: "Expr | None" + options: dict[str, "Expr"] + line: int + ns: str = "Buffer" + col: int = 0 + dups: tuple = () # option names that appeared more than once + + +Expr = IntLit | VarRef | Acquire | Move | BufferIntent # ---- statements ----------------------------------------------------------- @@ -177,9 +193,20 @@ class FnDecl: line: int +@dataclass +class PolicyDecl: + """policy Name { key = value; ... } — a named bundle of buffer defaults + (inline_bytes, max_bytes, mode, fallback, trace, counters, clear_on_release).""" + name: str + settings: dict[str, object] + line: int + dups: tuple = () # setting keys that appeared more than once + + @dataclass class Module: name: str resources: list[ResourceDecl] = field(default_factory=list) externs: list[ExternDecl] = field(default_factory=list) functions: list[FnDecl] = field(default_factory=list) + policies: list[PolicyDecl] = field(default_factory=list) diff --git a/ownlang/buffers.py b/ownlang/buffers.py new file mode 100644 index 00000000..04757192 --- /dev/null +++ b/ownlang/buffers.py @@ -0,0 +1,452 @@ +""" +Buffer storage policies — the stackalloc / scratch / pool / native line. + +A *buffer* is an owned resource (it is checked for release-exactly-once, escape, +and borrow conflicts just like any `acquire`d resource), but it additionally +carries an explicit **storage policy**. The policy is something the user states +as an intent; the checker proves the lifetime/ownership rules; the backend either +chooses or strictly honours the storage; codegen emits safe C#; and — the part +that matters — the choice is *logged* so nobody has to guess whether a "stack" +buffer quietly went to the heap. + +Modes (intent the user writes as `Buffer.(...)`): + + stack(N) / stack(size, max = M) + stack only. fallback to the heap is FORBIDDEN. cannot escape. the bound + must be statically known (a literal, or a `max =` guard for a dynamic size). + + scratch(size, inline = L, fallback = pool) + prefer the stack; fall back to ArrayPool when the request exceeds the + inline limit. local-only (may be stack-backed, so it cannot escape). + + pooled(size) + ArrayPool only. a movable owned resource (it may escape via consume/return). + Return is mandatory — enforced by the ownership checker. + + native(size) + unmanaged memory via NativeMemory. an unsafe owned resource. Free is + mandatory. movable. + + inline(N) + a fixed compile-time stack buffer. the most predictable mode. cannot escape. + +The single rule the whole design rests on: `stack` never falls into the heap; +`scratch` may, because the user explicitly allowed it. An API that lies about +where memory lives is not an abstraction. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + +from . import ast_nodes as A +from .diagnostics import Diagnostic + + +class BufferMode(Enum): + STACK = "stack" + SCRATCH = "scratch" + POOLED = "pooled" + NATIVE = "native" + INLINE = "inline" + + +MODE_NAMES = {m.value for m in BufferMode} + +# Recognized named options on a buffer intent, and recognized keys in a policy +# block. An unknown name (a typo like `fallbak`) must be rejected, not silently +# ignored — otherwise a misspelled `fallback = forbidden` quietly defaults to the +# heap, defeating the explicit storage guarantee. +VALID_OPTIONS = frozenset({ + "policy", "inline", "inline_bytes", "max", "max_bytes", + "fallback", "clear", "trace", "counters", +}) +VALID_POLICY_KEYS = frozenset({ + "inline_bytes", "max_bytes", "fallback", "clear_on_release", + "trace", "counters", +}) + +# Modes whose backing storage may live on the stack. A stack-backed buffer must +# not escape the function: it would be a dangling span the instant the frame +# pops. `scratch` is included because at runtime it *might* be the stack arm. +STACK_BACKED = {BufferMode.STACK, BufferMode.SCRATCH, BufferMode.INLINE} + +# Default inline limit for scratch when the user omits `inline =`. +DEFAULT_INLINE_BYTES = 1024 + +# Hard ceiling on how many bytes a stack-backed mode may reserve on the frame. +# Past this you are not optimising, you are inviting a stack overflow. Tunable. +MAX_STACK_BYTES = 4096 + + +@dataclass +class BufferInfo: + """Resolved, validated metadata for one buffer intent. Attached to the + owning Symbol and to the AcquireBuffer instruction, and read by codegen and + the report writer. Resolution is total: by the time this exists, any policy + contradiction has already been turned into a diagnostic.""" + mode: BufferMode + elem: str # element type, e.g. "byte" + # size: exactly one of const / var is set (or both None for a no-arg mode) + size_const: int | None + size_var: str | None + inline_bytes: int # inline capacity / stack bound, in elements + fallback_pool: bool # scratch: heap fallback allowed + fallback_forbidden: bool # stack: heap fallback explicitly forbidden + clear_on_release: bool # zero the bytes before returning/releasing + trace: bool # emit OwnTrace hooks + counters: bool # emit OwnCounters hooks + policy_name: str | None + line: int + + @property + def stack_backed(self) -> bool: + return self.mode in STACK_BACKED + + @property + def size_is_const(self) -> bool: + return self.size_const is not None + + @property + def escape_policy(self) -> str: + return "local-only" if self.stack_backed else "movable" + + def branches(self) -> list[dict]: + """The runtime backend branches, for the compile-time report.""" + if self.mode == BufferMode.SCRATCH and self.fallback_pool: + return [ + {"condition": f"size <= {self.inline_bytes}", "backend": "stackalloc"}, + {"condition": f"size > {self.inline_bytes}", "backend": "ArrayPool"}, + ] + if self.mode in (BufferMode.STACK, BufferMode.INLINE) or ( + self.mode == BufferMode.SCRATCH and not self.fallback_pool): + # a scratch that forbids the heap fallback is, at runtime, stack-only; + # the report must not advertise an ArrayPool branch that cannot occur. + return [{"condition": "always", "backend": "stackalloc"}] + if self.mode == BufferMode.POOLED: + return [{"condition": "always", "backend": "ArrayPool"}] + return [{"condition": "always", "backend": "NativeMemory"}] + + +@dataclass +class Policy: + """A named, reusable `policy { ... }` block of defaults.""" + name: str + settings: dict[str, object] = field(default_factory=dict) + line: int = 0 + dups: tuple = () # setting keys that appeared more than once + + +# -------------------------------------------------------------------------- +# Option parsing helpers (turn raw AST option values into Python values) +# -------------------------------------------------------------------------- + + +def _as_int(expr) -> int | None: + return expr.value if isinstance(expr, A.IntLit) else None + + +def _as_ident(expr) -> str | None: + return expr.name if isinstance(expr, A.VarRef) else None + + +# -------------------------------------------------------------------------- +# Resolution: intent + policies -> validated BufferInfo + diagnostics +# -------------------------------------------------------------------------- + + +def validate_policies(policies: dict[str, Policy]) -> list[Diagnostic]: + """Reject unknown keys in `policy` blocks (a typo like `fallbak` must not be + silently ignored). Reported once, at the policy's declaration.""" + diags: list[Diagnostic] = [] + for pol in policies.values(): + for key in pol.settings: + if key not in VALID_POLICY_KEYS: + diags.append(Diagnostic( + "OWN030", + f"unknown policy setting '{key}' in policy '{pol.name}'; " + f"expected one of {', '.join(sorted(VALID_POLICY_KEYS))}", + pol.line)) + for key in pol.dups: + diags.append(Diagnostic( + "OWN030", + f"duplicate policy setting '{key}' in policy '{pol.name}'", + pol.line)) + return diags + + +def resolve(intent: "A.BufferIntent", policies: dict[str, Policy] + ) -> tuple[BufferInfo, list[Diagnostic]]: + """Resolve one buffer intent against the available policies. Returns the + metadata plus any policy/bound diagnostics (OWN019/021/023). Always returns + a usable BufferInfo so later stages have something to lower.""" + diags: list[Diagnostic] = [] + line = intent.line + + mode = BufferMode(intent.mode) + opts = dict(intent.options) + + # reject misspelled / unknown option names before any defaults are applied, + # so e.g. `fallbak = forbidden` cannot silently fall back to the pool. + for key in intent.options: + if key not in VALID_OPTIONS: + diags.append(Diagnostic( + "OWN030", + f"unknown buffer option '{key}'; expected one of " + f"{', '.join(sorted(VALID_OPTIONS))}", line)) + # a repeated option (e.g. fallback = forbidden, fallback = pool) is a + # conflicting storage promise; reject it instead of letting the last win. + for key in intent.dups: + diags.append(Diagnostic( + "OWN030", f"duplicate buffer option '{key}'", line)) + + # Start from a referenced policy's defaults, then let inline options win. + base: dict[str, object] = {} + pol_name: str | None = None + pol_expr = opts.pop("policy", None) + if pol_expr is not None: + pol_name = _as_ident(pol_expr) + if pol_name is None: + # present but not a policy name (e.g. policy = 0): never fall through + # to defaults silently — that could bypass an intended policy. + diags.append(Diagnostic( + "OWN030", + f"invalid policy reference '{_fallback_token(pol_expr)}'; " + f"expected a policy name", line)) + elif pol_name in policies: + base = dict(policies[pol_name].settings) + else: + diags.append(Diagnostic( + "OWN030", f"undefined policy '{pol_name}'", line)) + + def opt_int(name: str, default: int) -> int: + # present-but-not-an-integer (e.g. inline = bogus) must fail safe and + # diagnose, not silently fall through to the default and change the policy. + if name in opts: + v = _as_int(opts[name]) + if v is not None: + return v + diags.append(Diagnostic( + "OWN030", + f"invalid '{name}' value '{_fallback_token(opts[name])}'; " + f"expected an integer", line)) + return default + if name in base: + bv = base[name] + if isinstance(bv, int) and not isinstance(bv, bool): + return bv + diags.append(Diagnostic( + "OWN030", + f"invalid '{name}' value '{bv}' in policy; expected an integer", + line)) + return default + return default + + def first_int(sources: list[tuple[bool, str]], label: str, + default: int) -> int: + """First *present* source, in priority order, as an integer. Only that + source is validated/diagnosed — lower-priority sources are ignored once a + higher-priority one is present, so an inline option overrides a policy + default even when the (now-irrelevant) policy value is malformed.""" + for from_opts, key in sources: + container = opts if from_opts else base + if key not in container: + continue + raw = container[key] + v = (_as_int(raw) if from_opts + else (raw if isinstance(raw, int) and not isinstance(raw, bool) + else None)) + if v is not None: + return v + where = "" if from_opts else " in policy" + diags.append(Diagnostic( + "OWN030", + f"invalid '{label}' value '{_fallback_token(raw)}'{where}; " + f"expected an integer", line)) + return default + return default + + # ---- size -------------------------------------------------------------- + size_const = _as_int(intent.size) if intent.size is not None else None + size_var = _as_ident(intent.size) if intent.size is not None else None + + # ---- per-mode resolution ---------------------------------------------- + inline_bytes = DEFAULT_INLINE_BYTES + fallback_pool = False + fallback_forbidden = False + + if mode in (BufferMode.STACK, BufferMode.INLINE): + fallback_forbidden = True + if size_const is not None: + inline_bytes = size_const + elif mode == BufferMode.INLINE: + # inline is a fixed compile-time stack buffer: the size MUST be an + # integer literal. A dynamic size (even with `max`) is `stack`, not + # `inline`. + diags.append(Diagnostic( + "OWN021", + "'inline' buffer requires a compile-time integer literal size; " + "use 'stack' with a 'max =' bound for a dynamic size", line)) + inline_bytes = MAX_STACK_BYTES + else: + # dynamic size: needs an explicit, integer max bound. + inline_bytes = MAX_STACK_BYTES # safe default while we validate + if "max" in opts and _as_int(opts["max"]) is None: + diags.append(Diagnostic( + "OWN030", + f"invalid 'max' value '{_fallback_token(opts['max'])}'; " + f"expected an integer", line)) + else: + mx_val = (_as_int(opts["max"]) if "max" in opts + else opt_int("max_bytes", -1)) + if mx_val < 0: + diags.append(Diagnostic( + "OWN021", + f"'{mode.value}' allocation of a dynamic size requires a " + f"statically known bound (add 'max = N')", line)) + else: + inline_bytes = mx_val + + elif mode == BufferMode.SCRATCH: + inline_bytes = first_int( + [(True, "inline"), (True, "inline_bytes"), (False, "inline_bytes")], + "inline", DEFAULT_INLINE_BYTES) + # distinguish "absent" (default to pool) from "present but malformed". + # A present-but-malformed value — a string typo (`forbiden`) OR a + # non-identifier (`fallback = 0`) — must fail safe and diagnose, never + # silently fall through to enabling the heap. + fb_present = "fallback" in opts or "fallback" in base + if fb_present: + raw = opts["fallback"] if "fallback" in opts else base["fallback"] + fb = _fallback_token(raw) + else: + fb = "pool" # scratch defaults to a pool fallback + fb_valid = fb in ("pool", "forbidden") + if fb_present and not fb_valid: + diags.append(Diagnostic( + "OWN030", + f"invalid fallback '{fb}' for scratch buffer; expected 'pool' " + f"or 'forbidden'", line)) + if fb == "pool": + fallback_pool = True + else: + # 'forbidden', or an invalid value handled fail-safe (no heap) + fallback_forbidden = True + # scratch with no heap fallback and a size that may exceed the inline + # limit cannot honour the "stack only" promise. + if fb_valid and (size_const is None or size_const > inline_bytes): + diags.append(Diagnostic( + "OWN023", + f"scratch buffer forbids a heap fallback but its size may " + f"exceed the inline limit of {inline_bytes}; use 'stack' " + f"with a 'max =' bound instead", line)) + + elif mode == BufferMode.POOLED: + fallback_pool = True + + # native: heap-via-unmanaged, nothing extra to resolve here. + + # ---- shared bound check on stack-backed modes ------------------------- + if mode in STACK_BACKED and inline_bytes > MAX_STACK_BYTES: + diags.append(Diagnostic( + "OWN019", + f"inline capacity {inline_bytes} bytes is too large for a " + f"stack-backed buffer (limit {MAX_STACK_BYTES}); use 'pooled' or " + f"raise the policy ceiling deliberately", line)) + + # ---- flags from options / policy -------------------------------------- + clear = _bool_flag(opts.get("clear"), base.get("clear_on_release"), + False, "clear_on_release", diags, line) + trace = _trace_flag(opts.get("trace"), base.get("trace"), True, diags, line) + counters = _bool_flag(opts.get("counters"), base.get("counters"), + True, "counters", diags, line) + + info = BufferInfo( + mode=mode, + elem="byte", + size_const=size_const, + size_var=size_var, + inline_bytes=inline_bytes, + fallback_pool=fallback_pool, + fallback_forbidden=fallback_forbidden, + clear_on_release=clear, + trace=trace, + counters=counters, + policy_name=pol_name, + line=line, + ) + return info, diags + + +def _fallback_token(v) -> str: + """Render a fallback value (an AST expr from an inline option, or a Python + value from a policy) as a display token for validation/diagnostics.""" + if isinstance(v, A.IntLit): + return str(v.value) + if isinstance(v, A.VarRef): + return v.name + if isinstance(v, bool): + return "true" if v else "false" + return str(v) + + +def _bool_flag(opt_expr, policy_val, default: bool, label: str, + diags, line: int) -> bool: + # a malformed boolean (a typo like `ture`, or any non-bool) must be rejected, + # not silently treated as the default — for a sensitive buffer that would + # quietly turn off clear-on-release. + if opt_expr is not None: + name = _as_ident(opt_expr) + if name == "true": + return True + if name == "false": + return False + diags.append(Diagnostic( + "OWN030", + f"invalid '{label}' value '{_fallback_token(opt_expr)}'; " + f"expected true or false", line)) + return default + if policy_val is not None: + if isinstance(policy_val, bool): + return policy_val + diags.append(Diagnostic( + "OWN030", + f"invalid '{label}' value '{policy_val}' in policy; expected true " + f"or false", line)) + return default + return default + + +_TRACE_ON = ("debug", "on", "true") +_TRACE_OFF = ("off", "none", "false") + + +def _trace_flag(opt_expr, policy_val, default: bool, diags, line: int) -> bool: + # `trace = debug` / `trace = off` / `trace = false`; on/off/none/true/false + # toggle the (Conditional) hooks. A malformed value is rejected, not assumed + # on. An inline option wins over the policy value. + if opt_expr is not None: + name = _as_ident(opt_expr) + if name in _TRACE_ON: + return True + if name in _TRACE_OFF: + return False + diags.append(Diagnostic( + "OWN030", + f"invalid 'trace' value '{_fallback_token(opt_expr)}'; expected one " + f"of debug/on/off/none/true/false", line)) + return default + if policy_val is not None: + if isinstance(policy_val, bool): + return policy_val + if policy_val in _TRACE_ON: + return True + if policy_val in _TRACE_OFF: + return False + diags.append(Diagnostic( + "OWN030", + f"invalid 'trace' value '{policy_val}' in policy; expected one of " + f"debug/on/off/none/true/false", line)) + return default + return default diff --git a/cfg.py b/ownlang/cfg.py similarity index 75% rename from cfg.py rename to ownlang/cfg.py index 751db25e..6577938e 100644 --- a/cfg.py +++ b/ownlang/cfg.py @@ -26,6 +26,7 @@ from . import ast_nodes as A from .ast_nodes import Effect from .diagnostics import Diagnostic +from .buffers import BufferInfo, Policy, resolve as resolve_buffer, MODE_NAMES # --------------------------------------------------------------------------- @@ -51,6 +52,18 @@ class Symbol: # for BORROW symbols: True if it is a mutable borrow (&mut / borrow_mut), # False if shared (&T / borrow). None for non-borrow symbols. borrow_is_mut: bool | None = None + # for OWNED buffer symbols: the resolved storage policy. None for ordinary + # `acquire`d resources. A stack-backed buffer (info.stack_backed) must not + # escape the function. + buffer: BufferInfo | None = None + # the declared/inferred type name (e.g. "int", "bool", a resource name), so a + # buffer size can be required to be an integer. None when unknown. + type_name: str | None = None + # a stable identity for the originating buffer (name#line). Set when a buffer + # is acquired and inherited across `move`, so a diagnostic about any alias can + # be attributed to the right buffer in the report (distinct from a same-named + # buffer in a sibling scope). + origin: str | None = None def __repr__(self) -> str: return f"<{self.name}:{self.kind.name}>" @@ -68,6 +81,16 @@ class Acquire: line: int +@dataclass +class AcquireBuffer: + """Acquire an owned buffer with an explicit storage policy. Carries the + resolved BufferInfo so analysis can apply escape rules and codegen can emit + the right backend (stackalloc / ArrayPool / NativeMemory) plus its logging.""" + sym: Symbol + info: BufferInfo + line: int + + @dataclass class MoveInto: dst: Symbol @@ -118,7 +141,8 @@ class Return: line: int -Instr = Acquire | MoveInto | Release | Use | Invoke | BorrowStart | BorrowEnd | Return +Instr = (Acquire | AcquireBuffer | MoveInto | Release | Use | Invoke + | BorrowStart | BorrowEnd | Return) # --------------------------------------------------------------------------- @@ -187,10 +211,12 @@ def collect_signatures(mod: A.Module) -> dict[str, Signature]: class _Builder: def __init__(self, fn: A.FnDecl, resource_names: set[str], - signatures: dict[str, Signature]): + signatures: dict[str, Signature], + policies: dict[str, Policy] | None = None): self.fn = fn self.resource_names = resource_names self.signatures = signatures + self.policies = policies or {} self.diags: list[Diagnostic] = [] self.blocks: list[Block] = [] self.scopes: list[dict[str, Symbol]] = [] @@ -242,6 +268,7 @@ def build(self) -> CFG: sym = self.declare(p.name, Kind.OWNED, p.line) else: sym = self.declare(p.name, Kind.PLAIN, p.line) + sym.type_name = p.type.name self.params.append(sym) entry = self.new_block("entry") @@ -306,6 +333,8 @@ def lower_let(self, st: A.Let, cur: Block) -> Block: sym = self.declare(st.name, Kind.OWNED, st.line) cur.instrs.append(Acquire(sym, rhs.resource, st.line)) return cur + if isinstance(rhs, A.BufferIntent): + return self.lower_buffer(st, rhs, cur) if isinstance(rhs, A.Move): src = self.lookup(rhs.var, rhs.line) if src is not None and src.kind != Kind.OWNED: @@ -315,6 +344,12 @@ def lower_let(self, st: A.Let, cur: Block) -> Block: src = None dst = self.declare(st.name, Kind.OWNED, st.line) if src is not None: + # a moved buffer keeps its storage policy AND its origin identity: + # a stack-backed buffer is still stack-backed after `move` (escape + # rules carry over), and diagnostics on the alias attribute to the + # original buffer in the report. + dst.buffer = src.buffer + dst.origin = src.origin cur.instrs.append(MoveInto(dst, src, st.line)) return cur if isinstance(rhs, A.VarRef): @@ -324,13 +359,61 @@ def lower_let(self, st: A.Let, cur: Block) -> Block: "OWN032", f"cannot copy owned resource '{src.name}' into '{st.name}'; " f"use 'move {src.name}' to transfer ownership", st.line)) - self.declare(st.name, Kind.PLAIN, st.line) + dst = self.declare(st.name, Kind.PLAIN, st.line) + if src is not None and src.kind == Kind.PLAIN: + dst.type_name = src.type_name # a copy keeps the value's type return cur if isinstance(rhs, A.IntLit): - self.declare(st.name, Kind.PLAIN, st.line) + dst = self.declare(st.name, Kind.PLAIN, st.line) + dst.type_name = "int" return cur raise AssertionError(f"unknown rhs {rhs!r}") + def lower_buffer(self, st: A.Let, rhs: A.BufferIntent, cur: Block) -> Block: + if rhs.ns != "Buffer": + self.diags.append(Diagnostic( + "OWN030", + f"unknown buffer namespace '{rhs.ns}'; buffer intents are " + f"written 'Buffer.(...)'", rhs.line)) + self.declare(st.name, Kind.OWNED, st.line) + return cur + if rhs.mode not in MODE_NAMES: + self.diags.append(Diagnostic( + "OWN030", + f"unknown buffer mode '{rhs.mode}'; expected one of " + f"{', '.join(sorted(MODE_NAMES))}", rhs.line)) + self.declare(st.name, Kind.OWNED, st.line) + return cur + # the size must resolve to an integer: an IntLit, or a plain `int` value. + # A bool/other plain, a borrow, or an owned resource as the size would + # lower to uncompilable C# (Rent(flag) / AsSpan(0, flag)). + if rhs.size is None: + self.diags.append(Diagnostic( + "OWN018", f"buffer '{st.name}' requires a size", rhs.line)) + elif isinstance(rhs.size, A.VarRef): + ssym = self.lookup(rhs.size.name, rhs.size.line) + if ssym is not None and ssym.kind != Kind.PLAIN: + self.diags.append(Diagnostic( + "OWN018", + f"buffer size '{rhs.size.name}' must be an integer, not " + f"{ssym.kind.name.lower()}", rhs.size.line)) + elif ssym is not None and ssym.type_name != "int": + # an unknown-typed plain (e.g. a copy of a borrow) is NOT an int; + # accepting it would lower to Rent(span)/AsSpan(0, span). + what = (f"it is '{ssym.type_name}'" if ssym.type_name + else "its type cannot be determined") + self.diags.append(Diagnostic( + "OWN018", + f"buffer size '{rhs.size.name}' must be an integer " + f"({what})", rhs.size.line)) + info, bdiags = resolve_buffer(rhs, self.policies) + self.diags.extend(bdiags) + sym = self.declare(st.name, Kind.OWNED, st.line) + sym.buffer = info + sym.origin = f"{st.name}#{rhs.line}:{rhs.col}" + cur.instrs.append(AcquireBuffer(sym, info, st.line)) + return cur + def lower_call(self, st: A.Call, cur: Block) -> Block: sig = self.signatures.get(st.callee) if sig is None: @@ -425,8 +508,15 @@ def lower_return(self, st: A.Return, cur: Block) -> Block | None: return None +def collect_policies(mod: A.Module) -> dict[str, Policy]: + return {p.name: Policy(p.name, dict(p.settings), p.line, p.dups) + for p in mod.policies} + + def build_cfg(fn: A.FnDecl, resource_names: set[str], - signatures: dict[str, Signature]) -> tuple[CFG, list[Diagnostic]]: - b = _Builder(fn, resource_names, signatures) + signatures: dict[str, Signature], + policies: dict[str, Policy] | None = None + ) -> tuple[CFG, list[Diagnostic]]: + b = _Builder(fn, resource_names, signatures, policies) cfg = b.build() return cfg, b.diags diff --git a/ownlang/codegen.py b/ownlang/codegen.py new file mode 100644 index 00000000..729b80f2 --- /dev/null +++ b/ownlang/codegen.py @@ -0,0 +1,726 @@ +""" +C# code generator for OwnLang. + +Two modes, chosen automatically per function: + +* **try/finally hoist** — when a function has no branches, no `move`, and no + owned `return`, every owned resource is acquired and released exactly once. + We lower it to the textbook exception-safe pattern: acquire, `try { ... }`, + `finally { release }`, nested for multiple resources. The checker has already + proven release-exactly-once; the `finally` additionally makes it hold across + C# exceptions. Note we do NOT emit a runtime "released?" flag: because the + release is hoisted *out* of the `try` (it is not also in the body), it runs + exactly once with no guard needed. A runtime flag would only make sense if we + didn't trust the static result — and if we don't trust it, we shouldn't ship. + +* **faithful inline** — for functions with branches or ownership transfer, + releases are emitted inline exactly where the source put them. Hoisting + releases out of arbitrary control flow into `finally` is real work, flagged as + a next step in the README rather than faked. + +Resource lowering: if a `resource` declares `emit_type` / `emit_acquire` / +`emit_release` / `emit_borrow` templates, those produce REAL .NET (e.g. +`ArrayPool.Shared.Rent/Return`, `byte[]`, `.AsSpan()`). Otherwise the +schematic `Resource.method()` form is used, with method names taken from the +declaration. `extern fn` calls lower to `Name(args)`; a borrow binding renders +as its C# local (the span / ref), an owned argument as its variable. +""" + +from __future__ import annotations + +from . import ast_nodes as A +from .buffers import BufferMode, Policy, resolve as resolve_buffer + + +class CodegenError(Exception): + pass + + +def _csharp_type(t: A.TypeRef) -> str: + # a borrowed Buffer is the buffer's view type: Span / ReadOnlySpan + # (the same view a buffer intent and `emit_borrow` produce), so a local helper + # `fn h(x: &mut Buffer)` lowers to one that accepts a buffer value by value. + if t.name == "Buffer" and t.borrowed: + return "Span" if t.mutable else "ReadOnlySpan" + base = {"int": "int", "bool": "bool"}.get(t.name, t.name) + if t.borrowed: + return ("ref " if t.mutable else "ref readonly ") + base + return base + + +class _FnGen: + def __init__(self, mod: A.Module, fn: A.FnDecl): + self.mod = mod + self.fn = fn + self.res = {r.name: r for r in mod.resources} + self.policies: dict[str, Policy] = { + p.name: Policy(p.name, dict(p.settings), p.line) for p in mod.policies + } + self.owned_resource: dict[str, str] = {} + # borrow binding name -> owner resource type (so calls know the C# view) + self.binding_owner_res: dict[str, str] = {} + # buffer local name -> resolved BufferInfo (so borrows/args render right) + self.buffer_vars: dict[str, object] = {} + # buffer local name -> cleanup lines, for buffers whose `release` is + # nested in branches (emitted at each release site, not in a finally) + self.buffer_cleanup: dict[str, list[str]] = {} + for p in fn.params: + if not p.type.borrowed and p.type.name in self.res: + self.owned_resource[p.name] = p.type.name + + # -- shape detection ---------------------------------------------------- + + def _is_simple(self) -> bool: + # Straight-line functions (no branch / move / owned-return) use the + # try/finally hoist, which nests buffers AND ordinary resources so each + # gets its own exception-safe finally. The hoist is only safe when: + # * every scope has a TOP-LEVEL `release` (a release nested in a borrow/if + # block, or a resource consumed by a call, cannot be hoisted into a + # finally without double-cleaning or emitting a stray release), and + # * scope lifetimes are LAMINAR (every pair nested or disjoint). + # Otherwise it falls back to faithful inline, which emits releases exactly + # where the source put them. + if _contains_branch_or_transfer(self.fn.body): + return False + if not _scopes_release_top_level(self.fn.body): + return False + if _scope_body_has_plain_let(self.fn.body): + # a plain local in a scope's body, used after the release, would be + # trapped in the hoisted try; emit faithfully inline instead. + return False + return _laminar_scopes(self.fn.body) + + # -- emit --------------------------------------------------------------- + + def emit(self) -> str: + ret = _csharp_type(self.fn.ret) if self.fn.ret else "void" + params = ", ".join(f"{_csharp_type(p.type)} {p.name}" for p in self.fn.params) + # a function that allocates a native buffer needs pointers, so the whole + # method is `unsafe` (cleaner than scoping each pointer in its own block, + # and it lets native buffers follow the same lifetime shapes as the rest). + unsafe = "unsafe " if _fn_has_native(self.fn.body) else "" + head = f"public static {unsafe}{ret} {self.fn.name}({params})" + if self._is_simple(): + body = self._emit_simple(self.fn.body) + else: + body = self._emit_inline(self.fn.body, indent=1) + return f"{head}\n{{\n{body}}}\n" + + # -- simple (try/finally hoist) ---------------------------------------- + + def _emit_simple(self, stmts: list[A.Stmt]) -> str: + return "".join(l + "\n" for l in self._emit_hoist(stmts, " ")) + + def _emit_hoist(self, stmts: list[A.Stmt], base: str) -> list[str]: + """Emit a straight-line sequence, nesting each openable resource (an + ordinary acquire or a buffer) in its own try/finally. Statements are + emitted in SOURCE ORDER, and each scope is split at its `release`: the + statements during its lifetime go inside the try, the statements after + the release are emitted after the finally (as siblings). So disjoint + sequential resources stay disjoint (a is returned before b is rented), + while overlapping ones nest (releases run LIFO). A scope with no cleanup + adds no try block.""" + ind = " " + out: list[str] = [] + i = 0 + while i < len(stmts): + st = stmts[i] + scope = self._scope_lowering(st) + if scope is not None: + prelude, fin = scope + out.extend(base + p for p in prelude) + after = stmts[i + 1:] + rel = self._find_release(after, 0, st.name) + body = after if rel is None else after[:rel] + tail = [] if rel is None else after[rel + 1:] + if fin: + out.append(f"{base}try") + out.append(f"{base}{{") + out.extend(self._emit_hoist(body, base + ind)) + out.append(f"{base}}}") + out.append(f"{base}finally") + out.append(f"{base}{{") + out.extend(base + ind + f for f in fin) + out.append(f"{base}}}") + else: + out.extend(self._emit_hoist(body, base)) + # statements after this scope's release are siblings, emitted + # after its finally so its lifetime ends at the source release. + out.extend(self._emit_hoist(tail, base)) + return out + # A release that reaches this top loop was NOT consumed by a scope: + # a scope handles its own release inside the recursion above (it is + # excluded from `body` and the function returns), so the only + # releases arriving here are for owned *parameters*, which open no + # scope. Emit them faithfully — skipping would leak the resource. + out.extend(self._stmt_inline(st, base)) + i += 1 + return out + + def _scope_lowering(self, st: A.Stmt) -> tuple[list[str], list[str]] | None: + """If `st` opens a resource (an `acquire` let or a buffer let), return its + (prelude, cleanup) lines; otherwise None.""" + if isinstance(st, A.Let) and isinstance(st.rhs, A.Acquire): + rt = st.rhs.resource + self.owned_resource[st.name] = rt + args_csv = ", ".join(self._arg(x) for x in st.rhs.args) + return ([f"{self._local_type(rt)} {st.name} = " + f"{self._acquire_expr(rt, args_csv)};"], + [self._release_stmt(st.name, rt)]) + if isinstance(st, A.Let) and isinstance(st.rhs, A.BufferIntent): + return self._buffer_lowering(st.name, st.rhs) + return None + + # -- faithful inline ---------------------------------------------------- + + def _emit_inline(self, stmts: list[A.Stmt], indent: int) -> str: + ind = " " * indent + out = self._emit_block(stmts, ind) + return "".join(l + "\n" for l in out) + + def _emit_block(self, stmts: list[A.Stmt], ind: str) -> list[str]: + """Emit a statement list, lowering each buffer let by its lifetime shape. + + A buffer gets the exception-safe try/finally only when it nests cleanly: + its `release` is straight-line at this level AND no other buffer is + acquired inside its body (which would make the lifetimes overlap in + non-LIFO order). Otherwise — overlapping lifetimes, or a release nested in + branches — it uses inline-release: the prelude here, the real cleanup + emitted at each `release` site (no hoist into finally; the same trade-off + the codegen already makes for branchy ordinary resources).""" + out: list[str] = [] + i = 0 + while i < len(stmts): + st = stmts[i] + if isinstance(st, A.Let) and isinstance(st.rhs, A.BufferIntent): + rest = stmts[i + 1:] + name = st.name + j = self._find_release(rest, 0, name) + if (j is not None and not _fn_has_buffer(rest[:j]) + and not _body_has_plain_let(rest[:j])): + out.extend(self._emit_buffer_scoped(name, st.rhs, rest[:j], ind)) + i += 1 + j + 1 # consume the let, its body, and its release + continue + if not _buffer_released(rest, name): + # leak or escape — both rejected by the checker first + # (OWN001 / OWN015 / OWN016 / OWN017); unreachable cleanly. + raise CodegenError( + f"buffer '{name}' is never released and does not escape " + f"cleanly; the checker should have rejected this") + out.extend(self._emit_buffer_inline(name, st.rhs, ind)) + i += 1 + continue + out.extend(self._stmt_inline(st, ind)) + i += 1 + return out + + @staticmethod + def _find_release(stmts: list[A.Stmt], start: int, name: str) -> int | None: + for k in range(start, len(stmts)): + st = stmts[k] + if isinstance(st, A.Release) and st.var == name: + return k + return None + + # -- buffer lowering (the stackalloc / scratch / pool / native line) ----- + + def _buffer_lowering(self, name: str, intent: A.BufferIntent + ) -> tuple[list[str], list[str]]: + """Lower one buffer to its (prelude, cleanup) C# lines. The prelude holds + the allocation + trace/counter hooks; the cleanup is the pool Return / + native Free / clear. How they are placed (try/finally vs inline) is the + caller's decision.""" + info, _ = resolve_buffer(intent, self.policies) + self.buffer_vars[name] = info + # a fresh declaration of this name shadows any stale moved-alias cleanup + # left over from an earlier (now out-of-scope) buffer of the same name. + self.buffer_cleanup.pop(name, None) + fn = self.fn.name + size = self._size_expr(info) + L = info.inline_bytes + pre: list[str] = [] # declarations + trace/counters, before the try + fin: list[str] = [] # cleanup, inside the finally / at release sites + scratch_pool = info.mode == BufferMode.SCRATCH and info.fallback_pool + # the OwnCounters are Scratch.* metrics — they answer "do scratch requests + # hit the stack?". Only scratch buffers touch them; pooled/native/stack + # allocations must not pollute the scratch hit/fallback counts. + sc = info.counters and info.mode == BufferMode.SCRATCH + + if info.mode in (BufferMode.STACK, BufferMode.INLINE) or ( + info.mode == BufferMode.SCRATCH and not info.fallback_pool): + if info.size_is_const: + if info.trace: + pre.append(f'OwnTrace.StackSelected("{fn}", "{name}", {size}, {L});') + if sc: + pre.append("OwnCounters.StackHit();") + if info.size_const == L: + pre.append(f"Span {name} = stackalloc byte[{L}];") + else: + # reserve the inline capacity but expose only the requested + # length (e.g. scratch(64, inline = 1024, fallback = forbidden)). + pre.append(f"Span {name}_backing = stackalloc byte[{L}];") + pre.append(f"Span {name} = {name}_backing[..{size}];") + else: + pre.append(f"if ((uint){size} > {L})") + pre.append(f" throw new ArgumentOutOfRangeException(nameof({size}));") + if info.trace: + pre.append(f'OwnTrace.StackSelected("{fn}", "{name}", {size}, {L});') + if sc: + pre.append("OwnCounters.StackHit();") + pre.append(f"Span {name}_backing = stackalloc byte[{L}];") + pre.append(f"Span {name} = {name}_backing[..{size}];") + if sc: + fin.append("OwnCounters.Release();") + if info.clear_on_release: + fin.append(f"{name}.Clear();") + + elif scratch_pool: + if not info.size_is_const: + # reject a negative size before any trace/counter runs: otherwise + # `size <= limit` takes the stack arm, logs a hit, and the [..size] + # slice throws before the try can balance it — corrupting metrics. + pre.append(f"if ({size} < 0)") + pre.append(f" throw new ArgumentOutOfRangeException(nameof({size}));") + pre.append(f"byte[]? {name}_rented = null;") + pre.append(f"Span {name}_backing = stackalloc byte[{L}];") + pre.append(f"Span {name};") + pre.append(f"if ({size} <= {L})") + pre.append("{") + if info.trace: + pre.append(f' OwnTrace.ScratchSelected("{fn}", "{name}", {size}, {L}, "stackalloc");') + if sc: + pre.append(" OwnCounters.StackHit();") + pre.append(f" {name} = {name}_backing[..{size}];") + pre.append("}") + pre.append("else") + pre.append("{") + if info.trace: + pre.append(f' OwnTrace.ScratchSelected("{fn}", "{name}", {size}, {L}, "ArrayPool");') + if sc: + pre.append(f" OwnCounters.PoolFallback({size});") + pre.append(f" {name}_rented = ArrayPool.Shared.Rent({size});") + pre.append(f" {name} = {name}_rented.AsSpan(0, {size});") + pre.append("}") + if sc: + fin.append("OwnCounters.Release();") + if info.clear_on_release: + fin.append(f"{name}.Clear();") + fin.append(f"if ({name}_rented is not null)") + fin.append(f" ArrayPool.Shared.Return({name}_rented);") + + elif info.mode == BufferMode.POOLED: + # pooled is not scratch: trace it, but do not touch the Scratch.* + # counters (that would report normal pool rents as scratch misses). + if info.trace: + pre.append(f'OwnTrace.PooledSelected("{fn}", "{name}", {size});') + pre.append(f"byte[] {name}_array = ArrayPool.Shared.Rent({size});") + pre.append(f"Span {name} = {name}_array.AsSpan(0, {size});") + if info.clear_on_release: + fin.append(f"{name}.Clear();") + fin.append(f"ArrayPool.Shared.Return({name}_array);") + + elif info.mode == BufferMode.NATIVE: + # the method is emitted `unsafe`, so the pointer needs no local block. + # The pointer is the backing (freed on release); the buffer is exposed + # as a Span view, so borrows/calls see the SAME logical type as + # pooled/stack/scratch — an `extern fn Fill(borrow_mut Buffer)` lowers + # to one C# signature (Span) regardless of storage mode. + if not info.size_is_const: + # a negative size would wrap when cast to nuint and request an + # enormous allocation; reject it before Alloc (and before clear). + pre.append(f"if ({size} < 0)") + pre.append(f" throw new ArgumentOutOfRangeException(nameof({size}));") + if info.trace: + pre.append(f'OwnTrace.NativeSelected("{fn}", "{name}", {size});') + pre.append(f"byte* {name}_ptr = (byte*)System.Runtime.InteropServices." + f"NativeMemory.Alloc((nuint){size});") + pre.append(f"Span {name} = new Span({name}_ptr, {size});") + if info.clear_on_release: + fin.append(f"{name}.Clear();") + fin.append(f"System.Runtime.InteropServices.NativeMemory.Free({name}_ptr);") + + return pre, fin + + def _emit_buffer_scoped(self, name: str, intent: A.BufferIntent, + body: list[A.Stmt], ind: str) -> list[str]: + """The buffer nests cleanly: wrap its body in an exception-safe + try/finally (this is the golden scratch shape).""" + pre, fin = self._buffer_lowering(name, intent) + lines = [ind + p for p in pre] + body_lines = self._emit_block(body, ind + " ") + if fin: + lines.append(f"{ind}try") + lines.append(f"{ind}{{") + lines.extend(body_lines) + lines.append(f"{ind}}}") + lines.append(f"{ind}finally") + lines.append(f"{ind}{{") + for f in fin: + lines.append(ind + " " + f) + lines.append(f"{ind}}}") + else: + # nothing to clean up (e.g. a stack buffer with no clear and no + # counters): the body runs straight, the frame reclaims the bytes. + for bl in body_lines: + lines.append(bl[4:] if bl.startswith(" ") else bl) + return lines + + def _emit_buffer_inline(self, name: str, intent: A.BufferIntent, + ind: str) -> list[str]: + """Overlapping or branchy lifetime: emit the prelude here and attach the + cleanup to each `release` site (handled by _stmt_inline). No try/finally — + the same exception-safety trade-off as branchy ordinary resources.""" + pre, fin = self._buffer_lowering(name, intent) + self.buffer_cleanup[name] = fin + return [ind + p for p in pre] + + def _size_expr(self, info) -> str: + if info.size_is_const: + return str(info.size_const) + if info.size_var: + return info.size_var + return "0" + + def _stmt_inline(self, st: A.Stmt, ind: str) -> list[str]: + if isinstance(st, A.Let): + # a (re)declaration of this name shadows any stale buffer-cleanup + # alias from an earlier same-named buffer in another scope. + self.buffer_cleanup.pop(st.name, None) + if isinstance(st.rhs, A.Acquire): + rt = st.rhs.resource + self.owned_resource[st.name] = rt + args_csv = ", ".join(self._arg(x) for x in st.rhs.args) + return [f"{ind}{self._local_type(rt)} {st.name} = {self._acquire_expr(rt, args_csv)};"] + if isinstance(st.rhs, A.Move): + # a moved buffer carries its identity to the new owner: copy the + # pending cleanup to the new name (do NOT remove the original — + # sibling branches that did not move still need it, and releasing + # the original after a move is use-after-move, rejected upstream). + if st.rhs.var in self.buffer_cleanup: + self.buffer_cleanup[st.name] = self.buffer_cleanup[st.rhs.var] + if st.rhs.var in self.buffer_vars: + self.buffer_vars[st.name] = self.buffer_vars[st.rhs.var] + self.owned_resource[st.name] = self.owned_resource.get(st.rhs.var, "") + return [f"{ind}var {st.name} = {st.rhs.var}; " + f"// ownership moved from {st.rhs.var}"] + if isinstance(st.rhs, A.IntLit): + return [f"{ind}var {st.name} = {st.rhs.value};"] + if isinstance(st.rhs, A.VarRef): + return [f"{ind}var {st.name} = {st.rhs.name};"] + if isinstance(st, A.Release): + if st.var in self.buffer_cleanup: + # a branchy buffer release: emit this buffer's real cleanup + # (pool Return / native Free / clear), not a generic Dispose. + return [ind + line for line in self.buffer_cleanup[st.var]] + rt = self.owned_resource.get(st.var, "") + return [f"{ind}{self._release_stmt(st.var, rt)}"] + if isinstance(st, A.Use): + return [f"{ind}Use({st.var});"] + if isinstance(st, A.Call): + return [f"{ind}{st.callee}({', '.join(self._arg(a) for a in st.args)});"] + if isinstance(st, A.BorrowBlock): + kind = "mutable" if st.kind == A.BorrowKind.MUT else "shared" + rt = self.owned_resource.get(st.owner, "") + self.binding_owner_res[st.binding] = rt + head = [f"{ind}{{ // {kind} borrow of {st.owner} as {st.binding}", + f"{ind} var {st.binding} = {self._borrow_expr(st.owner, rt)};"] + body = self._emit_block(st.body, ind + " ") + return head + body + [f"{ind}}}"] + if isinstance(st, A.If): + out = [f"{ind}if ({st.cond_text or 'cond'})", f"{ind}{{"] + out.extend(self._emit_block(st.then_body, ind + " ")) + out.append(f"{ind}}}") + if st.else_body: + out.append(f"{ind}else") + out.append(f"{ind}{{") + out.extend(self._emit_block(st.else_body, ind + " ")) + out.append(f"{ind}}}") + return out + if isinstance(st, A.Return): + return [f"{ind}return {st.var};" if st.var else f"{ind}return;"] + raise CodegenError(f"cannot codegen {st!r}") + + # -- template helpers --------------------------------------------------- + + def _local_type(self, resource: str) -> str: + r = self.res.get(resource) + if r and r.emit_type: + return r.emit_type + return "var" + + def _acquire_expr(self, resource: str, args_csv: str) -> str: + r = self.res.get(resource) + if r and r.emit_acquire: + return r.emit_acquire.replace("{args}", args_csv) + method = _member(r, "acquire") if r else "Create" + return f"{resource}.{method}({args_csv})" + + def _release_stmt(self, var: str, resource: str) -> str: + r = self.res.get(resource) + if r and r.emit_release: + return r.emit_release.replace("{0}", var) + ";" + method = _member(r, "release") if r else "Dispose" + return f"{var}.{method}();" + + def _borrow_expr(self, owner_var: str, resource: str) -> str: + r = self.res.get(resource) + if r and r.emit_borrow: + return r.emit_borrow.replace("{0}", owner_var) + return owner_var + + def _arg(self, e: A.Expr) -> str: + if isinstance(e, A.IntLit): + return str(e.value) + if isinstance(e, A.VarRef): + return e.name + return "/* expr */" + + +def _member(r: A.ResourceDecl, role: str) -> str: + for m in r.members: + if m.role == role: + return m.name + return "Create" if role == "acquire" else "Dispose" + + +def _contains_branch_or_transfer(stmts: list[A.Stmt]) -> bool: + for st in stmts: + if isinstance(st, A.If): + return True + if isinstance(st, A.Return) and st.var is not None: + return True + if isinstance(st, A.Let) and isinstance(st.rhs, A.Move): + return True + if isinstance(st, A.BorrowBlock): + if _contains_branch_or_transfer(st.body): + return True + return False + + +def _iter_stmts(stmts: list[A.Stmt]): + """Yield every statement in the tree, descending into if-branches and + borrow blocks.""" + for st in stmts: + yield st + if isinstance(st, A.If): + yield from _iter_stmts(st.then_body) + yield from _iter_stmts(st.else_body) + elif isinstance(st, A.BorrowBlock): + yield from _iter_stmts(st.body) + + +def _move_aliases(stmts: list[A.Stmt], name: str) -> set[str]: + """The set of names a buffer flows into via `let X = move `, starting + from its declaration name (transitive).""" + aliases = {name} + changed = True + while changed: + changed = False + for st in _iter_stmts(stmts): + if (isinstance(st, A.Let) and isinstance(st.rhs, A.Move) + and st.rhs.var in aliases and st.name not in aliases): + aliases.add(st.name) + changed = True + return aliases + + +def _buffer_released(stmts: list[A.Stmt], name: str) -> bool: + """Is the buffer released somewhere — possibly through a moved alias?""" + aliases = _move_aliases(stmts, name) + return any(isinstance(st, A.Release) and st.var in aliases + for st in _iter_stmts(stmts)) + + +def _fn_has_buffer(stmts: list[A.Stmt]) -> bool: + for st in stmts: + if isinstance(st, A.Let) and isinstance(st.rhs, A.BufferIntent): + return True + if isinstance(st, A.If): + if _fn_has_buffer(st.then_body) or _fn_has_buffer(st.else_body): + return True + if isinstance(st, A.BorrowBlock): + if _fn_has_buffer(st.body): + return True + return False + + +def _body_has_plain_let(stmts: list[A.Stmt]) -> bool: + """True if a plain local (`let x = `) is declared at this level. Such + a local declared inside a hoisted try would go out of scope after the finally, + so a scope whose body contains one is emitted inline (no try) instead.""" + return any(isinstance(s, A.Let) and isinstance(s.rhs, (A.IntLit, A.VarRef)) + for s in stmts) + + +def _scope_body_has_plain_let(stmts: list[A.Stmt]) -> bool: + """True if any top-level scope (acquire/buffer let) has a plain local declared + between its acquire and its release.""" + for i, st in enumerate(stmts): + if isinstance(st, A.Let) and isinstance(st.rhs, (A.Acquire, A.BufferIntent)): + rel = None + for k in range(i + 1, len(stmts)): + if isinstance(stmts[k], A.Release) and stmts[k].var == st.name: + rel = k + break + body = stmts[i + 1:rel] if rel is not None else stmts[i + 1:] + if _body_has_plain_let(body): + return True + return False + + +def _scopes_release_top_level(stmts: list[A.Stmt]) -> bool: + """True if every top-level owned scope (an acquire or buffer let) has a + matching top-level `release`. A scope whose release is nested in a borrow/if + block — or which is consumed by a call instead of released — cannot be safely + hoisted into a finally, so such functions use faithful inline instead.""" + released = {st.var for st in stmts if isinstance(st, A.Release)} + for st in stmts: + if isinstance(st, A.Let) and isinstance(st.rhs, (A.Acquire, A.BufferIntent)): + if st.name not in released: + return False + return True + + +def _laminar_scopes(stmts: list[A.Stmt]) -> bool: + """True if every pair of top-level resource lifetimes is nested or disjoint + (never partially overlapping). Only then is the try/finally hoist safe: a + partial overlap `let a; let b; release a; ... release b;` cannot be nested + without forcing b's release before its source point.""" + intervals: list[tuple[int, int]] = [] + for i, st in enumerate(stmts): + if isinstance(st, A.Let) and isinstance(st.rhs, (A.Acquire, A.BufferIntent)): + for k in range(i + 1, len(stmts)): + s2 = stmts[k] + if isinstance(s2, A.Release) and s2.var == st.name: + intervals.append((i, k)) + break + for a1, a2 in intervals: + for b1, b2 in intervals: + if a1 < b1 < a2 < b2: # partial overlap + return False + return True + + +def _fn_has_native(stmts: list[A.Stmt]) -> bool: + for st in stmts: + if (isinstance(st, A.Let) and isinstance(st.rhs, A.BufferIntent) + and st.rhs.mode == "native"): + return True + if isinstance(st, A.If): + if _fn_has_native(st.then_body) or _fn_has_native(st.else_body): + return True + if isinstance(st, A.BorrowBlock): + if _fn_has_native(st.body): + return True + return False + + +def _buffer_modes(mod: A.Module) -> set[str]: + modes: set[str] = set() + + def walk(stmts): + for st in stmts: + if isinstance(st, A.Let) and isinstance(st.rhs, A.BufferIntent): + modes.add(st.rhs.mode) + elif isinstance(st, A.If): + walk(st.then_body) + walk(st.else_body) + elif isinstance(st, A.BorrowBlock): + walk(st.body) + + for fn in mod.functions: + walk(fn.body) + return modes + + +def _usings(mod: A.Module) -> list[str]: + out = ["using System;"] + blob = " ".join( + (r.emit_acquire or "") + (r.emit_release or "") + (r.emit_type or "") + for r in mod.resources + ) + modes = _buffer_modes(mod) + if "ArrayPool" in blob or (modes & {"scratch", "pooled"}): + out.append("using System.Buffers;") + return out + + +# Runtime logging support emitted alongside any module that uses buffers. The +# two hooks are the runtime half of the design: a text trace of which backend a +# scratch/stack request actually selected, and counters answering "how often do +# we really hit the stack?". Both are [Conditional]: a normal Release build that +# defines neither symbol pays nothing — you do not want logging to become the +# new bottleneck on a hot path. +_RUNTIME_SUPPORT = '''\ +internal static class OwnTrace +{ + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void ScratchSelected(string function, string buffer, + int requestedBytes, int inlineLimit, string backend) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, " + + $"inline={inlineLimit}, backend={backend}"); + + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void StackSelected(string function, string buffer, + int requestedBytes, int inlineLimit) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, " + + $"inline={inlineLimit}, backend=stackalloc"); + + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void PooledSelected(string function, string buffer, int requestedBytes) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, backend=ArrayPool"); + + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void NativeSelected(string function, string buffer, int requestedBytes) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, backend=NativeMemory"); +} + +internal static class OwnCounters +{ + public static long ScratchStackHits; + public static long ScratchPoolFallbacks; + public static long ScratchPoolBytesRented; + public static long ScratchReleaseCount; + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void StackHit() + => System.Threading.Interlocked.Increment(ref ScratchStackHits); + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void PoolFallback(int bytes) + { + System.Threading.Interlocked.Increment(ref ScratchPoolFallbacks); + System.Threading.Interlocked.Add(ref ScratchPoolBytesRented, bytes); + } + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void Release() + => System.Threading.Interlocked.Increment(ref ScratchReleaseCount); +} +''' + + +def generate(mod: A.Module) -> str: + parts = [ + "// by OwnLang PoC. Ownership was checked at the .own", + "// level; this C# is the lowering. Do not hand-edit.", + ] + parts.extend(_usings(mod)) + parts.append("") + parts.append(f"public static class {mod.name}") + parts.append("{") + bodies = [_FnGen(mod, fn).emit() for fn in mod.functions] + indented = [] + for b in bodies: + indented.append("\n".join(" " + line if line else line + for line in b.splitlines())) + parts.append("\n\n".join(indented)) + parts.append("}") + if _buffer_modes(mod): + parts.append("") + parts.append(_RUNTIME_SUPPORT) + return "\n".join(parts) + "\n" diff --git a/diagnostics.py b/ownlang/diagnostics.py similarity index 76% rename from diagnostics.py rename to ownlang/diagnostics.py index b7cb3277..0a91355f 100644 --- a/diagnostics.py +++ b/ownlang/diagnostics.py @@ -40,6 +40,14 @@ class Severity(Enum): "OWN011": "mutable borrow while another mutable borrow is live", "OWN012": "shared borrow while a mutable borrow is live", "OWN013": "owner accessed while it is mutably borrowed", + # ---- buffer storage policies (stackalloc / scratch / pool / native) ---- + "OWN015": "stack-backed buffer cannot escape the current function", + "OWN016": "stack-backed buffer moved to a longer-lived owner", + "OWN017": "movable buffer escape is not supported by code generation (PoC limitation)", + "OWN018": "buffer size must be an integer", + "OWN019": "inline capacity too large for a stack-backed policy", + "OWN021": "stack allocation requires a statically known bound", + "OWN023": "scratch fallback forbidden but the size may exceed the inline limit", # ---- unsupported ---- "OWN020": "unsupported construct (out of scope for the MVP)", # ---- name resolution & structural ---- @@ -60,6 +68,9 @@ class Diagnostic: message: str line: int severity: Severity = Severity.ERROR + # for buffer diagnostics: a stable identity (name#line) of the buffer the + # diagnostic is about, so the report attributes it by symbol, not by name. + subject: str | None = None @property def title(self) -> str: diff --git a/lexer.py b/ownlang/lexer.py similarity index 98% rename from lexer.py rename to ownlang/lexer.py index 5c42a729..a700e63a 100644 --- a/lexer.py +++ b/ownlang/lexer.py @@ -37,6 +37,7 @@ class Tok(Enum): ELSE = auto() RETURN = auto() MUT = auto() + POLICY = auto() # emit-template keywords (only meaningful inside a resource body) EMIT_TYPE = auto() EMIT_ACQUIRE = auto() @@ -54,6 +55,7 @@ class Tok(Enum): SEMI = auto() AMP = auto() EQ = auto() + DOT = auto() ARROW = auto() EOF = auto() @@ -76,6 +78,7 @@ class Tok(Enum): "else": Tok.ELSE, "return": Tok.RETURN, "mut": Tok.MUT, + "policy": Tok.POLICY, "emit_type": Tok.EMIT_TYPE, "emit_acquire": Tok.EMIT_ACQUIRE, "emit_release": Tok.EMIT_RELEASE, @@ -197,6 +200,7 @@ def advance(k: int = 1) -> None: ";": Tok.SEMI, "&": Tok.AMP, "=": Tok.EQ, + ".": Tok.DOT, } if c in simple: advance() diff --git a/parser.py b/ownlang/parser.py similarity index 75% rename from parser.py rename to ownlang/parser.py index 2620d10a..c279bd56 100644 --- a/parser.py +++ b/ownlang/parser.py @@ -4,10 +4,11 @@ Grammar (informal): module := "module" IDENT item* - item := resource | extern | fn + item := resource | extern | fn | policy resource := "resource" IDENT "{" rmember* "}" rmember := ("acquire" | "release") IDENT | ("emit_type"|"emit_acquire"|"emit_release"|"emit_borrow") STRING + policy := "policy" IDENT "{" (IDENT "=" atom ";")* "}" extern := "extern" "fn" IDENT "(" eparams? ")" ("->" type)? ";" eparams := eparam ("," eparam)* eparam := ("borrow" | "borrow_mut" | "consume")? IDENT // IDENT = type name @@ -18,7 +19,11 @@ block := "{" stmt* "}" stmt := let | release | use | call | borrow | if | return let := "let" IDENT "=" rhs ";" - rhs := "acquire" IDENT "(" args? ")" | "move" IDENT | IDENT | INT + rhs := "acquire" IDENT "(" args? ")" | "move" IDENT + | bufferintent | IDENT | INT + bufferintent:= IDENT "." IDENT "(" bargs? ")" // e.g. Buffer.scratch(...) + bargs := barg ("," barg)* + barg := IDENT "=" atom | atom // named option | positional size release := "release" IDENT ";" use := "use" IDENT ";" call := IDENT "(" args? ")" ";" @@ -111,10 +116,43 @@ def parse_module(self) -> A.Module: mod.externs.append(self.parse_extern()) elif self.at(Tok.FN): mod.functions.append(self.parse_fn()) + elif self.at(Tok.POLICY): + mod.policies.append(self.parse_policy()) else: - raise ParseError("expected 'resource', 'extern' or 'fn'", self.cur) + raise ParseError( + "expected 'resource', 'extern', 'fn' or 'policy'", self.cur) return mod + # -- policies ----------------------------------------------------------- + + def parse_policy(self) -> A.PolicyDecl: + kw = self.eat(Tok.POLICY) + name = self.eat(Tok.IDENT).text + self.eat(Tok.LBRACE) + settings: dict[str, object] = {} + seen: list[str] = [] + while not self.at(Tok.RBRACE): + key = self.eat(Tok.IDENT).text + self.eat(Tok.EQ) + settings[key] = self._policy_value() + self.eat(Tok.SEMI) + seen.append(key) + self.eat(Tok.RBRACE) + dups = tuple(sorted({k for k in seen if seen.count(k) > 1})) + return A.PolicyDecl(name=name, settings=settings, line=kw.line, dups=dups) + + def _policy_value(self) -> object: + """A policy setting value: an int, or an identifier interpreted as a + bool (true/false) or a bare keyword string (pool/forbidden/debug/...).""" + if self.at(Tok.INT): + return int(self.eat(Tok.INT).text) + word = self.eat(Tok.IDENT).text + if word == "true": + return True + if word == "false": + return False + return word + # -- resources ---------------------------------------------------------- def parse_resource(self) -> A.ResourceDecl: @@ -260,8 +298,51 @@ def parse_rhs(self) -> A.Expr: kw = self.eat(Tok.MOVE) v = self.eat(Tok.IDENT) return A.Move(var=v.text, line=kw.line) + # buffer intent: Namespace "." mode "(" bargs? ")" e.g. Buffer.scratch(...) + if self.at(Tok.IDENT) and self.peek().kind == Tok.DOT: + return self.parse_buffer_intent() return self.parse_atom() + def parse_buffer_intent(self) -> A.BufferIntent: + ns = self.eat(Tok.IDENT) # namespace, conventionally "Buffer" + self.eat(Tok.DOT) + mode = self.eat(Tok.IDENT).text + self.eat(Tok.LPAREN) + size: A.Expr | None = None + options: dict[str, A.Expr] = {} + seen: list[str] = [] # option names in order (for dup detect) + first = True + if not self.at(Tok.RPAREN): + size, first = self._buffer_arg(options, seen, first) + while self.accept(Tok.COMMA): + got, first = self._buffer_arg(options, seen, first) + if got is not None: + size = got + self.eat(Tok.RPAREN) + dups = tuple(sorted({k for k in seen if seen.count(k) > 1})) + return A.BufferIntent(mode=mode, size=size, options=options, + line=ns.line, ns=ns.text, col=ns.col, dups=dups) + + def _buffer_arg(self, options: dict, seen: list, first: bool + ) -> tuple[A.Expr | None, bool]: + """Parse one buffer argument: a named option, or the leading positional + size. Returns (size_expr_or_None, still_first).""" + # named option: IDENT "=" atom. `policy` is a keyword token elsewhere but + # is also a valid option name here, so accept it too. + if (self.at(Tok.IDENT) or self.at(Tok.POLICY)) and self.peek().kind == Tok.EQ: + key = self.cur.text + self.pos += 1 + self.eat(Tok.EQ) + options[key] = self.parse_atom() + seen.append(key) + return None, first + atom = self.parse_atom() + if not first: + raise ParseError("only the leading size may be positional in a " + "buffer intent; later arguments must be named", + self.cur) + return atom, False + def parse_args(self) -> list[A.Expr]: args: list[A.Expr] = [] if not self.at(Tok.RPAREN): diff --git a/ownlang/report.py b/ownlang/report.py new file mode 100644 index 00000000..60578787 --- /dev/null +++ b/ownlang/report.py @@ -0,0 +1,113 @@ +""" +Compile-time buffer report — the first of the three logging surfaces. + +The whole point of the scratch/stack line is that it must not be magic. A buffer +that "smartly" chose the pool while the developer thought it lived on the stack +is exactly the kind of abstraction that lies. So at generation time we write down, +per buffer: the mode the user asked for, the inline limit, the fallback, the +escape policy, whether it is cleared on release, the runtime branches codegen +emitted, and which ownership checks held. It goes to stdout for a human and to +`.ownreport.json` for review tooling / CI. + +(The other two surfaces are runtime: the OwnTrace text hook and the OwnCounters, +both emitted into the generated C# by codegen.py under [Conditional] guards.) +""" + +from __future__ import annotations + +from . import ast_nodes as A +from .buffers import resolve as resolve_buffer, Policy, MODE_NAMES +from .diagnostics import Diagnostic + + +# Diagnostics that, if present for a given buffer, mean a specific check failed. +_CHECK_CODES = { + "noEscape": {"OWN015", "OWN016", "OWN017"}, + "releaseOnAllPaths": {"OWN001"}, + "noUseAfterRelease": {"OWN002", "OWN009"}, + "noActiveLoansAtRelease": {"OWN008"}, +} + + +def _walk_buffers(stmts: list[A.Stmt]): + """Yield (let_name, BufferIntent) for every buffer intent in a statement + tree, descending into if-branches and borrow blocks.""" + for st in stmts: + if isinstance(st, A.Let) and isinstance(st.rhs, A.BufferIntent): + yield st.name, st.rhs + elif isinstance(st, A.If): + yield from _walk_buffers(st.then_body) + yield from _walk_buffers(st.else_body) + elif isinstance(st, A.BorrowBlock): + yield from _walk_buffers(st.body) + + +def build_report(mod: A.Module, diags: list[Diagnostic]) -> dict: + policies: dict[str, Policy] = { + p.name: Policy(p.name, dict(p.settings), p.line) for p in mod.policies + } + # diagnostics are attributed to a buffer by its stable identity (name#line), + # carried on Diagnostic.subject — NOT by matching the name in the message, + # which would conflate same-named buffers in sibling scopes. Move-aliases + # already share the original buffer's subject (set in the checker). + by_subject: dict[str, set[str]] = {} + for d in diags: + if d.subject is not None: + by_subject.setdefault(d.subject, set()).add(d.code) + + entries: list[dict] = [] + for fn in mod.functions: + for name, intent in _walk_buffers(fn.body): + # skip a malformed intent (bad namespace or mode, e.g. Foo.stack / + # Buffer.bogus); the checker already reported OWN030, and resolving an + # unknown mode would throw. + if intent.ns != "Buffer" or intent.mode not in MODE_NAMES: + continue + info, _ = resolve_buffer(intent, policies) + mine_codes = by_subject.get(f"{name}#{intent.line}:{intent.col}", set()) + checks = { + check: not (codes & mine_codes) + for check, codes in _CHECK_CODES.items() + } + entries.append({ + "function": fn.name, + "buffer": name, + "mode": info.mode.value, + "inlineBytes": info.inline_bytes, + "fallback": ("ArrayPool" if info.fallback_pool + else ("NativeMemory" if info.mode.value == "native" + else "forbidden")), + "escapePolicy": info.escape_policy, + "clearOnRelease": info.clear_on_release, + "trace": info.trace, + "counters": info.counters, + "policy": info.policy_name, + "branches": info.branches(), + "checks": checks, + }) + return {"module": mod.name, "buffers": entries} + + +def render_report(report: dict) -> str: + lines: list[str] = [f"buffer report for module '{report['module']}'"] + if not report["buffers"]: + lines.append(" (no buffers)") + return "\n".join(lines) + for e in report["buffers"]: + lines.append("") + lines.append(f"Function: {e['function']}") + lines.append(f"Buffer: {e['buffer']}") + lines.append(f" Mode: {e['mode']}") + lines.append(f" InlineLimit: {e['inlineBytes']} bytes") + lines.append(f" Fallback: {e['fallback']}") + lines.append(f" EscapePolicy: {e['escapePolicy']}") + lines.append(f" ClearOnRelease: {str(e['clearOnRelease']).lower()}") + if e["policy"]: + lines.append(f" Policy: {e['policy']}") + lines.append(" Generated branches:") + for b in e["branches"]: + lines.append(f" {b['condition']:<18} -> {b['backend']}") + lines.append(" Checks:") + for k, v in e["checks"].items(): + lines.append(f" {'ok ' if v else 'FAIL'} {k}") + return "\n".join(lines) diff --git a/run_tests.py b/run_tests.py deleted file mode 100644 index 58ae2523..00000000 --- a/run_tests.py +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env python3 -""" -Zero-dependency regression suite for the OwnLang PoC. - -Run: python tests/run_tests.py -Each case lists the diagnostic codes it expects (order-independent). A case -passes iff the produced set of error codes equals the expected set. -""" - -from __future__ import annotations - -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - -from ownlang.parser import parse, ParseError # noqa: E402 -from ownlang.lexer import LexError # noqa: E402 -from ownlang.cfg import build_cfg, collect_signatures # noqa: E402 -from ownlang.analysis import analyze # noqa: E402 -from ownlang.diagnostics import Severity # noqa: E402 -from ownlang.codegen import generate # noqa: E402 - -PRELUDE = ( - "module M\n" - "resource Buffer { acquire rent release give }\n" - "resource Conn { acquire open release close }\n" - "extern fn Fill(borrow_mut Buffer);\n" - "extern fn Hash(borrow Buffer);\n" - "extern fn Store(consume Buffer);\n" -) - - -def codes(src: str) -> list[str]: - try: - mod = parse(src) - except (ParseError, LexError): - return ["OWN020"] - rnames = {r.name for r in mod.resources} - sigs = collect_signatures(mod) - cs: list[str] = [] - for fn in mod.functions: - cfg, d1 = build_cfg(fn, rnames, sigs) - d2 = analyze(cfg) - cs += [d.code for d in (d1 + d2) if d.severity == Severity.ERROR] - return cs - - -# (name, source-after-prelude, expected error codes) -CASES = [ - # ---- clean programs ---- - ("ok_basic", "fn f(){ let b = acquire Buffer(10); release b; }", []), - ("ok_move", "fn f(){ let a = acquire Buffer(1); let c = move a; release c; }", []), - ("ok_borrows", - "fn f(){ let b = acquire Buffer(1); borrow b as s { use s; } " - "borrow_mut b as m { use m; } release b; }", []), - ("ok_both_branches", - "fn f(){ let b = acquire Buffer(1); if (c) { release b; } else { release b; } }", []), - ("ok_return_owned", - "fn f() -> Buffer { let b = acquire Buffer(1); return b; }", []), - ("ok_borrowed_param_use", - "fn f(x: &Buffer) { use x; }", []), - ("ok_shared_read_while_shared", - "fn f(){ let b = acquire Buffer(1); borrow b as s { use b; } release b; }", []), - ("ok_nested_release_both_arms", - "fn f(){ let b = acquire Buffer(1); if (c) { use b; release b; } " - "else { use b; release b; } }", []), - # ---- clean programs exercising extern calls ---- - ("ok_extern_borrow_calls", - "fn f(){ let b = acquire Buffer(1); Fill(b); Hash(b); release b; }", []), - ("ok_extern_consume", - "fn f(){ let b = acquire Buffer(1); Store(b); }", []), - ("ok_call_in_borrow_block", - "fn f(){ let b = acquire Buffer(1); borrow_mut b as m { Fill(m); } " - "borrow b as r { Hash(r); } release b; }", []), - - # ---- single faults: linear ---- - ("leak", "fn f(){ let b = acquire Buffer(10); }", ["OWN001"]), - ("leak_one_branch", - "fn f(){ let b = acquire Buffer(1); if (c) { release b; } }", ["OWN001"]), - ("double_release", - "fn f(){ let b = acquire Buffer(1); release b; release b; }", ["OWN003"]), - ("use_after_release", - "fn f(){ let b = acquire Buffer(1); release b; use b; }", ["OWN002"]), - ("use_after_move", - "fn f(){ let a = acquire Buffer(1); let c = move a; use a; release c; }", - ["OWN005"]), - ("double_move", - "fn f(){ let a = acquire Buffer(1); let b = move a; let c = move a; " - "release b; release c; }", ["OWN005"]), - ("return_borrow", - "fn f(x: &Buffer) -> Buffer { return x; }", ["OWN004"]), - - # ---- single faults: loan / permission conflicts (now distinct codes) ---- - ("mut_while_shared", - "fn f(){ let b = acquire Buffer(1); borrow b as s { borrow_mut b as m " - "{ use m; } } release b; }", ["OWN006"]), - ("move_while_borrowed", - "fn f(){ let b = acquire Buffer(1); borrow b as s { let c = move b; " - "release c; } }", ["OWN007"]), - ("release_while_borrowed", - "fn f(){ let b = acquire Buffer(1); borrow b as s { release b; } }", - ["OWN008"]), - ("double_mut", - "fn f(){ let b = acquire Buffer(1); borrow_mut b as m1 { borrow_mut b as " - "m2 { use m2; } } release b; }", ["OWN011"]), - ("shared_while_mut", - "fn f(){ let b = acquire Buffer(1); borrow_mut b as m { borrow b as s " - "{ use s; } } release b; }", ["OWN012"]), - ("use_owner_while_mut", - "fn f(){ let b = acquire Buffer(1); borrow_mut b as m { use b; } release b; }", - ["OWN013"]), - - # ---- single faults: join / "maybe" ---- - ("use_after_maybe_release", - "fn f(){ let b = acquire Buffer(1); if (c) { release b; } use b; }", - ["OWN001", "OWN009"]), - ("use_after_maybe_move", - "fn f(){ let b = acquire Buffer(1); if (c) { let d = move b; release d; } " - "use b; }", ["OWN001", "OWN010"]), - - # ---- structural / resolution (renumbered) ---- - ("release_non_owned", - "fn f(x: &Buffer) { release x; }", ["OWN034"]), - ("move_non_owned", - "fn f(x: &Buffer) { let y = move x; }", ["OWN034"]), - ("borrow_non_owned", - "fn f(n: int) { borrow n as s { use s; } }", ["OWN034"]), - ("undefined", "fn f(){ use ghost; }", ["OWN030"]), - ("undefined_resource", "fn f(){ let x = acquire Nope(1); release x; }", ["OWN030"]), - ("copy_owned", - "fn f(){ let a = acquire Buffer(1); let b = a; release a; }", ["OWN032"]), - ("missing_return", - "fn f() -> Buffer { let b = acquire Buffer(1); release b; }", ["OWN033"]), - ("loop_rejected", "fn f(){ while (x) { use x; } }", ["OWN020"]), - ("async_rejected", "fn f(){ async { use x; } }", ["OWN020"]), - - # ---- extern boundary ---- - ("unknown_call", - "fn f(){ let b = acquire Buffer(1); Mystery(b); release b; }", ["OWN040"]), - ("arg_shared_where_mut", - "fn f(){ let b = acquire Buffer(1); borrow b as r { Fill(r); } release b; }", - ["OWN041"]), - ("arg_arity", - "fn f(){ let b = acquire Buffer(1); Fill(b, b); release b; }", ["OWN041"]), - ("arg_plain_to_resource", - "fn f(n: int){ Hash(n); }", ["OWN041"]), - ("consume_then_use", - "fn f(){ let b = acquire Buffer(1); Store(b); use b; }", ["OWN002"]), - ("consume_while_borrowed", - "fn f(){ let b = acquire Buffer(1); borrow b as r { Store(b); } }", ["OWN007"]), - - # ---- multiple faults in one function ---- - ("leak_and_uafr", - "fn f(){ let a = acquire Buffer(1); let b = acquire Buffer(2); " - "release b; use b; }", ["OWN001", "OWN002"]), -] - - -# Golden codegen smoke: a real ArrayPool-backed buffer must (a) check clean and -# (b) lower to genuine .NET — Rent/Return, try/finally, a Span view — with NO -# runtime "released?" flag (the release is hoisted out of the try, so it needs -# no guard). -GOLDEN = ( - "module PoolDemo\n" - "resource Buffer {\n" - " acquire rent\n" - " release give\n" - ' emit_type "byte[]"\n' - ' emit_acquire "ArrayPool.Shared.Rent({args})"\n' - ' emit_release "ArrayPool.Shared.Return({0})"\n' - ' emit_borrow "{0}.AsSpan()"\n' - "}\n" - "extern fn Fill(borrow_mut Buffer);\n" - "extern fn Hash(borrow Buffer);\n" - "fn process(size: int) {\n" - " let buf = acquire Buffer(size);\n" - " borrow_mut buf as bytes { Fill(bytes); }\n" - " borrow buf as view { Hash(view); }\n" - " release buf;\n" - "}\n" -) - - -def golden_smoke() -> list[str]: - """Returns a list of failure strings (empty == pass).""" - fails: list[str] = [] - cs = codes(GOLDEN) - errs = [c for c in cs if c.startswith("OWN")] - if errs: - fails.append(f"golden should check clean, got {sorted(set(errs))}") - cs_out = generate(parse(GOLDEN)) - must_contain = [ - "ArrayPool.Shared.Rent(size)", - "ArrayPool.Shared.Return(buf)", - "byte[] buf =", - "buf.AsSpan()", - "try", - "finally", - "Fill(bytes)", - "Hash(view)", - ] - for s in must_contain: - if s not in cs_out: - fails.append(f"golden C# missing: {s!r}") - # the Return must be inside finally and NOT also inside the try body - body = cs_out - if body.count("ArrayPool.Shared.Return(buf)") != 1: - fails.append("golden C# should Return exactly once (hoisted into finally)") - return fails - - -def run() -> int: - passed = 0 - failed = 0 - for name, body, expected in CASES: - got = sorted(set(codes(PRELUDE + body))) - want = sorted(set(expected)) - if got == want: - passed += 1 - else: - failed += 1 - print(f"FAIL {name}") - print(f" expected {want}") - print(f" got {got}") - - # codegen smoke: every clean case must generate without throwing - cg_total = len([c for c in CASES if not c[2]]) - cg_fail = 0 - for name, body, expected in CASES: - if expected: - continue - try: - generate(parse(PRELUDE + body)) - except Exception as e: # noqa: BLE001 - cg_fail += 1 - print(f"CODEGEN FAIL {name}: {type(e).__name__}: {e}") - - golden_fails = golden_smoke() - for f in golden_fails: - print(f"GOLDEN FAIL: {f}") - - total = passed + failed - print(f"\nanalysis: {passed}/{total} passed, {failed} failed") - print(f"codegen: {cg_total - cg_fail}/{cg_total} generated cleanly") - print(f"golden: {'PASS' if not golden_fails else 'FAIL'}") - return 1 if (failed or cg_fail or golden_fails) else 0 - - -if __name__ == "__main__": - raise SystemExit(run()) diff --git a/tests/buffer_scratch_program.cs.txt b/tests/buffer_scratch_program.cs.txt new file mode 100644 index 00000000..da0f9bcb --- /dev/null +++ b/tests/buffer_scratch_program.cs.txt @@ -0,0 +1,132 @@ +// Runnable golden for the OwnLang buffer line — a scratch buffer that prefers +// the stack and falls back to System.Buffers.ArrayPool. +// +// The ScratchDemo class body (parse) and the OwnTrace/OwnCounters classes are +// EMITTED VERBATIM by: +// python -m ownlang emit buffer_scratch.own +// Fill/Hash/Main are hand-written host code. This proves the buffer model lowers +// to real, runnable .NET — it is not executed in the PoC sandbox (no .NET SDK). +// +// Build & run (see the runtime trace + counters): +// dotnet run -p:DefineConstants="OWNSHARP_TRACE;OWNSHARP_COUNTERS" +using System; +using System.Buffers; + +public static class ScratchDemo +{ + public static void parse(int size) + { + if (size < 0) + throw new ArgumentOutOfRangeException(nameof(size)); + byte[]? tmp_rented = null; + Span tmp_backing = stackalloc byte[1024]; + Span tmp; + if (size <= 1024) + { + OwnTrace.ScratchSelected("parse", "tmp", size, 1024, "stackalloc"); + OwnCounters.StackHit(); + tmp = tmp_backing[..size]; + } + else + { + OwnTrace.ScratchSelected("parse", "tmp", size, 1024, "ArrayPool"); + OwnCounters.PoolFallback(size); + tmp_rented = ArrayPool.Shared.Rent(size); + tmp = tmp_rented.AsSpan(0, size); + } + try + { + { // mutable borrow of tmp as bytes + var bytes = tmp; + Fill(bytes); + } + { // shared borrow of tmp as view + var view = tmp; + Hash(view); + } + } + finally + { + OwnCounters.Release(); + if (tmp_rented is not null) + ArrayPool.Shared.Return(tmp_rented); + } + } + + // ---- hand-written host (an `extern fn` is a promise the host keeps) ---- + // Borrow params are noescape: these must not retain the span past the call. + static void Fill(Span b) + { + for (int i = 0; i < b.Length; i++) b[i] = (byte)(i & 0xFF); + } + + static void Hash(ReadOnlySpan b) + { + int h = 17; + foreach (var x in b) h = unchecked(h * 31 + x); + Console.WriteLine($"hash={h} len={b.Length}"); + } + + public static void Main() + { + parse(64); // <= inline limit -> stackalloc arm (no heap) + parse(4096); // > inline limit -> ArrayPool arm (real Rent/Return) + // Visible only when built with -p:DefineConstants="OWNSHARP_COUNTERS". + Console.WriteLine( + $"stackHits={OwnCounters.ScratchStackHits} " + + $"poolFallbacks={OwnCounters.ScratchPoolFallbacks} " + + $"poolBytesRented={OwnCounters.ScratchPoolBytesRented}"); + Console.WriteLine("ok"); + } +} + +internal static class OwnTrace +{ + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void ScratchSelected(string function, string buffer, + int requestedBytes, int inlineLimit, string backend) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, " + + $"inline={inlineLimit}, backend={backend}"); + + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void StackSelected(string function, string buffer, + int requestedBytes, int inlineLimit) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, " + + $"inline={inlineLimit}, backend=stackalloc"); + + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void PooledSelected(string function, string buffer, int requestedBytes) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, backend=ArrayPool"); + + [System.Diagnostics.Conditional("OWNSHARP_TRACE")] + public static void NativeSelected(string function, string buffer, int requestedBytes) + => System.Diagnostics.Trace.WriteLine( + $"[OwnSharp] {function}.{buffer}: requested={requestedBytes}, backend=NativeMemory"); +} + +internal static class OwnCounters +{ + public static long ScratchStackHits; + public static long ScratchPoolFallbacks; + public static long ScratchPoolBytesRented; + public static long ScratchReleaseCount; + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void StackHit() + => System.Threading.Interlocked.Increment(ref ScratchStackHits); + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void PoolFallback(int bytes) + { + System.Threading.Interlocked.Increment(ref ScratchPoolFallbacks); + System.Threading.Interlocked.Add(ref ScratchPoolBytesRented, bytes); + } + + [System.Diagnostics.Conditional("OWNSHARP_COUNTERS")] + public static void Release() + => System.Threading.Interlocked.Increment(ref ScratchReleaseCount); +} + diff --git a/tests/run_tests.py b/tests/run_tests.py new file mode 100644 index 00000000..027fa6d9 --- /dev/null +++ b/tests/run_tests.py @@ -0,0 +1,950 @@ +#!/usr/bin/env python3 +""" +Zero-dependency regression suite for the OwnLang PoC. + +Run: python tests/run_tests.py +Each case lists the diagnostic codes it expects (order-independent). A case +passes iff the produced set of error codes equals the expected set. +""" + +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from ownlang.parser import parse, ParseError # noqa: E402 +from ownlang.lexer import LexError # noqa: E402 +from ownlang.cfg import (build_cfg, collect_signatures, # noqa: E402 + collect_policies) +from ownlang.analysis import analyze # noqa: E402 +from ownlang.diagnostics import Severity # noqa: E402 +from ownlang.codegen import generate # noqa: E402 +from ownlang.buffers import validate_policies # noqa: E402 +from ownlang.report import build_report # noqa: E402 + +PRELUDE = ( + "module M\n" + "resource Buffer { acquire rent release give }\n" + "resource Conn { acquire open release close }\n" + "extern fn Fill(borrow_mut Buffer);\n" + "extern fn Hash(borrow Buffer);\n" + "extern fn Store(consume Buffer);\n" +) + + +def codes(src: str) -> list[str]: + try: + mod = parse(src) + except (ParseError, LexError): + return ["OWN020"] + rnames = {r.name for r in mod.resources} + sigs = collect_signatures(mod) + pols = collect_policies(mod) + cs: list[str] = [d.code for d in validate_policies(pols) + if d.severity == Severity.ERROR] + for fn in mod.functions: + cfg, d1 = build_cfg(fn, rnames, sigs, pols) + d2 = analyze(cfg) + cs += [d.code for d in (d1 + d2) if d.severity == Severity.ERROR] + return cs + + +# (name, source-after-prelude, expected error codes) +CASES = [ + # ---- clean programs ---- + ("ok_basic", "fn f(){ let b = acquire Buffer(10); release b; }", []), + ("ok_move", "fn f(){ let a = acquire Buffer(1); let c = move a; release c; }", []), + ("ok_borrows", + "fn f(){ let b = acquire Buffer(1); borrow b as s { use s; } " + "borrow_mut b as m { use m; } release b; }", []), + ("ok_both_branches", + "fn f(){ let b = acquire Buffer(1); if (c) { release b; } else { release b; } }", []), + ("ok_return_owned", + "fn f() -> Buffer { let b = acquire Buffer(1); return b; }", []), + ("ok_borrowed_param_use", + "fn f(x: &Buffer) { use x; }", []), + ("ok_shared_read_while_shared", + "fn f(){ let b = acquire Buffer(1); borrow b as s { use b; } release b; }", []), + ("ok_nested_release_both_arms", + "fn f(){ let b = acquire Buffer(1); if (c) { use b; release b; } " + "else { use b; release b; } }", []), + # ---- clean programs exercising extern calls ---- + ("ok_extern_borrow_calls", + "fn f(){ let b = acquire Buffer(1); Fill(b); Hash(b); release b; }", []), + ("ok_extern_consume", + "fn f(){ let b = acquire Buffer(1); Store(b); }", []), + ("ok_call_in_borrow_block", + "fn f(){ let b = acquire Buffer(1); borrow_mut b as m { Fill(m); } " + "borrow b as r { Hash(r); } release b; }", []), + + # ---- single faults: linear ---- + ("leak", "fn f(){ let b = acquire Buffer(10); }", ["OWN001"]), + ("leak_one_branch", + "fn f(){ let b = acquire Buffer(1); if (c) { release b; } }", ["OWN001"]), + ("double_release", + "fn f(){ let b = acquire Buffer(1); release b; release b; }", ["OWN003"]), + ("use_after_release", + "fn f(){ let b = acquire Buffer(1); release b; use b; }", ["OWN002"]), + ("use_after_move", + "fn f(){ let a = acquire Buffer(1); let c = move a; use a; release c; }", + ["OWN005"]), + ("double_move", + "fn f(){ let a = acquire Buffer(1); let b = move a; let c = move a; " + "release b; release c; }", ["OWN005"]), + ("return_borrow", + "fn f(x: &Buffer) -> Buffer { return x; }", ["OWN004"]), + + # ---- single faults: loan / permission conflicts (now distinct codes) ---- + ("mut_while_shared", + "fn f(){ let b = acquire Buffer(1); borrow b as s { borrow_mut b as m " + "{ use m; } } release b; }", ["OWN006"]), + ("move_while_borrowed", + "fn f(){ let b = acquire Buffer(1); borrow b as s { let c = move b; " + "release c; } }", ["OWN007"]), + ("release_while_borrowed", + "fn f(){ let b = acquire Buffer(1); borrow b as s { release b; } }", + ["OWN008"]), + ("double_mut", + "fn f(){ let b = acquire Buffer(1); borrow_mut b as m1 { borrow_mut b as " + "m2 { use m2; } } release b; }", ["OWN011"]), + ("shared_while_mut", + "fn f(){ let b = acquire Buffer(1); borrow_mut b as m { borrow b as s " + "{ use s; } } release b; }", ["OWN012"]), + ("use_owner_while_mut", + "fn f(){ let b = acquire Buffer(1); borrow_mut b as m { use b; } release b; }", + ["OWN013"]), + + # ---- single faults: join / "maybe" ---- + ("use_after_maybe_release", + "fn f(){ let b = acquire Buffer(1); if (c) { release b; } use b; }", + ["OWN001", "OWN009"]), + ("use_after_maybe_move", + "fn f(){ let b = acquire Buffer(1); if (c) { let d = move b; release d; } " + "use b; }", ["OWN001", "OWN010"]), + + # ---- structural / resolution (renumbered) ---- + ("release_non_owned", + "fn f(x: &Buffer) { release x; }", ["OWN034"]), + ("move_non_owned", + "fn f(x: &Buffer) { let y = move x; }", ["OWN034"]), + ("borrow_non_owned", + "fn f(n: int) { borrow n as s { use s; } }", ["OWN034"]), + ("undefined", "fn f(){ use ghost; }", ["OWN030"]), + ("undefined_resource", "fn f(){ let x = acquire Nope(1); release x; }", ["OWN030"]), + ("copy_owned", + "fn f(){ let a = acquire Buffer(1); let b = a; release a; }", ["OWN032"]), + ("missing_return", + "fn f() -> Buffer { let b = acquire Buffer(1); release b; }", ["OWN033"]), + ("loop_rejected", "fn f(){ while (x) { use x; } }", ["OWN020"]), + ("async_rejected", "fn f(){ async { use x; } }", ["OWN020"]), + + # ---- extern boundary ---- + ("unknown_call", + "fn f(){ let b = acquire Buffer(1); Mystery(b); release b; }", ["OWN040"]), + ("arg_shared_where_mut", + "fn f(){ let b = acquire Buffer(1); borrow b as r { Fill(r); } release b; }", + ["OWN041"]), + ("arg_arity", + "fn f(){ let b = acquire Buffer(1); Fill(b, b); release b; }", ["OWN041"]), + ("arg_plain_to_resource", + "fn f(n: int){ Hash(n); }", ["OWN041"]), + ("consume_then_use", + "fn f(){ let b = acquire Buffer(1); Store(b); use b; }", ["OWN002"]), + ("consume_while_borrowed", + "fn f(){ let b = acquire Buffer(1); borrow b as r { Store(b); } }", ["OWN007"]), + + # ---- multiple faults in one function ---- + ("leak_and_uafr", + "fn f(){ let a = acquire Buffer(1); let b = acquire Buffer(2); " + "release b; use b; }", ["OWN001", "OWN002"]), + + # ---- buffer storage policies: clean ---- + ("buf_scratch_ok", + "fn f(n: int){ let b = Buffer.scratch(n, inline = 1024); " + "borrow_mut b as m { Fill(m); } release b; }", []), + ("buf_stack_const_ok", + "fn f(){ let b = Buffer.stack(256); borrow_mut b as m { Fill(m); } " + "release b; }", []), + ("buf_stack_dyn_bounded_ok", + "fn f(n: int){ let b = Buffer.stack(n, max = 1024); release b; }", []), + ("buf_inline_ok", + "fn f(){ let b = Buffer.inline(64); release b; }", []), + ("buf_pooled_local_ok", + "fn f(n: int){ let b = Buffer.pooled(n); borrow_mut b as m { Fill(m); } " + "release b; }", []), + ("buf_branchy_release_ok", + "fn f(n: int){ let b = Buffer.pooled(n); if (c) { release b; } " + "else { release b; } }", []), + ("buf_overlapping_fifo_ok", + "fn f(n: int){ let a = Buffer.pooled(n); let b = Buffer.pooled(n); " + "release a; release b; }", []), + ("buf_overlapping_lifo_ok", + "fn f(n: int){ let a = Buffer.pooled(n); let b = Buffer.pooled(n); " + "release b; release a; }", []), + ("buf_scratch_bad_fallback", + "fn f(n: int){ let b = Buffer.scratch(n, fallback = forbiden); " + "release b; }", ["OWN030"]), + ("buf_scratch_nonident_fallback", + "fn f(n: int){ let b = Buffer.scratch(n, fallback = 0); release b; }", + ["OWN030"]), + ("buf_move_release_ok", + "fn f(n: int){ let a = Buffer.pooled(n); let b = move a; release b; }", []), + ("buf_bad_namespace", + "fn f(n: int){ let b = Foo.stack(n, max = 1024); release b; }", ["OWN030"]), + ("buf_move_escapes", + "fn f(n: int) -> Buffer { let a = Buffer.stack(n, max = 100); " + "let b = move a; return b; }", ["OWN015"]), + ("buf_sibling_move_ok", + "fn f(n: int){ let a = Buffer.pooled(n); if (c) { let b = move a; " + "release b; } else { release a; } }", []), + ("buf_disjoint_sequential_ok", + "fn f(n: int){ let a = Buffer.pooled(n); release a; " + "let b = Buffer.pooled(n); release b; }", []), + ("buf_bad_policy_ref", + "fn f(n: int){ let b = Buffer.scratch(n, policy = 0); release b; }", + ["OWN030"]), + ("buf_bad_inline_bound", + "fn f(n: int){ let b = Buffer.scratch(n, inline = bogus); release b; }", + ["OWN030"]), + ("buf_bad_max_bound", + "fn f(n: int){ let b = Buffer.stack(n, max = bogus); release b; }", + ["OWN030"]), + ("buf_unknown_option", + "fn f(n: int){ let b = Buffer.scratch(n, fallbak = forbidden); " + "release b; }", ["OWN030"]), + ("buf_unknown_policy_setting", + "policy P { fallbak = forbidden; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = P); release b; }", + ["OWN030"]), + ("buf_size_not_int_bool", + "fn f(flag: bool){ let b = Buffer.pooled(flag); release b; }", ["OWN018"]), + ("buf_size_not_int_owned", + "fn f(){ let r = acquire Conn(1); let b = Buffer.pooled(r); " + "release b; release r; }", ["OWN018"]), + ("buf_inline_dynamic_rejected", + "fn f(n: int){ let b = Buffer.inline(n, max = 1024); release b; }", + ["OWN021"]), + ("buf_size_borrow_temp", + "fn f(x: &Buffer){ let n = x; let b = Buffer.pooled(n); release b; }", + ["OWN018"]), + ("buf_local_after_release_ok", + "fn f(n: int){ let b = Buffer.pooled(n); let x = 1; release b; " + "let y = x; }", []), + ("buf_inline_literal_ok", + "fn f(){ let b = Buffer.inline(256); release b; }", []), + ("buf_policy_bad_clear", + "policy Sensitive { clear_on_release = ture; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = Sensitive); release b; }", + ["OWN030"]), + ("buf_bad_counters", + "fn f(n: int){ let b = Buffer.scratch(n, counters = ture); release b; }", + ["OWN030"]), + ("buf_bad_trace", + "fn f(n: int){ let b = Buffer.scratch(n, trace = ture); release b; }", + ["OWN030"]), + ("buf_policy_bools_ok", + "policy S { clear_on_release = true; counters = false; trace = off; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = S); release b; }", []), + ("buf_duplicate_option", + "fn f(n: int){ let b = Buffer.scratch(n, fallback = forbidden, " + "fallback = pool); release b; }", ["OWN030"]), + ("buf_duplicate_policy_setting", + "policy P { inline_bytes = 512; inline_bytes = 1024; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = P); release b; }", + ["OWN030"]), + ("buf_inline_override_ignores_bad_policy", + "policy P { inline_bytes = bogus; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = P, inline = 128); " + "release b; }", []), + ("buf_bad_policy_inline_no_override", + "policy P { inline_bytes = bogus; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = P); release b; }", + ["OWN030"]), + ("buf_alias_redecl_ok", + "fn f(n: int){ let a = Buffer.pooled(n); if (c) { let b = move a; " + "release b; } else { release a; } let b = acquire Conn(); release b; }", + []), + ("res_partial_overlap_ok", + "fn f(){ let a = acquire Buffer(1); let c = acquire Conn(1); " + "release a; use c; release c; }", []), + ("buf_nested_release_ok", + "fn f(n: int){ let a = acquire Conn(1); let b = Buffer.pooled(n); " + "borrow a as s { release b; } release a; }", []), + ("buf_local_helper_ok", + "fn helper(x: &mut Buffer){ use x; } " + "fn f(n: int){ let b = Buffer.pooled(n); helper(b); release b; }", []), + ("buf_native_ok", + "fn f(n: int){ let b = Buffer.native(n); release b; }", []), + ("buf_policy_ok", + "policy P { inline_bytes = 512; fallback = pool; } " + "fn f(n: int){ let b = Buffer.scratch(n, policy = P); release b; }", []), + + # ---- buffer storage policies: faults ---- + ("buf_stack_dyn_unbounded", + "fn f(n: int){ let b = Buffer.stack(n); release b; }", ["OWN021"]), + ("buf_stack_too_large", + "fn f(){ let b = Buffer.stack(1000000); release b; }", ["OWN019"]), + ("buf_scratch_escapes_return", + "fn f(n: int) -> Buffer { let b = Buffer.scratch(n); return b; }", + ["OWN015"]), + ("buf_stack_escapes_return", + "fn f() -> Buffer { let b = Buffer.stack(64); return b; }", ["OWN015"]), + ("buf_scratch_escapes_consume", + "fn f(n: int){ let b = Buffer.scratch(n); Store(b); }", ["OWN016"]), + ("buf_pooled_escapes_return", + "fn f(n: int) -> Buffer { let b = Buffer.pooled(n); return b; }", + ["OWN017"]), + ("buf_pooled_escapes_consume", + "fn f(n: int){ let b = Buffer.pooled(n); Store(b); }", ["OWN017"]), + ("buf_native_escapes_return", + "fn f(n: int) -> Buffer { let b = Buffer.native(n); return b; }", + ["OWN017"]), + ("buf_scratch_forbid_dynamic", + "fn f(n: int){ let b = Buffer.scratch(n, fallback = forbidden); " + "release b; }", ["OWN023"]), + ("buf_scratch_leak", + "fn f(n: int){ let b = Buffer.scratch(n); }", ["OWN001"]), + ("buf_unknown_mode", + "fn f(n: int){ let b = Buffer.bogus(n); release b; }", ["OWN030"]), + ("buf_release_while_borrowed", + "fn f(n: int){ let b = Buffer.scratch(n); borrow b as s { release b; } }", + ["OWN008"]), + ("buf_moved_then_used", + "fn f(n: int){ let a = Buffer.pooled(n); let b = move a; use a; " + "release b; }", ["OWN005"]), +] + + +# Golden codegen smoke: a real ArrayPool-backed buffer must (a) check clean and +# (b) lower to genuine .NET — Rent/Return, try/finally, a Span view — with NO +# runtime "released?" flag (the release is hoisted out of the try, so it needs +# no guard). +GOLDEN = ( + "module PoolDemo\n" + "resource Buffer {\n" + " acquire rent\n" + " release give\n" + ' emit_type "byte[]"\n' + ' emit_acquire "ArrayPool.Shared.Rent({args})"\n' + ' emit_release "ArrayPool.Shared.Return({0})"\n' + ' emit_borrow "{0}.AsSpan()"\n' + "}\n" + "extern fn Fill(borrow_mut Buffer);\n" + "extern fn Hash(borrow Buffer);\n" + "fn process(size: int) {\n" + " let buf = acquire Buffer(size);\n" + " borrow_mut buf as bytes { Fill(bytes); }\n" + " borrow buf as view { Hash(view); }\n" + " release buf;\n" + "}\n" +) + + +def golden_smoke() -> list[str]: + """Returns a list of failure strings (empty == pass).""" + fails: list[str] = [] + cs = codes(GOLDEN) + errs = [c for c in cs if c.startswith("OWN")] + if errs: + fails.append(f"golden should check clean, got {sorted(set(errs))}") + cs_out = generate(parse(GOLDEN)) + must_contain = [ + "ArrayPool.Shared.Rent(size)", + "ArrayPool.Shared.Return(buf)", + "byte[] buf =", + "buf.AsSpan()", + "try", + "finally", + "Fill(bytes)", + "Hash(view)", + ] + for s in must_contain: + if s not in cs_out: + fails.append(f"golden C# missing: {s!r}") + # the Return must be inside finally and NOT also inside the try body + body = cs_out + if body.count("ArrayPool.Shared.Return(buf)") != 1: + fails.append("golden C# should Return exactly once (hoisted into finally)") + return fails + + +BUFFER_GOLDEN = ( + "module ScratchDemo\n" + "policy DefaultScratch { inline_bytes = 1024; fallback = pool; " + "trace = debug; counters = true; clear_on_release = false; }\n" + "extern fn Fill(borrow_mut Buffer);\n" + "extern fn Hash(borrow Buffer);\n" + "fn parse(size: int) {\n" + " let tmp = Buffer.scratch(size, inline = 1024, fallback = pool);\n" + " borrow_mut tmp as bytes { Fill(bytes); }\n" + " borrow tmp as view { Hash(view); }\n" + " release tmp;\n" + "}\n" +) + + +def buffer_smoke() -> list[str]: + """A scratch buffer must check clean, lower to the stack-first/pool-fallback + pattern with the trace + counter hooks, ship the [Conditional] runtime + support, and produce a compile-time report that names the chosen backends.""" + fails: list[str] = [] + cs = [c for c in codes(BUFFER_GOLDEN) if c.startswith("OWN")] + if cs: + fails.append(f"buffer golden should check clean, got {sorted(set(cs))}") + out = generate(parse(BUFFER_GOLDEN)) + must_contain = [ + "Span tmp_backing = stackalloc byte[1024];", + "if (size <= 1024)", + 'OwnTrace.ScratchSelected("parse", "tmp", size, 1024, "stackalloc");', + 'OwnTrace.ScratchSelected("parse", "tmp", size, 1024, "ArrayPool");', + "OwnCounters.StackHit();", + "OwnCounters.PoolFallback(size);", + "ArrayPool.Shared.Rent(size)", + "ArrayPool.Shared.Return(tmp_rented)", + "try", + "finally", + "internal static class OwnTrace", + "internal static class OwnCounters", + 'Conditional("OWNSHARP_TRACE")', + 'Conditional("OWNSHARP_COUNTERS")', + ] + for s in must_contain: + if s not in out: + fails.append(f"buffer C# missing: {s!r}") + # the pool Return must run exactly once, guarded by the rented null-check + if out.count("ArrayPool.Shared.Return(tmp_rented)") != 1: + fails.append("buffer C# should Return the rented array exactly once") + + # compile-time report + mod = parse(BUFFER_GOLDEN) + rep = build_report(mod, []) + if not rep["buffers"]: + fails.append("buffer report produced no entries") + else: + e = rep["buffers"][0] + if e["mode"] != "scratch" or e["inlineBytes"] != 1024: + fails.append(f"buffer report has wrong mode/inline: {e}") + if e["escapePolicy"] != "local-only": + fails.append(f"scratch report should be local-only, got {e['escapePolicy']}") + backends = {b["backend"] for b in e["branches"]} + if backends != {"stackalloc", "ArrayPool"}: + fails.append(f"scratch report branches wrong: {backends}") + + # the committed runnable golden must stay in sync with the emitter and stay + # a real, compilable ArrayPool program (a human can `dotnet run` it). + golden_path = os.path.join(os.path.dirname(__file__), + "buffer_scratch_program.cs.txt") + if os.path.exists(golden_path): + prog = open(golden_path, encoding="utf-8").read() + for s in ("public static void parse(int size)", + "if (size < 0)", + "ArrayPool.Shared.Rent(size)", + "ArrayPool.Shared.Return(tmp_rented)", + "internal static class OwnTrace", + "internal static class OwnCounters", + "public static void Main()"): + if s not in prog: + fails.append(f"runnable golden missing: {s!r}") + # the negative-size guard must precede the trace/counter hooks + if "if (size < 0)" in prog and "ScratchSelected" in prog: + if prog.index("if (size < 0)") > prog.index("ScratchSelected"): + fails.append("runnable golden guard must precede the trace hook") + # its emitted parse body must match what the emitter produces today + if "tmp = tmp_backing[..size];" not in prog: + fails.append("runnable golden parse body drifted from the emitter") + else: + fails.append("runnable golden buffer_scratch_program.cs.txt is missing") + return fails + + +def escape_and_length_smoke() -> list[str]: + """Regression guards for the PR #2 review: + - the checker rejects an escaping movable (pooled/native) buffer rather than + letting codegen emit C# that leaks the rent or fails to compile (OWN017); + - a constant scratch smaller than its inline limit exposes the requested + length, not the reservation; + - a fallback-forbidden scratch reports as stack-only.""" + fails: list[str] = [] + + pooled_ret = ("module M\n" + "fn f(n: int) -> Buffer { let b = Buffer.pooled(n); return b; }\n") + if "OWN017" not in codes(pooled_ret): + fails.append("escaping pooled (return) must be rejected with OWN017") + + pooled_consume = ("module M\nextern fn Store(consume Buffer);\n" + "fn f(n: int){ let b = Buffer.pooled(n); Store(b); }\n") + if "OWN017" not in codes(pooled_consume): + fails.append("escaping pooled (consume) must be rejected with OWN017") + + native_ret = ("module M\n" + "fn f(n: int) -> Buffer { let b = Buffer.native(n); return b; }\n") + if "OWN017" not in codes(native_ret): + fails.append("escaping native (return) must be rejected with OWN017") + + # a locally-released pooled buffer is fine and DOES Return to the pool + pooled_local = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn f(n: int){ let b = Buffer.pooled(n); " + "borrow_mut b as m { Fill(m); } release b; }\n") + if codes(pooled_local): + fails.append(f"local pooled buffer should check clean, got {codes(pooled_local)}") + if "ArrayPool.Shared.Return" not in generate(parse(pooled_local)): + fails.append("local pooled buffer must Return to the pool") + + scratch_const = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn f(){ let b = Buffer.scratch(64, inline = 1024, " + "fallback = forbidden); borrow_mut b as m { Fill(m); } " + "release b; }\n") + out = generate(parse(scratch_const)) + if "b_backing[..64]" not in out: + fails.append("constant forbidden-fallback scratch must expose length 64") + + rep = build_report(parse(scratch_const), []) + e = rep["buffers"][0] + if e["fallback"] != "forbidden": + fails.append(f"forbidden scratch report fallback should be 'forbidden', got {e['fallback']}") + backends = {b["backend"] for b in e["branches"]} + if backends != {"stackalloc"}: + fails.append(f"forbidden scratch report should be stack-only, got {backends}") + return fails + + +def branchy_and_malformed_smoke() -> list[str]: + """Two follow-up review guards: + - a buffer released inside branches is NOT an escape: codegen must emit the + real pool cleanup at each release site, never a generic Dispose on a Span; + - the report must skip a malformed buffer mode (Buffer.bogus) instead of + throwing, leaving the checker's OWN030 to stand.""" + fails: list[str] = [] + + branchy = ("module M\n" + "fn f(n: int){ let b = Buffer.pooled(n); " + "if (c) { release b; } else { release b; } }\n") + if codes(branchy): + fails.append(f"branchy buffer release should check clean, got {codes(branchy)}") + out = generate(parse(branchy)) + if "Dispose()" in out: + fails.append("branchy buffer release must not emit a generic Dispose()") + if out.count("ArrayPool.Shared.Return(b_array)") != 2: + fails.append("branchy pooled release must Return at both release sites") + if "ArrayPool.Shared.Rent(n)" not in out: + fails.append("branchy pooled buffer must still Rent once") + + bogus = parse("module M\nfn f(n: int){ let b = Buffer.bogus(n); release b; }\n") + try: + rep = build_report(bogus, []) + except Exception as e: # noqa: BLE001 + fails.append(f"report crashed on a malformed buffer mode: {type(e).__name__}: {e}") + else: + if rep["buffers"]: + fails.append("report should skip an unresolved buffer mode") + + # overlapping buffer lifetimes released in non-LIFO (FIFO) order must lower + # without a CodegenError, and both arrays must be returned to the pool. + fifo = ("module M\nfn f(n: int){ let a = Buffer.pooled(n); " + "let b = Buffer.pooled(n); release a; release b; }\n") + if codes(fifo): + fails.append(f"FIFO overlapping buffers should check clean, got {codes(fifo)}") + try: + out = generate(parse(fifo)) + except Exception as e: # noqa: BLE001 + fails.append(f"FIFO overlapping buffers crashed codegen: {type(e).__name__}: {e}") + else: + if out.count("ArrayPool.Shared.Return(a_array)") != 1: + fails.append("FIFO: a must be returned to the pool exactly once") + if out.count("ArrayPool.Shared.Return(b_array)") != 1: + fails.append("FIFO: b must be returned to the pool exactly once") + + # a misspelled forbidden-fallback must be diagnosed, never silently pooled + bad_fb = ("module M\nfn f(n: int){ " + "let b = Buffer.scratch(n, fallback = forbiden); release b; }\n") + if "OWN030" not in codes(bad_fb): + fails.append("misspelled scratch fallback must produce OWN030") + + # an inline override must skip a malformed policy default (override wins) and + # the effective inline limit must be the override + ov = ("module M\npolicy P { inline_bytes = bogus; }\n" + "fn f(n: int){ let b = Buffer.scratch(n, policy = P, inline = 128); " + "release b; }\n") + if codes(ov): + fails.append(f"inline override should ignore bad policy default, got {codes(ov)}") + if "stackalloc byte[128]" not in generate(parse(ov)): + fails.append("inline override (128) must be the effective inline limit") + + # a non-identifier fallback (fallback = 0) must likewise fail safe: OWN030 + # AND no ArrayPool fallback enabled (it must not silently heap-allocate). + from ownlang.buffers import resolve as _resolve + from ownlang.ast_nodes import BufferIntent, IntLit + intent = BufferIntent(mode="scratch", size=IntLit(8, 1), + options={"fallback": IntLit(0, 1)}, line=1) + info, idiags = _resolve(intent, {}) + if "OWN030" not in {d.code for d in idiags}: + fails.append("non-identifier scratch fallback must produce OWN030") + if info.fallback_pool: + fails.append("non-identifier scratch fallback must not enable the pool") + + # the report's noEscape check must agree with the OWN017 checker diagnostic + esc = parse("module M\nfn f(n: int) -> Buffer { let b = Buffer.pooled(n); " + "return b; }\n") + if _report_check(esc, "noEscape"): + fails.append("report noEscape must be false for an OWN017-rejected buffer") + + # a buffer released through a moved alias must lower (not raise) and the + # cleanup must attach to the moved-to name's release. + moved = ("module M\nfn f(n: int){ let a = Buffer.pooled(n); " + "let b = move a; release b; }\n") + if codes(moved): + fails.append(f"move-then-release buffer should check clean, got {codes(moved)}") + try: + out = generate(parse(moved)) + except Exception as e: # noqa: BLE001 + fails.append(f"move-then-release buffer crashed codegen: {type(e).__name__}: {e}") + else: + if out.count("ArrayPool.Shared.Return(a_array)") != 1: + fails.append("moved buffer must still Return its original backing once") + + # a wrong namespace (Foo.stack) must be diagnosed, not silently lowered + bad_ns = "module M\nfn f(n: int){ let b = Foo.stack(n, max = 1024); release b; }\n" + if "OWN030" not in codes(bad_ns): + fails.append("non-Buffer namespace must produce OWN030") + + # an escape reported on a moved-to alias must fail the buffer's noEscape check + moved_esc = parse("module M\nfn f(n: int) -> Buffer { " + "let a = Buffer.stack(n, max = 100); let b = move a; " + "return b; }\n") + if _report_check(moved_esc, "noEscape"): + fails.append("report noEscape must be false when a moved alias escapes") + return fails + + +def _report_check(mod, check): + """Run the full checker over a parsed module and return the named report + check (True/False) for its first buffer.""" + from ownlang.cfg import build_cfg, collect_signatures, collect_policies + rn = {r.name for r in mod.resources} + sg = collect_signatures(mod) + pl = collect_policies(mod) + diags = [] + for fn in mod.functions: + cfg, d1 = build_cfg(fn, rn, sg, pl) + diags += d1 + analyze(cfg) + rep = build_report(mod, diags) + return rep["buffers"][0]["checks"][check] if rep["buffers"] else None + + +def nesting_native_trace_smoke() -> list[str]: + """Three follow-up review guards: + - an ordinary resource inside a buffer's lifetime keeps its own finally + (exception from the body must not leak it while the buffer is returned); + - a native buffer with a dynamic size guards against a negative request + before NativeMemory.Alloc; + - a policy `trace = false` is honoured (no OwnTrace, report trace:false).""" + fails: list[str] = [] + + # Issue 1: buffer + ordinary resource each get their own finally + mix = ("module M\nresource Conn { acquire open release close }\n" + "extern fn Work(borrow_mut Buffer);\n" + "fn f(n: int){ let b = Buffer.scratch(n); let c = acquire Conn(); " + "Work(b); release c; release b; }\n") + if codes(mix): + fails.append(f"buffer+resource should check clean, got {codes(mix)}") + out = generate(parse(mix)) + if out.count("finally") != 2: + fails.append(f"buffer+resource must produce two finally blocks, got {out.count('finally')}") + if "c.close();" not in out: + fails.append("Conn must be closed") + elif out.index("c.close();") > out.index("ArrayPool.Shared.Return"): + fails.append("Conn finally must be nested inside the buffer's try (close before Return)") + + # Issue 2: native dynamic size guards against a negative request + nat_dyn = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn g(n: int){ let b = Buffer.native(n); " + "borrow_mut b as m { Fill(m); } release b; }\n") + out = generate(parse(nat_dyn)) + if "if (n < 0)" not in out or "ArgumentOutOfRangeException(nameof(n))" not in out: + fails.append("native dynamic size must guard against a negative request") + nat_const = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn g(){ let b = Buffer.native(256); " + "borrow_mut b as m { Fill(m); } release b; }\n") + if "< 0" in generate(parse(nat_const)): + fails.append("native constant size should not emit a negative guard") + + # Issue 3: a policy trace = false disables tracing + from ownlang.buffers import resolve as _resolve, Policy + from ownlang.ast_nodes import BufferIntent, VarRef + intent = BufferIntent(mode="scratch", size=VarRef("n", 1), + options={"policy": VarRef("Quiet", 1)}, line=1) + pol = {"Quiet": Policy("Quiet", {"trace": False, "counters": True})} + info, _ = _resolve(intent, pol) + if info.trace: + fails.append("policy trace = false must disable tracing") + quiet = ("module M\npolicy Quiet { trace = false; }\n" + "extern fn Fill(borrow_mut Buffer);\n" + "fn h(n: int){ let b = Buffer.scratch(n, policy = Quiet); " + "borrow_mut b as m { Fill(m); } release b; }\n") + hbody = generate(parse(quiet)).split("public static void h")[1].split("internal static")[0] + if "OwnTrace" in hbody: + fails.append("policy trace = false must emit no OwnTrace calls") + return fails + + +def ordering_counters_smoke() -> list[str]: + """Three follow-up review guards: + - a moved buffer's cleanup survives in sibling branches (a `move` in one + branch must not strip the original's cleanup from the other); + - pooled buffers do not emit the Scratch.* counters (they would corrupt the + stack-hit-rate metric); + - simple-mode keeps a buffer prelude after earlier plain statements it can + depend on (no Rent(n) before `var n = 64;`).""" + fails: list[str] = [] + + # Issue 1: move in one branch must not strip cleanup from the sibling branch + sib = ("module M\nfn f(n: int){ let a = Buffer.pooled(n); " + "if (c) { let b = move a; release b; } else { release a; } }\n") + if codes(sib): + fails.append(f"sibling-branch move should check clean, got {codes(sib)}") + out = generate(parse(sib)) + if "Dispose()" in out: + fails.append("sibling-branch release must return to the pool, not Dispose") + if out.count("ArrayPool.Shared.Return(a_array)") != 2: + fails.append("both branches must return a_array to the pool") + + # Issue 2: pooled buffers must not touch the Scratch.* counters (check the + # function body only — the OwnCounters class itself is always emitted) + pooled = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn p(n: int){ let b = Buffer.pooled(n); " + "borrow_mut b as m { Fill(m); } release b; }\n") + pbody = generate(parse(pooled)).split("public static void p")[1].split("internal static")[0] + if "OwnCounters" in pbody: + fails.append("pooled buffer must not emit Scratch.* counters") + # ...but scratch still must (sanity that the gate is mode-specific) + scratch = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn s(n: int){ let b = Buffer.scratch(n); " + "borrow_mut b as m { Fill(m); } release b; }\n") + sbody = generate(parse(scratch)).split("public static void s")[1].split("internal static")[0] + if "OwnCounters.StackHit()" not in sbody: + fails.append("scratch buffer must still emit Scratch.* counters") + + # Issue 3: a plain statement before a buffer stays before its prelude + order = "module M\nfn q(){ let n = 64; let b = Buffer.pooled(n); release b; }\n" + out = generate(parse(order)) + if "var n = 64;" not in out or "Rent(n)" not in out: + fails.append("ordering smoke missing expected lines") + elif out.index("var n = 64;") > out.index("Rent(n)"): + fails.append("buffer prelude must come after the plain statement it depends on") + + # disjoint sequential buffers: a must be returned before b is rented (its + # source release point must not be swallowed into b's lifetime) + disj = ("module M\nextern fn Use1(borrow_mut Buffer);\n" + "fn d(n: int){ let a = Buffer.pooled(n); Use1(a); release a; " + "let b = Buffer.pooled(n); Use1(b); release b; }\n") + out = generate(parse(disj)) + if "ArrayPool.Shared.Return(a_array)" not in out or "b_array = ArrayPool" not in out: + fails.append("disjoint smoke missing expected lines") + elif out.index("Return(a_array)") > out.index("b_array = ArrayPool"): + fails.append("buffer a must be returned before buffer b is rented") + + # a plain local declared in a buffer's body and used after the release must + # not be trapped in a hoisted try (it would go out of C# scope); the buffer + # is lowered inline (no try) so the local stays at function scope. + aftrel = ("module M\nfn f(n: int){ let b = Buffer.pooled(n); let x = 1; " + "release b; let y = x; }\n") + if codes(aftrel): + fails.append(f"local-after-release should check clean, got {codes(aftrel)}") + body = generate(parse(aftrel)).split("public static void f")[1].split("internal static")[0] + if "try" in body: + fails.append("a buffer with a plain local in its body must not be hoisted") + if "var x = 1;" not in body or "var y = x;" not in body: + fails.append("local-after-release smoke missing expected lines") + elif body.index("var x = 1;") > body.index("var y = x;"): + fails.append("x must be declared before y") + + # a negative scratch size is rejected BEFORE any trace/counter runs + negsc = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn g(n: int){ let b = Buffer.scratch(n); " + "borrow_mut b as m { Fill(m); } release b; }\n") + out = generate(parse(negsc)) + if "if (n < 0)" not in out: + fails.append("scratch dynamic size must guard against a negative request") + elif out.index("if (n < 0)") > out.index("ScratchSelected"): + fails.append("scratch negative guard must run before any trace/counter") + + # partially-overlapping lifetimes (a released while b still live) must NOT be + # hoisted into nested try/finally — that would force b's release before its + # source point. They fall back to faithful inline (release where written). + overlap = ("module M\nresource A { acquire oa release ca }\n" + "resource B { acquire ob release cb }\n" + "fn f(){ let a = acquire A(); let b = acquire B(); " + "release a; use b; release b; }\n") + out = generate(parse(overlap)) + if "Use(b)" not in out or "b.cb()" not in out: + fails.append("overlap smoke missing expected lines") + elif out.index("Use(b)") > out.index("b.cb()"): + fails.append("partial overlap must not emit use-after-release for b") + if "a.ca()" in out and out.index("a.ca()") > out.index("Use(b)"): + fails.append("a must be released at its source point, before use b") + + # a buffer whose release is nested in another owner's borrow block must NOT + # be hoisted (that would double-clean: a stray Dispose plus the finally). + nested = ("module M\nresource Conn { acquire open release close }\n" + "fn f(n: int){ let a = acquire Conn(1); let b = Buffer.pooled(n); " + "borrow a as s { release b; } release a; }\n") + if codes(nested): + fails.append(f"nested buffer release should check clean, got {codes(nested)}") + out = generate(parse(nested)) + if "Dispose()" in out: + fails.append("nested buffer release must not emit a generic Dispose()") + if out.count("ArrayPool.Shared.Return(b_array)") != 1: + fails.append("nested buffer release must return b_array exactly once") + + # native buffers expose a Span view (so a borrow/call sees the same + # logical type as pooled/stack), and free the backing pointer on release + native = ("module M\nextern fn Fill(borrow_mut Buffer);\n" + "fn g(n: int){ let b = Buffer.native(n); " + "borrow_mut b as m { Fill(m); } release b; }\n") + out = generate(parse(native)) + if "new Span(b_ptr, n)" not in out: + fails.append("native buffer must expose a Span view for borrows/calls") + if "NativeMemory.Free(b_ptr)" not in out: + fails.append("native release must free the backing pointer") + if "var m = b;" not in out: + fails.append("native borrow must bind the span view, not the raw pointer") + + # a moved buffer's cleanup alias must not leak onto a later same-named, but + # unrelated, resource declared in another scope + redecl = ("module M\nresource Conn { acquire open release close }\n" + "fn f(n: int){ let a = Buffer.pooled(n); " + "if (c) { let b = move a; release b; } else { release a; } " + "let b = acquire Conn(); release b; }\n") + if codes(redecl): + fails.append(f"alias redeclaration should check clean, got {codes(redecl)}") + out = generate(parse(redecl)) + if "b.close();" not in out: + fails.append("redeclared Conn must be closed, not treated as the buffer alias") + if out.count("ArrayPool.Shared.Return(a_array)") != 2: + fails.append("a_array must be returned exactly once per branch (no stale alias)") + return fails + + +def helper_and_report_smoke() -> list[str]: + """Two review guards: + - a local helper with a `&mut Buffer` / `&Buffer` param lowers to a + Span / ReadOnlySpan signature, so passing a buffer value (a + Span) compiles; + - the report attributes diagnostics by buffer identity (name#line), so two + same-named buffers in sibling scopes are not conflated.""" + fails: list[str] = [] + + helper = ("module M\nfn helper(x: &mut Buffer){ use x; }\n" + "fn view(y: &Buffer){ use y; }\n" + "fn f(n: int){ let b = Buffer.pooled(n); helper(b); view(b); " + "release b; }\n") + if codes(helper): + fails.append(f"local Buffer helper should check clean, got {codes(helper)}") + out = generate(parse(helper)) + if "public static void helper(Span x)" not in out: + fails.append("&mut Buffer param must lower to Span") + if "public static void view(ReadOnlySpan y)" not in out: + fails.append("&Buffer param must lower to ReadOnlySpan") + if "ref Buffer" in out or "ref readonly Buffer" in out: + fails.append("Buffer borrow params must not lower to ref Buffer") + + # same name in sibling scopes: only the leaking buffer fails releaseOnAllPaths + sib = ("module M\nfn f(n: int){ if (c) { let b = Buffer.pooled(n); } " + "else { let b = Buffer.pooled(n); release b; } }\n") + mod = parse(sib) + rn = {r.name for r in mod.resources} + sg = collect_signatures(mod) + pl = collect_policies(mod) + diags = list(validate_policies(pl)) + for fn in mod.functions: + cfg, d1 = build_cfg(fn, rn, sg, pl) + diags += d1 + analyze(cfg) + rep = build_report(mod, diags) + flags = sorted(e["checks"]["releaseOnAllPaths"] for e in rep["buffers"]) + if flags != [False, True]: + fails.append(f"sibling same-name buffers must be attributed separately, got {flags}") + return fails + + +def run() -> int: + passed = 0 + failed = 0 + for name, body, expected in CASES: + got = sorted(set(codes(PRELUDE + body))) + want = sorted(set(expected)) + if got == want: + passed += 1 + else: + failed += 1 + print(f"FAIL {name}") + print(f" expected {want}") + print(f" got {got}") + + # codegen smoke: every clean case must generate without throwing + cg_total = len([c for c in CASES if not c[2]]) + cg_fail = 0 + for name, body, expected in CASES: + if expected: + continue + try: + generate(parse(PRELUDE + body)) + except Exception as e: # noqa: BLE001 + cg_fail += 1 + print(f"CODEGEN FAIL {name}: {type(e).__name__}: {e}") + + golden_fails = golden_smoke() + for f in golden_fails: + print(f"GOLDEN FAIL: {f}") + + buffer_fails = buffer_smoke() + for f in buffer_fails: + print(f"BUFFER FAIL: {f}") + + escape_fails = escape_and_length_smoke() + for f in escape_fails: + print(f"ESCAPE FAIL: {f}") + + branchy_fails = branchy_and_malformed_smoke() + for f in branchy_fails: + print(f"BRANCHY FAIL: {f}") + + nest_fails = nesting_native_trace_smoke() + for f in nest_fails: + print(f"NESTING FAIL: {f}") + + order_fails = ordering_counters_smoke() + for f in order_fails: + print(f"ORDERING FAIL: {f}") + + helper_fails = helper_and_report_smoke() + for f in helper_fails: + print(f"HELPER FAIL: {f}") + + total = passed + failed + print(f"\nanalysis: {passed}/{total} passed, {failed} failed") + print(f"codegen: {cg_total - cg_fail}/{cg_total} generated cleanly") + print(f"golden: {'PASS' if not golden_fails else 'FAIL'}") + print(f"buffer: {'PASS' if not buffer_fails else 'FAIL'}") + print(f"escape: {'PASS' if not escape_fails else 'FAIL'}") + print(f"branchy: {'PASS' if not branchy_fails else 'FAIL'}") + print(f"nesting: {'PASS' if not nest_fails else 'FAIL'}") + print(f"ordering: {'PASS' if not order_fails else 'FAIL'}") + print(f"helper: {'PASS' if not helper_fails else 'FAIL'}") + + # Content-level codegen assertions + property fuzzer: these inspect the + # generated C# itself (release placement/count, declaration order), catching + # lowerings that are silently wrong rather than ones that merely throw. + import test_codegen # noqa: E402 (lives in this directory) + cc_rc = test_codegen.run() + import test_codegen_props # noqa: E402 + pf_rc = test_codegen_props.run(iterations=3000, seed=1234) + + return 1 if (failed or cg_fail or golden_fails or buffer_fails + or escape_fails or branchy_fails or nest_fails + or order_fails or helper_fails or cc_rc or pf_rc) else 0 + + +if __name__ == "__main__": + raise SystemExit(run()) diff --git a/tests/test_codegen.py b/tests/test_codegen.py new file mode 100644 index 00000000..e323a341 --- /dev/null +++ b/tests/test_codegen.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +""" +Codegen-focused regression suite for the OwnLang PoC. + +The suite in ``run_tests.py`` proves the *analyzer* assigns the right diagnostic +codes, and its codegen smoke only checks that ``generate`` does not throw. That +is not enough: every codegen bug found so far produced **valid-looking C# that +silently does the wrong thing** (a double-free, a leak, a use-before-declaration) +without raising. So this file asserts on the *content* of the generated C#. + +Each case first re-checks that the source is clean at the ownership level (we +only lower programs the checker accepted), then asserts structural facts about +the emitted C#. + +Run: python tests/run_tests.py (invokes these too) + python tests/test_codegen.py (standalone) +""" + +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from ownlang.parser import parse # noqa: E402 +from ownlang.cfg import build_cfg, collect_signatures # noqa: E402 +from ownlang.analysis import analyze # noqa: E402 +from ownlang.diagnostics import Severity # noqa: E402 +from ownlang.codegen import generate # noqa: E402 + + +# Schematic prelude: resource Buffer renders as Buffer.rent(...) / x.give(). +SCHEMATIC = ( + "module M\n" + "resource Buffer { acquire rent release give }\n" + "extern fn Fill(borrow_mut Buffer);\n" + "extern fn Hash(borrow Buffer);\n" + "extern fn Store(consume Buffer);\n" +) + +# ArrayPool prelude: the same Buffer lowered to real .NET via emit_* templates. +ARRAYPOOL = ( + "module M\n" + "resource Buffer {\n" + " acquire rent\n" + " release give\n" + ' emit_type "byte[]"\n' + ' emit_acquire "ArrayPool.Shared.Rent({args})"\n' + ' emit_release "ArrayPool.Shared.Return({0})"\n' + ' emit_borrow "{0}.AsSpan()"\n' + "}\n" + "extern fn Fill(borrow_mut Buffer);\n" + "extern fn Hash(borrow Buffer);\n" + "extern fn Store(consume Buffer);\n" +) + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + + +def _errors(src: str) -> list[str]: + mod = parse(src) + rnames = {r.name for r in mod.resources} + sigs = collect_signatures(mod) + out: list[str] = [] + for fn in mod.functions: + cfg, d1 = build_cfg(fn, rnames, sigs) + d2 = analyze(cfg) + out += [d.code for d in (d1 + d2) if d.severity == Severity.ERROR] + return out + + +class Check: + """A tiny fluent assertion helper over the generated C# of one program.""" + + def __init__(self, name: str, prelude: str, fn_src: str): + self.name = name + self.fails: list[str] = [] + src = prelude + fn_src + errs = _errors(src) + if errs: + self.fails.append(f"source is not clean, got {sorted(set(errs))}") + self.cs = "" + else: + self.cs = generate(parse(src)) + + # -- assertions --------------------------------------------------------- + + def has(self, needle: str) -> "Check": + if needle not in self.cs: + self.fails.append(f"expected to contain {needle!r}") + return self + + def lacks(self, needle: str) -> "Check": + if needle in self.cs: + self.fails.append(f"expected NOT to contain {needle!r}") + return self + + def count(self, needle: str, n: int) -> "Check": + got = self.cs.count(needle) + if got != n: + self.fails.append(f"expected {needle!r} x{n}, got x{got}") + return self + + def before(self, a: str, b: str) -> "Check": + """`a` must appear, and its first occurrence must precede `b`'s.""" + ia, ib = self.cs.find(a), self.cs.find(b) + if ia < 0: + self.fails.append(f"ordering: {a!r} missing") + elif ib < 0: + self.fails.append(f"ordering: {b!r} missing") + elif ia >= ib: + self.fails.append(f"ordering: expected {a!r} before {b!r}") + return self + + def release_is_hoisted(self, give: str) -> "Check": + """`give` (a release call) must sit inside a `finally`, exactly once, + and not also be duplicated in the `try` body.""" + self.has("try").has("finally").count(give, 1) + gi = self.cs.find(give) + fi = self.cs.rfind("finally", 0, gi) + if fi < 0: + self.fails.append(f"{give!r} is not inside a finally block") + return self + + +_REGISTRY: list[Check] = [] + + +def case(name: str, prelude: str, fn_src: str) -> Check: + c = Check(name, prelude, fn_src) + _REGISTRY.append(c) + return c + + +# --------------------------------------------------------------------------- +# Bug C — double-free: an acquired resource that ESCAPES via a consume-call +# must NOT get a hoisted `finally` release. +# --------------------------------------------------------------------------- + +case("consume_no_release", SCHEMATIC, + "fn f(){ let b = acquire Buffer(1); Store(b); }") \ + .has("Store(b);").lacks(".give(").lacks("finally") + +case("consume_among_releases", SCHEMATIC, + "fn f(){ let a = acquire Buffer(1); let b = acquire Buffer(2); " + "release a; Store(b); }") \ + .has("Store(b);").has("a.give();").lacks("b.give(") + +case("arraypool_consume_no_return", ARRAYPOOL, + "fn f(){ let b = acquire Buffer(1); Store(b); }") \ + .has("ArrayPool.Shared.Rent(1)") \ + .has("Store(b);") \ + .lacks("ArrayPool.Shared.Return") + + +# --------------------------------------------------------------------------- +# Bug A — leak: a released owned PARAMETER must actually be released (it has no +# matching acquire, so the old hoist dropped it entirely). +# --------------------------------------------------------------------------- + +case("param_release", SCHEMATIC, "fn f(b: Buffer){ release b; }") \ + .count("b.give();", 1) + +case("param_release_with_acquire", SCHEMATIC, + "fn f(b: Buffer){ let a = acquire Buffer(1); release a; release b; }") \ + .count("a.give();", 1).count("b.give();", 1) + +case("arraypool_param_release", ARRAYPOOL, "fn f(b: Buffer){ release b; }") \ + .count("ArrayPool.Shared.Return(b)", 1) + +# A param that is consumed (not released) plus a locally-acquired resource: the +# local still hoists into a try/finally, the param is consumed inline, and the +# param is NOT released. +case("acquire_hoists_param_consumed", SCHEMATIC, + "fn f(b: Buffer){ let a = acquire Buffer(1); release a; Store(b); }") \ + .release_is_hoisted("a.give();").has("Store(b);").lacks("b.give(") + + +# --------------------------------------------------------------------------- +# Bug B — use-before-declaration: an acquire whose argument is a preceding local +# must not be hoisted above that local. +# --------------------------------------------------------------------------- + +case("acquire_arg_local_order", SCHEMATIC, + "fn f(){ let n = 5; let a = acquire Buffer(n); release a; }") \ + .before("var n = 5;", "Buffer.rent(n)").has("a.give();") + +# But an acquire whose argument is a parameter is free to hoist. +case("acquire_arg_param_hoists", SCHEMATIC, + "fn f(sz: int){ let a = acquire Buffer(sz); release a; }") \ + .release_is_hoisted("a.give();").has("Buffer.rent(sz)") + + +# --------------------------------------------------------------------------- +# Hoist path stays correct for the straight-line cases it was designed for. +# --------------------------------------------------------------------------- + +case("hoist_single", SCHEMATIC, + "fn f(){ let b = acquire Buffer(10); release b; }") \ + .release_is_hoisted("b.give();").before("Buffer.rent(10)", "try") + +# Two resources with crossing lifetimes are not laminar, so this codegen +# emits them inline (faithful release at each site) rather than nested +# try/finally. Either way each is released exactly once. +case("two_resources_each_released_once", SCHEMATIC, + "fn f(){ let a = acquire Buffer(1); let b = acquire Buffer(2); " + "release a; release b; }") \ + .count("a.give();", 1).count("b.give();", 1) + +# Borrows inside a hoisted body render via emit_borrow and stay inside the try. +case("hoist_with_borrows", SCHEMATIC, + "fn f(){ let b = acquire Buffer(1); borrow_mut b as m { Fill(m); } " + "borrow b as r { Hash(r); } release b; }") \ + .release_is_hoisted("b.give();").has("Fill(m)").has("Hash(r)") \ + .before("Fill(m)", "b.give();") + + +# --------------------------------------------------------------------------- +# Inline path: move / owned-return / branches. +# --------------------------------------------------------------------------- + +case("move_inline", SCHEMATIC, + "fn f(){ let a = acquire Buffer(1); let c = move a; release c; }") \ + .has("// ownership moved from a").count("c.give();", 1).lacks("a.give(") \ + .lacks("finally") + +case("owned_return_inline", SCHEMATIC, + "fn f() -> Buffer { let b = acquire Buffer(1); return b; }") \ + .has("return b;").lacks(".give(").lacks("finally") \ + .has("public static Buffer f(") + +case("branch_release_both_arms", SCHEMATIC, + "fn f(){ let b = acquire Buffer(1); if (c) { release b; } " + "else { release b; } }") \ + .has("if (c)").has("else").count("b.give();", 2).lacks("finally") + +case("branch_release_one_arm_consume_other", SCHEMATIC, + "fn f(){ let b = acquire Buffer(1); if (c) { release b; } " + "else { Store(b); } }") \ + .count("b.give();", 1).has("Store(b);") + + +# --------------------------------------------------------------------------- +# emit_* templates → real .NET, and signature type rendering. +# --------------------------------------------------------------------------- + +case("arraypool_golden", ARRAYPOOL, + "fn process(size: int){ let buf = acquire Buffer(size); " + "borrow_mut buf as bytes { Fill(bytes); } " + "borrow buf as view { Hash(view); } release buf; }") \ + .has("using System.Buffers;") \ + .has("byte[] buf = ArrayPool.Shared.Rent(size);") \ + .release_is_hoisted("ArrayPool.Shared.Return(buf);") \ + .has("buf.AsSpan()").has("Fill(bytes)").has("Hash(view)") + +# A borrowed Buffer lowers to its span view (Span / ReadOnlySpan), +# the same view emit_borrow and buffer intents produce. +case("sig_shared_borrow_param", SCHEMATIC, "fn f(x: &Buffer){ use x; }") \ + .has("public static void f(ReadOnlySpan x)") + +case("sig_mut_borrow_param", SCHEMATIC, "fn g(x: &mut Buffer){ Fill(x); }") \ + .has("public static void g(Span x)") + +case("sig_plain_param", SCHEMATIC, "fn f(n: int){ }") \ + .has("public static void f(int n)") + +# Two functions land in the same static class. +case("two_functions", SCHEMATIC, + "fn a(){ let b = acquire Buffer(1); release b; } " + "fn c(){ let d = acquire Buffer(2); release d; }") \ + .has("static void a(").has("static void c(") + + +# --------------------------------------------------------------------------- +# runner +# --------------------------------------------------------------------------- + + +def run() -> int: + passed = failed = 0 + for c in _REGISTRY: + if c.fails: + failed += 1 + print(f"CODEGEN FAIL {c.name}") + for f in c.fails: + print(f" {f}") + else: + passed += 1 + total = passed + failed + print(f"codegen content: {passed}/{total} passed, {failed} failed") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(run()) diff --git a/tests/test_codegen_props.py b/tests/test_codegen_props.py new file mode 100644 index 00000000..0335a110 --- /dev/null +++ b/tests/test_codegen_props.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +""" +Property-based codegen fuzzer for the OwnLang PoC. + +Hand-written cases only cover the bugs we already thought of. This file instead +generates many random *clean* programs and, for every one the checker accepts, +asserts invariants about the generated C# that are computed **independently from +the AST** — a separate, much simpler implementation than codegen — so a bug in +codegen cannot be masked by the same bug in the oracle. + +Invariants checked on every clean program: + + P1 generation does not throw. + P2 `try` and `finally` counts are equal, and all braces balance. + P3 release accounting: for every variable v, the number of `release v;` + statements in the source equals the number of release-calls on v in the + generated C#. A dropped owned-parameter release makes this 1 != 0 + (leak); a `finally` release of a consumed/returned resource makes it + 0 != 1 (double-free / use-after-escape). + P4 declaration-before-use: every local C# introduces (`var X =`) appears + textually before any other use of X — catches an `acquire Buffer(n)` + being hoisted above `var n = ...;`. + +The generator is constructive: each owned resource is given exactly one "fate" +(release / consume / move-then-release / return), so programs are clean by +construction; the checker is still applied as a filter and non-clean draws are +skipped. Coverage is asserted at the end (both codegen paths and each fate must +actually be exercised), so the run can never pass vacuously. + +Run: python tests/test_codegen_props.py [iterations] [seed] + python tests/run_tests.py (invokes a fixed-seed run) +""" + +from __future__ import annotations + +import os +import random +import re +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from ownlang import ast_nodes as A # noqa: E402 +from ownlang.parser import parse # noqa: E402 +from ownlang.cfg import build_cfg, collect_signatures # noqa: E402 +from ownlang.analysis import analyze # noqa: E402 +from ownlang.diagnostics import Severity # noqa: E402 +from ownlang.codegen import generate # noqa: E402 + + +PRELUDE = ( + "module M\n" + "resource Buffer { acquire rent release give }\n" + "extern fn Fill(borrow_mut Buffer);\n" + "extern fn Hash(borrow Buffer);\n" + "extern fn Store(consume Buffer);\n" +) + + +# --------------------------------------------------------------------------- +# random clean-by-construction program generator +# --------------------------------------------------------------------------- + + +class Gen: + def __init__(self, rng: random.Random): + self.rng = rng + self.n = 0 + self.plain_locals: list[str] = [] # int locals usable as acquire args + self.coverage: set[str] = set() + + def fresh(self, p: str) -> str: + self.n += 1 + return f"{p}{self.n}" + + def fn(self) -> tuple[str, set[str]]: + """Return (function source, coverage-tags hit).""" + self.coverage = set() + params: list[str] = [] # (name, owned?) + owned: list[str] = [] # names of owned values still needing a fate + lines: list[str] = [] + + # parameters: a mix of plain ints, borrows, and owned resources. + n_params = self.rng.randint(0, 2) + psig: list[str] = [] + for _ in range(n_params): + kind = self.rng.choice(["int", "owned", "shared", "mut"]) + nm = self.fresh("p") + if kind == "int": + psig.append(f"{nm}: int") + self.plain_locals.append(nm) + elif kind == "owned": + psig.append(f"{nm}: Buffer") + owned.append(nm) + self.coverage.add("owned_param") + elif kind == "shared": + psig.append(f"{nm}: &Buffer") + else: + psig.append(f"{nm}: &mut Buffer") + + # an optional leading plain local, sometimes used as an acquire arg. + if self.rng.random() < 0.5: + nm = self.fresh("k") + lines.append(f" let {nm} = {self.rng.randint(0, 99)};") + self.plain_locals.append(nm) + + # acquire a few owned locals. + for _ in range(self.rng.randint(0, 3)): + nm = self.fresh("a") + arg = self._acquire_arg() + lines.append(f" let {nm} = acquire Buffer({arg});") + owned.append(nm) + + # some borrow blocks / temporary-borrow calls over live owned values. + if owned: + for _ in range(self.rng.randint(0, 2)): + v = self.rng.choice(owned) + stmt = self.rng.choice(["borrow", "borrow_mut", "Hash", "Fill", "use"]) + b = self.fresh("s") + if stmt == "borrow": + lines.append(f" borrow {v} as {b} {{ use {b}; }}") + elif stmt == "borrow_mut": + lines.append(f" borrow_mut {v} as {b} {{ Fill({b}); }}") + elif stmt == "Hash": + lines.append(f" Hash({v});") + elif stmt == "Fill": + lines.append(f" Fill({v});") + else: + lines.append(f" use {v};") + + # choose a return value (owned) sometimes. + ret = "" + returned: str | None = None + if owned and self.rng.random() < 0.3: + returned = self.rng.choice(owned) + ret = " -> Buffer" + self.coverage.add("return") + + # give every owned value exactly one fate. + for v in owned: + if v == returned: + continue + self._emit_fate(v, lines) + + if returned is not None: + lines.append(f" return {returned};") + + name = self.fresh("f") + sig = ", ".join(psig) + body = "\n".join(lines) + src = f"fn {name}({sig}){ret} {{\n{body}\n}}\n" + return src, set(self.coverage) + + def _acquire_arg(self) -> str: + if self.plain_locals and self.rng.random() < 0.5: + self.coverage.add("acquire_arg_local") + return self.rng.choice(self.plain_locals) + return str(self.rng.randint(0, 99)) + + def _emit_fate(self, v: str, lines: list[str]) -> None: + fate = self.rng.choice(["release", "release", "consume", "move", "branch"]) + if fate == "release": + lines.append(f" release {v};") + self.coverage.add("release") + elif fate == "consume": + lines.append(f" Store({v});") + self.coverage.add("consume") + elif fate == "move": + w = self.fresh("m") + lines.append(f" let {w} = move {v};") + lines.append(f" release {w};") + self.coverage.add("move") + else: # branch: release on both arms (clean), exercises the inline path + lines.append(f" if (cond) {{ release {v}; }} else {{ release {v}; }}") + self.coverage.add("branch") + + +# --------------------------------------------------------------------------- +# independent oracle (reads the AST, not the codegen) +# --------------------------------------------------------------------------- + + +def _walk(stmts: list, fn) -> None: + for st in stmts: + fn(st) + if isinstance(st, A.If): + _walk(st.then_body, fn) + _walk(st.else_body, fn) + elif isinstance(st, A.BorrowBlock): + _walk(st.body, fn) + + +def source_release_counts(fn: A.FnDecl) -> dict[str, int]: + counts: dict[str, int] = {} + + def visit(st) -> None: + if isinstance(st, A.Release): + counts[st.var] = counts.get(st.var, 0) + 1 + + _walk(fn.body, visit) + return counts + + +def all_names(fn: A.FnDecl) -> set[str]: + names = {p.name for p in fn.params} + + def visit(st) -> None: + if isinstance(st, A.Let): + names.add(st.name) + elif isinstance(st, A.BorrowBlock): + names.add(st.binding) + + _walk(fn.body, visit) + return names + + +# --------------------------------------------------------------------------- +# invariant checks against generated C# +# --------------------------------------------------------------------------- + + +def check_invariants(fn: A.FnDecl, cs: str) -> list[str]: + fails: list[str] = [] + + # Strip `// ...` line comments first: codegen annotates borrows and moves + # with the source name (`// mutable borrow of a as s`, `// ownership moved + # from a`), and those mentions are not C# uses of the variable. + cs = re.sub(r"//[^\n]*", "", cs) + + # P2: try/finally balance and brace balance. + if cs.count("try") != cs.count("finally"): + fails.append(f"try({cs.count('try')}) != finally({cs.count('finally')})") + if cs.count("{") != cs.count("}"): + fails.append("unbalanced braces") + + # P3: per-variable release accounting. + src = source_release_counts(fn) + for name in all_names(fn): + want = src.get(name, 0) + got = len(re.findall(rf"\b{re.escape(name)}\.give\(", cs)) + if want != got: + fails.append(f"release count for {name!r}: source={want} emitted={got}") + + # P4: declaration-before-use for codegen-introduced locals. + for m in re.finditer(r"(?:var|byte\[\])\s+(\w+)\s*=", cs): + nm = m.group(1) + first = re.search(rf"\b{re.escape(nm)}\b", cs) + if first and first.start() < m.start(): + fails.append(f"{nm!r} used at {first.start()} before declaration at {m.start()}") + + return fails + + +def _is_clean(src: str) -> bool: + mod = parse(src) + rnames = {r.name for r in mod.resources} + sigs = collect_signatures(mod) + for fn in mod.functions: + cfg, d1 = build_cfg(fn, rnames, sigs) + d2 = analyze(cfg) + if any(d.severity == Severity.ERROR for d in (d1 + d2)): + return False + return True + + +# --------------------------------------------------------------------------- +# runner +# --------------------------------------------------------------------------- + + +def run(iterations: int = 4000, seed: int = 1234) -> int: + rng = random.Random(seed) + clean = 0 + hoist = inline = 0 + coverage: set[str] = set() + failures: list[tuple[str, list[str]]] = [] + + for _ in range(iterations): + gen = Gen(rng) + fn_src, cov = gen.fn() + src = PRELUDE + fn_src + try: + if not _is_clean(src): + continue + except Exception: # noqa: BLE001 (malformed draw -> skip) + continue + clean += 1 + coverage |= cov + + mod = parse(src) + try: + cs = generate(mod) # P1: must not throw + except Exception as e: # noqa: BLE001 + failures.append((fn_src, [f"generate threw {type(e).__name__}: {e}"])) + continue + + if "try" in cs: + hoist += 1 + else: + inline += 1 + + fails = check_invariants(mod.functions[0], cs) + if fails: + failures.append((fn_src, fails)) + + # report + for fn_src, fails in failures[:10]: + print("PROP FAIL ----------------------------------------------------") + print(fn_src.rstrip()) + for f in fails: + print(f" {f}") + + print(f"property fuzz: {clean} clean programs checked " + f"({hoist} hoist, {inline} inline), {len(failures)} failed") + + # the run must not be vacuous: both paths and every fate must be exercised. + need = {"release", "consume", "move", "branch", "owned_param", + "acquire_arg_local", "return"} + missing = need - coverage + coverage_ok = clean > 200 and hoist > 0 and inline > 0 and not missing + if not coverage_ok: + print(f"COVERAGE WEAK: clean={clean} hoist={hoist} inline={inline} " + f"missing={sorted(missing)}") + + return 1 if (failures or not coverage_ok) else 0 + + +if __name__ == "__main__": + it = int(sys.argv[1]) if len(sys.argv) > 1 else 4000 + sd = int(sys.argv[2]) if len(sys.argv) > 2 else 1234 + raise SystemExit(run(it, sd))