From 33184321b3562390cbf382471ba9d1cb8b033c38 Mon Sep 17 00:00:00 2001 From: gHashTag Date: Thu, 17 Sep 2026 13:07:10 +0700 Subject: [PATCH] Feed the swarm from CI, with fuel that does not run out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The feeder ran on a laptop scheduled task and fired twice between 18:03 and 05:43 instead of 36 times: the machine was asleep, and the bees idled all night. It also fed from a pool that is nearly empty — 24 files of empty function bodies left, about fifteen minutes of swarm work. This runs the feeder on GitHub's cron every five minutes, and feeds it from the project's own goal instead: 870 hand-written Python, Rust, TypeScript and C files that have no .t27 counterpart. bootstrap/ is excluded — it is stage-0 and compiler.rs sits under the FROZEN_HASH seal. The criteria it writes are structural and cannot express correctness; the review-side oracle remains the judge of that. Refs #3977 Co-Authored-By: Claude Code --- .github/workflows/queen-feed.yml | 57 +++++++ tools/queen/queen_feed_port.py | 282 +++++++++++++++++++++++++++++++ 2 files changed, 339 insertions(+) create mode 100644 .github/workflows/queen-feed.yml create mode 100755 tools/queen/queen_feed_port.py diff --git a/.github/workflows/queen-feed.yml b/.github/workflows/queen-feed.yml new file mode 100644 index 0000000000..2ec79f4099 --- /dev/null +++ b/.github/workflows/queen-feed.yml @@ -0,0 +1,57 @@ +# Keep the swarm's queue full from GitHub's cron, not from a laptop. +# +# Why this exists: the feeder ran on a local scheduled task every 20 minutes and +# fired TWICE between 18:03 and 05:43 on 2026-09-16/17 instead of 36 times - the +# machine was asleep. The swarm burns ~10 issues in 7 minutes across 10 lanes, +# so a night of that is a night of idle bees regardless of how much fuel exists. +# +# This job needs no compiler: the port feeder only walks the checkout and reads +# the issue list, so a plain runner with python3 and gh is enough. +name: queen-feed + +on: + schedule: + - cron: '*/5 * * * *' # GitHub's floor is 5 minutes and it may run late + workflow_dispatch: + inputs: + target: + description: 'Backlog depth to maintain' + default: '60' + limit: + description: 'Max issues to create in one run' + default: '10' + +# Least privilege: this job creates issues and reads the repository. Nothing else. +permissions: + contents: read + issues: write + +concurrency: + group: queen-feed + cancel-in-progress: false # two feeders racing would double-create + +jobs: + feed: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Feed the swarm up to the backlog target + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + QUEEN_WORK: ${{ github.workspace }} + QUEEN_NO_SYNC: '1' + QUEEN_LOG: ${{ runner.temp }}/feed.log + run: | + python3 tools/queen/queen_feed_port.py \ + --top-up "${{ github.event.inputs.target || 60 }}" \ + --limit "${{ github.event.inputs.limit || 10 }}" + - name: Upload the log + if: always() + uses: actions/upload-artifact@v4 + with: + name: feed-log + path: ${{ runner.temp }}/feed.log + if-no-files-found: ignore diff --git a/tools/queen/queen_feed_port.py b/tools/queen/queen_feed_port.py new file mode 100755 index 0000000000..b4df070aae --- /dev/null +++ b/tools/queen/queen_feed_port.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""queen_feed_port.py - the inexhaustible fuel line: hand-written code -> .t27. + +The "empty bodies" line (queen_feed.py) is nearly dry: 24 uncovered files left, +about fifteen minutes of swarm work, and then zero forever. Measured 2026-09-17: +the bees consume ~10 issues in 7 minutes across 10 lanes, while the feeder can +supply at most 10 per 20 minutes. The deficit is structural, not a tuning knob. + +This line has a different bottom. The project's own rule - stated in +tools/oracle/run.sh - is that even helper files are authored in `.t27`; the +repository nevertheless holds ~1400 hand-written source files in Python, Rust, +TypeScript and C. Each one is a port: re-author the logic as `.t27`, let `t27c` +emit the language again. That is the stated goal of the project, and as fuel it +outlasts any cadence the swarm can burn it at. + +What this script does NOT do: judge whether the port is correct. The criteria +below are structural - the file exists, it declares the same function names, the +codegen emits no panic, the parser recovers nothing, every function carries a +test. Structural criteria cannot express correctness (LOOP_STATE cycle 2: a +sha256 Sigma0 with the wrong operator satisfied all six of its criteria). The +correctness gate is the review-side oracle, which compiles the generated Zig and +runs its tests. This script feeds; the oracle judges. + +Usage: + queen_feed_port.py --dry-run [--limit N] write bodies to work/port-issues/ + queen_feed_port.py --limit N create up to N issues + queen_feed_port.py --top-up TARGET [--limit N] + create only as many as it takes to bring the number of READY issues + (open, unclaimed, carrying a boundary) up to TARGET. This is the + anti-idle mode: it maintains a backlog instead of reacting to an empty + one, which is what --when-idle in queen_feed.py could never do - by the + time it fires, the lanes are already idle. +""" +import argparse, json, os, re, subprocess, sys, urllib.request, datetime + +HERE = os.path.dirname(os.path.abspath(__file__)) +# In CI the checkout IS the work tree: set QUEEN_WORK to it and QUEEN_NO_SYNC=1. +# That is what lets this run on GitHub Actions every five minutes instead of on +# a laptop that sleeps - measured 2026-09-17: the 20-minute local cron fired +# twice between 18:03 and 05:43 instead of 36 times, and the swarm sat idle for +# the rest of the night with nothing to eat. +WORK = os.environ.get("QUEEN_WORK") or os.path.join(HERE, "work", "t27-master") +OUTD = os.path.join(HERE, "work", "port-issues") +LOG = os.environ.get("QUEEN_LOG") or os.path.join(HERE, "feed.log") +REPO = "gHashTag/t27" +STATUS = "https://trios-agent-server-production.up.railway.app/queen/status" +CHUNK = 8 + +# `bootstrap/` holds the stage-0 compiler; `bootstrap/src/compiler.rs` is under +# the FROZEN_HASH seal and may only be touched by a human ceremony (FROZEN.md +# §5). Porting it is the endgame of the project, not a bee's turn. +# `target/`, `gen/`, `external/` and `node_modules/` are not hand-written. +SKIP_DIRS = ("bootstrap/", "target/", "gen/", "external/", "node_modules/", + ".git/", "outputs/", "build/", "datasets/", "test-ledger/") +# Ported first: the helpers the project's own rule already says should be .t27. +PRIORITY = ("tools/", "scripts/", "cli/", "rings/", "conformance/") + +LANGS = { + ".py": ("Python", r'(?m)^def\s+([A-Za-z_]\w*)\s*\('), + ".rs": ("Rust", r'(?m)^\s*pub\s+fn\s+([A-Za-z_]\w*)\s*[(<]'), + ".ts": ("TypeScript", r'(?m)^\s*export\s+(?:async\s+)?function\s+([A-Za-z_]\w*)\s*[(<]'), + ".c": ("C", r'(?m)^[A-Za-z_][\w \t*]*\s\*?([A-Za-z_]\w*)\s*\([^;]*\)\s*\{'), +} + +def log(msg): + line = f"{datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')} {msg}" + print(line) + with open(LOG, "a") as f: f.write(line + "\n") + +def sync_master(): + if os.environ.get("QUEEN_NO_SYNC") == "1": + r = subprocess.run(["git", "log", "-1", "--format=%h"], cwd=WORK, capture_output=True, text=True) + return r.stdout.strip() or "checkout" + if not os.path.isdir(os.path.join(WORK, ".git")): + os.makedirs(os.path.dirname(WORK), exist_ok=True) + subprocess.run(["git", "clone", "-q", "--depth", "1", "--single-branch", "-b", "master", + f"/{REPO}", WORK], check=True, timeout=900) + else: + subprocess.run(["git", "fetch", "-q", "--depth", "1", "origin", "master"], cwd=WORK, check=True, timeout=300) + subprocess.run(["git", "reset", "-q", "--hard", "origin/master"], cwd=WORK, check=True) + r = subprocess.run(["git", "log", "-1", "--format=%h"], cwd=WORK, capture_output=True, text=True) + return r.stdout.strip() + +def candidates(): + """Every hand-written source file that has no .t27 counterpart yet.""" + out = [] + for d, dirs, fs in os.walk(WORK): + rel_d = os.path.relpath(d, WORK).replace("\\", "/") + "/" + if rel_d == "./": rel_d = "" + if any(rel_d.startswith(s) or ("/" + s) in ("/" + rel_d) for s in SKIP_DIRS): + dirs[:] = []; continue + for f in fs: + ext = os.path.splitext(f)[1] + if ext not in LANGS: continue + rel = os.path.relpath(os.path.join(d, f), WORK).replace("\\", "/") + if any(rel.startswith(s) for s in SKIP_DIRS): continue + if os.path.exists(os.path.join(WORK, dst_for(rel))): continue + out.append(rel) + # The project's own rule names helpers first; after that, smallest first, so + # a turn finishes inside its budget and the queue keeps moving. + def key(rel): + pri = next((i for i, p in enumerate(PRIORITY) if rel.startswith(p)), len(PRIORITY)) + try: size = os.path.getsize(os.path.join(WORK, rel)) + except OSError: size = 1 << 30 + return (pri, size, rel) + return sorted(out, key=key) + +def dst_for(rel): + """tools/check_catalog_count.py -> specs/port/tools/check_catalog_count.t27""" + return "specs/port/" + os.path.splitext(rel)[0] + ".t27" + +def functions(rel, text): + lang, pat = LANGS[os.path.splitext(rel)[1]] + seen, out = set(), [] + for m in re.finditer(pat, text): + name = m.group(1) + if name in seen: continue + seen.add(name) + line = text[:m.start()].count("\n") + 1 + sig = " ".join(text[m.start():text.find("\n", m.start())].split())[:160] + out.append((name, sig, line)) + return lang, out + +def open_boundaries(): + out = subprocess.run(["gh", "issue", "list", "--repo", REPO, "--state", "open", "--limit", "1000", + "--json", "number,title,body"], capture_output=True, text=True, timeout=180) + if out.returncode != 0: + raise SystemExit("gh issue list failed: " + out.stderr[:300]) + issues = json.loads(out.stdout) + covered = set() + ready = 0 + for it in issues: + body = it.get("body") or "" + m = re.search(r"(?ims)^##\s*boundary\s*$(.*?)(?=^#|\Z)", body) + section = m.group(1) if m else "" + found = re.findall(r"specs/[\w./-]+\.t27", section) or re.findall(r"specs/[\w./-]+\.t27", it["title"]) + covered.update(found) + if found and re.search(r"(?i)acceptance criteria", body): + ready += 1 + return covered, ready + +def dispatchable_now(): + """What the Queen itself says it can pick, and why it cannot pick the rest. + + The number that matters for a backlog target is not "open issues carrying a + boundary" - 811 issues are open and the Queen skipped 828 candidates last + tick: 547 without a boundary, 153 already completed but still open, 94 + claimed, 17 held by a file another bee has, 17 incomplete. Counting the + first number as backlog would report a full queue while the lanes sit + empty, which is the failure this whole mode exists to prevent.""" + try: + d = json.load(urllib.request.urlopen(STATUS, timeout=30)) + except Exception as e: + log(f"status unreachable: {e}"); return None + w = d.get("workers") or {} + lt = d.get("lastTick") or {} + ss = lt.get("skipSummary") or {} + d["_blocked"] = sum((ss.get(k) or {}).get("count") or 0 + for k in ("claimed", "completed", "fileConflict", "incompleteSpec")) + log(f"status: queue={(d.get('queue') or {}).get('state')} " + f"workers active={w.get('active')}/{w.get('capacity')} " + f"refusal={lt.get('refusal')} blocked={d['_blocked']} " + f"(claimed={(ss.get('claimed') or {}).get('count')}, " + f"completed={(ss.get('completed') or {}).get('count')}, " + f"fileConflict={(ss.get('fileConflict') or {}).get('count')})") + return d + +HEAD = ("**`.t27` is the hand-authored source language.** `t27c` compiles it out to C, Rust, " + "Verilog and Zig. You are writing source, not compiler output, and not prose about an " + "implementation. The goal of this project is that every hand-written file in the " + "repository is authored in `.t27` and the other languages are generated from it - this " + "issue is one file of that migration.\n") + +def body_for(rel, dst, lang, fns, nlines, part=None, parts=None): + names = [f[0] for f in fns] + n = len(fns) + if part: + title = f"Port part {part} of {parts} of {rel} to {dst} ({n} functions)" + else: + title = f"Port {rel} ({lang}, {n} function{'s' if n != 1 else ''}) to {dst}" + L = [f"# {title}\n", "## Context\n", + f"`{rel}` is {nlines} lines of hand-written {lang}. Re-author its logic as `{dst}` so " + f"that `t27c` generates the equivalent. The {lang} file stays where it is - this issue " + f"adds the `.t27` source it should have been written in.\n", HEAD] + if part: + L.append(f"**All {parts} parts name the same file, so they share one boundary and cannot " + f"run at the same time.** The swarm holds `{dst}` for whoever has it.\n") + L += ["## Read the original first\n", "```", "# from the repository root", f"$ wc -l {rel}", + f"$ sed -n '1,120p' {rel}", "```\n", + "## What to write\n", + f"Create `{dst}` with one `.t27` function per {lang} function below. Keep the names " + f"exactly as they are - the name is how the port is checked. Quoted verbatim from the " + f"original:\n"] + for i, (nm, sig, line) in enumerate(fns, 1): + L.append(f"{i}. line {line} - `{sig}`") + L += ["", f"Add a `test` block for each of the {n} functions, asserting on the behaviour you " + f"read in the original. A ported body with nothing asserting on it is a claim, not a " + f"result - and the review compiles the generated Zig and runs exactly those tests.\n", + "## Acceptance criteria\n", + f"- 1. `test -f {dst} && echo present` prints `present` (today the file does not exist)", + f"- 2. `grep -cE '^\\s*(pub )?fn ({'|'.join(names)})\\(' {dst}` prints `{n}` - every " + f"function above is ported under its own name", + f"- 3. `./target/release/t27c gen {dst} 2>&1 | grep -c 'not yet implemented'` prints `0`, " + f"and `./target/release/t27c gen {dst} | wc -l` prints more than `{max(10, n * 3)}` - an " + f"absent or empty file makes the grep print `0` on its own, so both halves are required", + f"- 4. with criterion 3 satisfied, `./target/release/t27c parse {dst} 2>&1 | grep -E " + f"'^(recovery-events|declarations-swallowed|lexer-discarded-chars):'` reports 0, 0 and 0", + f"- 5. `grep -c '^\\s*test \"' {dst}` prints at least `{n}`", + "", "Use `./target/release/t27c` from the repository root, not the `t27c` on PATH: the PATH build prints no parse " + "metrics at all, so criterion 4 greps nothing and an empty result reads as a pass.\n", + "## Boundary\n", dst] + return title, "\n".join(L) + "\n" + +def build(rel): + path = os.path.join(WORK, rel) + try: + text = open(path, encoding="utf-8", errors="replace").read() + except OSError: + return [] + lang, fns = functions(rel, text) + if not fns: return [] # nothing with a name to check: not spec-able + dst = dst_for(rel) + nlines = text.count("\n") + 1 + if len(fns) <= CHUNK: + return [body_for(rel, dst, lang, fns, nlines)] + chunks = [fns[i:i + CHUNK] for i in range(0, len(fns), CHUNK)] + return [body_for(rel, dst, lang, ch, nlines, i + 1, len(chunks)) for i, ch in enumerate(chunks)] + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--limit", type=int, default=8) + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--top-up", type=int, default=0, + help="maintain this many READY issues; create only the shortfall") + ap.add_argument("--max-functions", type=int, default=40, + help="skip a source file with more functions than this (it is a module, not a turn)") + a = ap.parse_args() + + st = dispatchable_now() + sha = sync_master() + covered, ready = open_boundaries() + if a.top_up: + # Subtract what the Queen told us it cannot pick. If the status endpoint + # is unreachable we do NOT fall back to the raw count - that is the + # number that reads full while the swarm starves; we stop instead. + if st is None: + log("status unreachable - refusing to size a backlog blind"); return + free = max(0, ready - st["_blocked"]) + shortfall = max(0, a.top_up - free) + log(f"ready backlog {ready} - {st['_blocked']} blocked = {free} dispatchable, " + f"target {a.top_up} -> shortfall {shortfall}") + if shortfall == 0: + log("backlog full - nothing added"); return + a.limit = min(a.limit, shortfall) + + cands = [c for c in candidates() if dst_for(c) not in covered] + log(f"master {sha}: {len(cands)} hand-written files have no .t27 counterpart and no open issue") + os.makedirs(OUTD, exist_ok=True) + made = 0 + for rel in cands: + if made >= a.limit: break + issues = build(rel) + if not issues: + continue + if len(issues) * CHUNK > a.max_functions and len(issues) > 1: + continue # too big to be honest work for one swarm pass + for title, body in issues: + if made >= a.limit: break + slug = re.sub(r'[^a-z0-9]+', '-', title.lower()).strip('-')[:120] + p = os.path.join(OUTD, slug + ".md") + open(p, "w").write(body) + if a.dry_run: + log(f"dry-run: {title}"); made += 1; continue + r = subprocess.run(["gh", "issue", "create", "--repo", REPO, "--title", title, + "--body-file", p], capture_output=True, text=True, timeout=120) + if r.returncode != 0: + log(f"create FAILED for {title}: {r.stderr.strip()[:200]}"); continue + log(f"created {r.stdout.strip()} {title}"); made += 1 + log(f"done: {made} port issue(s) {'would be ' if a.dry_run else ''}created") + +if __name__ == "__main__": + main()