From 13b50630dbb5e89f34afa04d72be8df4632d3bda Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Sun, 16 Aug 2026 06:18:43 -0500 Subject: [PATCH 01/33] feat(sprint): W0 phase timing + real-server TLS probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Groundwork for the "real public HTTPS server at 48 MHz" sprint. No 6502 code changes; both additions are host-side measurement tools. tools/probe_server_tls.py (new) Sends a byte-faithful replica of the ClientHello src/tls_handshake.s emits and reports the three things that decide reachability: whether the server accepts our very narrow offer (one cipher suite, one group, ECDSA only), how large its records are against the 548 B TLS_REC_BUF_MAX, and how long it tolerates a silent client. Measured 2026-08-15: github.com 529 B / 90 s, browserleaks.com 529 B / 45 s, en.wikipedia.org 529 B / >100 s (leaf 1636 B exceeds cert_buf), cloudflare.com 2714 B / 15 s. Normal OpenSSL servers DO honor max_fragment_length, so their records already fit; Cloudflare ignores it AND enforces a hard 15 s connect-to-first-request deadline, which is why it is out of scope. tools/uci/rig_https_local.py Opt-in phase timing via PHASE_TIMING=1 (PHASE_POLL tunes the interval); default behaviour is unchanged. Writes phase_timing.txt to the artifact dir alongside a per-marker cost rollup. Deliberately does NOT reuse tests/rig_vice_https_macos.py's "latest progress needle on screen" heuristic: measured here it yielded 3 transitions across a 42 s handshake, because the screen scrolls and the markers repeat once per encrypted record. Taking the last non-empty line is immune to both. tls_state ($A84A) is not a usable alternative either — it sits under BASIC ROM, so a DMA read returns ROM bytes. W0 result (U64E @ 48 MHz, comb, local listener, PASS/200): client Finished at 31.1 s, of which the server sees 24.7 s of silence — 3.6x headroom against github.com, 45% against browserleaks.com. X25519 is now co-dominant with the verify (8.0 s vs 16.3 s), not a minor term. Co-Authored-By: Claude Opus 5 (1M context) --- tools/probe_server_tls.py | 181 +++++++++++++++++++++++++++++++++++ tools/uci/rig_https_local.py | 88 ++++++++++++++++- 2 files changed, 268 insertions(+), 1 deletion(-) create mode 100755 tools/probe_server_tls.py diff --git a/tools/probe_server_tls.py b/tools/probe_server_tls.py new file mode 100755 index 00000000..4eb3a855 --- /dev/null +++ b/tools/probe_server_tls.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Probe a real HTTPS server for c64-https compatibility. No C64, no hardware. + +Sends a byte-faithful replica of the ClientHello src/tls_handshake.s actually +emits and reports the three things that decide whether the C64 can talk to a +given server: + + 1. Does it accept our very narrow ClientHello at all? We offer exactly one + cipher suite (TLS_CHACHA20_POLY1305_SHA256), one group (X25519) and two + ECDSA signature algorithms. No RSA path exists in the client. + 2. How large are its records? src/tls_record_io.s rejects any record whose + length field exceeds TLS_REC_BUF_MAX (548). We ask for small records via + max_fragment_length=512 (RFC 6066); servers are free to ignore it. + 3. How long will it wait for a silent client? The C64 goes quiet for tens of + seconds computing the X25519 shared secret and the ECDSA verify, and the + server's handshake timer is running the whole time. + +Measured 2026-08-15 (see the sprint plan and the real-server-tls-tolerance +memory note): + + server max record silence tolerance leaf + github.com 529 B 90 s 1010 B + browserleaks.com 529 B 45 s 952 B + en.wikipedia.org 529 B >100 s 1636 B (> cert_buf) + cloudflare.com 2714 B 15 s 981 B (out of scope) + +These are live third-party servers. Re-run this before relying on any of those +numbers. + +Usage: + tools/probe_server_tls.py github.com browserleaks.com + tools/probe_server_tls.py --no-mfl cloudflare.com # omit MFL extension + tools/probe_server_tls.py --cap 120 example.com # silence-probe cap +""" +from __future__ import annotations + +import argparse +import os +import socket +import struct +import sys +import threading +import time + +# src/tls_record_io.s:43 — the client hard-rejects anything larger. +TLS_REC_BUF_MAX = 548 +# src/der_decode.s — leaf certificate staging buffer. +CERT_BUF = 1536 + + +def client_hello(hostname: str, send_mfl: bool = True) -> bytes: + """Rebuild src/tls_handshake.s's ClientHello byte-for-byte. + + The key share is arbitrary bytes: we never complete the handshake, we only + need the server to commit to a flight. + """ + kx = bytes(range(32)) + body = b"\x03\x03" + os.urandom(32) # legacy_version + random + body += b"\x00" # session_id_length = 0 + body += b"\x00\x02\x13\x03" # TLS_CHACHA20_POLY1305_SHA256 + body += b"\x01\x00" # compression: null + + ext = b"\x00\x2b\x00\x03\x02\x03\x04" # supported_versions + ext += b"\x00\x0a\x00\x04\x00\x02\x00\x1d" # supported_groups + ext += b"\x00\x0d\x00\x06\x00\x04\x04\x03\x05\x03" # sigalgs 0403/0503 + ext += b"\x00\x33\x00\x26\x00\x24\x00\x1d\x00\x20" + kx # key_share x25519 + hn = hostname.encode() + ext += (b"\x00\x00" + struct.pack(">H", len(hn) + 5) + + struct.pack(">H", len(hn) + 3) + b"\x00" + + struct.pack(">H", len(hn)) + hn) # SNI + if send_mfl: + ext += b"\x00\x01\x00\x01\x01" # MFL = 512 + + body += struct.pack(">H", len(ext)) + ext + hs = b"\x01" + struct.pack(">I", len(body))[1:] + body + return b"\x16\x03\x01" + struct.pack(">H", len(hs)) + hs + + +def split_records(buf: bytes) -> list[tuple[int, int]]: + out, off = [], 0 + while off + 5 <= len(buf): + typ = buf[off] + ln = struct.unpack(">H", buf[off + 3:off + 5])[0] + out.append((typ, ln)) + off += 5 + ln + return out + + +def probe(host: str, port: int, send_mfl: bool, cap: float) -> dict: + r: dict = {"host": host} + try: + s = socket.create_connection((host, port), timeout=15) + t0 = time.time() + s.sendall(client_hello(host, send_mfl)) + s.settimeout(4) + buf = b"" + try: + while True: + chunk = s.recv(16384) + if not chunk: + break + buf += chunk + except socket.timeout: + pass + recs = split_records(buf) + r["flight"] = len(buf) + r["records"] = len(recs) + r["max_record"] = max((ln for t, ln in recs if t in (22, 23)), default=0) + r["alert"] = any(t == 21 for t, _ in recs) + if not buf: + r["tolerance"] = "no reply" + elif r["alert"]: + r["tolerance"] = "REJECTED our ClientHello" + else: + # Go silent, exactly as the C64 does while it computes. + s.settimeout(5) + r["tolerance"] = f">{cap:.0f}s" + while time.time() - t0 < cap: + try: + if not s.recv(4096): + r["tolerance"] = f"{time.time() - t0:.0f}s" + break + except socket.timeout: + continue + except OSError: + r["tolerance"] = f"{time.time() - t0:.0f}s (RST)" + break + s.close() + except Exception as exc: # noqa: BLE001 - report, don't raise + r["tolerance"] = f"ERROR {type(exc).__name__}: {exc}" + return r + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("hosts", nargs="+") + ap.add_argument("--port", type=int, default=443) + ap.add_argument("--no-mfl", action="store_true", + help="omit max_fragment_length (shows unfragmented sizes)") + ap.add_argument("--cap", type=float, default=100.0, + help="seconds to stay silent before giving up (default 100)") + args = ap.parse_args() + + results: dict[str, dict] = {} + threads = [threading.Thread( + target=lambda h: results.__setitem__( + h, probe(h, args.port, not args.no_mfl, args.cap)), + args=(h,)) for h in args.hosts] + for t in threads: + t.start() + for t in threads: + t.join() + + print(f"max_fragment_length: {'omitted' if args.no_mfl else 'sent (512)'}; " + f"silence cap {args.cap:.0f}s\n") + print(f"{'host':24s} {'flight':>8s} {'#rec':>5s} {'max rec':>8s} " + f"{'fits 548':>9s} tolerance") + print("-" * 78) + worst = 0 + for h in args.hosts: + r = results.get(h, {}) + mx = r.get("max_record", 0) + fits = "-" if not mx else ("YES" if mx <= TLS_REC_BUF_MAX else "NO") + if fits == "NO": + worst += 1 + print(f"{h:24s} {r.get('flight', 0):8d} {r.get('records', 0):5d} " + f"{mx:8d} {fits:>9s} {r.get('tolerance', '?')}") + + print(f"\nRecords over {TLS_REC_BUF_MAX} B are rejected outright by " + f"src/tls_record_io.s.") + print("Tolerance is how long the server waits with the client silent — the " + "C64's\nbudget for the X25519 shared secret plus the ECDSA verify.") + if worst: + print(f"\n{worst} server(s) exceed the record limit and need the " + "large-record path.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/uci/rig_https_local.py b/tools/uci/rig_https_local.py index 2151c75a..33c54d6c 100644 --- a/tools/uci/rig_https_local.py +++ b/tools/uci/rig_https_local.py @@ -634,6 +634,68 @@ def _decode_screen_ram(data: bytes) -> str: return '\n'.join(lines) +# --- Phase timing (W0 sprint spike, opt-in via PHASE_TIMING=1) --------------- +# The handshake prints a progress marker per phase (src/boot.s:1153-1161, +# printed from src/tls13.s), so timestamping markers yields a per-phase +# wall-clock breakdown without touching any 6502 code. +# +# The obvious approach — tests/rig_vice_https_macos.py's "latest progress +# needle anywhere on screen" — does NOT work on this rig, measured +# 2026-08-15: it produced 3 transitions across a 42 s handshake. Two reasons, +# both specific to running the full flight rather than VICE's slower pace: +# 1. the screen SCROLLS, so needle position stops tracking print order; +# 2. markers REPEAT — ENC1/RX/GOT2/GOT/DEC/PROC cycle once per encrypted +# record — so "which needle is furthest right" oscillates. +# Taking the last non-empty line instead is exactly "what was printed most +# recently", which is immune to both. +# +# Note tls_state ($A84A) is NOT a usable alternative here: it sits in +# SHADOW_BSS under the BASIC ROM, and a DMA read of $A000-$BFFF returns ROM +# bytes unless $01 banks it out. Screen RAM at $0400 is always RAM. +def _current_phase(screen: str) -> str: + for line in reversed(screen.split("\n")): + s = line.strip() + if s: + return s + return "(blank)" + + +def _format_phase_log(phase_log: "list[tuple[str, float]]") -> str: + """Render the phase log as a timeline plus a per-marker cost rollup. + + Markers repeat (one ENC1/RX/GOT2/GOT/DEC/PROC cycle per encrypted + record), so the rollup — total wall-clock attributed to each marker and + how many times it was entered — is usually the more readable half. + """ + if not phase_log: + return "(no phase transitions observed)" + + out = ["--- timeline ---", + f"{'marker':24s} {'t (s)':>9s} {'delta (s)':>11s}", "-" * 47] + prev = 0.0 + totals: "dict[str, list]" = {} + for i, (name, t) in enumerate(phase_log): + d = t - prev + out.append(f"{name:24s} {t:9.1f} {d:11.1f}") + # Attribute this interval to the marker that was showing DURING it, + # i.e. the previous entry — not to the one that just appeared. + if i > 0: + owner = phase_log[i - 1][0] + e = totals.setdefault(owner, [0.0, 0]) + e[0] += d + e[1] += 1 + prev = t + + out += ["", "--- cost by marker (descending) ---", + f"{'marker':24s} {'total (s)':>11s} {'entered':>9s}", "-" * 47] + for name, (secs, n) in sorted(totals.items(), key=lambda kv: -kv[1][0]): + out.append(f"{name:24s} {secs:11.1f} {n:9d}") + out.append("") + out.append(f"last marker reached: {phase_log[-1][0]} at " + f"{phase_log[-1][1]:.1f}s") + return "\n".join(out) + + def _dump_diag(transport: Ultimate64Transport, labels: dict[str, int]) -> None: def r8(name: str) -> int: @@ -1564,8 +1626,12 @@ def main() -> int: deadline = time.time() + SENTINEL_POLL_TIMEOUT last_progress = -1 start = time.time() + phase_timing = os.environ.get("PHASE_TIMING", "0") == "1" + phase_poll = float(os.environ.get("PHASE_POLL", "0.5")) + phase_log: "list[tuple[str, float]]" = [] + cur_phase = None while time.time() < deadline: - time.sleep(0.5) + time.sleep(phase_poll if phase_timing else 0.5) blob = transport.read_memory(SENTINEL_ADDR, 2) sentinel = blob[0] progress = blob[1] @@ -1573,9 +1639,29 @@ def main() -> int: elapsed = time.time() - start print(f" [{elapsed:6.1f}s] progress=0x{progress:02X}") last_progress = progress + if phase_timing: + scr = _decode_screen_ram( + bytes(transport.read_memory(0x0400, 1000))) + phase = _current_phase(scr) + if cur_phase is None: + cur_phase = phase + elif phase != cur_phase: + cur_phase = phase + phase_log.append((phase, time.time() - start)) + print(f" phase +{phase_log[-1][1]:7.1f}s {phase}") if sentinel == SENTINEL_VALUE: print(" sentinel set — routine complete") break + if phase_timing: + table = _format_phase_log(phase_log) + print("\n=== Phase timing (W0) ===") + print(table) + try: + (run_dir / "phase_timing.txt").write_text( + f"target={host_str.decode(errors='replace')} " + f"turbo={TURBO_MHZ}MHz poll={phase_poll}s\n\n{table}\n") + except Exception as exc: # diagnostics must never fail the run + print(f" (could not write phase_timing.txt: {exc})") else: print(f"TIMEOUT: sentinel not set after " f"{SENTINEL_POLL_TIMEOUT:.0f}s " From ac176debad7c28f5534f6e55458d64256a0953c8 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:28:48 -0400 Subject: [PATCH 02/33] feat(w3): build-time HTTPS_HOST target override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HTTPS GET target (host data, SNI copy, DNS lookup) was hardcoded to www.foo.bar in src/boot.s. Add `make HTTPS_HOST=` mirroring the HTTPS_PORT idiom. ca65's -D is numeric-only, so the string travels via a generated build/https_host.inc (already on the -I build path), with a content-compared FORCE rule so a changed host rebuilds boot.o without `make clean` — on a change the rule also deletes boot.o, because Make 3.81's 1-second mtime resolution otherwise leaves the previous host's object counting as up to date (measured). The link step keeps the repo-wide same-second caveat; comment documents it. Default build proven byte-identical (b574b344cdca...) with the default string; en.wikipedia.org/github.com overrides verified to embed and to revert cleanly; UCI comb variant links. A .assert refuses hosts longer than the 63-char SNI copy guard at build time. http_host_foo renamed to http_host_target (no external consumers). Co-Authored-By: Claude Fable 5 --- Makefile | 30 ++++++++++++++++++++++++++++++ src/boot.s | 31 ++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 9e49935e..bac2cf19 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,13 @@ endif ifdef HTTPS_PORT CA65FLAGS += -D HTTPS_PORT=$(HTTPS_PORT) endif +# Optional HTTPS target-host override (src/boot.s defaults to www.foo.bar). +# `make HTTPS_HOST=github.com` retargets the GET + SNI + DNS lookup. +# ca65's -D accepts numeric values only, so the hostname travels via a +# generated include (build/https_host.inc, reached through `-I build`). +# The generator rule is content-compared: changing HTTPS_HOST= rebuilds +# boot.o with no `make clean`, and an unchanged value touches nothing. +HTTPS_HOST ?= www.foo.bar # VIC-II blanking around the CPU-bound crypto primitives (src/vic.s). # ON by default and that is what ships; `make VIC_BLANK=0` degrades # vic_blank/vic_unblank to bare RTS so the badline tax can be measured @@ -400,6 +407,29 @@ build/%.o: src/%.s # this object, and never requests $(IP65_BIN).) build/net/ip65/ip65_blob.o: $(IP65_BIN) +# src/boot.s pulls the HTTPS target hostname from this generated include +# (see the HTTPS_HOST block near the top). FORCE makes the rule run on +# every make so a changed HTTPS_HOST= is noticed; the content compare +# leaves the file's mtime alone when the value is unchanged, so boot.o +# is NOT rebuilt in that case. On a change it also deletes boot.o +# outright: this Make (3.81 on macOS) compares mtimes at 1-second +# resolution, so two `make HTTPS_HOST=...` runs inside the same second +# would otherwise leave the previous host's boot.o counting as up to +# date — measured, not hypothetical. The LINK step keeps the repo-wide +# same-second caveat (see "make clean when you change flags" in +# CLAUDE.md): scripted back-to-back host switches inside one second can +# skip the relink, so check the PRG hash when a build matters. Deleting +# $(PRG) here does NOT fix that — make's cached stat of the goal leaves +# it "up to date" and the file simply vanishes (measured too). +build/https_host.inc: FORCE + @mkdir -p build + @printf '.define HTTPS_HOST_STR "%s"\n' '$(HTTPS_HOST)' > $@.tmp + @if cmp -s $@.tmp $@; then rm -f $@.tmp; else mv $@.tmp $@ && rm -f build/boot.o; fi + +FORCE: + +build/boot.o: build/https_host.inc + # Phase C.3: c64-nist-curves sibling archive (libs/nistcurves/ submodule). # Phase 1.5 split: produces TWO archives, one per overlay half. The # script writes both with a single invocation; the second target is a diff --git a/src/boot.s b/src/boot.s index 40e5e736..e355755d 100644 --- a/src/boot.s +++ b/src/boot.s @@ -11,6 +11,12 @@ HTTPS_PORT = 443 .endif + ; HTTPS target host. `make HTTPS_HOST=github.com` writes the + ; string into build/https_host.inc (ca65 -D is numeric-only, so + ; a string override must travel via a generated include). The + ; default is www.foo.bar and the default build byte-identical. + .include "https_host.inc" + ; ---- exports: entry + print helpers ---- .export start .export main_loop @@ -93,8 +99,8 @@ ; ---- exports: hostnames / path data ---- .export http_host_zimmers .export http_host_zimmers_len - .export http_host_foo - .export http_host_foo_len + .export http_host_target + .export http_host_target_len .export http_path_root ; ---- exports: local BSS ---- @@ -519,11 +525,11 @@ do_https_get: jsr print_string ; --- set HTTP host/path/port --- - lda #http_host_foo + lda #>http_host_target sta http_host_ptr+1 - lda #http_host_foo_len + lda #http_host_target_len sta http_host_len lda #http_host_foo + lda #http_host_target jsr net_dns_resolve bcc @dns_ok @@ -1194,10 +1200,13 @@ http_host_zimmers: .byte 0 http_host_zimmers_len = 15 -http_host_foo: - .byte "www.foo.bar" +http_host_target: + .byte HTTPS_HOST_STR .byte 0 -http_host_foo_len = 11 +http_host_target_len = * - http_host_target - 1 + ; the SNI copy loop below guards at 63 chars; refuse at build + ; time rather than truncating at runtime + .assert http_host_target_len <= 63, error, "HTTPS_HOST exceeds the 63-char SNI guard" http_path_root: .byte "/" From 805961a62644b9394194bd8223605ebcf3572e38 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:35:00 -0400 Subject: [PATCH 03/33] fix(uci): rig_https_local TIMEOUT path lost its while/else binding in W0 The W0 phase-timing commit (13b5063) inserted an 'if phase_timing:' block between the sentinel poll loop and its 'else:', silently rebinding the else from the while (runs only on exhaustion) to the if. Result at 13b5063: a successful default-mode run (PHASE_TIMING unset) reported TIMEOUT and exited 1, and a PHASE_TIMING=1 run that genuinely timed out skipped the TIMEOUT diagnostics entirely. Track the break explicitly (sentinel_seen) and route the TIMEOUT path off that flag, independent of phase timing. Co-Authored-By: Claude Fable 5 --- tools/uci/rig_https_local.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/uci/rig_https_local.py b/tools/uci/rig_https_local.py index 33c54d6c..8ce272fe 100644 --- a/tools/uci/rig_https_local.py +++ b/tools/uci/rig_https_local.py @@ -1630,6 +1630,7 @@ def main() -> int: phase_poll = float(os.environ.get("PHASE_POLL", "0.5")) phase_log: "list[tuple[str, float]]" = [] cur_phase = None + sentinel_seen = False while time.time() < deadline: time.sleep(phase_poll if phase_timing else 0.5) blob = transport.read_memory(SENTINEL_ADDR, 2) @@ -1651,7 +1652,14 @@ def main() -> int: print(f" phase +{phase_log[-1][1]:7.1f}s {phase}") if sentinel == SENTINEL_VALUE: print(" sentinel set — routine complete") + sentinel_seen = True break + # NOTE: this used to be the `else:` of the while-loop (while/else — + # runs only when the loop exhausts without break). The W0 phase-timing + # commit inserted the `if phase_timing:` block between the loop and + # its `else`, silently rebinding the `else` to the `if`: a successful + # default-mode run then reported TIMEOUT, and a PHASE_TIMING=1 run + # never took the TIMEOUT path at all. Track the break explicitly. if phase_timing: table = _format_phase_log(phase_log) print("\n=== Phase timing (W0) ===") @@ -1662,7 +1670,7 @@ def main() -> int: f"turbo={TURBO_MHZ}MHz poll={phase_poll}s\n\n{table}\n") except Exception as exc: # diagnostics must never fail the run print(f" (could not write phase_timing.txt: {exc})") - else: + if not sentinel_seen: print(f"TIMEOUT: sentinel not set after " f"{SENTINEL_POLL_TIMEOUT:.0f}s " f"(progress=0x{last_progress:02X})", file=sys.stderr) From 21580371e8a032609665657bd01fbf5a81da3db2 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:35:10 -0400 Subject: [PATCH 04/33] =?UTF-8?q?feat(http):=20W4=20=E2=80=94=20chunked=20?= =?UTF-8?q?transfer=20decoding=20+=20real-server=20response=20robustness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real web servers (github.com, browserleaks.com, en.wikipedia.org) send responses the local listener never did: Transfer-Encoding: chunked with no Content-Length, and header blocks far larger than http_resp_buf that span many TLS records. Previously chunk framing landed raw in the body buffer. - check_transfer_encoding: case-insensitive 'Transfer-Encoding: chunked' header match (same single-SP-after-colon shape as check_content_length), sets http_chunked. - http_state_body_chunked: strips chunk framing byte-by-byte with persistent sub-state (hex size lines incl. chunk extensions, per-chunk CRLF, terminal 0/trailers/blank-line), copying only payload into http_resp_buf. Terminates C=0 on the terminal chunk exactly like the Content-Length match path. - Truncate-at-capacity: past 512 B payload is consumed and discarded (http_resp_len caps at the captured prefix) and the parser still finds the terminal chunk, so oversized bodies end cleanly instead of falling to the ~65k-poll timeout. - Header scanning was already incremental (persistent line-state across records, 31-byte line cap); now proven against >2.5 KB header blocks split across 14 spans by new tests. - Memory: body-state handler moved out of CODE into new HTTP_AUX_CODE segment (UCI: LOADER, 588 B free pre-W4; ip65: CRYPTO_OVERLAY — ip65 LOADER had only 8 B free). CODE shrank 64 B on both backends. New BSS: 4 B (http_chunked, http_chunk_state, http_chunk_rem). PRG sizes unchanged (47,105 B ip65 / 62,977 B uci-comb). - tools/test_http.py: span-mode drive (the TLS input path) + 5 new vector groups: chunked single-span, 5-span chunked w/ split header name + split chunk data + uppercase hex + chunk extension + split terminal CRLF, 700 B chunked truncation, 2.6 KB headers w/ buried Content-Length, chunked + large headers. 48/48 (was 27). Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 18 +- cfg/c64-https-ip65.cfg | 4 + cfg/c64-https-uci-onchip.cfg | 4 + cfg/c64-https-uci.cfg | 4 + src/http.s | 315 ++++++++++++++++++++++++++++++----- tools/test_http.py | 258 ++++++++++++++++++++++++++++ 6 files changed, 558 insertions(+), 45 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c4be9697..4a85e99c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -789,7 +789,23 @@ Five latent bugs and three new ones were cleared to get here: the colon. Once `http_resp_len` equals `http_content_length`, state_body returns C=0 and `http_get` exits cleanly. When the header is absent the parser falls back to the previous - keep-polling behaviour (preserves chunked/streaming paths). + keep-polling behaviour (preserves streaming paths). + W4 added `Transfer-Encoding: chunked` support on top: a matching + header (same case-insensitive single-SP matcher shape) sets + `http_chunked`, and the body state routes through + `http_state_body_chunked` (src/http.s), which strips chunk framing + — hex size lines incl. extensions, per-chunk CRLF, terminal + `0\r\n\r\n` with optional trailers — copying only payload into + `http_resp_buf` (truncate-at-capacity at 512 B: the buffer holds + the prefix, the rest is consumed and discarded so the terminal + chunk still terminates cleanly with C=0). Chunks >64 KB would wrap + the 16-bit `http_chunk_rem` and desync to the poll-timeout + fallback; real servers chunk at 8-32 KB. The body-state handler + moved out of `CODE` into the new `HTTP_AUX_CODE` segment (UCI: + LOADER; ip65: CRYPTO_OVERLAY — ip65's LOADER had 8 B free), which + is also where the header-state reset (`http_hdr_init`) and the + `Transfer-Encoding` matcher live. Covered by the chunked / + large-header span-mode vectors in `tools/test_http.py`. - `net_tcp_set_recv_cb` is an RTS stub (no callers in-tree). - Boot banner line 03 still says "rr-net" under ip65 build even though Phase 2 made it backend-aware — this is correct/expected diff --git a/cfg/c64-https-ip65.cfg b/cfg/c64-https-ip65.cfg index 06b0fde8..5c729e51 100644 --- a/cfg/c64-https-ip65.cfg +++ b/cfg/c64-https-ip65.cfg @@ -188,6 +188,10 @@ SEGMENTS { # below $A000. TLS_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; CRYPTO_AUX_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; + # W4: http.s body-state / chunked-transfer parser. Lives with the + # other resident code in the reclaimed overlay slot because LOADER + # is packed on ip65 (8 B free); under UCI it rides LOADER instead. + HTTP_AUX_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; CRYPTO_CODE: load = CRYPTO_RESIDENT, type = ro; CRYPTO_RODATA: load = CRYPTO_RESIDENT, type = ro; # libs/nistcurves segment names (c64-lib-contract / cfa9085+). diff --git a/cfg/c64-https-uci-onchip.cfg b/cfg/c64-https-uci-onchip.cfg index f258fbba..e1e14fd7 100644 --- a/cfg/c64-https-uci-onchip.cfg +++ b/cfg/c64-https-uci-onchip.cfg @@ -180,6 +180,10 @@ SEGMENTS { EXEHDR: load = LOADER, type = ro; STARTUP: load = LOADER, type = ro, optional = yes; CODE: load = LOADER, type = ro; + # W4: http.s body-state / chunked-transfer parser. Rides LOADER + # here (588 B free pre-W4); under ip65 LOADER is packed so the + # segment maps to CRYPTO_OVERLAY there instead. + HTTP_AUX_CODE: load = LOADER, type = ro, optional = yes; RODATA: load = CRYPTO_OVERLAY, type = ro; # onchip-comb variant: relieve CRYPTO_HOT INIT: load = LOADER, type = ro, optional = yes; diff --git a/cfg/c64-https-uci.cfg b/cfg/c64-https-uci.cfg index d5110a8e..442fb34f 100644 --- a/cfg/c64-https-uci.cfg +++ b/cfg/c64-https-uci.cfg @@ -166,6 +166,10 @@ SEGMENTS { EXEHDR: load = LOADER, type = ro; STARTUP: load = LOADER, type = ro, optional = yes; CODE: load = LOADER, type = ro; + # W4: http.s body-state / chunked-transfer parser. Rides LOADER + # here (588 B free pre-W4); under ip65 LOADER is packed so the + # segment maps to CRYPTO_OVERLAY there instead. + HTTP_AUX_CODE: load = LOADER, type = ro, optional = yes; RODATA: load = CRYPTO_HOT, type = ro; INIT: load = LOADER, type = ro, optional = yes; diff --git a/src/http.s b/src/http.s index 8e842d12..e29ab500 100644 --- a/src/http.s +++ b/src/http.s @@ -434,7 +434,7 @@ http_recv_response: beq @state_status cmp #1 beq @jmp_headers - jmp @state_body + jmp http_state_body @jmp_headers: jmp @state_headers @@ -523,11 +523,10 @@ http_recv_response: lda #0 sta http_hdr_match sta http_line_idx - ; Reset Content-Length sentinel to $FFFF ("unknown") — will be - ; overwritten by check_content_length if the header is seen. - lda #$ff - sta http_content_length - sta http_content_length+1 + ; Reset the per-response header-derived state (Content-Length + ; sentinel to $FFFF "unknown", chunked flag + chunk parser state + ; to 0). Lives in HTTP_AUX_CODE — LOADER is packed on ip65. + jsr http_hdr_init jmp @state_headers ; ----- state 1: read header lines into http_line_buf until an empty line ----- @@ -556,6 +555,7 @@ http_recv_response: lda http_line_idx beq @hdr_end ; empty line -> end of headers jsr check_content_length + jsr check_transfer_encoding lda #0 sta http_line_idx jmp @state_headers @@ -570,23 +570,141 @@ http_recv_response: lda #0 sta http_resp_len sta http_resp_len+1 - jmp @state_body + jmp http_state_body + +@not_done: + sec + rts + +; ============================================================================= +; W4: body state + chunked-transfer support. +; +; Everything below lives in HTTP_AUX_CODE, NOT in CODE: the ip65 +; LOADER region is packed (8 B free pre-W4), so the body-state handler +; was moved out of CODE and the chunked machinery added alongside it. +; The segment maps to LOADER under UCI (588 B free) and to the +; resident CRYPTO_OVERLAY slot under ip65 (see the cfgs). +; ============================================================================= + .segment "HTTP_AUX_CODE" + +; ----------------------------------------------------------------------------- +; http_hdr_init - reset per-response header-derived parser state. +; Called at the status-line -> headers transition. +; ----------------------------------------------------------------------------- +http_hdr_init: + ; Content-Length sentinel $FFFF = "unknown" + lda #$ff + sta http_content_length + sta http_content_length+1 + lda #0 + sta http_chunked + sta http_chunk_state + sta http_chunk_rem + sta http_chunk_rem+1 + rts + +; ----------------------------------------------------------------------------- +; check_transfer_encoding - inspect http_line_buf[0..http_line_idx) for +; "Transfer-Encoding: chunked" (case-insensitive, single SP after the +; colon — mirrors check_content_length). Sets http_chunked=1 on match. +; Lenient on trailing bytes: real servers send exactly "chunked". +; Clobbers: A, X +; ----------------------------------------------------------------------------- +check_transfer_encoding: + lda http_line_idx + cmp #26 ; len("transfer-encoding: chunked") + bcc @cte_rts + ldx #25 +@cte_cmp: + lda http_line_buf,x + ora #$20 ; fold A..Z -> a..z (':',' ','-' unchanged) + cmp te_pattern,x + bne @cte_rts + dex + bpl @cte_cmp + lda #1 + sta http_chunked +@cte_rts: + rts + +te_pattern: + .byte "transfer-encoding: chunked" + +; ----------------------------------------------------------------------------- +; body_append - append byte in A to http_resp_buf if there is room. +; http_resp_buf is 512 B; once http_resp_len reaches 512 the byte is +; discarded and the length no longer advances (truncate-at-capacity: +; http_resp_len reports the captured prefix length). +; Clobbers: A, Y, zp_ptr. Preserves X. +; ----------------------------------------------------------------------------- +body_append: + ldy http_resp_len+1 + cpy #$02 + bcs @ba_full ; >= 512: discard + ; store byte at http_resp_buf + http_resp_len + pha + clc + lda #http_resp_buf + adc http_resp_len+1 + sta zp_ptr+1 + pla + ldy #0 + sta (zp_ptr),y + inc http_resp_len + bne @ba_done + inc http_resp_len+1 +@ba_done: +@ba_full: + rts + +; ----------------------------------------------------------------------------- +; hex_digit - convert ASCII hex digit in A to its value. +; Output: C=0 + value 0..15 in A, or C=1 if A is not a hex digit. +; Preserves X, Y. +; ----------------------------------------------------------------------------- +hex_digit: + cmp #'0' + bcc @hd_no + cmp #'9'+1 + bcc @hd_num + ora #$20 ; fold A-F -> a-f + cmp #'a' + bcc @hd_no + cmp #'f'+1 + bcs @hd_no + sbc #'a'-11 ; C=0 here: A - ('a'-10) + clc + rts +@hd_num: + and #$0f + clc + rts +@hd_no: + sec + rts ; ----- state 2: read body bytes ----- -; RFC 7230 behaviour: we treat an empty ring as "body still arriving" -; (return @not_done, letting the caller poll for more TLS records) instead -; of declaring the response complete. Only two events end body reads: -; 1. buffer full (512 B safety cap — http_resp_buf is 512 B), or -; 2. caller-side poll-timeout in http_get's @recv_loop (the caller gives -; up after ~65 k empty ticks with no data). -; Previously this state treated "ring empty this instant" as "body done", -; which clipped the response to zero bytes whenever the HTTP response -; arrived in a second TLS record after the headers — a real situation -; under UCI because the server emits NewSessionTicket handshake records -; (app-key phase) ahead of the actual HTTP response record, so the very -; first post-handshake TLS record the parser sees contains only the -; status line + headers and the body lands in the next record. -@state_body: +; RFC 7230 behaviour: we treat an empty input as "body still arriving" +; (return C=1, letting the caller poll for more TLS records) instead +; of declaring the response complete. Termination events: +; 1. http_resp_len reaches Content-Length (identity encoding), or +; 2. the terminal chunk 0\r\n\r\n is consumed (chunked encoding), or +; 3. buffer full at 512 B (identity encoding only — chunked keeps +; consuming/discarding so it can find the terminal chunk), or +; 4. caller-side poll-timeout in http_recv_body's @recv_loop (the +; caller gives up after ~65 k empty ticks with no data — the +; "accept whatever we have" fallback for responses with neither +; Content-Length nor chunked framing; we send Connection: close, +; so the peer's close ends those). +http_state_body: + lda http_chunked + beq @plain + jmp http_state_body_chunked + +@plain: ; Content-Length termination: http_content_length defaults to $FFFF ; ("unknown"), overwritten by check_content_length if the header ; was seen. $FFFF will never match because the buffer-full check @@ -598,37 +716,134 @@ http_recv_response: bne @body_read lda http_resp_len+1 cmp http_content_length+1 - beq @done + beq @sb_done @body_read: jsr http_in_byte - bcs @not_done ; no byte right now -> poll more - ; store byte using zp_ptr = http_resp_buf + http_resp_len - pha ; save received byte - clc - lda #http_resp_buf - adc http_resp_len+1 - sta zp_ptr+1 - pla ; restore byte - ldy #0 - sta (zp_ptr),y - ; increment 16-bit length - inc http_resp_len - bne @body_check_full - inc http_resp_len+1 -@body_check_full: + bcs @sb_not_done ; no byte right now -> poll more + jsr body_append ; check if buffer full (512 bytes = $0200) lda http_resp_len+1 cmp #$02 - bcs @done ; high byte >= 2 means >= 512 - jmp @state_body ; keep reading (also re-checks Content-Length) + bcs @sb_done ; high byte >= 2 means >= 512 + jmp @plain ; keep reading (also re-checks Content-Length) -@not_done: +@sb_not_done: sec rts -@done: +@sb_done: + clc + rts + +; ----------------------------------------------------------------------------- +; http_state_body_chunked - strip Transfer-Encoding: chunked framing. +; +; Sub-state in http_chunk_state: +; 0 = accumulating hex chunk-size digits (http_chunk_rem, 16-bit) +; 1 = skipping the rest of the size line (chunk extensions) until LF +; 2 = copying chunk payload (http_chunk_rem bytes) via body_append +; 3 = skipping the CRLF that follows chunk payload, until LF +; 4 = terminal chunk seen: skipping trailer lines; an empty line ends +; the response (http_line_idx is reused as a line-nonempty flag — +; header parsing is over, so the buffer index is free) +; +; A chunk larger than 65535 B would wrap http_chunk_rem and desync the +; framing; real servers chunk at their buffer size (8-32 KB). A desync +; degrades to the caller's poll-timeout fallback, never to a hang. +; ----------------------------------------------------------------------------- +http_state_body_chunked: + jsr http_in_byte + bcc @cb_have + rts ; C=1 from http_in_byte: no data -> not done +@cb_have: + ldx http_chunk_state + beq @cs_size + dex + beq @cs_skip + dex + beq @cs_data + dex + beq @cs_crlf + jmp @cs_final + +@cs_size: + cmp #$0a ; LF ends the size line + beq @size_eol + jsr hex_digit + bcs @to_skip ; non-hex (CR, ';', extension) -> state 1 + ; http_chunk_rem = http_chunk_rem*16 + digit + pha + asl http_chunk_rem + rol http_chunk_rem+1 + asl http_chunk_rem + rol http_chunk_rem+1 + asl http_chunk_rem + rol http_chunk_rem+1 + asl http_chunk_rem + rol http_chunk_rem+1 + pla + ora http_chunk_rem + sta http_chunk_rem + jmp http_state_body_chunked +@to_skip: + lda #1 + sta http_chunk_state + jmp http_state_body_chunked + +@cs_skip: + cmp #$0a + bne @cb_loop +@size_eol: + ; size line complete: 0 -> terminal chunk, else payload follows + lda http_chunk_rem + ora http_chunk_rem+1 + beq @final_enter + lda #2 + sta http_chunk_state + jmp http_state_body_chunked +@final_enter: + lda #4 + sta http_chunk_state + lda #0 + sta http_line_idx ; trailer-line-nonempty flag + jmp http_state_body_chunked + +@cs_data: + jsr body_append ; append-if-room (discards past 512) + ; 16-bit decrement of http_chunk_rem + lda http_chunk_rem + bne :+ + dec http_chunk_rem+1 +: dec http_chunk_rem + lda http_chunk_rem + ora http_chunk_rem+1 + bne @cb_loop + lda #3 + sta http_chunk_state + jmp http_state_body_chunked + +@cs_crlf: + cmp #$0a + bne @cb_loop + lda #0 ; back to size parsing (rem is already 0) + sta http_chunk_state + jmp http_state_body_chunked + +@cs_final: + cmp #$0a + beq @fin_eol + cmp #$0d + beq @cb_loop ; CR ignored + inc http_line_idx ; trailer line has content + jmp http_state_body_chunked +@fin_eol: + lda http_line_idx + beq @cb_done ; empty line after terminal chunk -> done + lda #0 + sta http_line_idx +@cb_loop: + jmp http_state_body_chunked + +@cb_done: clc rts @@ -816,6 +1031,18 @@ http_crlf: http_bg_idx: .res 1 ; build_get write cursor http_bg_src: .res 1 ; build_get source index temp ; Parser input source (issue #72 span-input redesign — see http_in_byte) +; Exported so tools/test_http.py can drive the parser in span mode, +; the same input mode the TLS path uses. + .export http_in_mode + .export http_in_ptr + .export http_in_len http_in_mode: .res 1 ; 0 = TCP ring (plain HTTP), 1 = span (TLS) http_in_ptr: .res 2 ; span read cursor http_in_len: .res 2 ; span bytes remaining (16-bit) +; W4 chunked-transfer state (see http_state_body_chunked) + .export http_chunked + .export http_chunk_state + .export http_chunk_rem +http_chunked: .res 1 ; 1 = Transfer-Encoding: chunked seen +http_chunk_state: .res 1 ; chunk parser sub-state (0..4) +http_chunk_rem: .res 2 ; bytes remaining in current chunk diff --git a/tools/test_http.py b/tools/test_http.py index e9f9207d..6106aacc 100755 --- a/tools/test_http.py +++ b/tools/test_http.py @@ -165,6 +165,239 @@ def _reset_parser(transport, labels): write_bytes(transport, line_idx, [0]) write_bytes(transport, hdr_match, [0]) write_bytes(transport, resp_len, [0, 0]) + # Ring input mode by default (span tests flip to 1 per span). + write_bytes(transport, labels.address("http_in_mode"), [0]) + + +# --- span-mode input (the TLS path's input mode) ------------------------- +# The TLS path hands each decrypted record to http_recv_response as a +# linear span (http_in_mode=1). These helpers drive the parser exactly +# that way, one call per "record", which is what lets the tests cover +# headers/chunks spanning many TLS records. + +SPAN_SCRATCH = 0xC800 # inside TCP_BUF, above the test scratch at $C000-$C3xx +CARRY_STUB = 0xC300 # carry-latching JSR stub +CARRY_LATCH = 0xC2F0 + + +def _feed_span(transport, labels, data, timeout=30.0): + """Feed one span to http_recv_response. + + Returns the carry flag after the call: 1 = parser needs more data, + 0 = response complete. + """ + assert len(data) <= 0x700, "span too large for scratch area" + write_bytes(transport, SPAN_SCRATCH, data) + write_bytes(transport, labels.address("http_in_mode"), [1]) + write_bytes(transport, labels.address("http_in_ptr"), + [SPAN_SCRATCH & 0xFF, SPAN_SCRATCH >> 8]) + write_bytes(transport, labels.address("http_in_len"), + [len(data) & 0xFF, len(data) >> 8]) + recv = labels.address("http_recv_response") + stub = bytes([ + 0x20, recv & 0xFF, recv >> 8, # JSR http_recv_response + 0xA9, 0x00, # LDA #0 + 0x69, 0x00, # ADC #0 (A = carry) + 0x8D, CARRY_LATCH & 0xFF, CARRY_LATCH >> 8, # STA latch + 0x60, # RTS + ]) + write_bytes(transport, CARRY_LATCH, [0xEE]) # poison the latch + write_bytes(transport, CARRY_STUB, stub) + jsr(transport, CARRY_STUB, timeout=timeout) + return read_bytes(transport, CARRY_LATCH, 1)[0] + + +def _run_span_response(transport, labels, spans): + """Reset the parser and feed a list of spans. + + Asserts carry=1 after every span except the last. Returns the carry + after the final span (0 expected for a cleanly terminated response). + """ + _reset_parser(transport, labels) + carry = None + for i, span in enumerate(spans): + carry = _feed_span(transport, labels, span) + if i < len(spans) - 1 and carry != 1: + return ("early", i, carry) + return carry + + +def _check_response(transport, labels, want_status, want_body, prefix=None): + """Common assertions: status, resp_len, body bytes. + + If prefix is not None, expect resp_len == len(prefix) and the buffer + to hold prefix (the truncate-at-capacity case); otherwise expect the + full want_body. + """ + passed = 0 + failed = 0 + expect = prefix if prefix is not None else want_body + + status_bytes = read_bytes(transport, labels.address("http_status"), 2) + status = status_bytes[0] | (status_bytes[1] << 8) + if status == want_status: + print(f" PASS: http_status = {status}") + passed += 1 + else: + print(f" FAIL: http_status = {status}, expected {want_status}") + failed += 1 + + resp_len_bytes = read_bytes(transport, labels.address("http_resp_len"), 2) + resp_len = resp_len_bytes[0] | (resp_len_bytes[1] << 8) + if resp_len == len(expect): + print(f" PASS: http_resp_len = {resp_len}") + passed += 1 + else: + print(f" FAIL: http_resp_len = {resp_len}, expected {len(expect)}") + failed += 1 + + body = read_bytes(transport, labels.address("http_resp_buf"), + min(resp_len, 512)) + if body == expect[:512]: + print(f" PASS: body matches ({len(expect[:512])} bytes)") + passed += 1 + else: + mism = [(i, body[i], expect[i]) + for i in range(min(len(body), len(expect))) + if body[i] != expect[i]] + print(f" FAIL: body mismatch — {len(mism)} byte(s) differ, " + f"len got {len(body)} vs {len(expect[:512])}") + for idx, got, exp in mism[:5]: + print(f" offset {idx}: got ${got:02X}, expected ${exp:02X}") + failed += 1 + return passed, failed + + +def _chunked(body, sizes, ext=b""): + """Encode body as chunked with the given chunk sizes.""" + out = b"" + pos = 0 + for n in sizes: + out += format(n, "x").encode() + ext + b"\r\n" + body[pos:pos+n] + b"\r\n" + pos += n + assert pos == len(body) + out += b"0\r\n\r\n" + return out + + +def test_chunked_basic(transport, labels): + """Chunked response, everything in one span, lowercase hex sizes.""" + body = b"Hello!!!" + resp = (b"HTTP/1.1 200 OK\r\n" + b"Transfer-Encoding: chunked\r\n" + b"\r\n" + b"5\r\nHello\r\n3\r\n!!!\r\n0\r\n\r\n") + carry = _run_span_response(transport, labels, [resp]) + if carry != 0: + print(f" FAIL: parser did not complete (carry={carry})") + return 0, 1 + print(" PASS: parser completed on terminal chunk") + p, f = _check_response(transport, labels, 200, body) + chunked = read_bytes(transport, labels.address("http_chunked"), 1)[0] + if chunked == 1: + print(" PASS: http_chunked flag set") + p += 1 + else: + print(f" FAIL: http_chunked = {chunked}, expected 1") + f += 1 + return p + 1, f + + +def test_chunked_multi_record(transport, labels): + """Chunks + headers split across spans: header name split mid-word, + chunk data split mid-chunk, uppercase hex size, chunk extension, + terminal-chunk CRLF split between spans.""" + body = b"0123456789" + bytes(range(0x41, 0x51)) # 10 + 16 = 26 bytes + wire = (b"HTTP/1.1 200 OK\r\n" + b"Transfer-Enco") + wire2 = (b"ding: chunked\r\n" + b"X-Other: yes\r\n" + b"\r\n" + b"A\r\n" + body[:4]) + wire3 = body[4:10] + b"\r\n" + b"10;ext=1\r\n" + body[10:18] + wire4 = body[18:26] + b"\r\n" + b"0\r\n" + wire5 = b"\r\n" + carry = _run_span_response(transport, labels, + [wire, wire2, wire3, wire4, wire5]) + if carry != 0: + print(f" FAIL: parser did not complete cleanly (result={carry})") + return 0, 1 + print(" PASS: 5-span chunked response completed on terminal chunk") + p, f = _check_response(transport, labels, 200, body) + return p + 1, f + + +def test_chunked_truncation(transport, labels): + """Chunked body larger than the 512 B http_resp_buf: the parser must + fill the buffer, keep consuming/discarding, and still terminate + cleanly on the terminal chunk with http_resp_len = 512.""" + body = bytes((i * 7 + 3) & 0xFF for i in range(700)) + wire = (b"HTTP/1.1 200 OK\r\n" + b"Transfer-Encoding: chunked\r\n" + b"\r\n" + _chunked(body, [100] * 7)) + # split into ~400-byte spans (multi-record chunks) + spans = [wire[i:i+400] for i in range(0, len(wire), 400)] + carry = _run_span_response(transport, labels, spans) + if carry != 0: + print(f" FAIL: parser did not complete cleanly (result={carry})") + return 0, 1 + print(f" PASS: {len(spans)}-span oversized chunked response completed") + p, f = _check_response(transport, labels, 200, body, prefix=body[:512]) + return p + 1, f + + +def _big_headers(count=24): + """A >1.5 KB header block: filler headers, several lines longer than + the 32-byte line buffer, Content-Length-lookalike prefixes, mixed + case.""" + hdrs = b"HTTP/1.1 200 OK\r\n" + hdrs += b"Server: test-rig/1.0\r\n" + # a CSP-style monster line (~200 bytes) + hdrs += (b"Content-Security-Policy: default-src 'none'; " + + b"script-src " + b"https://example.invalid " * 6 + + b"; style-src 'unsafe-inline'\r\n") + # a header whose 31-byte truncated prefix must NOT match Content-Length + hdrs += b"Content-Length-Hint: not-a-real-header-9999\r\n" + for i in range(count): + hdrs += (f"X-Filler-Header-{i:02d}: ".encode() + + b"v" * 40 + b"\r\n") + return hdrs + + +def test_large_headers_content_length(transport, labels): + """Header block far larger than http_resp_buf, split across many + spans, with Content-Length buried mid-block.""" + body = b"Hello" + hdrs = _big_headers() + hdrs += b"Content-Length: 5\r\n" + hdrs += _big_headers(8)[len(b"HTTP/1.1 200 OK\r\n"):] # more filler after + wire = hdrs + b"\r\n" + body + assert len(hdrs) > 1500, f"header block only {len(hdrs)} bytes" + spans = [wire[i:i+200] for i in range(0, len(wire), 200)] + carry = _run_span_response(transport, labels, spans) + if carry != 0: + print(f" FAIL: parser did not complete cleanly (result={carry})") + return 0, 1 + print(f" PASS: {len(spans)}-span large-header response completed " + f"({len(hdrs)} header bytes)") + p, f = _check_response(transport, labels, 200, body) + return p + 1, f + + +def test_chunked_large_headers(transport, labels): + """Chunked + large headers combined — the realistic github.com shape.""" + body = b"chunked page" + hdrs = _big_headers(16) + hdrs += b"Transfer-Encoding: chunked\r\n" + wire = hdrs + b"\r\n" + _chunked(body, [16, 9]) + spans = [wire[i:i+250] for i in range(0, len(wire), 250)] + carry = _run_span_response(transport, labels, spans) + if carry != 0: + print(f" FAIL: parser did not complete cleanly (result={carry})") + return 0, 1 + print(f" PASS: {len(spans)}-span chunked+large-header response completed") + p, f = _check_response(transport, labels, 200, body) + return p + 1, f def _run_parser_loop(transport, labels, timeout=30.0): @@ -408,6 +641,31 @@ def run_tests(transport, labels, verbose=False): total_passed += p total_failed += f + print("\n--- Chunked: basic (single span) ---") + p, f = test_chunked_basic(transport, labels) + total_passed += p + total_failed += f + + print("\n--- Chunked: multi-record spans ---") + p, f = test_chunked_multi_record(transport, labels) + total_passed += p + total_failed += f + + print("\n--- Chunked: >512 B truncation ---") + p, f = test_chunked_truncation(transport, labels) + total_passed += p + total_failed += f + + print("\n--- Large headers + Content-Length ---") + p, f = test_large_headers_content_length(transport, labels) + total_passed += p + total_failed += f + + print("\n--- Chunked + large headers ---") + p, f = test_chunked_large_headers(transport, labels) + total_passed += p + total_failed += f + return total_passed, total_failed From 981ebe399859874b3a4ba8558bc6c740e7da9811 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:35:11 -0400 Subject: [PATCH 05/33] feat(uci): W5 live-server HTTPS rig (rig_https_live.py) TLS 1.3 handshake + GET against a real public server (HTTPS_TARGET, default github.com) through the UCI backend. No inline listener, no repo certs; pass criteria are C64-side only: sentinel + http_get C=0 + http_status == EXPECT_STATUS (default 200). Follows every established hardware convention: turbo set BEFORE reset/run_prg with the read-skip-write pattern, preflight_reu() under the DeviceLock right after enable_uci (the comb PRG claims REU bank 2), all DMA addresses from build_policy_and_arbiter_with_overlay_carveout(), and a rig_ name so pytest never collects it. C64_INIT_WAIT defaults to 75 s on a comb build (ec_precompute_256 boot pass), detected from build/labels.txt via the banks equate + ec_precompute symbol union. PHASE_TIMING defaults ON here (opt-in in the local rig) so live runs produce the W0 phase table; artifacts are kept on PASS by default. The target reaches the C64 via http_host_ptr DMA -> http_get, whose prologue copies the hostname into tls_hostname for SNI, so the rig works with today's PRG and does not depend on W3's HTTPS_HOST build define. en.wikipedia.org is selectable but warned about: its 1636 B leaf exceeds the 1536 B cert_buf. Hardware-free coverage: --selfcheck exercises env handling, the comb init-wait derivation, and the trampoline's opcode structure (verified against both synthetic labels and a real comb build's labels.txt). Co-Authored-By: Claude Fable 5 --- tools/uci/rig_https_live.py | 724 ++++++++++++++++++++++++++++++++++++ 1 file changed, 724 insertions(+) create mode 100644 tools/uci/rig_https_live.py diff --git a/tools/uci/rig_https_live.py b/tools/uci/rig_https_live.py new file mode 100644 index 00000000..9f6fa4df --- /dev/null +++ b/tools/uci/rig_https_live.py @@ -0,0 +1,724 @@ +#!/usr/bin/env python3 +"""W5 LIVE HTTPS: full TLS 1.3 handshake + GET against a REAL public server +through the UCI backend on a real U64E / C64 Ultimate. + +This is the live-server counterpart of ``rig_https_local.py``: there is no +inline listener, no repo cert, and no server-side evidence — the target is a +third-party host on the open internet, so **pass criteria come from C64-side +state only**: the sentinel fires, ``http_get`` returns C=0 (the TLS 1.3 +handshake completed and the response parsed), and ``http_status`` equals the +expected status (200 — both sprint targets return 200 at ``/``). + +Targets (2026-08-21 re-probe, ``tools/probe_server_tls.py``, 2 runs each): + + target max record silence tolerance leaf flight + github.com 529 B 90 s (2/2) 1010 B 11 recs + browserleaks.com 529 B 45 s (2/2) 952 B 12 recs + en.wikipedia.org 529 B ~120 s (2/2) 1636 B 14 recs + +en.wikipedia.org's leaf (1636 B) exceeds the 1536 B ``cert_buf``; the rig +warns loudly when it is selected but does not refuse — the run is how the +"certificate too large" error path gets exercised once W2 lands. + +How the target reaches the C64: this rig programs ``http_host_ptr`` / +``http_host_len`` over DMA and calls ``http_get``, whose prologue copies the +hostname into ``tls_hostname`` for SNI (src/http.s:95-111) and resolves it +via UCI firmware DNS — so any hostname works with today's PRG, unmodified. +The PRG's *menu-driven* 'G' path is a different story: its host is baked in +(``www.foo.bar``, src/boot.s) until W3's ``make HTTPS_HOST=`` / +``HTTPS_HOST_LEN`` build defines land. This rig does not depend on W3. + +Environment variables: + U64_HOST — device IP (default 192.168.1.81) + HTTPS_TARGET — hostname to dial (default github.com) + HTTPS_TARGET_PORT — TCP port (default 443) + EXPECT_STATUS — expected http_status (default 200) + TURBO_MHZ — C64 CPU MHz (default 48); timeouts auto-scale + by 48/TURBO_MHZ exactly like rig_https_local + C64_INIT_WAIT — boot/auto-init wait seconds. Default is derived + from build/labels.txt: 75 for a comb build (the + ec_precompute_256 boot pass is ~45 s at 48 MHz — + the local rig's 22 s default would poll a machine + still precomputing), 22 otherwise. + SENTINEL_POLL_TIMEOUT — C64-side completion budget (default 300 * scale) + PHASE_TIMING — default **1** here (opt-in in the local rig): a + live run is a milestone and the W0 phase table is + half its value. Set 0 to disable. + PHASE_POLL — phase-poll interval seconds (default 0.5) + UCI_DEBUG_DIR — artifact base dir (default /tmp/uci_https_live_debug) + KEEP_DEBUG_ON_PASS — default **1** here (0 in the local rig): live-run + artifacts (phase table, TLS snapshot) are the + deliverable, keep them even on PASS. + TURBO_SETTLE — settle after a real turbo config write (default 3.0) + C64_SKIP_REU_PREFLIGHT— skip the REU preflight (see _reu_preflight.py) + +Hardware conventions honored (see CLAUDE.md "UCI rig scripts"): + * turbo is set BEFORE reset/run_prg, with the read-skip-write pattern + (a mid-session or even redundant config write drops the next UCI + command — UCI_ERR_NO_SOCKET on the first TCP_CONNECT); + * ``preflight_reu()`` runs under the DeviceLock right after enable_uci + (the comb build claims REU bank 2 — LIB_NISTCURVES_REU_BANKS_USED=$04 — + so a device with the REU disabled must be refused before the run); + * every DMA address comes from ``build_policy_and_arbiter_with_overlay_ + carveout()`` — nothing is hardcoded; + * the filename is ``rig_*.py`` so pytest never collects it. + +Deliberately NOT carried over from rig_https_local: the 6510 bus-stream +DebugCapture. Live failures are expected to be protocol/timing-shaped +(server RST during a crypto stall, cert too large), which the phase table, +``tls_state_dump.json``, ``ring.bin`` and the screen dump diagnose; run +rig_https_local for bus-level post-mortems. + +Offline structural check (no hardware, no DeviceLock): + ./rig_https_live.py --selfcheck +""" +from __future__ import annotations + +import os +import sys +import time +from pathlib import Path + +from c64_test_harness.backends.device_lock import DeviceLock, DeviceLockTimeout +from c64_test_harness.backends.ultimate64 import Ultimate64Transport +from c64_test_harness.backends.ultimate64_client import Ultimate64Client +from c64_test_harness.backends.ultimate64_helpers import ( + set_turbo_mhz, + runner_health_check, + Ultimate64RunnerStuckError, + CAT_U64_SPECIFIC, + cpu_speed_enum, +) +from c64_test_harness.uci_network import enable_uci, disable_uci +from c64_test_harness.keyboard import send_text +from c64_test_harness.labels import Labels + +from _memory_policy import build_policy_and_arbiter_with_overlay_carveout +from _reu_preflight import ReuPreflightError, preflight_reu + +# Pure helpers shared with the local rig (all side-effect free; importing +# the module only evaluates env defaults, it drives no hardware). +from rig_https_local import ( + _create_run_dir, + _current_phase, + _decode_screen_ram, + _dump_diag, + _dump_ring, + _dump_tls_state_snapshot, + _format_phase_log, + _prune_old_run_dirs, + _write_run_info, +) + +HOST = os.environ.get("U64_HOST", "192.168.1.81") +REPO_ROOT = Path(__file__).resolve().parents[2] +PRG_PATH = REPO_ROOT / "build" / "c64-https.prg" +LABELS_PATH = REPO_ROOT / "build" / "labels.txt" + +HTTPS_TARGET = os.environ.get("HTTPS_TARGET", "github.com") +HTTPS_TARGET_PORT = int(os.environ.get("HTTPS_TARGET_PORT", "443")) +EXPECT_STATUS = int(os.environ.get("EXPECT_STATUS", "200")) + +TURBO_MHZ = int(os.environ.get("TURBO_MHZ", "48")) +_TIMEOUT_SCALE = max(1.0, 48.0 / float(TURBO_MHZ)) +SENTINEL_POLL_TIMEOUT = float( + os.environ.get("SENTINEL_POLL_TIMEOUT", str(300.0 * _TIMEOUT_SCALE)) +) + +SENTINEL_VALUE = 0xC4 + +UCI_DEBUG_BASE_DIR = Path( + os.environ.get("UCI_DEBUG_DIR", "/tmp/uci_https_live_debug") +) +UCI_DEBUG_KEEP = 5 +# Live-run divergence from the local rig: artifacts are kept on PASS by +# default — the phase table IS the milestone record. +UCI_DEBUG_KEEP_ON_PASS = os.environ.get("KEEP_DEBUG_ON_PASS", "1") != "0" + +# 2026-08-21 re-probe values; see the module docstring table. Used only to +# print margin context and to warn — never to gate a run. +_KNOWN_TARGETS: dict[str, dict] = { + "github.com": {"tolerance_s": 90, "leaf_b": 1010, "records": 11}, + "browserleaks.com": {"tolerance_s": 45, "leaf_b": 952, "records": 12}, + "en.wikipedia.org": {"tolerance_s": 120, "leaf_b": 1636, "records": 14}, +} +CERT_BUF_BYTES = 1536 # src/der_decode.s cert_buf capacity + +# Comb marker: manifest equate bit for REU bank 2 (Lim-Lee anchor table). +_BANKS_EQUATE = "LIB_NISTCURVES_REU_BANKS_USED" +_COMB_BANK_MASK = 0x04 + + +def _load_labels() -> dict[str, int]: + return dict(Labels.from_file(LABELS_PATH)) + + +def default_init_wait(labels: dict[str, int]) -> float: + """Boot-wait default for the *linked* build. + + A comb build runs ``ec_precompute_256`` at boot (~45 s at 48 MHz), so + the local rig's 22 s default would start driving a machine that is + still precomputing. Detection is a union, mirroring _reu_preflight's + philosophy: the manifest equate's bank-2 bit OR the precompute symbol — + renaming either upstream cannot silently misclassify. + """ + banks = labels.get(_BANKS_EQUATE) + if banks is not None and (banks & _COMB_BANK_MASK): + return 75.0 + if any(name.startswith("ec_precompute") for name in labels): + return 75.0 + return 22.0 + + +def build_live_routine(labels: dict[str, int], *, + routine_addr: int, + host_str_addr: int, + path_str_addr: int, + sentinel_addr: int, + progress_addr: int, + carry_flag_addr: int, + host_len: int, + port: int) -> bytes: + """Emit the 6502 trampoline that calls http_get for a live hostname. + + Identical in shape to rig_https_local's routine, but parameterised + (no module-global addresses) and with the host length baked in — the + hostname is known up front, so there is no patch offset. + """ + if not (1 <= host_len <= 63): + raise ValueError(f"host_len {host_len} out of range 1..63 " + "(tls_hostname copy guards at 63)") + code = bytearray() + + def emit(*bs: int) -> None: + code.extend(bs) + + def emit_lda_imm(v: int) -> None: + emit(0xA9, v & 0xFF) + + def emit_sta_abs(addr: int) -> None: + emit(0x8D, addr & 0xFF, (addr >> 8) & 0xFF) + + def emit_lda_abs(addr: int) -> None: + emit(0xAD, addr & 0xFF, (addr >> 8) & 0xFF) + + def emit_jsr(addr: int) -> None: + emit(0x20, addr & 0xFF, (addr >> 8) & 0xFF) + + def emit_progress(step: int) -> None: + emit_lda_imm(step) + emit_sta_abs(progress_addr) + + http_get = labels["http_get"] + http_host_ptr = labels["http_host_ptr"] + http_host_len = labels["http_host_len"] + http_path_ptr = labels["http_path_ptr"] + http_path_len = labels["http_path_len"] + http_port = labels["http_port"] + net_init = labels["net_init"] + tcp_recv_head = labels["tcp_recv_head"] + tcp_recv_tail = labels["tcp_recv_tail"] + + # 0) Bank BASIC ROM OUT so $A000-$BFFF is RAM + emit_lda_abs(0x0001) + emit(0x29, 0xFE) # AND #$FE + emit_sta_abs(0x0001) + + # Clear markers + emit_lda_imm(0x00) + emit_sta_abs(sentinel_addr) + emit_sta_abs(progress_addr) + emit_sta_abs(carry_flag_addr) + + emit_progress(0x01) + + # Re-init UCI (ensure idle state after auto-init) + emit_jsr(net_init) + + # Zero the TCP ring head/tail to flush stale boot-poll data + emit_lda_imm(0x00) + emit_sta_abs(tcp_recv_head) + emit_sta_abs(tcp_recv_head + 1) + emit_sta_abs(tcp_recv_tail) + emit_sta_abs(tcp_recv_tail + 1) + + emit_progress(0x02) + + # http_host_ptr = host_str_addr; http_host_len = host_len + emit_lda_imm(host_str_addr & 0xFF) + emit_sta_abs(http_host_ptr) + emit_lda_imm((host_str_addr >> 8) & 0xFF) + emit_sta_abs(http_host_ptr + 1) + emit_lda_imm(host_len) + emit_sta_abs(http_host_len) + + # http_path_ptr = path_str_addr; http_path_len = 1 ("/") + emit_lda_imm(path_str_addr & 0xFF) + emit_sta_abs(http_path_ptr) + emit_lda_imm((path_str_addr >> 8) & 0xFF) + emit_sta_abs(http_path_ptr + 1) + emit_lda_imm(1) + emit_sta_abs(http_path_len) + + # http_port (16-bit LE) + emit_lda_imm(port & 0xFF) + emit_sta_abs(http_port) + emit_lda_imm((port >> 8) & 0xFF) + emit_sta_abs(http_port + 1) + + emit_progress(0x03) + + # Call http_get — the REAL TLS 1.3 + HTTP code path. + emit_jsr(http_get) + + # Store carry (0=success, 1=failure) via PHP/PLA + emit(0x08) # PHP + emit(0x68) # PLA + emit_sta_abs(carry_flag_addr) + + emit_progress(0x04) + + # Write sentinel + emit_lda_imm(SENTINEL_VALUE) + emit_sta_abs(sentinel_addr) + + emit_progress(0x05) + + # Park CPU + park = routine_addr + len(code) + emit(0x4C, park & 0xFF, (park >> 8) & 0xFF) # JMP park + + return bytes(code) + + +def _print_target_context(target: str, port: int) -> None: + info = _KNOWN_TARGETS.get(target) + print(f"\nLive target : {target}:{port}") + if info is None: + print(" (not in the probed target table — no tolerance/leaf data; " + "run tools/probe_server_tls.py against it first)") + return + print(f" silence tolerance : ~{info['tolerance_s']} s " + f"(2026-08-21 probe; live servers change — re-probe before " + f"trusting)") + print(f" leaf cert : {info['leaf_b']} B " + f"(cert_buf = {CERT_BUF_BYTES} B)") + print(f" server flight : ~{info['records']} records, all <= 529 B") + if info["leaf_b"] > CERT_BUF_BYTES: + print(f" *** WARNING: leaf exceeds cert_buf by " + f"{info['leaf_b'] - CERT_BUF_BYTES} B — expect the client to " + f"abort at Certificate (this exercises the W2 'certificate " + f"too large' path; it cannot PASS until cert_buf is raised)") + + +def main() -> int: + if not PRG_PATH.is_file(): + print(f"ERROR: PRG not found at {PRG_PATH}", file=sys.stderr) + print("Run: make BACKEND=uci USE_NISTCURVES_ONCHIP_COMB=1", + file=sys.stderr) + return 2 + if not LABELS_PATH.is_file(): + print("ERROR: labels.txt not found", file=sys.stderr) + print("Run: make BACKEND=uci USE_NISTCURVES_ONCHIP_COMB=1", + file=sys.stderr) + return 2 + + labels = _load_labels() + required = [ + "http_get", "http_host_ptr", "http_host_len", + "http_path_ptr", "http_path_len", "http_port", + "net_init", "net_initialized", "uci_socket_id", + "net_last_error", "net_tcp_state", + "tcp_recv_head", "tcp_recv_tail", + "http_resp_buf", "http_resp_len", "http_status", + "tls_state", "tls_last_state", + ] + missing = [n for n in required if n not in labels] + if missing: + print(f"ERROR: missing labels: {missing}", file=sys.stderr) + return 2 + for n in sorted(required): + print(f" {n:22s} = ${labels[n]:04X}") + + init_wait = float(os.environ.get("C64_INIT_WAIT", + str(default_init_wait(labels)))) + + # DMA addresses from the arbiter — never hardcoded. Same carveout the + # local rig uses (the comb build's overlay landings fill CRYPTO_OVERLAY; + # the NET_CODE zero-fill tail is the safe scratch region). + memory_policy, arbiter = build_policy_and_arbiter_with_overlay_carveout( + LABELS_PATH, PRG_PATH, + ) + routine_addr = arbiter.alloc(256, name="trampoline") + host_str_addr = arbiter.alloc(64, name="host_str") + path_str_addr = arbiter.alloc(64, name="path_str") + sentinel_addr = arbiter.alloc(1, name="sentinel") + progress_addr = arbiter.alloc(1, name="progress") + carry_flag_addr = arbiter.alloc(1, name="carry_flag") + print(f"\nMemoryPolicy reserved {len(memory_policy.reserved_regions)}" + f" region(s); arbiter allocations:") + for base, last, note in arbiter.allocations: + print(f" ${base:04X}-${last:04X} {note}") + + _print_target_context(HTTPS_TARGET, HTTPS_TARGET_PORT) + + host_bytes = HTTPS_TARGET.encode("ascii") + routine_bytes = build_live_routine( + labels, + routine_addr=routine_addr, + host_str_addr=host_str_addr, + path_str_addr=path_str_addr, + sentinel_addr=sentinel_addr, + progress_addr=progress_addr, + carry_flag_addr=carry_flag_addr, + host_len=len(host_bytes), + port=HTTPS_TARGET_PORT, + ) + print(f"Routine size : {len(routine_bytes)} bytes @ ${routine_addr:04X}") + print(f"C64_INIT_WAIT : {init_wait:.0f} s " + f"(x{_TIMEOUT_SCALE:.0f} timeout scale at {TURBO_MHZ} MHz)") + + host_str = host_bytes + b"\x00" + path_str = b"/\x00" + prg = PRG_PATH.read_bytes() + + lock = DeviceLock(HOST) + try: + lock.acquire_or_raise(timeout=120.0) + except DeviceLockTimeout as exc: + print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) + return 2 + info = lock.read_info() + print(f"Acquired DeviceLock({HOST}); holder info: {info!r}") + + removed = _prune_old_run_dirs(UCI_DEBUG_BASE_DIR, UCI_DEBUG_KEEP) + for d in removed: + print(f"Pruning old debug artifacts: {d}") + run_dir = _create_run_dir(UCI_DEBUG_BASE_DIR) + print(f"Debug artifacts dir: {run_dir}") + + client: Ultimate64Client | None = None + uci_enabled = False + outcome = "UNKNOWN" + exit_code = 1 + run_start = time.time() + try: + client = Ultimate64Client(host=HOST, timeout=15.0) + transport = Ultimate64Transport(host=HOST, timeout=15.0, client=client) + transport.memory_policy = memory_policy + + print("Enabling UCI...") + enable_uci(client) + uci_enabled = True + + # Wedged-runner pre-detect (see rig_https_local for the rationale). + try: + runner_health_check(client) + except Ultimate64RunnerStuckError as exc: + print(f"[fatal] runner wedged at {HOST}: {exc}", file=sys.stderr) + print("[fatal] supervisor: investigate / authorize recover()", + file=sys.stderr) + return 3 + + # REU preflight (issue #97) — under the DeviceLock, right after + # enable_uci, before anything long-running. The comb build claims + # REU bank 2, so this is not optional for the sprint's own PRG. + try: + preflight_reu(client, LABELS_PATH) + except ReuPreflightError as exc: + print(str(exc), file=sys.stderr) + return 4 + + # --- Set turbo BEFORE reset/run_prg, skipping a redundant write --- + # (the config WRITE itself glitches the UCI bridge and drops the + # next TCP_CONNECT — see rig_https_local's comment block; the shape + # of the REST response is also documented there). + try: + cat = client.get_config_category(CAT_U64_SPECIFIC) + inner = cat.get(CAT_U64_SPECIFIC, cat) + cur_speed = inner.get("CPU Speed") + cur_turbo = inner.get("Turbo Control") + except Exception as exc: # probe is best-effort + print(f" (turbo state probe failed: {exc}; writing anyway)") + cur_speed = cur_turbo = None + + want_speed = str(cpu_speed_enum(TURBO_MHZ)) + if str(cur_speed) == want_speed and cur_turbo == "Manual": + print(f"Turbo already {TURBO_MHZ} MHz (Manual) — skipping " + "config write") + else: + print(f"Setting turbo to {TURBO_MHZ} MHz " + f"(from {cur_turbo}/{cur_speed})...") + set_turbo_mhz(client, TURBO_MHZ) + time.sleep(float(os.environ.get("TURBO_SETTLE", "3.0"))) + + print("Resetting machine...") + client.reset() + time.sleep(2.5) + + print("run_prg(PRG)...") + client.run_prg(prg) + print(f"Waiting {init_wait * _TIMEOUT_SCALE:.0f} s for auto-init " + "(entropy, REU init, comb precompute, DHCP)...") + time.sleep(init_wait * _TIMEOUT_SCALE) + + init_flag = transport.read_memory(labels["net_initialized"], 1)[0] + print(f"net_initialized = ${init_flag:02X}") + if init_flag == 0: + print("WARNING: net_initialized is 0 — auto-init may have failed") + + # Quit PRG main_loop back to BASIC + print("Sending 'Q' to exit PRG main_loop...") + send_text(transport, "q\r") + time.sleep(2.0 * _TIMEOUT_SCALE) + + # DMA-write the routine + data + CHUNK = 64 + for i in range(0, len(routine_bytes), CHUNK): + transport.write_memory(routine_addr + i, + routine_bytes[i:i + CHUNK]) + transport.write_memory(host_str_addr, host_str.ljust(64, b"\x00")) + transport.write_memory(path_str_addr, path_str.ljust(8, b"\x00")) + transport.write_memory(sentinel_addr, bytes(16)) + + # Trigger via SYS + sys_line = f"sys{routine_addr}\r" + print(f"Triggering: {sys_line.strip()}") + send_text(transport, sys_line) + + # --- Poll sentinel, with phase timing (default ON for live runs) --- + deadline = time.time() + SENTINEL_POLL_TIMEOUT + last_progress = -1 + start = time.time() + phase_timing = os.environ.get("PHASE_TIMING", "1") == "1" + phase_poll = float(os.environ.get("PHASE_POLL", "0.5")) + phase_log: list[tuple[str, float]] = [] + cur_phase = None + sentinel_seen = False + while time.time() < deadline: + time.sleep(phase_poll if phase_timing else 0.5) + blob = transport.read_memory(sentinel_addr, 2) + sentinel = blob[0] + progress = blob[1] + if progress != last_progress: + elapsed = time.time() - start + print(f" [{elapsed:6.1f}s] progress=0x{progress:02X}") + last_progress = progress + if phase_timing: + scr = _decode_screen_ram( + bytes(transport.read_memory(0x0400, 1000))) + phase = _current_phase(scr) + if cur_phase is None: + cur_phase = phase + elif phase != cur_phase: + cur_phase = phase + phase_log.append((phase, time.time() - start)) + print(f" phase +{phase_log[-1][1]:7.1f}s {phase}") + if sentinel == SENTINEL_VALUE: + print(" sentinel set — routine complete") + sentinel_seen = True + break + + if phase_timing: + table = _format_phase_log(phase_log) + print("\n=== Phase timing (live) ===") + print(table) + try: + (run_dir / "phase_timing.txt").write_text( + f"target={HTTPS_TARGET}:{HTTPS_TARGET_PORT} " + f"turbo={TURBO_MHZ}MHz poll={phase_poll}s\n\n{table}\n") + except Exception as exc: # diagnostics must never fail the run + print(f" (could not write phase_timing.txt: {exc})") + + # --- Diagnostics (both outcomes) --- + _dump_diag(transport, labels) + try: + _dump_ring(transport, labels, run_dir) + _dump_tls_state_snapshot(transport, labels, run_dir) + print(f" snapshots -> {run_dir}") + except Exception as exc: + print(f"WARNING: snapshot dump failed: {exc}") + + carry = transport.read_memory(carry_flag_addr, 1)[0] & 0x01 + status_raw = transport.read_memory(labels["http_status"], 2) + http_status = status_raw[0] | (status_raw[1] << 8) + resp_len_raw = transport.read_memory(labels["http_resp_len"], 2) + resp_len = resp_len_raw[0] | (resp_len_raw[1] << 8) + read_len = min(resp_len, 200) if resp_len > 0 else 200 + resp_data = bytes(transport.read_memory(labels["http_resp_buf"], + read_len)) + screen = bytes(transport.read_memory(0x0400, 1000)) + screen_text = _decode_screen_ram(screen) + + print(f"\nhttp_get carry = {carry} (0=success, 1=failure)") + print(f"http_status = {http_status}") + print(f"http_resp_len = {resp_len}") + print(f"http_resp_buf = {resp_data[:100]!r}") + print("\n--- screen RAM ---") + for line in screen_text.split("\n"): + if line.strip(): + print(f" {line}") + try: + (run_dir / "screen.txt").write_text(screen_text + "\n") + except Exception: + pass + + # --- Pass criteria: C64-side state ONLY --- + problems: list[str] = [] + if not sentinel_seen: + problems.append( + f"sentinel not set within {SENTINEL_POLL_TIMEOUT:.0f}s " + f"(progress=0x{last_progress:02X})") + else: + if carry != 0: + problems.append( + "http_get returned C=1 — TLS handshake or HTTP exchange " + "failed (see tls_state_dump.json)") + if http_status != EXPECT_STATUS: + problems.append( + f"http_status is {http_status}, expected {EXPECT_STATUS}") + + if problems: + print(f"\nFAIL against {HTTPS_TARGET}:", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + outcome = "TIMEOUT" if not sentinel_seen else "FAIL" + exit_code = 1 + return exit_code + + print(f"\nPASS: TLS 1.3 handshake completed against {HTTPS_TARGET} " + f"and http_status={http_status}") + outcome = "PASS" + exit_code = 0 + return exit_code + + finally: + try: + _write_run_info( + run_dir / "run_info.txt", + outcome=outcome, + duration=time.time() - run_start, + exit_code=exit_code, + extra={ + "target": f"{HTTPS_TARGET}:{HTTPS_TARGET_PORT}", + "turbo_mhz": TURBO_MHZ, + "host": HOST, + }, + ) + except Exception as exc: + print(f"WARNING: run_info write failed: {exc}") + print(f"\nDebug artifacts: {run_dir}") + if outcome == "PASS" and not UCI_DEBUG_KEEP_ON_PASS: + import shutil + try: + shutil.rmtree(run_dir) + print("(removed on PASS; KEEP_DEBUG_ON_PASS=1 to retain)") + except Exception as exc: + print(f"WARNING: failed to remove PASS run dir: {exc}") + + if uci_enabled and client is not None: + print("\nDisabling UCI...") + try: + disable_uci(client) + except Exception as exc: + print(f"WARNING: disable_uci failed: {exc}") + lock.release() + print(f"Released DeviceLock({HOST})") + + +# --------------------------------------------------------------------------- +# Offline structural self-check — no hardware, no DeviceLock, no REST. +# --------------------------------------------------------------------------- + +def _selfcheck() -> int: + failures: list[str] = [] + + def check(name: str, cond: bool, detail: str = "") -> None: + status = "ok" if cond else "FAIL" + print(f" [{status}] {name}" + (f" — {detail}" if detail else "")) + if not cond: + failures.append(name) + + print("rig_https_live selfcheck (offline)") + + # 1) env handling + check("HTTPS_TARGET default/env", + HTTPS_TARGET == os.environ.get("HTTPS_TARGET", "github.com")) + check("port parses as int", isinstance(HTTPS_TARGET_PORT, int)) + check("timeout scale >= 1", _TIMEOUT_SCALE >= 1.0, + f"TURBO_MHZ={TURBO_MHZ} -> x{_TIMEOUT_SCALE:g}") + + # 2) init-wait derivation + check("comb via banks equate ($04) -> 75", + default_init_wait({_BANKS_EQUATE: 0x04}) == 75.0) + check("comb via ec_precompute_256 symbol -> 75", + default_init_wait({"ec_precompute_256": 0x8000}) == 75.0) + check("onchip (banks=0, no symbol) -> 22", + default_init_wait({_BANKS_EQUATE: 0}) == 22.0) + check("reu profile (banks=$03) -> 22", + default_init_wait({_BANKS_EQUATE: 0x03}) == 22.0) + + # 3) trampoline structure against synthetic labels + fake_labels = { + "http_get": 0x6123, "http_host_ptr": 0xA801, "http_host_len": 0xA803, + "http_path_ptr": 0xA804, "http_path_len": 0xA806, + "http_port": 0xA807, "net_init": 0x2001, + "tcp_recv_head": 0xA810, "tcp_recv_tail": 0xA812, + } + addrs = dict(routine_addr=0x3C00, host_str_addr=0x3D00, + path_str_addr=0x3D40, sentinel_addr=0x3D80, + progress_addr=0x3D81, carry_flag_addr=0x3D82) + host = "github.com" + code = build_live_routine(fake_labels, **addrs, + host_len=len(host), port=443) + check("banks ROM out first", + code[:8] == bytes([0xAD, 0x01, 0x00, 0x29, 0xFE, 0x8D, 0x01, 0x00])) + jsr_http_get = bytes([0x20, 0x23, 0x61]) + check("JSR http_get present", jsr_http_get in code) + check("PHP/PLA carry latch after http_get", + code[code.index(jsr_http_get) + 3: + code.index(jsr_http_get) + 5] == b"\x08\x68") + check("host_len baked in", + bytes([0xA9, len(host), 0x8D, 0x03, 0xA8]) in code) + check("port 443 stored LE", + bytes([0xA9, 443 & 0xFF, 0x8D, 0x07, 0xA8, 0xA9, 443 >> 8]) in code) + check("sentinel value written", + bytes([0xA9, SENTINEL_VALUE]) in code) + # The park JMP targets its own address (spin in place), which is the + # routine base plus the code length *before* the 3-byte JMP itself. + park = addrs["routine_addr"] + len(code) - 3 + check("parks with JMP self", + code[-3:] == bytes([0x4C, park & 0xFF, (park >> 8) & 0xFF])) + check("routine fits its 256 B slot", len(code) <= 256, + f"{len(code)} bytes") + + # 4) host-length guard + try: + build_live_routine(fake_labels, **addrs, host_len=64, port=443) + check("host_len > 63 rejected", False) + except ValueError: + check("host_len > 63 rejected", True) + + # 5) real build, if one exists (structural only — no hardware) + if LABELS_PATH.is_file(): + labels = _load_labels() + have = [n for n in ("http_get", "http_host_ptr", "net_init") + if n in labels] + check("real labels.txt loads", len(have) == 3, + f"{LABELS_PATH}") + print(f" (real build init wait would default to " + f"{default_init_wait(labels):.0f} s)") + else: + print(f" (no {LABELS_PATH} — real-build check skipped)") + + if failures: + print(f"\nSELFCHECK FAIL: {len(failures)} check(s): {failures}") + return 1 + print("\nSELFCHECK PASS") + return 0 + + +if __name__ == "__main__": + if "--selfcheck" in sys.argv[1:]: + raise SystemExit(_selfcheck()) + raise SystemExit(main()) From 438f7cc80327fe7dcac5b607134686c5fa09a872 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:35:21 -0400 Subject: [PATCH 06/33] =?UTF-8?q?feat(https=5Fe2e):=20p256-chain=20listene?= =?UTF-8?q?r=20profile=20=E2=80=94=20padded=203-cert=20chain=20bench?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New cert profile 'p256-chain' (kwarg or HTTPS_LISTENER_CERT_PROFILE=, mirroring the p384 selector): the existing P-256 test leaf, unchanged, plus two padded throwaway intermediates minted by chain_certs.py with the stdlib-only gen_certs toolkit. Chain total ~3.0 KB (real targets measured 2026-08-21: github 2719 B, browserleaks 3430 B, wikipedia 4114 B), which forces the server flight into 11 records of <=529 B against the C64's ClientHello — the real-record-count overhead becomes benchable with no internet egress, while the C64 still verifies the same leaf. Padding is one non-critical private-OID extension per intermediate (CHAIN_PAD_BYTES tunes it); linkage is deliberately fake — nothing on either side parses the intermediates, byte/record count is what is under test. The chain file regenerates whenever the leaf does. Host-side selftest (chain_certs.py --selftest, loopback, no hardware): stdlib ssl client confirms TLS 1.3, all 3 certs served, leaf identical to server.pem, HTTP 200; then the byte-faithful C64 ClientHello replica (probe_server_tls) confirms MFL is honored — max record 529 B, 11-record flight. PASSES on this host. Co-Authored-By: Claude Fable 5 --- tools/https_e2e/chain_certs.py | 315 ++++++++++++++++++++++++++++++ tools/https_e2e/ensure_certs.py | 26 ++- tools/https_e2e/https_listener.py | 7 +- 3 files changed, 341 insertions(+), 7 deletions(-) create mode 100644 tools/https_e2e/chain_certs.py diff --git a/tools/https_e2e/chain_certs.py b/tools/https_e2e/chain_certs.py new file mode 100644 index 00000000..c6628451 --- /dev/null +++ b/tools/https_e2e/chain_certs.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +"""Padded 3-cert chain for the local HTTPS listener — real-record-count bench. + +Why this exists (real-server sprint, bench de-risk) +--------------------------------------------------- +The local e2e listener serves ONE small self-signed cert, so its encrypted +flight is ~4 records. The real sprint targets send 3-4 certs totalling +2.7-4.1 KB, which — with max_fragment_length honored — arrives as 11-14 +records (2026-08-21 probe: github 11, browserleaks 12, en.wikipedia.org 14). +Every extra record costs the C64 a ChaCha20+Poly1305 decrypt, a SHA-256 +transcript update, and UCI ``net_poll`` round-trips at ~40 ms each. This +module makes that overhead measurable on the bench, with no internet egress: + + leaf — the EXISTING P-256 test cert (certs/server.pem), unchanged, + so the C64 still verifies CertificateVerify against it; + + two padded throwaway "intermediates" minted here, sized so the whole + chain lands in the real-target 2.7-3.5 KB window. + +The intermediates are deliberately fake: self-signed, unrelated keys, no +issuer linkage to the leaf. Nothing on either side checks linkage — Python's +``load_cert_chain`` sends the extra certs verbatim, and the C64 client +copies only the FIRST CertificateEntry into ``cert_buf`` (W2) and discards +the rest without parsing them. What is under test is the *byte count and +record count*, not PKI. Do not deploy any of this anywhere real. + +Padding mechanism: each intermediate carries one non-critical private-OID +extension (1.3.6.1.4.1.55555.1) whose OCTET STRING is ``pad_bytes`` of +0x5A. That is DER-legal, ignored by every TLS stack, and lets the chain be +sized to the byte. + +Usage +----- +As a listener profile (mirrors the existing ``cert_profile="p384"`` shape): + + from https_listener import start_https_listener + h = start_https_listener(host, port, cert_profile="p256-chain") + +or via the env var the listener already honors: + + HTTPS_LISTENER_CERT_PROFILE=p256-chain python3 ... (any listener user) + +Files: ``certs/server-chain.pem`` (leaf + 2 intermediates) served with the +existing ``certs/server.key``. The chain file is rebuilt whenever it is +missing or older than ``server.pem`` (leaf regeneration invalidates it). + +``CHAIN_PAD_BYTES`` (env or kwarg) tunes the per-intermediate padding; +the default lands the chain at ~3.2 KB, mid-window. + +Host-side selftest (no C64, no hardware): + + python3 tools/https_e2e/chain_certs.py --selftest + +serves the chain on loopback, checks with a stdlib ``ssl`` client that all +3 certificates are sent and the leaf is byte-identical to server.pem, then +replays the C64's ClientHello replica (tools/probe_server_tls.py) to +confirm max_fragment_length is honored: every flight record <= 548 B +(``TLS_REC_BUF_MAX``) and the flight is >= 8 records. +""" +from __future__ import annotations + +import datetime +import os +import secrets +import sys +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +REPO_ROOT = _HERE.parents[1] +GEN_DIR = REPO_ROOT / "tools" / "package" / "listener" +TOOLS_DIR = REPO_ROOT / "tools" + +CHAIN_PEM_NAME = "server-chain.pem" + +#: Per-intermediate padding (bytes of the private-OID extension payload). +#: Two intermediates at (~420 B base + pad) plus the ~480 B leaf lands the +#: chain around 3.2 KB — the middle of the real-target 2.7-3.5 KB window. +DEFAULT_PAD_BYTES = int(os.environ.get("CHAIN_PAD_BYTES", "950")) + +#: Private enterprise arc — payload is opaque padding, never interpreted. +_PAD_OID = "1.3.6.1.4.1.55555.1" + + +def _gen_certs(): + """Import the stdlib-only DER/cert toolkit the listener package ships.""" + if str(GEN_DIR) not in sys.path: + sys.path.insert(0, str(GEN_DIR)) + import gen_certs # noqa: PLC0415 + return gen_certs + + +def build_padded_intermediate(cn: str, pad_bytes: int) -> bytes: + """Mint one self-signed throwaway P-256 cert of ~(420 + pad_bytes) B. + + Returns the DER. The key is generated and discarded — these certs are + padding, not signers of anything. + """ + g = _gen_certs() + crv = g.CURVES["p256"] + + d = secrets.randbelow(crv.n - 1) + 1 + qx, qy = g._mul(d, (crv.gx, crv.gy), crv.p, crv.a) + pub_point = (b"\x04" + qx.to_bytes(crv.size, "big") + + qy.to_bytes(crv.size, "big")) + + name = g._seq(g._set(g._seq(g._oid(g.OID_COMMON_NAME), g._utf8(cn)))) + now = datetime.datetime.now(datetime.timezone.utc) + validity = g._seq( + g._utctime(now - datetime.timedelta(minutes=5)), + g._utctime(now + datetime.timedelta(days=3650)), + ) + sig_alg = g._seq(g._oid(g.OID_ECDSA_SHA256)) + spki = g._seq( + g._seq(g._oid(g.OID_EC_PUBLIC_KEY), g._oid(g.OID_PRIME256V1)), + g._bitstring(pub_point), + ) + # One non-critical private-OID extension carrying opaque padding. + extensions = g._explicit(3, g._seq( + g._seq(g._oid(_PAD_OID), g._octetstring(b"\x5a" * pad_bytes)), + )) + tbs = g._seq( + g._explicit(0, g._int(2)), + g._int(secrets.randbits(159) | 1), + sig_alg, + name, # issuer == subject (throwaway self-signed) + validity, + name, + spki, + extensions, + ) + signature = g._ecdsa_sign(crv.hasher(tbs).digest(), d, crv) + return g._seq(tbs, sig_alg, g._bitstring(signature)) + + +def ensure_chain_certs(certs_dir: Path | None = None, + force: bool = False, + pad_bytes: int | None = None, + quiet: bool = False) -> tuple[Path, Path]: + """Return (chain_pem, key) for the p256-chain profile, building if stale. + + The leaf pair is ensured first (delegates to ensure_certs "p256"); the + chain file is (re)built when absent, older than the leaf, or *force*. + """ + if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) + from ensure_certs import ensure_certs # noqa: PLC0415 + + leaf_pem, key_path = ensure_certs("p256", certs_dir, force=force, + quiet=quiet) + chain_pem = leaf_pem.parent / CHAIN_PEM_NAME + + stale = ( + force + or not chain_pem.is_file() + or chain_pem.stat().st_mtime < leaf_pem.stat().st_mtime + ) + if not stale: + return chain_pem, key_path + + pad = DEFAULT_PAD_BYTES if pad_bytes is None else pad_bytes + g = _gen_certs() + leaf_bytes = leaf_pem.read_bytes() + ders = [build_padded_intermediate(f"C64 Test Padding Intermediate {i}", + pad) + for i in (1, 2)] + pems = [g._pem("CERTIFICATE", der) for der in ders] + chain_pem.write_bytes(leaf_bytes + b"".join(pems)) + + if not quiet: + import base64 # noqa: PLC0415 + leaf_der_len = len(base64.b64decode("".join( + line for line in leaf_bytes.decode().splitlines() + if not line.startswith("-")))) + total = leaf_der_len + sum(len(d) for d in ders) + print(f"wrote {chain_pem}") + print(f" leaf = {leaf_der_len} B DER (certs/server.pem, " + f"unchanged — the C64 still verifies it)") + for i, der in enumerate(ders, 1): + print(f" intermediate{i} = {len(der)} B DER (padded, throwaway)") + print(f" chain total = {total} B DER " + f"(real targets: 2719-4114 B)") + return chain_pem, key_path + + +# --------------------------------------------------------------------------- +# Selftest — host-only proof that the chain serves and MFL keeps records +# under TLS_REC_BUF_MAX. No C64, no hardware, loopback only. +# --------------------------------------------------------------------------- + +def _selftest() -> int: + import base64 + import socket + import ssl + + if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) + if str(TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(TOOLS_DIR)) + from https_listener import start_https_listener, stop_https_listener + import probe_server_tls + + failures: list[str] = [] + + def check(name: str, cond: bool, detail: str = "") -> None: + print(f" [{'ok' if cond else 'FAIL'}] {name}" + + (f" — {detail}" if detail else "")) + if not cond: + failures.append(name) + + print("chain_certs selftest (host-only, loopback)") + chain_pem, key_path = ensure_chain_certs() + leaf_der = base64.b64decode("".join( + line for line in + (chain_pem.parent / "server.pem").read_text().splitlines() + if not line.startswith("-"))) + + handle = start_https_listener(host="127.0.0.1", port=0, + cert_profile="p256-chain") + # A raw probe below deliberately abandons its handshake; keep the + # resulting per-connection SSL errors out of the selftest output. + handle.server.handle_error = lambda *a: None + port = handle.server.server_port + print(f" listener on 127.0.0.1:{port} serving {handle.cert_path}") + try: + # --- 1. stdlib ssl client: chain content ------------------------- + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + ctx.minimum_version = ssl.TLSVersion.TLSv1_3 + with socket.create_connection(("127.0.0.1", port), timeout=10) as s: + with ctx.wrap_socket(s, server_hostname="www.foo.bar") as tls: + check("TLS 1.3 negotiated", tls.version() == "TLSv1.3", + str(tls.version())) + peer_leaf = tls.getpeercert(binary_form=True) + check("served leaf == certs/server.pem", + peer_leaf == leaf_der, + f"{len(peer_leaf)} B") + if hasattr(tls, "get_unverified_chain"): + chain = tls.get_unverified_chain() or [] + # Depending on the Python build this yields Certificate + # objects or raw DER bytes — normalize to DER. + ders = [c if isinstance(c, (bytes, bytearray)) + else c.public_bytes(ssl._ssl.ENCODING_DER) + for c in chain] + total = sum(len(d) for d in ders) + check("3 certificates served", len(ders) == 3, + f"per-cert {[len(d) for d in ders]}") + check("chain total in 2.7-3.5 KB window", + 2700 <= total <= 3500, f"{total} B") + else: + print(" (get_unverified_chain unavailable on this " + "Python — chain count unchecked)") + tls.sendall(b"GET / HTTP/1.0\r\nHost: www.foo.bar\r\n\r\n") + resp = b"" + try: + while True: + b = tls.recv(4096) + if not b: + break + resp += b + except (ssl.SSLError, OSError): + pass + check("HTTP 200 over the chain", + resp.startswith(b"HTTP/1.0 200"), resp[:30].decode( + "ascii", "replace")) + + # --- 2. C64 ClientHello replica: record sizing ------------------- + # probe_server_tls sends the byte-faithful CH (single suite, X25519, + # MFL=512) and splits the server flight into records. cap=1 keeps + # the silent-tolerance leg short — sizing is all we want here. + r = probe_server_tls.probe("127.0.0.1", port, send_mfl=True, cap=1) + check("server accepted the C64's narrow ClientHello", + not r.get("alert", True) and r.get("flight", 0) > 0, + f"tolerance={r.get('tolerance')}") + check("MFL honored: max record <= 548 B " + "(TLS_REC_BUF_MAX; ~529 B expected)", + 0 < r.get("max_record", 0) <= probe_server_tls.TLS_REC_BUF_MAX, + f"max={r.get('max_record')} B") + check("flight >= 8 records (real targets: 11-14)", + r.get("records", 0) >= 8, + f"{r.get('records')} records, {r.get('flight')} B flight") + finally: + stop_https_listener(handle) + + if failures: + print(f"\nSELFTEST FAIL: {failures}") + return 1 + print("\nSELFTEST PASS") + return 0 + + +def main(argv: list[str] | None = None) -> int: + import argparse + p = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--selftest", action="store_true", + help="serve the chain on loopback and verify it " + "(stdlib ssl client + C64 ClientHello replica)") + p.add_argument("--force", action="store_true", + help="rebuild the chain file even if fresh") + p.add_argument("--pad-bytes", type=int, default=None, + help=f"per-intermediate padding (default " + f"{DEFAULT_PAD_BYTES}, or CHAIN_PAD_BYTES env)") + args = p.parse_args(argv) + + if args.selftest: + return _selftest() + chain, key = ensure_chain_certs(force=args.force, + pad_bytes=args.pad_bytes) + print(f"chain: {chain}\nkey: {key}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/https_e2e/ensure_certs.py b/tools/https_e2e/ensure_certs.py index 521331a6..225477e0 100644 --- a/tools/https_e2e/ensure_certs.py +++ b/tools/https_e2e/ensure_certs.py @@ -23,10 +23,14 @@ different curve — so the in-tree tests now have no third-party dependency for certs either. -Two profiles, matching the listener's ``cert_profile`` selector: +Three profiles, matching the listener's ``cert_profile`` selector: - p256 (default) -> certs/server.pem + certs/server.key - p384 -> certs/server-p384.pem + certs/server-p384.key + p256 (default) -> certs/server.pem + certs/server.key + p384 -> certs/server-p384.pem + certs/server-p384.key + p256-chain -> certs/server-chain.pem + certs/server.key + (the p256 leaf + 2 padded throwaway intermediates, + ~3.2 KB total — the real-record-count bench; see + chain_certs.py) Usable directly, too: @@ -43,12 +47,14 @@ CN = "www.foo.bar" SANS = ["foo.bar", "www.foo.bar"] -PROFILES = ("p256", "p384") +PROFILES = ("p256", "p384", "p256-chain") -# Filenames per profile — must stay in step with gen_certs.CURVE_PROFILES. +# Filenames per profile — must stay in step with gen_certs.CURVE_PROFILES +# (and, for the chain profile, chain_certs.CHAIN_PEM_NAME + the p256 key). _FILENAMES = { "p256": ("server.pem", "server.key"), "p384": ("server-p384.pem", "server-p384.key"), + "p256-chain": ("server-chain.pem", "server.key"), } @@ -76,6 +82,16 @@ def ensure_certs(profile: str = "p256", """ cert_path, key_path = cert_paths(profile, certs_dir) + if profile == "p256-chain": + # Delegates to chain_certs, which ensures the p256 leaf pair first + # and handles its own staleness (the chain must be rebuilt whenever + # the leaf regenerates, not only when the chain file is absent). + here = str(Path(__file__).resolve().parent) + if here not in sys.path: + sys.path.insert(0, here) + from chain_certs import ensure_chain_certs # noqa: PLC0415 + return ensure_chain_certs(certs_dir, force=force, quiet=quiet) + if cert_path.is_file() and key_path.is_file() and not force: return cert_path, key_path diff --git a/tools/https_e2e/https_listener.py b/tools/https_e2e/https_listener.py index cf42bea2..165f2871 100644 --- a/tools/https_e2e/https_listener.py +++ b/tools/https_e2e/https_listener.py @@ -4,11 +4,14 @@ returns a fixed 200 OK with a short body. The server runs in a daemon thread so the test can drive VICE in the main thread. -The TLS layer uses a self-signed ECDSA certificate. Two cert profiles +The TLS layer uses a self-signed ECDSA certificate. Three cert profiles are available: "p256" (default) -- P-256 / ecdsa-with-SHA256 "p384" -- P-384 / ecdsa-with-SHA384 + "p256-chain" -- the p256 leaf + 2 padded throwaway intermediates + (~3.2 KB chain, 11+ flight records at MFL 512) — + the real-record-count bench; see chain_certs.py Both are generated lazily on first use (see ensure_certs.py) into the gitignored tools/https_e2e/certs/ directory, and reused thereafter. @@ -49,7 +52,7 @@ _CERT_PROFILE_ENV = "HTTPS_LISTENER_CERT_PROFILE" _DEFAULT_CERT_PROFILE = "p256" -_CERT_PROFILES = ("p256", "p384") +_CERT_PROFILES = ("p256", "p384", "p256-chain") DEFAULT_RESPONSE_BODY = "HELLO FROM HTTPS TEST SERVER" From 8c577d562bf75ec079fd621fafe3a58018cca25f Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:35:00 -0400 Subject: [PATCH 07/33] fix(uci): rig_https_local TIMEOUT path lost its while/else binding in W0 The W0 phase-timing commit (13b5063) inserted an 'if phase_timing:' block between the sentinel poll loop and its 'else:', silently rebinding the else from the while (runs only on exhaustion) to the if. Result at 13b5063: a successful default-mode run (PHASE_TIMING unset) reported TIMEOUT and exited 1, and a PHASE_TIMING=1 run that genuinely timed out skipped the TIMEOUT diagnostics entirely. Track the break explicitly (sentinel_seen) and route the TIMEOUT path off that flag, independent of phase timing. Co-Authored-By: Claude Fable 5 --- tools/uci/rig_https_local.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/uci/rig_https_local.py b/tools/uci/rig_https_local.py index 33c54d6c..8ce272fe 100644 --- a/tools/uci/rig_https_local.py +++ b/tools/uci/rig_https_local.py @@ -1630,6 +1630,7 @@ def main() -> int: phase_poll = float(os.environ.get("PHASE_POLL", "0.5")) phase_log: "list[tuple[str, float]]" = [] cur_phase = None + sentinel_seen = False while time.time() < deadline: time.sleep(phase_poll if phase_timing else 0.5) blob = transport.read_memory(SENTINEL_ADDR, 2) @@ -1651,7 +1652,14 @@ def main() -> int: print(f" phase +{phase_log[-1][1]:7.1f}s {phase}") if sentinel == SENTINEL_VALUE: print(" sentinel set — routine complete") + sentinel_seen = True break + # NOTE: this used to be the `else:` of the while-loop (while/else — + # runs only when the loop exhausts without break). The W0 phase-timing + # commit inserted the `if phase_timing:` block between the loop and + # its `else`, silently rebinding the `else` to the `if`: a successful + # default-mode run then reported TIMEOUT, and a PHASE_TIMING=1 run + # never took the TIMEOUT path at all. Track the break explicitly. if phase_timing: table = _format_phase_log(phase_log) print("\n=== Phase timing (W0) ===") @@ -1662,7 +1670,7 @@ def main() -> int: f"turbo={TURBO_MHZ}MHz poll={phase_poll}s\n\n{table}\n") except Exception as exc: # diagnostics must never fail the run print(f" (could not write phase_timing.txt: {exc})") - else: + if not sentinel_seen: print(f"TIMEOUT: sentinel not set after " f"{SENTINEL_POLL_TIMEOUT:.0f}s " f"(progress=0x{last_progress:02X})", file=sys.stderr) From 27c53eec070267e0996151e02acfd22583ab6b53 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:38:38 -0400 Subject: [PATCH 08/33] refactor(tls): route handshake-message reads through a ZP base pointer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three encrypted-handshake handlers hardcoded tls_rec_buf as the message base (lda tls_rec_buf,y / tls_rec_buf+N in tls_cert.s, the tls_rec_buf+4 verify_data compare in tls_keyschedule.s). Introduce tls_hs_ptr at $3E-$3F — inside ZP_CRYPTO but past the ECDSA bignum claim ($22-$3C) and nistcurves_zp_ptr2 ($3D), so it survives across ecdsa_verify inside tls_handle_cert_verify — and read the message through (tls_hs_ptr),y instead. Behavior is identical: each handler resets the pointer to tls_rec_buf at entry via the new tls_hs_ptr_reset (CRYPTO_CODE — the NET_CODE region hosting TLS_CODE under UCI has ~8 B of slack), which also keeps the direct-call test drivers (test_finished_verify.py, test_tls_p384_negotiation.py) working unchanged. tls_verify_finished moves to CRYPTO_CODE for the same budget reason; TLS_CODE nets out 38 bytes smaller. This is the mechanical precursor to the W1 streaming deframer, which will point tls_hs_ptr at tls_rec_buf+offset (in-place dispatch) or at a carry buffer. Verified: UCI comb, ip65 default and ip65 onchip all link; VICE regression on the ip65 build: run_all_tests 266/266, test_finished_verify 18/18, test_tls_handshake 21/21, test_tls_p384_negotiation PASS, test_x509 PASS. Co-Authored-By: Claude Fable 5 --- src/constants.inc | 12 ++++++ src/tls_cert.s | 86 ++++++++++++++++++++++++++++++------------- src/tls_keyschedule.s | 22 ++++++++--- 3 files changed, 90 insertions(+), 30 deletions(-) diff --git a/src/constants.inc b/src/constants.inc index 21672667..9c37141f 100644 --- a/src/constants.inc +++ b/src/constants.inc @@ -106,6 +106,18 @@ x25_prev_bit = $38 ; 1 byte — previous k_t for swap x25_byte_idx = $39 ; 1 byte — byte index in scalar x25_bit_mask = $3a ; 1 byte — current bit mask +; --- TLS handshake-message base pointer (W1 streaming deframer) --- +; Points at the current handshake message; the message handlers in +; tls_cert.s / tls_keyschedule.s read the message through this pointer +; instead of hardcoding tls_rec_buf, so the streaming deframer can +; dispatch a message at any offset (tls_rec_buf + N, or a carry buffer). +; $3E-$3F is inside ZP_CRYPTO ($22-$3F) but past the ECDSA bignum claim +; ($22-$3C) and nistcurves_zp_ptr2 ($3D); ECDSA never touches $3E-$3F, +; so the pointer survives across ecdsa_verify inside +; tls_handle_cert_verify. Do NOT relocate to $36-$37: that pair looks +; free but sits inside fe25519's declared $2C-$37 claim. +tls_hs_ptr = $3e ; 2 bytes ($3E-$3F) + ; --- General pointers (shared, save/restore around ip65) --- zp_ptr = $fb ; 2 bytes ($FB-$FC) zp_temp = $fd ; 1 byte diff --git a/src/tls_cert.s b/src/tls_cert.s index 42cb42de..63f690eb 100644 --- a/src/tls_cert.s +++ b/src/tls_cert.s @@ -18,15 +18,20 @@ ; out of tls_rec_buf to avoid the 8-bit copy loop that ; truncated records >=256 B like the 352 B Certificate.) ; constants.inc: TLS_HS_CERTIFICATE, TLS_HS_CERT_VERIFY, -; zp_ptr, zp_count, zp_tmp1, zp_tmp2 +; zp_ptr, zp_count, zp_tmp1, zp_tmp2, tls_hs_ptr ; -; ZP usage: zp_ptr ($FB-$FC), zp_count ($FE), zp_tmp1 ($02), zp_tmp2 ($03) +; ZP usage: zp_ptr ($FB-$FC), zp_count ($FE), zp_tmp1 ($02), zp_tmp2 ($03), +; tls_hs_ptr ($3E-$3F) — base pointer to the current handshake +; message. Both handlers reset it to tls_rec_buf at entry via +; tls_hs_ptr_reset (W1: the streaming deframer will set it to +; an arbitrary base instead and the resets become conditional). .include "constants.inc" .export tls_handle_certificate .export x509_extract_pubkey .export tls_handle_cert_verify + .export tls_hs_ptr_reset .export cv_sig_scheme ; Phase 4b: exported for negotiation tests .import tls_rec_buf @@ -59,21 +64,42 @@ ; failure happened. .import tls_recv_sub_progress +; ============================================================================= +; tls_hs_ptr_reset - point tls_hs_ptr at tls_rec_buf +; +; The pre-deframer contract: every handshake message starts at +; tls_rec_buf[0]. Handlers call this at entry so that direct callers +; (tools/test_tls_p384_negotiation.py, tools/test_finished_verify.py — +; both drive handlers over DMA with no dispatcher in the loop) keep +; working unchanged. Lives in CRYPTO_CODE, not TLS_CODE: the NET_CODE +; region that hosts TLS_CODE under UCI has ~8 bytes of slack. +; Clobbers: A +; ============================================================================= + .segment "CRYPTO_CODE" +tls_hs_ptr_reset: + lda #tls_rec_buf + sta tls_hs_ptr+1 + rts + .segment "TLS_CODE" ; ============================================================================= ; tls_handle_certificate - Process TLS 1.3 Certificate message ; -; Input: tls_rec_buf contains the Certificate handshake message +; Input: (tls_hs_ptr) points at the Certificate handshake message +; (reset to tls_rec_buf at entry — see tls_hs_ptr_reset) ; tls_rec_len = message length ; Output: C=0 success (leaf cert pubkey extracted to ecdsa_pubkey_x/y) ; C=1 error (bad format, unsupported key type) ; ============================================================================= tls_handle_certificate: + jsr tls_hs_ptr_reset ldy #0 ; --- Verify handshake type = 11 (Certificate) --- - lda tls_rec_buf + lda (tls_hs_ptr),y cmp #TLS_HS_CERTIFICATE beq @cert_type_ok jmp @cert_error @@ -87,42 +113,42 @@ tls_handle_certificate: ; --- certificate_request_context length [4] --- ; For server Certificate, this is always 0 - lda tls_rec_buf,y + lda (tls_hs_ptr),y bne @cert_error ; non-zero context length = unexpected iny ; Y=5 ; --- certificate_list length [5-7] (24-bit, skip high byte) --- ; High byte must be 0 (certs < 64K) - lda tls_rec_buf,y + lda (tls_hs_ptr),y bne @cert_error ; cert list > 65535 bytes iny ; Y=6 - lda tls_rec_buf,y ; cert_list_len high byte (of 16-bit) + lda (tls_hs_ptr),y ; cert_list_len high byte (of 16-bit) sta cert_list_len_hi iny ; Y=7 - lda tls_rec_buf,y ; cert_list_len low byte + lda (tls_hs_ptr),y ; cert_list_len low byte sta cert_list_len_lo iny ; Y=8 ; --- First CertificateEntry --- ; cert_data length [8-10] (24-bit) - lda tls_rec_buf,y ; high byte (must be 0) + lda (tls_hs_ptr),y ; high byte (must be 0) bne @cert_error iny ; Y=9 - lda tls_rec_buf,y + lda (tls_hs_ptr),y sta cert_data_len_hi iny ; Y=10 - lda tls_rec_buf,y + lda (tls_hs_ptr),y sta cert_data_len_lo iny ; Y=11 - ; cert_data starts at tls_rec_buf + Y + ; cert_data starts at (tls_hs_ptr) + Y ; Store pointer to cert data tya clc - adc #tls_rec_buf + adc tls_hs_ptr+1 sta cert_data_ptr+1 ; Save cert_data start offset for extension skipping later @@ -156,10 +182,10 @@ tls_handle_certificate: ; Read extensions length (2 bytes) at cert_parse_pos ; For simplicity, assume offset < 256 for now (typical small certs) ldy zp_tmp1 - lda tls_rec_buf,y + lda (tls_hs_ptr),y sta cert_ext_len_hi iny - lda tls_rec_buf,y + lda (tls_hs_ptr),y sta cert_ext_len_lo iny @@ -527,18 +553,21 @@ x509_extract_pubkey: ; ============================================================================= ; tls_handle_cert_verify - Process TLS 1.3 CertificateVerify message ; -; Input: tls_rec_buf contains the CertificateVerify handshake message +; Input: (tls_hs_ptr) points at the CertificateVerify handshake message +; (reset to tls_rec_buf at entry — see tls_hs_ptr_reset) ; tls_transcript (32 bytes) = current transcript hash ; Server's public key already in ecdsa_pubkey_x/y ; Output: C=0 signature valid, C=1 invalid ; ============================================================================= tls_handle_cert_verify: + jsr tls_hs_ptr_reset ; stage marker: entered handler lda #$20 sta tls_recv_sub_progress ; --- Verify handshake type = 15 (CertificateVerify) --- - lda tls_rec_buf + ldy #0 + lda (tls_hs_ptr),y cmp #TLS_HS_CERT_VERIFY beq @cv_type_ok jmp @cv_error @@ -552,12 +581,14 @@ tls_handle_cert_verify: ; P-384 — actual P-384 verify dispatch is filled in by Phase 4a ; through the existing ecdsa_verify entry (curve_id-switched). ; cv_sig_scheme := high byte - $04, so 0 = P-256, 1 = P-384. - lda tls_rec_buf+5 + ldy #5 + lda (tls_hs_ptr),y cmp #$03 beq :+ jmp @cv_error ; low byte must be 03 for both : - lda tls_rec_buf+4 + dey ; Y=4 + lda (tls_hs_ptr),y sec sbc #$04 cmp #2 @@ -585,18 +616,23 @@ tls_handle_cert_verify: @cv_p256_path: ; --- Read signature length [6-7] (big-endian) --- - lda tls_rec_buf+6 ; high byte (expect 0) + ldy #6 + lda (tls_hs_ptr),y ; high byte (expect 0) beq :+ jmp @cv_error ; signature > 255 bytes : - lda tls_rec_buf+7 ; low byte + iny ; Y=7 + lda (tls_hs_ptr),y ; low byte sta cv_sig_len ; --- Parse DER signature into ecdsa_sig_r/s --- - ; Signature data starts at tls_rec_buf+8 - lda #<(tls_rec_buf+8) + ; Signature data starts at (tls_hs_ptr)+8 + lda tls_hs_ptr + clc + adc #8 sta zp_ptr - lda #>(tls_rec_buf+8) + lda tls_hs_ptr+1 + adc #0 sta zp_ptr+1 lda cv_sig_len sta zp_count diff --git a/src/tls_keyschedule.s b/src/tls_keyschedule.s index bad1cf4c..ad1ca095 100644 --- a/src/tls_keyschedule.s +++ b/src/tls_keyschedule.s @@ -55,7 +55,7 @@ .import tls_app_write_iv .import tls_app_read_key .import tls_app_read_iv -.import tls_rec_buf +.import tls_hs_ptr_reset .import input_buffer ; HMAC BSS state (data.asm) @@ -709,13 +709,23 @@ tls_compute_finished: ; ; Input: tls_s_hs_secret = server handshake traffic secret ; tls_transcript = transcript hash up to (but not including) Finished -; tls_rec_buf+4 = received verify_data (32 bytes, after 4-byte HS header) +; (tls_hs_ptr)+4 = received verify_data (32 bytes, after 4-byte HS +; header). The pointer is reset to tls_rec_buf at entry (see +; tls_hs_ptr_reset) so direct callers — tools/test_finished_verify.py +; drives this routine over DMA with no dispatcher in the loop — +; keep the historical tls_rec_buf+4 contract unchanged. ; Output: C=0 if match (server Finished is valid) ; C=1 if mismatch (verification failed) ; ; Computes expected verify_data and compares with received (constant-time). +; +; Placed in CRYPTO_CODE rather than TLS_CODE: the NET_CODE region that +; hosts TLS_CODE under UCI is within a few bytes of full, and this +; routine has no locality tie to the rest of the key schedule. ; ============================================================================= +.segment "CRYPTO_CODE" tls_verify_finished: + jsr tls_hs_ptr_reset ; Set hkdf_prk = server handshake traffic secret ldx #31 @vf_c1: @@ -727,16 +737,18 @@ tls_verify_finished: ; Compute expected Finished jsr tls_compute_finished - ; Constant-time comparison of tls_verify_data vs tls_rec_buf+4 + ; Constant-time comparison of tls_verify_data vs (tls_hs_ptr)+4 ; Accumulate differences in A (OR of all XOR bytes) lda #0 sta zp_tmp1 ; clear accumulator tax ; X = index + ldy #4 ; received verify_data offset @vf_cmp: - lda tls_verify_data,x - eor tls_rec_buf+4,x + lda (tls_hs_ptr),y + eor tls_verify_data,x ora zp_tmp1 ; accumulate differences sta zp_tmp1 + iny inx cpx #32 bne @vf_cmp From 766a11ee27b646a252aff150fd443cf3080afcfe Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:46:14 -0400 Subject: [PATCH 09/33] feat(tls): W1 streaming handshake-message deframer (UCI) Real servers honoring max_fragment_length deliver the encrypted flight as 11-14 records where handshake messages no longer align with records: Certificate spans ~6 records, and CertificateVerify + Finished share the tail record. The old tls_recv_encrypted dispatched exactly one message per record from tls_rec_buf[0]. New src/tls_deframe.s tracks the current message type, its 24-bit declared length and bytes consumed, and per decrypted record feeds the plaintext through: - message wholly inside the record -> dispatched IN PLACE, zero copy (tls_hs_ptr = tls_rec_buf + offset). This covers 100% of the local listener's traffic, so the existing e2e is the control. - small message spanning records -> assembled into a 280 B carry buffer, dispatched via the pointer. Oversize spanning messages are an explicit error, never an overflow. - Certificate spanning records -> rejected for now; the W2 streaming consumer lands next. Transcript discipline now holds per MESSAGE rather than per record: when a header completes the running hash is snapshotted (so CertVerify and Finished verify against the transcript excluding their own message), then the header and every body chunk are folded as consumed. Fold-as-you-go is what the streaming Certificate will require; for the other paths it is equivalent to the old fold-after-dispatch. tls_recv_encrypted still returns after ONE dispatched message, so the tls_connect state machine and its screen markers are unchanged; a record's leftover bytes are consumed on the next call before any new record is fetched. Placement: code in a new TLS_DEFRAME_CODE segment routed to CRYPTO_OVERLAY by both UCI cfgs (CRYPTO_HOT is 446 B short of holding it under the plain-uci REU archive); resident there, mutually exclusive with the overlay-embed flags exactly as LIB_NISTCURVES_MUL_CODE already is. State + carry buffer (300 B) in NET_BSS_TAIL. Gated on TLS_STREAM_DEFRAME - default ON under BACKEND=uci, OFF under ip65 (no code or BSS headroom there); the ip65 PRG is byte-identical across this commit (f557658a...). New VICE coverage: tools/test_tls_deframe.py (BACKEND=uci build, DMA-driven) - 18/18: multi-message records, splits at body/header/ header-boundary offsets, split server Finished accepted AND a corrupted split Finished rejected (per-message transcript proof), unknown-type and oversize-spanning error paths, transcript compared against a Python SHA-256 of the exact fed bytes after every scenario. All three UCI profiles (plain / onchip / comb) link with the deframer. Co-Authored-By: Claude Fable 5 --- Makefile | 17 ++ cfg/c64-https-uci-onchip.cfg | 7 + cfg/c64-https-uci.cfg | 9 + src/tls13.s | 82 ++++++ src/tls_cert.s | 10 + src/tls_deframe.s | 526 +++++++++++++++++++++++++++++++++++ src/tls_keyschedule.s | 5 + tools/test_tls_deframe.py | 377 +++++++++++++++++++++++++ 8 files changed, 1033 insertions(+) create mode 100644 src/tls_deframe.s create mode 100644 tools/test_tls_deframe.py diff --git a/Makefile b/Makefile index 9e49935e..e257ebc3 100644 --- a/Makefile +++ b/Makefile @@ -88,6 +88,23 @@ endif ifeq ($(VIC_BLANK),0) CA65FLAGS += -D NO_VIC_BLANK=1 endif +# W1 streaming handshake-message deframer (src/tls_deframe.s + the +# TLS_STREAM_DEFRAME paths in src/tls13.s / tls_cert.s / +# tls_keyschedule.s). Lets the client process a real server's encrypted +# flight where handshake messages do not align with TLS records (one +# message spanning several records; several messages sharing one +# record). Default ON under UCI, OFF under ip65: the ip65 image has +# neither the code headroom nor any BSS slack for the deframer state + +# carry buffer (~330 B). Override with TLS_STREAM_DEFRAME=0|1 — +# forcing it on under ip65 is expected to overflow the link. +ifeq ($(BACKEND),uci) +TLS_STREAM_DEFRAME ?= 1 +else +TLS_STREAM_DEFRAME ?= 0 +endif +ifeq ($(TLS_STREAM_DEFRAME),1) +CA65FLAGS += -D TLS_STREAM_DEFRAME=1 +endif # Lazy (=) so the USE_NISTCURVES_ONCHIP_COMB block below can retarget # CFG to the cfg variant after this line. LD65FLAGS = -C $(CFG) -Ln build/labels.txt -m build/c64-https.map --dbgfile build/c64-https.dbg diff --git a/cfg/c64-https-uci-onchip.cfg b/cfg/c64-https-uci-onchip.cfg index f258fbba..3a814550 100644 --- a/cfg/c64-https-uci-onchip.cfg +++ b/cfg/c64-https-uci-onchip.cfg @@ -229,6 +229,13 @@ SEGMENTS { OVERLAY_P256: load = CRYPTO_OVERLAY, type = ro, optional = yes; OVERLAY_P384: load = CRYPTO_OVERLAY, type = ro, optional = yes; + # W1 streaming handshake-message deframer (src/tls_deframe.s, + # TLS_STREAM_DEFRAME builds — default under BACKEND=uci). Same + # routing as the base uci cfg: CRYPTO_OVERLAY, resident, mutually + # exclusive with the overlay-embed flags (already forbidden in + # combination with the onchip profiles by the Makefile). + TLS_DEFRAME_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; + # Phase C.5: sibling c64-x25519 rodata tables (mul38, sqr_lo/hi, # a24_b0..b3 — ~2 KB) AND the sibling's page-aligned BSS buffers # (fe25519_tmp1..4, x25_*, mul_dma_lo/hi/carry — ~1.5 KB) ride diff --git a/cfg/c64-https-uci.cfg b/cfg/c64-https-uci.cfg index d5110a8e..167bb74d 100644 --- a/cfg/c64-https-uci.cfg +++ b/cfg/c64-https-uci.cfg @@ -215,6 +215,15 @@ SEGMENTS { OVERLAY_P256: load = CRYPTO_OVERLAY, type = ro, optional = yes; OVERLAY_P384: load = CRYPTO_OVERLAY, type = ro, optional = yes; + # W1 streaming handshake-message deframer (src/tls_deframe.s, + # TLS_STREAM_DEFRAME builds — default under BACKEND=uci). Routed to + # CRYPTO_OVERLAY because CRYPTO_HOT cannot absorb its ~700 B under + # the REU-profile archive. Resident code: mutually exclusive at + # runtime with the overlay-embed flags that page images into this + # region (EMBED_P256_OVERLAY / USE_OVERLAY_P384_EMBED), exactly as + # LIB_NISTCURVES_MUL_CODE already is under the onchip profile. + TLS_DEFRAME_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; + # Phase C.5: sibling c64-x25519 rodata tables (mul38, sqr_lo/hi, # a24_b0..b3 — ~2 KB) AND the sibling's page-aligned BSS buffers # (fe25519_tmp1..4, x25_*, mul_dma_lo/hi/carry — ~1.5 KB) ride diff --git a/src/tls13.s b/src/tls13.s index 15687c88..9efc7d15 100644 --- a/src/tls13.s +++ b/src/tls13.s @@ -92,6 +92,13 @@ .import tls_handle_certificate .import tls_handle_cert_verify +.ifdef TLS_STREAM_DEFRAME +; --- W1 streaming handshake-message deframer (tls_deframe.s) --- +.import tls_deframe_init +.import tls_deframe_new_record +.import tls_deframe_pump +.endif + ; --- Networking (net.s) --- .import net_poll @@ -186,6 +193,13 @@ tls_connect: ldy #>keys_ok_msg jsr print_string +.ifdef TLS_STREAM_DEFRAME + ; Reset the streaming deframer. The ServerHello record has been + ; fully consumed; the encrypted handshake messages begin with + ; the next record fetched by tls_recv_encrypted. + jsr tls_deframe_init +.endif + ; --- receive EncryptedExtensions (encrypted) --- lda #TLS_STATE_ENCRYPTED_EXT sta tls_state @@ -519,6 +533,72 @@ tls_recv_server_hello: ; tls_derive_traffic_keys - in tls_keyschedule.s ; tls_verify_finished - in tls_keyschedule.s +.ifdef TLS_STREAM_DEFRAME +; ============================================================================= +; tls_recv_encrypted — streaming variant (W1 deframer, tls_deframe.s). +; +; Handshake messages no longer line up with records in either direction +; (a real server's Certificate spans ~6 records; CertificateVerify and +; Finished share the tail record), so instead of dispatching "the one +; message in this record" we pump the deframer: it consumes the current +; record's unconsumed bytes first and asks for another record only when +; it runs dry. Returns once ONE handshake message has been dispatched +; successfully, so tls_connect's state machine and progress markers are +; unchanged. Transcript discipline (snapshot before dispatch, fold per +; MESSAGE — see the pre-deframer comment in the .else branch below, and +; the fold-as-you-go note in tls_deframe.s) lives inside the deframer. +; +; Output: C=0 one message dispatched, C=1 timeout/error +; ============================================================================= +tls_recv_encrypted: + lda #enc1_msg + jsr print_string + lda #0 + sta enc_timeout + sta enc_timeout+1 + lda #rx_msg + jsr print_string +@df_pump: + jsr tls_deframe_pump + bcc @df_msg_done + cmp #0 + beq @enc_wait ; A=0: record exhausted — fetch the next + jmp @enc_error ; A!=0: deframe/handler error +@enc_wait: + jsr net_poll + jsr tls_record_recv_and_decrypt + bcc @enc_got_record + inc enc_timeout + bne @enc_wait + inc enc_timeout+1 + bne @enc_wait + ; timeout + sec + rts +@enc_got_record: + lda #got2_msg + jsr print_string + ; verify inner content type is handshake + lda tls_rec_type + cmp #TLS_CT_HANDSHAKE + bne @enc_error + jsr tls_deframe_new_record + jmp @df_pump +@df_msg_done: + lda #proc_msg + jsr print_string + clc + rts +@enc_error: + sec + rts + +.else ; !TLS_STREAM_DEFRAME — one message per record (ip65 path) + ; ============================================================================= ; tls_recv_encrypted - receive encrypted handshake msg, decrypt, dispatch ; Output: C=0 success, C=1 timeout/error @@ -651,6 +731,8 @@ tls_recv_encrypted: sec rts +.endif ; TLS_STREAM_DEFRAME + ; ============================================================================= ; tls_send_finished - compute client Finished, encrypt, send ; Output: C=0 success, C=1 failure diff --git a/src/tls_cert.s b/src/tls_cert.s index 63f690eb..032be830 100644 --- a/src/tls_cert.s +++ b/src/tls_cert.s @@ -95,7 +95,12 @@ tls_hs_ptr_reset: ; C=1 error (bad format, unsupported key type) ; ============================================================================= tls_handle_certificate: +.ifndef TLS_STREAM_DEFRAME + ; Non-streaming build: every message starts at tls_rec_buf. + ; Under TLS_STREAM_DEFRAME the deframer sets tls_hs_ptr before + ; dispatch (in-place offset or carry buffer) — do not reset it. jsr tls_hs_ptr_reset +.endif ldy #0 ; --- Verify handshake type = 11 (Certificate) --- @@ -560,7 +565,12 @@ x509_extract_pubkey: ; Output: C=0 signature valid, C=1 invalid ; ============================================================================= tls_handle_cert_verify: +.ifndef TLS_STREAM_DEFRAME + ; Non-streaming build: every message starts at tls_rec_buf. + ; Under TLS_STREAM_DEFRAME the deframer sets tls_hs_ptr before + ; dispatch (in-place offset or carry buffer) — do not reset it. jsr tls_hs_ptr_reset +.endif ; stage marker: entered handler lda #$20 sta tls_recv_sub_progress diff --git a/src/tls_deframe.s b/src/tls_deframe.s new file mode 100644 index 00000000..588b9dca --- /dev/null +++ b/src/tls_deframe.s @@ -0,0 +1,526 @@ +; tls_deframe.s — W1 streaming handshake-message deframer +; +; Sits between the TLS record layer and the handshake-message handlers. +; A real server's encrypted flight does not align handshake messages +; with TLS records: with max_fragment_length honored the flight arrives +; as 11-14 records where one message (Certificate, ~2.7 KB) spans many +; records and several messages (CertificateVerify + Finished) share one +; record. The pre-deframer dispatch in tls13.s assumed exactly one +; message per record, starting at tls_rec_buf[0]. +; +; Contract (consumed by the TLS_STREAM_DEFRAME path in tls13.s): +; +; tls_deframe_init — once per handshake, before the first +; encrypted message is received. Marks the +; current record (the ServerHello) exhausted +; and clears all message state. +; tls_deframe_new_record — after tls_record_recv_and_decrypt leaves a +; fresh handshake-plaintext record in +; tls_rec_buf / tls_rec_len. +; tls_deframe_pump — consume bytes from the current record. +; Returns: +; C=0 one complete handshake message +; was dispatched successfully +; C=1, A=0 record exhausted — fetch the +; next record and pump again +; C=1, A!=0 error (code also stored in +; df_last_err) +; +; Routing per message (design settled 2026-08-16 — see the sprint plan): +; - message wholly inside the current record -> dispatch IN PLACE, +; zero copy: tls_hs_ptr = tls_rec_buf + offset. This covers 100% of +; the local-listener traffic, so the existing e2e is the control. +; - message spanning records (EE / CertVerify / Finished, all small) +; -> assembled into df_carry_buf, dispatched from there. +; - Certificate spanning records -> W2 streaming consumer +; (tls_cert_stream_* below): the leaf is copied into cert_buf, the +; rest of the chain is counted and discarded without buffering. +; +; Transcript discipline (RFC 8446 §4.4.*, historically the most +; bug-prone path here — see "Summary of recent fixes" items 1/2/4 in +; CLAUDE.md): the handlers need tls_transcript to hold the hash of +; everything BEFORE the current message. The pre-deframer code +; snapshotted before dispatch and folded the record after. Per MESSAGE +; that becomes: +; 1. when a message header completes, snapshot the running hash +; (tls_transcript_hash — non-destructive) BEFORE folding any of +; this message's bytes, +; 2. fold the header + each body chunk into the running hash as it is +; consumed (fold-as-you-go: required for the streaming Certificate, +; whose bytes are not retained; equivalent for the other paths), +; 3. dispatch the handler once the message is complete — it sees the +; pre-message snapshot in tls_transcript, exactly as before. +; +; ZP usage: zp_ptr/zp_count (transcript + copy scratch, recomputed after +; every external call), zp_tmp1/zp_tmp2 (carry/cert copy destination +; pointer), tls_hs_ptr ($3E-$3F, set before each dispatch). +; +; Memory: code in its own TLS_DEFRAME_CODE segment, routed to the +; CRYPTO_OVERLAY region by both UCI cfgs — CRYPTO_HOT cannot absorb it +; under the plain-uci cfg (446 B short with the REU-profile archive). +; CRYPTO_OVERLAY is free in every default build; the overlay-embed +; flags (EMBED_P256_OVERLAY / USE_OVERLAY_P384_EMBED) that stage +; runtime-swapped images there are mutually exclusive with a resident +; deframer, exactly as they already are with LIB_NISTCURVES_MUL_CODE. +; State + carry buffer live in the NET_BSS_TAIL region (~378 B free +; under UCI comb). +; The whole module is gated on TLS_STREAM_DEFRAME (UCI-only for now: +; ip65 has neither the code nor the BSS headroom — see the Makefile). + +.include "constants.inc" + +.ifdef TLS_STREAM_DEFRAME + +.export tls_deframe_init +.export tls_deframe_new_record +.export tls_deframe_pump +; State exported for the VICE test harness (tools/test_tls_deframe.py) +; and hardware post-mortems. +.export df_mode +.export df_hdr +.export df_hdr_have +.export df_msg_rem +.export df_rec_off +.export df_last_err +.export df_carry_buf + +.import tls_rec_buf +.import tls_rec_len +.import tls_transcript_hash +.import tls_transcript_update +.import tls_handle_certificate +.import tls_handle_cert_verify +.import tls_verify_finished +.import tls_recv_sub_progress + +; Carry buffer: 4-byte handshake header + body. EncryptedExtensions, +; CertificateVerify and Finished are all well under 256 B; Certificate +; never comes here (streamed). 280 total fits NET_BSS_TAIL's slack. +DF_CARRY_SIZE = 280 +DF_CARRY_MAX_BODY = DF_CARRY_SIZE - 4 + +; df_last_err codes (A on a C=1 return; 0 means "need more data") +DF_ERR_HDR_LEN = $01 ; 24-bit length high byte non-zero +DF_ERR_TOO_BIG = $02 ; spanning non-Certificate message > carry cap +DF_ERR_DISPATCH = $03 ; message handler returned C=1 +DF_ERR_TYPE = $04 ; unknown handshake message type + +; df_mode values +DF_MODE_HDR = 0 ; collecting the 4-byte message header +DF_MODE_CARRY = 1 ; assembling body into df_carry_buf +DF_MODE_STREAM = 2 ; streaming Certificate body (W2) + +.segment "TLS_DEFRAME_CODE" + +; ============================================================================= +; tls_deframe_init — reset all deframer state; mark current record consumed +; (tls_rec_len still holds the ServerHello record length at call time). +; ============================================================================= +tls_deframe_init: + lda #0 + sta df_mode + sta df_hdr_have + sta df_hdr_split + sta df_last_err + lda tls_rec_len + sta df_rec_off + lda tls_rec_len+1 + sta df_rec_off+1 + rts + +; ============================================================================= +; tls_deframe_new_record — a fresh decrypted handshake record sits in +; tls_rec_buf / tls_rec_len; start consuming it from offset 0. +; ============================================================================= +tls_deframe_new_record: + lda #0 + sta df_rec_off + sta df_rec_off+1 + rts + +; ============================================================================= +; tls_deframe_pump — consume bytes from the current record until one +; message completes (dispatch it) or the record is exhausted. +; See the contract in the file header for the C/A return protocol. +; ============================================================================= +tls_deframe_pump: + ; df_avail = tls_rec_len - df_rec_off + sec + lda tls_rec_len + sbc df_rec_off + sta df_avail + lda tls_rec_len+1 + sbc df_rec_off+1 + sta df_avail+1 + + lda df_mode + beq @hdr_phase + cmp #DF_MODE_CARRY + bne :+ + jmp df_carry_body +: jmp df_stream_body + +; --- header phase: collect up to 4 header bytes ---------------------------- +@hdr_phase: +@hdr_loop: + lda df_hdr_have + cmp #4 + bcs @hdr_complete + ; any record bytes left? + lda df_avail + ora df_avail+1 + bne :+ + ; record exhausted mid-header (or before one started) + lda df_hdr_have + beq @hdr_need_data + lda #1 + sta df_hdr_split +@hdr_need_data: + jmp df_need_data +: + jsr df_rec_cursor ; zp_ptr = tls_rec_buf + df_rec_off + ldy #0 + lda (zp_ptr),y + ldx df_hdr_have + sta df_hdr,x + inc df_hdr_have + ; df_rec_off++ + inc df_rec_off + bne :+ + inc df_rec_off+1 +: ; df_avail-- + lda df_avail + bne :+ + dec df_avail+1 +: dec df_avail + jmp @hdr_loop + +; --- header complete: validate, snapshot, fold, route ---------------------- +@hdr_complete: + lda #$30 + sta tls_recv_sub_progress + + ; 24-bit length high byte must be 0 (no handshake message we + ; accept exceeds 64 KB; the record layer caps well below that) + lda df_hdr+1 + beq :+ + lda #DF_ERR_HDR_LEN + jmp df_fail +: + lda df_hdr+2 + sta df_msg_len+1 + sta df_msg_rem+1 + lda df_hdr+3 + sta df_msg_len + sta df_msg_rem + + ; Snapshot the running transcript hash BEFORE any byte of this + ; message is folded — handlers (CertVerify, Finished) verify + ; against the transcript EXCLUDING their own message. + jsr tls_transcript_hash + + ; Fold the 4 header bytes into the running transcript. + lda #df_hdr + sta zp_ptr+1 + lda #4 + sta zp_count + lda #0 + sta zp_count+1 + jsr tls_transcript_update + + ; Route: in place iff the header was not split across records + ; AND the whole body is inside the current record. + lda df_hdr_split + bne @route_spanning + lda df_avail+1 + cmp df_msg_len+1 + bcc @route_spanning + bne @in_place + lda df_avail + cmp df_msg_len + bcc @route_spanning + +; --- in-place dispatch: zero copy ----------------------------------------- +@in_place: + lda #$31 + sta tls_recv_sub_progress + jsr df_rec_cursor ; zp_ptr = body start + ; message start (incl. header) = body - 4 + lda zp_ptr + sec + sbc #4 + sta tls_hs_ptr + lda zp_ptr+1 + sbc #0 + sta tls_hs_ptr+1 + ; fold the body into the running transcript + lda df_msg_len + sta zp_count + lda df_msg_len+1 + sta zp_count+1 + jsr tls_transcript_update + ; consume the body + clc + lda df_rec_off + adc df_msg_len + sta df_rec_off + lda df_rec_off+1 + adc df_msg_len+1 + sta df_rec_off+1 + jmp df_dispatch + +; --- spanning message: carry buffer (or W2 Certificate stream) ------------- +@route_spanning: + lda df_hdr ; message type + cmp #TLS_HS_CERTIFICATE + bne @route_carry + jmp df_stream_begin + +@route_carry: + ; must fit: body <= DF_CARRY_MAX_BODY + lda df_msg_len+1 + cmp #>DF_CARRY_MAX_BODY + bcc @carry_fits + bne @carry_too_big + lda df_msg_len + cmp #<(DF_CARRY_MAX_BODY+1) + bcc @carry_fits +@carry_too_big: + lda #DF_ERR_TOO_BIG + jmp df_fail +@carry_fits: + ; carry[0..3] = header + ldx #3 +: lda df_hdr,x + sta df_carry_buf,x + dex + bpl :- + lda #4 + sta df_carry_off + lda #0 + sta df_carry_off+1 + lda #DF_MODE_CARRY + sta df_mode + ; fall through into the body consumer + +; --- carry-mode body consumer ---------------------------------------------- +df_carry_body: + lda df_avail + ora df_avail+1 + bne :+ + jmp df_need_data +: + jsr df_chunk_min ; df_chunk = min(df_avail, df_msg_rem) + + ; fold the chunk (from the record) into the running transcript + jsr df_rec_cursor + lda df_chunk + sta zp_count + lda df_chunk+1 + sta zp_count+1 + jsr tls_transcript_update + + ; copy chunk: (tls_rec_buf + df_rec_off) -> (df_carry_buf + df_carry_off) + jsr df_rec_cursor + clc + lda #df_carry_buf + adc df_carry_off+1 + sta zp_tmp2 + jsr df_copy_chunk + + ; df_carry_off += chunk + clc + lda df_carry_off + adc df_chunk + sta df_carry_off + lda df_carry_off+1 + adc df_chunk+1 + sta df_carry_off+1 + + jsr df_consume_chunk ; rec_off += chunk, msg_rem -= chunk + + lda df_msg_rem + ora df_msg_rem+1 + beq :+ + jmp df_need_data ; record exhausted (chunk == avail) +: + ; message complete — dispatch from the carry buffer + lda #$32 + sta tls_recv_sub_progress + lda #df_carry_buf + sta tls_hs_ptr+1 + jmp df_dispatch + +; ============================================================================= +; df_dispatch — (tls_hs_ptr) points at a complete message (incl. 4-byte +; header, type = df_hdr[0]). Resets per-message state, runs the handler. +; ============================================================================= +df_dispatch: + lda #0 + sta df_mode + sta df_hdr_have + sta df_hdr_split + + lda df_hdr ; message type + cmp #TLS_HS_ENCRYPTED_EXT + beq @disp_ok ; accept, nothing to extract for MVP + cmp #TLS_HS_CERTIFICATE + beq @disp_cert + cmp #TLS_HS_CERT_VERIFY + beq @disp_cv + cmp #TLS_HS_FINISHED + beq @disp_fin + lda #DF_ERR_TYPE + jmp df_fail +@disp_cert: + jsr tls_handle_certificate + jmp @disp_check +@disp_cv: + jsr tls_handle_cert_verify + jmp @disp_check +@disp_fin: + ; Verify server Finished with the pre-Finished transcript + ; snapshot taken at header-complete time above. + jsr tls_verify_finished +@disp_check: + bcs @disp_fail +@disp_ok: + clc + rts +@disp_fail: + lda #DF_ERR_DISPATCH + ; fall through to df_fail + +; --- shared exits ----------------------------------------------------------- +df_fail: + sta df_last_err + sec + rts + +df_need_data: + lda #0 + sec + rts + +; ============================================================================= +; W2 streaming Certificate consumer — placeholder in W1: a Certificate +; that spans records is rejected until the streaming consumer lands. +; ============================================================================= +df_stream_begin: +df_stream_body: + lda #DF_ERR_TOO_BIG + jmp df_fail + +; ============================================================================= +; Helpers +; ============================================================================= + +; zp_ptr = tls_rec_buf + df_rec_off. Clobbers A. +df_rec_cursor: + clc + lda #tls_rec_buf + adc df_rec_off+1 + sta zp_ptr+1 + rts + +; df_chunk = min(df_avail, df_msg_rem). Clobbers A. +df_chunk_min: + lda df_avail+1 + cmp df_msg_rem+1 + bcc @use_avail + bne @use_rem + lda df_avail + cmp df_msg_rem + bcc @use_avail +@use_rem: + lda df_msg_rem + sta df_chunk + lda df_msg_rem+1 + sta df_chunk+1 + rts +@use_avail: + lda df_avail + sta df_chunk + lda df_avail+1 + sta df_chunk+1 + rts + +; Consume df_chunk record bytes into the current message: +; df_rec_off += df_chunk, df_avail -= df_chunk, df_msg_rem -= df_chunk. +df_consume_chunk: + clc + lda df_rec_off + adc df_chunk + sta df_rec_off + lda df_rec_off+1 + adc df_chunk+1 + sta df_rec_off+1 + sec + lda df_avail + sbc df_chunk + sta df_avail + lda df_avail+1 + sbc df_chunk+1 + sta df_avail+1 + sec + lda df_msg_rem + sbc df_chunk + sta df_msg_rem + lda df_msg_rem+1 + sbc df_chunk+1 + sta df_msg_rem+1 + rts + +; Copy df_chunk bytes from (zp_ptr) to (zp_tmp1). Clobbers A,X,Y and +; advances both pointers' high bytes across pages. +df_copy_chunk: + ldx df_chunk+1 + beq @tail +@page: ldy #0 +: lda (zp_ptr),y + sta (zp_tmp1),y + iny + bne :- + inc zp_ptr+1 + inc zp_tmp2 + dex + bne @page +@tail: lda df_chunk + beq @done + ldy #0 +: lda (zp_ptr),y + sta (zp_tmp1),y + iny + cpy df_chunk + bne :- +@done: rts + +; ============================================================================= +; Deframer state — NET_BSS_TAIL region (UCI: ~378 B free under comb) +; ============================================================================= +.segment "NET_BSS_TAIL" + +df_mode: .res 1 ; DF_MODE_* +df_hdr: .res 4 ; handshake header: type + 24-bit length +df_hdr_have: .res 1 ; header bytes collected (0-4) +df_hdr_split: .res 1 ; 1 = header began in an earlier record +df_msg_len: .res 2 ; declared body length +df_msg_rem: .res 2 ; body bytes still unconsumed +df_rec_off: .res 2 ; consumed offset into tls_rec_buf +df_avail: .res 2 ; unconsumed bytes in current record +df_chunk: .res 2 ; current chunk length +df_carry_off: .res 2 ; write offset into df_carry_buf +df_last_err: .res 1 ; DF_ERR_* of the last failure +df_carry_buf: .res DF_CARRY_SIZE + +.endif ; TLS_STREAM_DEFRAME diff --git a/src/tls_keyschedule.s b/src/tls_keyschedule.s index ad1ca095..4135a1d8 100644 --- a/src/tls_keyschedule.s +++ b/src/tls_keyschedule.s @@ -725,7 +725,12 @@ tls_compute_finished: ; ============================================================================= .segment "CRYPTO_CODE" tls_verify_finished: +.ifndef TLS_STREAM_DEFRAME + ; Non-streaming build: every message starts at tls_rec_buf. + ; Under TLS_STREAM_DEFRAME the deframer sets tls_hs_ptr before + ; dispatch (in-place offset or carry buffer) — do not reset it. jsr tls_hs_ptr_reset +.endif ; Set hkdf_prk = server handshake traffic secret ldx #31 @vf_c1: diff --git a/tools/test_tls_deframe.py b/tools/test_tls_deframe.py new file mode 100644 index 00000000..875c7d4f --- /dev/null +++ b/tools/test_tls_deframe.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +"""test_tls_deframe.py - W1 streaming handshake-message deframer coverage. + +Drives ``tls_deframe_init`` / ``tls_deframe_new_record`` / +``tls_deframe_pump`` (src/tls_deframe.s) directly over DMA in VICE, +feeding synthetic handshake plaintext "records" into tls_rec_buf at +record/message alignments a real server produces but the local listener +never does: + + - several messages sharing one record + - a message body split across records (carry-buffer path) + - a message HEADER split across records + - a split server Finished, positive and negative — proves the + per-MESSAGE transcript discipline: tls_verify_finished must see the + running hash EXCLUDING the Finished message itself, even when that + message arrived in two records + - a spanning non-Certificate message larger than the carry buffer + (explicit error, never an overflow) + - W2: a Certificate spanning records — leaf streamed into cert_buf, + remaining chain entries discarded, pubkey extracted; and the + explicit "certificate too large" error path + +The deframer is UCI-only (TLS_STREAM_DEFRAME — see the Makefile), so +this test builds ``BACKEND=uci``. The UCI build boots to the menu in +VICE (net_init fails non-fatally without UCI hardware) exactly like +tools/test_x25519.py's BACKEND=uci mode. + +After each successful scenario the running transcript is finalized +(tls_transcript_hash) and compared against a Python SHA-256 of every +handshake byte fed — the deframer must fold exactly the message bytes, +once each, in order, whatever the record framing was. + +Env: + C64_SKIP_BUILD=1 reuse the already-built PRG (must be BACKEND=uci) + +Requires: Python 3.10+, c64_test_harness, VICE x64sc +""" + +from __future__ import annotations + +import hashlib +import hmac +import os +import struct +import subprocess +import sys + +from c64_test_harness import ( + Labels, + ViceInstanceManager, + read_bytes, + write_bytes, + jsr, + wait_for_text, +) + +PROJECT_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from _vice_helpers import default_vice_config # noqa: E402 + +PRG_PATH = os.path.join(PROJECT_ROOT, "build", "c64-https.prg") +LABELS_PATH = os.path.join(PROJECT_ROOT, "build", "labels.txt") + +REQUIRED_LABELS = [ + "tls_deframe_init", "tls_deframe_new_record", "tls_deframe_pump", + "df_last_err", "df_carry_buf", + "tls_rec_buf", "tls_rec_len", + "tls_transcript_init", "tls_transcript_hash", "tls_transcript", + "tls_s_hs_secret", +] + +# Handshake message types +HS_EE = 8 +HS_CERT = 11 +HS_CV = 15 +HS_FIN = 20 + +# deframer error codes (src/tls_deframe.s DF_ERR_*) +ERR_HDR_LEN = 1 +ERR_TOO_BIG = 2 +ERR_DISPATCH = 3 +ERR_TYPE = 4 +ERR_CERT_FMT = 5 +ERR_CERT_TOO_BIG = 6 + +TLS_HS_PTR = 0x3E # ZP base pointer (src/constants.inc) + +# Cassette buffer (see test_finished_verify.py for the layout rationale). +# The stub is 13 bytes ($0340-$034C), so the latches sit at $0355/$0356 — +# clear of the stub and of the harness trampolines at $0334 and $0360. +STUB_ADDR = 0x0340 +C_LATCH = 0x0355 +A_LATCH = 0x0356 + + +# --------------------------------------------------------------------------- +# Python reference crypto (same helpers as test_finished_verify.py) +# --------------------------------------------------------------------------- + +def hkdf_expand_label(secret: bytes, label: bytes, context: bytes, + length: int) -> bytes: + assert length <= 32 + info = struct.pack(">H", length) + info += bytes([6 + len(label)]) + b"tls13 " + label + info += bytes([len(context)]) + context + return hmac.new(secret, info + b"\x01", hashlib.sha256).digest()[:length] + + +def finished_verify_data(traffic_secret: bytes, transcript: bytes) -> bytes: + finished_key = hkdf_expand_label(traffic_secret, b"finished", b"", 32) + return hmac.new(finished_key, transcript, hashlib.sha256).digest() + + +def hs_msg(msg_type: int, body: bytes) -> bytes: + return bytes([msg_type]) + len(body).to_bytes(3, "big") + body + + +SECRET = hashlib.sha256(b"c64-https lane A deframe secret").digest() + + +# --------------------------------------------------------------------------- +# C64 plumbing +# --------------------------------------------------------------------------- + +def install_stub(transport, pump_addr: int) -> None: + """jsr pump; latch A (sta abs leaves flags alone), then latch carry.""" + stub = bytes([ + 0x20, pump_addr & 0xFF, pump_addr >> 8, # jsr tls_deframe_pump + 0x8D, A_LATCH & 0xFF, A_LATCH >> 8, # sta A_LATCH + 0xA9, 0x00, # lda #0 + 0x2A, # rol a (carry -> bit 0) + 0x8D, C_LATCH & 0xFF, C_LATCH >> 8, # sta C_LATCH + 0x60, # rts + ]) + write_bytes(transport, STUB_ADDR, stub) + if read_bytes(transport, STUB_ADDR, len(stub)) != stub: + raise RuntimeError("pump stub readback mismatch") + + +class Rig: + def __init__(self, transport, labels): + self.t = transport + self.l = labels + + def reset(self): + """Fresh transcript + deframer state (record marked consumed).""" + write_bytes(self.t, self.l["tls_rec_len"], b"\x00\x00") + jsr(self.t, self.l["tls_transcript_init"], timeout=30.0) + jsr(self.t, self.l["tls_deframe_init"], timeout=30.0) + + def feed_record(self, data: bytes): + write_bytes(self.t, self.l["tls_rec_buf"], data) + write_bytes(self.t, self.l["tls_rec_len"], + struct.pack(" next record + events.append(("err", a)) + return events + else: + raise RuntimeError("pump livelock (no progress)") + return events + + def transcript(self) -> bytes: + jsr(self.t, self.l["tls_transcript_hash"], timeout=60.0) + return read_bytes(self.t, self.l["tls_transcript"], 32) + + def hs_ptr(self) -> int: + p = read_bytes(self.t, TLS_HS_PTR, 2) + return p[0] | (p[1] << 8) + + +# --------------------------------------------------------------------------- +# Cases +# --------------------------------------------------------------------------- + +def run_cases(transport, labels): + rig = Rig(transport, labels) + passed = failed = 0 + fed_ok = [] # messages whose bytes should be in transcript + + def check(name, cond, detail=""): + nonlocal passed, failed + if cond: + print(f" PASS {name}") + passed += 1 + else: + print(f" FAIL {name} {detail}") + failed += 1 + + def check_transcript(name, *msgs): + expect = hashlib.sha256(b"".join(msgs)).digest() + got = rig.transcript() + check(name + " (transcript)", got == expect, + f"got {got.hex()} want {expect.hex()}") + + # --- 1. single message, wholly in one record (in-place, zero copy) --- + m1 = hs_msg(HS_EE, b"\x00\x00") + rig.reset() + ev = rig.drive([m1]) + check("in-place single EE", ev == ["msg"], f"events={ev}") + check("in-place dispatch pointer = tls_rec_buf", + rig.hs_ptr() == labels["tls_rec_buf"], + f"ptr=${rig.hs_ptr():04X}") + check_transcript("in-place single EE", m1) + + # --- 2. two messages sharing one record --- + m2 = hs_msg(HS_EE, b"\x00\x0b\x00\x01\x02") + rig.reset() + ev = rig.drive([m1 + m2]) + check("two messages, one record", ev == ["msg", "msg"], f"events={ev}") + check_transcript("two messages, one record", m1, m2) + + # --- 3. body split across two records (carry path) --- + m3 = hs_msg(HS_EE, bytes(range(40))) + rig.reset() + ev = rig.drive([m3[:14], m3[14:]]) + check("body split across records", ev == ["msg"], f"events={ev}") + check("carry dispatch pointer = df_carry_buf", + rig.hs_ptr() == labels["df_carry_buf"], + f"ptr=${rig.hs_ptr():04X}") + check_transcript("body split across records", m3) + + # --- 4. header split across two records --- + rig.reset() + ev = rig.drive([m3[:2], m3[2:]]) + check("header split across records", ev == ["msg"], f"events={ev}") + check_transcript("header split across records", m3) + + # --- 5. record boundary exactly after the header --- + rig.reset() + ev = rig.drive([m3[:4], m3[4:]]) + check("boundary at header end", ev == ["msg"], f"events={ev}") + check_transcript("boundary at header end", m3) + + # --- 6. split server Finished, correct verify_data --- + # The pre-Finished transcript is sha256(m1): the deframer snapshots + # it when the Finished header completes, so tls_verify_finished must + # accept a verify_data computed over exactly that. + write_bytes(transport, labels["tls_s_hs_secret"], SECRET) + pre = hashlib.sha256(m1).digest() + fin_good = hs_msg(HS_FIN, finished_verify_data(SECRET, pre)) + rig.reset() + ev = rig.drive([m1, fin_good[:20], fin_good[20:]]) + check("split Finished accepted (per-message transcript)", + ev == ["msg", "msg"], f"events={ev}") + check_transcript("split Finished accepted", m1, fin_good) + + # --- 6b. same Finished sharing one record with EE (in-place) --- + rig.reset() + ev = rig.drive([m1 + fin_good]) + check("EE+Finished sharing one record", ev == ["msg", "msg"], + f"events={ev}") + + # --- 7. split server Finished, corrupted verify_data --- + bad_vd = bytearray(finished_verify_data(SECRET, pre)) + bad_vd[7] ^= 0x40 + fin_bad = hs_msg(HS_FIN, bytes(bad_vd)) + rig.reset() + ev = rig.drive([m1, fin_bad[:20], fin_bad[20:]]) + check("split Finished REJECTED on bad verify_data", + ev == ["msg", ("err", ERR_DISPATCH)], f"events={ev}") + + # --- 8. unknown handshake type --- + rig.reset() + ev = rig.drive([hs_msg(0x63, b"\x00" * 5)]) + check("unknown handshake type rejected", + ev == [("err", ERR_TYPE)], f"events={ev}") + + # --- 9. spanning non-Certificate message beyond the carry cap --- + big = hs_msg(HS_EE, bytes(300)) + rig.reset() + ev = rig.drive([big[:100], big[100:]]) + check("oversize spanning non-Certificate rejected", + ev == [("err", ERR_TOO_BIG)], f"events={ev}") + + return passed, failed + + +def main() -> int: + os.chdir(PROJECT_ROOT) + + if os.environ.get("C64_SKIP_BUILD"): + print("\n=== Building (skipped: C64_SKIP_BUILD set) ===") + else: + print("\n=== Building (BACKEND=uci — deframer is UCI-only) ===") + subprocess.run(["make", "clean"], capture_output=True) + result = subprocess.run(["make", "BACKEND=uci"], + capture_output=True, text=True) + if result.returncode != 0: + print(f"Build failed:\n{result.stderr}") + return 1 + print(" Build OK") + + if not os.path.exists(PRG_PATH): + print(f"FATAL: {PRG_PATH} not found") + return 1 + + labels = Labels.from_file(LABELS_PATH) + missing = [n for n in REQUIRED_LABELS if labels.address(n) is None] + if missing: + print(f"FATAL: required label(s) not found: {', '.join(missing)}") + print("(is this an ip65 build? the deframer needs BACKEND=uci)") + return 1 + + print("\n=== Labels ===") + for name in REQUIRED_LABELS: + print(f" {name:<24} = ${labels[name]:04X}") + + print("\n=== Starting VICE ===") + config = default_vice_config( + prg_path=PRG_PATH, + warp=True, + ntsc=True, + sound=False, + ) + + with ViceInstanceManager(config=config) as mgr: + inst = mgr.acquire() + transport = inst.transport + print(f" VICE PID={inst.pid}, port={inst.port}") + + print(" Waiting for main menu...") + grid = wait_for_text(transport, "Q=QUIT", timeout=180.0, verbose=False) + if grid is None: + print("FATAL: Main menu did not appear") + mgr.release(inst) + return 1 + print(" Main menu ready") + + install_stub(transport, labels["tls_deframe_pump"]) + + print("\n=== tls_deframe ===") + try: + passed, failed = run_cases(transport, labels) + finally: + mgr.release(inst) + + total = passed + failed + print("\n" + "=" * 60) + print(f" Passed: {passed}/{total}") + print(f" Failed: {failed}/{total}") + if failed == 0: + print(f"\n [+] TLS deframe: ALL {total} TESTS PASSED") + else: + print(f"\n [-] TLS deframe: {failed} TEST(S) FAILED") + print("=" * 60) + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) From 1897af8f80cd43bcb82e0534f7f64d40b03ee936 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:46:35 -0400 Subject: [PATCH 10/33] test(deframer): VICE acceptance suite + evil_listener re-framing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lane D of the real-server sprint. Independent acceptance gate for the streaming handshake-message deframer (W1/W2), written without sight of the implementation — expectations derive from RFC 8446 reassembly. tools/test_tls_deframer.py (new): Drives the public tls_recv_encrypted seam over genuinely AEAD-encrypted records primed into the TCP receive ring (net_poll patched to RTS). Oracles are implementation-independent: leaf pubkey extraction proves a Certificate was fully reassembled+parsed; the running transcript proves per-message folding; a cert_buf+1536 sentinel proves the oversized-leaf guard fires instead of overflowing. Current build: plumbing 2/2 PASS (rig proven), deframer 0/9 (xfail as expected pre-deframer). Exit 2 = gate not yet met, plumbing OK. Covers: two-messages-per-record, message split across records incl. at each of the 4 header-byte offsets, mid-body, exact boundary, many-record cert, and oversized-leaf clean error. tools/https_e2e/evil_listener.py: Add RECORD_FRAME re-framing (onepermsg|mfl512|pathological), orthogonal to good/bad_finished. onepermsg kept byte-for-byte (rig_https_bad_finished unaffected). --selftest now covers all 6 combos: OpenSSL accepts every framing in good and rejects bad_finished, proving the wire bytes are valid TLS and framing is the only variable. Co-Authored-By: Claude Fable 5 --- tools/https_e2e/evil_listener.py | 139 +++++++- tools/test_tls_deframer.py | 588 +++++++++++++++++++++++++++++++ 2 files changed, 709 insertions(+), 18 deletions(-) create mode 100644 tools/test_tls_deframer.py diff --git a/tools/https_e2e/evil_listener.py b/tools/https_e2e/evil_listener.py index f2d0488c..e7b010c1 100644 --- a/tools/https_e2e/evil_listener.py +++ b/tools/https_e2e/evil_listener.py @@ -41,6 +41,30 @@ records what the client actually did, in ``client_accepted_finished``: a client that goes on to send its own Finished did not check ours. +Record framing (``record_frame`` / ``RECORD_FRAME`` env) +------------------------------------------------------- +Orthogonal to *mode*. Controls how the encrypted **handshake content stream** +(EncryptedExtensions || Certificate || CertificateVerify || Finished) is cut +into TLS records — the exact axis the streaming deframer (sprint W1/W2) has to +handle. The message bytes, transcript, keys and signatures are identical across +all framings; only the record boundaries move: + +``"onepermsg"`` (default) + One handshake message per record — the historical behavior, kept + byte-for-byte so the existing ``rig_https_bad_finished.py`` is unaffected. + +``"mfl512"`` + The whole handshake stream re-chunked into <=512-byte record fragments, + the way a real server honoring ``max_fragment_length`` fragments it. Messages + span records; a record carries multiple/partial messages. This is the + real-world github.com / browserleaks.com pattern. + +``"pathological"`` + Adversarial fragmentation: tiny records, splits *inside* the 4-byte message + header, and message boundaries mid-record. Maximum stress for the deframer; + still perfectly valid TLS (Python's own ``ssl`` client reassembles it, which + ``--selftest`` proves). + Self-validation --------------- ``python3 tools/https_e2e/evil_listener.py --selftest`` runs both modes against @@ -185,6 +209,36 @@ def _plaintext_record(content_type: int, payload: bytes) -> bytes: return bytes([content_type, 0x03, 0x03]) + struct.pack(">H", len(payload)) + payload +VALID_RECORD_FRAMES = ("onepermsg", "mfl512", "pathological") + + +def frame_handshake_stream(stream: bytes, mode: str) -> list[bytes]: + """Cut the handshake content *stream* into record-payload fragments. + + Returns a list of byte-slices whose concatenation is exactly *stream*; each + slice becomes one TLS record. ``"onepermsg"`` is handled by the caller (it + frames per message, not per stream) and is rejected here. + """ + if mode == "mfl512": + return [stream[i:i + 512] for i in range(0, len(stream), 512)] or [b""] + if mode == "pathological": + # A fixed, deterministic pattern of awkward sizes. Small leading pieces + # guarantee a split inside the first message's 4-byte header; the mix of + # sizes scatters message boundaries across records. Deterministic so a + # failure reproduces. + pattern = [1, 2, 3, 1, 250, 7, 511, 13, 400, 1] + out = [] + pos = 0 + i = 0 + while pos < len(stream): + size = pattern[i % len(pattern)] + out.append(stream[pos:pos + size]) + pos += size + i += 1 + return out or [b""] + raise ValueError(f"frame_handshake_stream does not handle mode {mode!r}") + + class RecordReader: """Reassembles TLS records from a stream socket.""" @@ -290,11 +344,18 @@ class EvilTls13Server: def __init__(self, cert_path: str, key_path: str, *, mode: str = "good", - body: str = DEFAULT_BODY): + body: str = DEFAULT_BODY, + record_frame: str = "onepermsg"): if mode not in ("good", "bad_finished"): raise ValueError(f"unknown mode {mode!r}") + if record_frame not in VALID_RECORD_FRAMES: + raise ValueError( + f"unknown record_frame {record_frame!r}; " + f"expected one of {VALID_RECORD_FRAMES}" + ) self.mode = mode self.body = body + self.record_frame = record_frame with open(cert_path, "rb") as f: pem = f.read() @@ -308,6 +369,7 @@ def __init__(self, cert_path: str, key_path: str, *, self.result: dict = { "mode": mode, + "record_frame": record_frame, "listening": False, "client_hello_seen": False, "server_flight_sent": False, @@ -400,27 +462,45 @@ def serve_one(self, sock: socket.socket, timeout: float) -> dict: s_keys = TrafficKeys(s_hs) c_keys = TrafficKeys(c_hs) - def send_hs(msg: bytes) -> None: - # One handshake message per record: the C64 client's - # tls_recv_encrypted dispatches on tls_rec_buf[0] and handles - # exactly one message per decrypted record. - sock.sendall(s_keys.encrypt(msg + bytes([CT_HANDSHAKE]))) + # The handshake content stream, accumulated in message order. Under + # "onepermsg" each message is sent as its own record as it is produced + # (the historical path, byte-for-byte). Under any other framing the + # whole stream is buffered here and re-cut into records after the + # Finished — the message bytes, transcript and signatures are identical + # either way; only record boundaries move. + hs_stream = bytearray() + + def emit(msg: bytes) -> None: + if self.record_frame == "onepermsg": + # One handshake message per record: the pre-deframer C64 client + # dispatches on tls_rec_buf[0] and handles exactly one message + # per decrypted record. + sock.sendall(s_keys.encrypt(msg + bytes([CT_HANDSHAKE]))) + else: + hs_stream.extend(msg) ee = _handshake(HS_ENCRYPTED_EXTENSIONS, b"\x00\x00") - send_hs(ee) + emit(ee) transcript += ee cert = self._certificate() - send_hs(cert) + emit(cert) transcript += cert cv = self._certificate_verify(_sha256(transcript)) - send_hs(cv) + emit(cv) transcript += cv fin, corrupted = self._finished(s_hs, _sha256(transcript)) - send_hs(fin) + emit(fin) self.result["finished_corrupted"] = corrupted + + if self.record_frame != "onepermsg": + fragments = frame_handshake_stream(bytes(hs_stream), self.record_frame) + for frag in fragments: + sock.sendall(s_keys.encrypt(frag + bytes([CT_HANDSHAKE]))) + self.result["record_count"] = len(fragments) + self.result["server_flight_sent"] = True # Fold the Finished we actually SENT. A client that accepts the @@ -526,8 +606,16 @@ def send_hs(msg: bytes) -> None: def serve_one_connection(srv: socket.socket, cert_path: str, key_path: str, *, mode: str, body: str, timeout: float, - result: dict) -> None: - """Accept exactly one connection and run the flight. Fills *result*.""" + result: dict, record_frame: str | None = None) -> None: + """Accept exactly one connection and run the flight. Fills *result*. + + *record_frame* selects handshake record framing (see module docstring). When + ``None`` it falls back to the ``RECORD_FRAME`` environment variable, then to + ``"onepermsg"`` — so an out-of-band rig can pick the framing without the + caller threading a new argument. + """ + if record_frame is None: + record_frame = os.environ.get("RECORD_FRAME", "onepermsg") conn = None try: srv.settimeout(timeout) @@ -535,7 +623,8 @@ def serve_one_connection(srv: socket.socket, cert_path: str, key_path: str, *, result["listening"] = True conn, addr = srv.accept() result["client_addr"] = addr - server = EvilTls13Server(cert_path, key_path, mode=mode, body=body) + server = EvilTls13Server(cert_path, key_path, mode=mode, body=body, + record_frame=record_frame) result.update(server.result) result["client_addr"] = addr result["listening"] = True @@ -569,8 +658,20 @@ def _selftest() -> int: cert_path, key_path = _ensure_certs_p256() failures = 0 - for mode, expect in (("good", "handshake completes"), - ("bad_finished", "client rejects")): + # Every (mode, record_frame) combination must behave correctly against + # Python's own OpenSSL client. The re-framings are only useful as a C64 + # deframer test if OpenSSL — which reassembles records faithfully — accepts + # them in `good` and still rejects the corrupted Finished; that proves the + # bytes on the wire are valid TLS and the framing, not a protocol error, is + # what the C64 must cope with. + cases = [ + (mode, frame, expect) + for frame in VALID_RECORD_FRAMES + for mode, expect in (("good", "handshake completes"), + ("bad_finished", "client rejects")) + ] + + for mode, record_frame, expect in cases: srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) srv.bind(("127.0.0.1", 0)) @@ -581,7 +682,7 @@ def _selftest() -> int: target=serve_one_connection, args=(srv, cert_path, key_path), kwargs=dict(mode=mode, body=DEFAULT_BODY, timeout=20.0, - result=result), + result=result, record_frame=record_frame), daemon=True, ) t.start() @@ -613,10 +714,12 @@ def _selftest() -> int: t.join(timeout=25.0) verdict = "PASS" if ok else "FAIL" - print(f" {verdict}: mode={mode:<13} expect {expect}") + print(f" {verdict}: frame={record_frame:<12} mode={mode:<13} " + f"expect {expect}") + print(f" records : {result.get('record_count', '1/msg')}") print(f" client saw : {detail}") - print(f" server saw : {result}") if not ok: + print(f" server saw : {result}") failures += 1 print() diff --git a/tools/test_tls_deframer.py b/tools/test_tls_deframer.py new file mode 100644 index 00000000..11cb76c2 --- /dev/null +++ b/tools/test_tls_deframer.py @@ -0,0 +1,588 @@ +#!/usr/bin/env python3 +"""test_tls_deframer.py - VICE acceptance suite for the streaming handshake +deframer (sprint W1/W2). + +WHAT THIS IS +------------ +An *independent* acceptance gate for the record/message deframer that a +parallel lane is adding to ``src/tls13.s`` / ``src/tls_record_io.s`` / +``src/tls_cert.s``. It was written without sight of that implementation: +every expectation here is derived from RFC 8446 handshake reassembly and the +sprint plan, so a build that satisfies this suite satisfies the spec, not one +particular author's code. + +THE PROBLEM THE DEFRAMER SOLVES (RFC 8446 §5.1) +---------------------------------------------- +Handshake messages are ``TYPE(1) || LEN(3) || BODY``, packed back-to-back in +the handshake content stream. TLS records fragment that stream *arbitrarily*: + + (a) one record can carry two whole messages (EncryptedExtensions + the start + of Certificate is the real-world pattern; CertificateVerify + Finished + likewise), + (b) one message can span several records, split at *any* offset — including + inside the 4-byte message header, + (c) the Certificate message (~2.7 KB, multi-cert chain) spans ~6 records; + only the first (leaf) cert must be captured, the rest counted+discarded, + (d) a leaf certificate larger than ``cert_buf`` (1536 B) must produce a clean + "certificate too large" error, never a buffer overflow. + +The current code (``tls13.s`` ~555-660) assumes exactly one message per record. +Every scenario below that is marked ``[deframer]`` therefore FAILS on the +pre-deframer build and is EXPECTED to — that is the whole point of an +acceptance gate written ahead of the code. Scenarios marked ``[plumbing]`` +exercise only the one-message-per-record path that already works, so they pass +today and prove the rig itself (ring priming, decrypt, dispatch, transcript, +cert parsing) is sound. + +HOW IT DRIVES THE REAL SEAM +--------------------------- +No internal deframer symbol is named here, so the suite cannot rot when the +implementation names things differently. It drives the *public* entry the +deframer sits behind — ``tls_recv_encrypted`` — exactly as ``tls_connect`` +does, over genuinely AEAD-encrypted records: + + 1. ``net_poll`` is patched to a bare ``RTS`` so the routine's built-in NIC + pump is inert; every byte comes from the pre-primed TCP receive ring. + 2. Encrypted TLS records (ChaCha20-Poly1305 under a fixed handshake read key) + are written straight into ``tcp_recv_buf`` and the ring head/tail set, so + ``tls_record_recv_and_decrypt`` decrypts them the same way it decrypts + records off the wire. + 3. ``tls_recv_encrypted`` is called until the flight is drained. + +ORACLES (implementation-independent) +------------------------------------ + * ``ecdsa_pubkey_x`` / ``ecdsa_pubkey_y`` == the leaf cert's public key. + This is the load-bearing discriminator: it is populated only if the + Certificate message was *fully reassembled and parsed*. A build that + dispatches only the first message of a two-message record, or mis-handles a + Certificate split across records, leaves it at its poison value. + * the running transcript (read via ``tls_transcript_hash``) == + SHA-256(concatenated handshake messages). Proves every message's bytes + were folded regardless of record framing. + * for the oversized-leaf case, a sentinel at ``cert_buf + 1536`` survives and + no valid pubkey is extracted — proves the size guard fired instead of + overflowing. + +Both crypto-verified messages (CertificateVerify, Finished) are intentionally +absent from this suite: their transcript discipline needs a self-consistent +real handshake and is the job of the end-to-end re-framing test +(``tools/https_e2e/evil_listener.py`` ``RECORD_FRAME=...`` + the live rig). +Here we use EncryptedExtensions (accepted + folded, no crypto) and Certificate +(parsed + folded, no signature check), which is all the deframer's *framing* +logic needs. + +Vectors are fully deterministic (fixed key/iv/seq, repo test cert), so a +failure reproduces byte-for-byte. + +Usage: + python3 tools/test_tls_deframer.py [--verbose] + +Env: + C64_SKIP_BUILD=1 reuse the already-built PRG + +Requires: Python 3.10+, c64_test_harness, VICE x64sc, cryptography +""" + +from __future__ import annotations + +import hashlib +import os +import subprocess +import sys + +from cryptography.hazmat.primitives.ciphers.aead import ChaCha20Poly1305 + +from c64_test_harness import ( + Labels, + ViceInstanceManager, + read_bytes, + write_bytes, + jsr, + wait_for_text, +) + +PROJECT_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from _vice_helpers import default_vice_config # noqa: E402 + +PRG_PATH = os.path.join(PROJECT_ROOT, "build", "c64-https.prg") +LABELS_PATH = os.path.join(PROJECT_ROOT, "build", "labels.txt") + +VERBOSE = False + +# --- TLS constants (mirror src/constants.inc) --- +TLS_CT_HANDSHAKE = 22 +TLS_CT_APPLICATION = 23 +TLS_HS_ENCRYPTED_EXT = 8 +TLS_HS_CERTIFICATE = 11 +TLS_STATE_CERTIFICATE = 4 # >= ENCRYPTED_EXT(3): decrypt with hs read keys + +# cert_buf capacity from the sprint plan / src/der_decode.s:588. +CERT_BUF_MAX = 1536 + +# Fixed handshake read key/iv/seq so every record is reproducible. +HS_READ_KEY = bytes((i * 7 + 3) & 0xFF for i in range(32)) +HS_READ_IV = bytes((i * 11 + 5) & 0xFF for i in range(12)) + +POISON = 0xA5 +SENTINEL = 0x5A + +REQUIRED_LABELS = [ + "tls_recv_encrypted", + "net_poll", + "tcp_recv_buf", "tcp_recv_head", "tcp_recv_tail", + "tls_rec_buf", "tls_rec_len", "tls_rec_type", + "tls_recv_state", "tls_recv_count", + "tls_state", + "tls_hs_read_key", "tls_hs_read_iv", "tls_read_seq", + "tls_transcript", "tls_transcript_init", "tls_transcript_hash", + "ecdsa_pubkey_x", "ecdsa_pubkey_y", + "cert_buf", + "sqtab_init", +] + +RTS = 0x60 + + +# --------------------------------------------------------------------------- +# Handshake-message + record construction +# --------------------------------------------------------------------------- + +def _u24(n: int) -> bytes: + return bytes([(n >> 16) & 0xFF, (n >> 8) & 0xFF, n & 0xFF]) + + +def hs_message(msg_type: int, body: bytes) -> bytes: + """One handshake message: TYPE(1) || LEN(3) || BODY.""" + return bytes([msg_type]) + _u24(len(body)) + body + + +def encrypted_extensions() -> bytes: + # Empty EncryptedExtensions: a 2-byte (zero-length) extensions block. + return hs_message(TLS_HS_ENCRYPTED_EXT, b"\x00\x00") + + +def certificate_message(cert_der: bytes) -> bytes: + """A TLS 1.3 Certificate message carrying one entry (matches the shape the + server fixture emits): context(0) || cert_list_len(24) || + [ cert_data_len(24) || cert_data || ext_len(16)=0 ].""" + entry = _u24(len(cert_der)) + cert_der + b"\x00\x00" + body = b"\x00" + _u24(len(entry)) + entry + return hs_message(TLS_HS_CERTIFICATE, body) + + +def encrypt_record(inner_plaintext: bytes, seq: int) -> bytes: + """Build one TLSCiphertext record (5-byte header + ct + tag). + + inner_plaintext already includes its trailing inner content-type byte. + """ + padded_seq = b"\x00" * 4 + seq.to_bytes(8, "big") + nonce = bytes(a ^ b for a, b in zip(HS_READ_IV, padded_seq)) + total = len(inner_plaintext) + 16 + aad = bytes([TLS_CT_APPLICATION, 0x03, 0x03, (total >> 8) & 0xFF, total & 0xFF]) + ct = ChaCha20Poly1305(HS_READ_KEY).encrypt(nonce, inner_plaintext, aad) + return aad + ct + + +def records_from_chunks(chunks: list[bytes]) -> bytes: + """Encrypt each handshake-stream *chunk* as its own record, seq 0,1,2,... + + Each record's inner plaintext = chunk || CT_HANDSHAKE, exactly as the wire + carries a fragment of the handshake content stream. + """ + out = bytearray() + for i, chunk in enumerate(chunks): + out += encrypt_record(chunk + bytes([TLS_CT_HANDSHAKE]), i) + return bytes(out), len(chunks) + + +def chunk_bytes(stream: bytes, sizes: list[int]) -> list[bytes]: + """Slice *stream* into pieces of the given sizes (last piece takes the + remainder if sizes run short).""" + out = [] + pos = 0 + for s in sizes: + if pos >= len(stream): + break + out.append(stream[pos:pos + s]) + pos += s + if pos < len(stream): + out.append(stream[pos:]) + return out + + +# --------------------------------------------------------------------------- +# C64 plumbing +# --------------------------------------------------------------------------- + +def prime_ring(transport, labels, record_bytes: bytes) -> None: + """Load encrypted records into the TCP receive ring and set head/tail.""" + n = len(record_bytes) + assert n < 0x1000, f"flight {n} B exceeds the 4 KB ring" + write_bytes(transport, labels["tcp_recv_buf"], record_bytes) + # head = 0, tail = n (both 16-bit little-endian) + write_bytes(transport, labels["tcp_recv_head"], [0, 0]) + write_bytes(transport, labels["tcp_recv_tail"], [n & 0xFF, (n >> 8) & 0xFF]) + + +def reset_recv_state(transport, labels) -> None: + write_bytes(transport, labels["tls_recv_state"], [0]) + write_bytes(transport, labels["tls_recv_count"], [0, 0]) + write_bytes(transport, labels["tls_read_seq"], b"\x00" * 8) + write_bytes(transport, labels["tls_state"], [TLS_STATE_CERTIFICATE]) + + +def drive_flight(transport, labels, record_bytes, n_records): + """Prime the ring and pump tls_recv_encrypted until the flight drains. + + Returns the list of carry flags observed (diagnostic only; the oracle is + end-state, so a build whose call granularity differs is still judged fairly). + """ + reset_recv_state(transport, labels) + prime_ring(transport, labels, record_bytes) + carries = [] + # An upper bound on calls under any sane design: never more than one call + # per record, plus slack for a drain call. Capped so a wedged build cannot + # spin the suite forever. + max_calls = min(n_records + 2, 8) + for _ in range(max_calls): + regs = jsr(transport, labels["tls_recv_encrypted"], timeout=120.0) + carry = None + if regs: + for k in ("FL", "P", "FLAGS", "SR"): + if k in regs: + carry = regs[k] & 0x01 + break + carries.append(carry) + return carries + + +def read_pubkey(transport, labels): + x = read_bytes(transport, labels["ecdsa_pubkey_x"], 32) + y = read_bytes(transport, labels["ecdsa_pubkey_y"], 32) + return x, y + + +def read_transcript(transport, labels): + """Snapshot the running transcript hash into tls_transcript and read it.""" + jsr(transport, labels["tls_transcript_hash"], timeout=60.0) + return read_bytes(transport, labels["tls_transcript"], 32) + + +# --------------------------------------------------------------------------- +# Scenarios +# --------------------------------------------------------------------------- + +class Scenario: + def __init__(self, name, kind, chunks, expect_pubkey, expect_transcript, + expect_overflow_safe=False): + self.name = name + self.kind = kind # "plumbing" or "deframer" + self.chunks = chunks + self.expect_pubkey = expect_pubkey # (x, y) or None + self.expect_transcript = expect_transcript # bytes or None + self.expect_overflow_safe = expect_overflow_safe + + +def build_scenarios(cert_der, pubkey_xy): + ee = encrypted_extensions() + cert = certificate_message(cert_der) + px, py = pubkey_xy + + scenarios = [] + + # --- plumbing: one message per record (works today, proves the rig) --- + stream = ee + cert + scenarios.append(Scenario( + "plumbing/one_msg_per_record", + "plumbing", + [ee, cert], + (px, py), + hashlib.sha256(stream).digest(), + )) + + # A lone Certificate in a single record — the simplest reassembly-free + # cert parse. Also passes today. + scenarios.append(Scenario( + "plumbing/cert_single_record", + "plumbing", + [cert], + (px, py), + hashlib.sha256(cert).digest(), + )) + + # --- deframer (a): two whole messages in one record --- + scenarios.append(Scenario( + "deframer/two_msgs_one_record", + "deframer", + [ee + cert], + (px, py), + hashlib.sha256(ee + cert).digest(), + )) + + # --- deframer (b): one message split across records --- + # mid-body split of the Certificate message. + scenarios.append(Scenario( + "deframer/cert_split_mid_body", + "deframer", + chunk_bytes(cert, [200]), + (px, py), + hashlib.sha256(cert).digest(), + )) + + # split INSIDE the 4-byte handshake header, at each of offsets 1/2/3. + for k in (1, 2, 3): + scenarios.append(Scenario( + f"deframer/cert_split_in_header@{k}", + "deframer", + chunk_bytes(cert, [k]), + (px, py), + hashlib.sha256(cert).digest(), + )) + + # split exactly on the header/body boundary (offset 4). + scenarios.append(Scenario( + "deframer/cert_split_header_body_boundary", + "deframer", + chunk_bytes(cert, [4]), + (px, py), + hashlib.sha256(cert).digest(), + )) + + # exact message-boundary case: a record ends mid-flight exactly at the EE + # message boundary, and the Certificate spans two records. + half = len(cert) // 2 + scenarios.append(Scenario( + "deframer/msg_boundary_and_cert_span", + "deframer", + [ee + cert[:half], cert[half:]], + (px, py), + hashlib.sha256(ee + cert).digest(), + )) + + # certificate spanning many small records (the real ~6-record pattern). + scenarios.append(Scenario( + "deframer/cert_many_records", + "deframer", + chunk_bytes(cert, [64, 64, 64, 64, 64]), + (px, py), + hashlib.sha256(cert).digest(), + )) + + # --- deframer (d): oversized leaf must error, not overflow --- + # A single-entry Certificate whose cert_data exceeds cert_buf (1536 B). + big_leaf = b"\x30\x82" + (CERT_BUF_MAX + 200).to_bytes(2, "big") + \ + bytes((i * 3) & 0xFF for i in range(CERT_BUF_MAX + 160)) + big_cert = certificate_message(big_leaf) + scenarios.append(Scenario( + "deframer/oversized_leaf_errors", + "deframer", + chunk_bytes(big_cert, [480, 480, 480, 480]), + None, # must NOT extract a pubkey + None, # transcript not asserted + expect_overflow_safe=True, + )) + + return scenarios + + +# --------------------------------------------------------------------------- +# Runner +# --------------------------------------------------------------------------- + +def run_scenario(transport, labels, sc): + """Return (ok, detail).""" + # Poison the observables so a stale value from a prior scenario cannot be + # mistaken for a fresh extraction. + write_bytes(transport, labels["ecdsa_pubkey_x"], bytes([POISON]) * 32) + write_bytes(transport, labels["ecdsa_pubkey_y"], bytes([POISON]) * 32) + write_bytes(transport, labels["tls_transcript"], bytes([POISON]) * 32) + # Overflow sentinel just past cert_buf's capacity. + write_bytes(transport, labels["cert_buf"] + CERT_BUF_MAX, bytes([SENTINEL])) + + # Fresh running transcript so the fold accumulates exactly this flight. + jsr(transport, labels["tls_transcript_init"], timeout=60.0) + write_bytes(transport, labels["tls_hs_read_key"], HS_READ_KEY) + write_bytes(transport, labels["tls_hs_read_iv"], HS_READ_IV) + + record_bytes, n_records = records_from_chunks(sc.chunks) + carries = drive_flight(transport, labels, record_bytes, n_records) + + problems = [] + + if sc.expect_pubkey is not None: + gx, gy = read_pubkey(transport, labels) + ex, ey = sc.expect_pubkey + if gx != ex or gy != ey: + problems.append( + f"pubkey mismatch (Certificate not fully reassembled/parsed)\n" + f" want x {ex.hex()}\n got x {gx.hex()}\n" + f" want y {ey.hex()}\n got y {gy.hex()}" + ) + + if sc.expect_transcript is not None: + got_t = read_transcript(transport, labels) + if got_t != sc.expect_transcript: + problems.append( + f"transcript mismatch (a message's bytes were not folded)\n" + f" want {sc.expect_transcript.hex()}\n" + f" got {got_t.hex()}" + ) + + if sc.expect_overflow_safe: + sentinel = read_bytes(transport, labels["cert_buf"] + CERT_BUF_MAX, 1)[0] + if sentinel != SENTINEL: + problems.append( + f"BUFFER OVERFLOW: sentinel at cert_buf+{CERT_BUF_MAX} was " + f"clobbered (${sentinel:02X}, expected ${SENTINEL:02X}) — the " + f"oversized leaf was copied past cert_buf" + ) + gx, gy = read_pubkey(transport, labels) + if gx != bytes([POISON]) * 32 or gy != bytes([POISON]) * 32: + problems.append( + "oversized leaf still produced a pubkey — the size guard did " + "not fire before extraction" + ) + + detail = "" + if problems: + detail = "\n " + "\n ".join(problems) + if VERBOSE: + detail += f"\n carries={carries} records={n_records}" + return (not problems), detail + + +def run_tests(transport, labels, cert_der, pubkey_xy): + print("\n Initializing sqtab (Poly1305 multiply table)...") + jsr(transport, labels["sqtab_init"], timeout=60.0) + + # Neutralize the NIC pump: every byte comes from the primed ring. + write_bytes(transport, labels["net_poll"], [RTS]) + print(" net_poll patched to RTS (ring-driven, no live NIC)") + + scenarios = build_scenarios(cert_der, pubkey_xy) + + plumb_pass = plumb_fail = defr_pass = defr_fail = 0 + for sc in scenarios: + ok, detail = run_scenario(transport, labels, sc) + verdict = "PASS" if ok else "FAIL" + tag = f"[{sc.kind}]" + print(f" {verdict} {tag:<11} {sc.name}{detail}") + if sc.kind == "plumbing": + if ok: + plumb_pass += 1 + else: + plumb_fail += 1 + else: + if ok: + defr_pass += 1 + else: + defr_fail += 1 + + return plumb_pass, plumb_fail, defr_pass, defr_fail + + +# --------------------------------------------------------------------------- +# Cert fixture +# --------------------------------------------------------------------------- + +def load_cert_fixture(): + """Return (cert_der, (pubkey_x, pubkey_y)) from the repo P-256 test cert.""" + from cryptography.x509 import load_pem_x509_certificate + from cryptography.hazmat.primitives import serialization + + sys.path.insert(0, os.path.join(PROJECT_ROOT, "tools", "https_e2e")) + from ensure_certs import ensure_certs # noqa: PLC0415 + + cert_path, _ = ensure_certs("p256") + pem = open(cert_path, "rb").read() + cert = load_pem_x509_certificate(pem) + der = cert.public_bytes(serialization.Encoding.DER) + nums = cert.public_key().public_numbers() + return der, (nums.x.to_bytes(32, "big"), nums.y.to_bytes(32, "big")) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main() -> int: + global VERBOSE + os.chdir(PROJECT_ROOT) + + if "--verbose" in sys.argv: + VERBOSE = True + + if os.environ.get("C64_SKIP_BUILD"): + print("\n=== Building (skipped: C64_SKIP_BUILD set) ===") + else: + print("\n=== Building ===") + subprocess.run(["make", "clean"], capture_output=True, cwd=PROJECT_ROOT) + result = subprocess.run(["make"], capture_output=True, text=True, + cwd=PROJECT_ROOT) + if result.returncode != 0: + print(f"Build failed:\n{result.stderr}") + return 1 + print(" Build OK") + + if not os.path.exists(PRG_PATH): + print(f"FATAL: {PRG_PATH} not found") + return 1 + + labels = Labels.from_file(LABELS_PATH) + missing = [n for n in REQUIRED_LABELS if labels.address(n) is None] + if missing: + print(f"FATAL: required label(s) not found: {', '.join(missing)}") + return 1 + + cert_der, pubkey_xy = load_cert_fixture() + print(f" Test cert: {len(cert_der)} B DER, " + f"pubkey x={pubkey_xy[0][:6].hex()}...") + + config = default_vice_config(prg_path=PRG_PATH, warp=True, ntsc=True, + sound=False) + print("\n=== Starting VICE ===") + with ViceInstanceManager(config=config) as mgr: + inst = mgr.acquire() + transport = inst.transport + print(f" VICE PID={inst.pid}, port={inst.port}") + + grid = wait_for_text(transport, "Q=QUIT", timeout=120.0, verbose=False) + if grid is None: + print("FATAL: Main menu did not appear") + mgr.release(inst) + return 1 + print(" Main menu ready") + + print("\n=== Deframer scenarios ===") + try: + pp, pf, dp, df = run_tests(transport, labels, cert_der, pubkey_xy) + finally: + mgr.release(inst) + + print("\n" + "=" * 64) + print("RESULTS") + print("=" * 64) + print(f" plumbing (must pass on ANY build): {pp}/{pp + pf} passed") + print(f" deframer (W1/W2 acceptance gate): {dp}/{dp + df} passed") + print() + if pf: + print(" [-] PLUMBING FAILED — the rig itself is broken; deframer") + print(" results below are not trustworthy. Fix plumbing first.") + print("=" * 64) + return 1 + if df: + print(f" [x] {df} deframer scenario(s) not yet satisfied.") + print(" EXPECTED on the pre-deframer build; this is the gate the") + print(" streaming-deframer lane must turn green.") + print("=" * 64) + # Non-zero so CI/callers see the gate is not yet met, but plumbing is OK. + return 2 + print(" [+] Deframer acceptance gate: ALL scenarios PASSED") + print("=" * 64) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 24430cae49dfaf448fe0494fc6222618d7a11b70 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:54:46 -0400 Subject: [PATCH 11/33] feat(tls): W2 streaming Certificate consumer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A real server's Certificate message (~2.7 KB chain) spans ~6 records and cannot be dispatched from a buffer. The deframer now parses it incrementally as its bytes arrive: certificate_request_context (must be 0), certificate_list length, then per CertificateEntry the 24-bit cert_data length, the data, and the extensions. Only the FIRST entry (the leaf) is copied — into cert_buf ($A000, 1536 B) — and the moment it completes the public key is extracted through the new tls_cert_stream_finish (src/tls_cert.s), which runs the same x509_extract_pubkey the in-place path uses against the staged copy and sets cert_buf_len. Every later chain entry is counted and discarded without buffering. Message bytes are folded into the running transcript in bulk per record chunk (fold-as-you-go — nothing is retained to fold later). A Certificate wholly inside one record still dispatches in place through tls_handle_certificate, untouched — the local-listener e2e path is byte-for-byte the same. Error paths are explicit, never overflows: - DF_ERR_CERT_TOO_BIG ($06): leaf > 1536 B (en.wikipedia.org's leaf is 1636 B). Checked when the entry length completes, BEFORE any copy — cert_buf is not written at all (proven by test). - DF_ERR_CERT_FMT ($05): non-zero request context, entry >= 64 KB, truncated message, or a complete leaf with no usable EC key. Coverage: tools/test_tls_deframe.py grows to 41/41 — streamed 2-entry chain split at awkward offsets (leaf staged byte-exact in cert_buf, pubkey == the generated cert's coordinates, transcript == Python SHA-256 of the fed bytes), streamed chain followed by a trailing message in the tail record, in-place single-record Certificate, oversize/1536-boundary/bad-context/junk-leaf error paths. Lane D's independent RFC-derived acceptance gate (tools/test_tls_deframer.py, on the sprint branch) passes 8/9 plumbing+deframer scenarios; the one miss is a fixture artifact — its overflow sentinel at cert_buf+1536 sits on tls_rec_buf[0] ($A600), which the record layer overwrites legitimately on any build. ip65 remains gated off and its PRG byte-identical (f557658a...); all three UCI profiles link. Co-Authored-By: Claude Fable 5 --- src/tls_cert.s | 31 +++ src/tls_deframe.s | 402 +++++++++++++++++++++++++++++++++++++- tools/test_tls_deframe.py | 149 ++++++++++++++ 3 files changed, 574 insertions(+), 8 deletions(-) diff --git a/src/tls_cert.s b/src/tls_cert.s index 032be830..6eaac97f 100644 --- a/src/tls_cert.s +++ b/src/tls_cert.s @@ -83,6 +83,37 @@ tls_hs_ptr_reset: sta tls_hs_ptr+1 rts +.ifdef TLS_STREAM_DEFRAME +; ============================================================================= +; tls_cert_stream_finish - finalize the W2 streaming Certificate path +; +; The deframer (tls_deframe.s) has copied the complete leaf certificate +; into cert_buf. Run the same public-key extraction the in-place +; tls_handle_certificate path uses, against the staged copy. +; +; Input: A/X = leaf certificate length (lo/hi); leaf DER in cert_buf +; Output: C=0 pubkey extracted to ecdsa_pubkey_x/y (+ curve id set), +; C=1 extraction failed (no EC pubkey / unsupported curve) +; cert_buf_len = leaf length +; ============================================================================= + .import cert_buf + .import cert_buf_len + + .export tls_cert_stream_finish + + .segment "TLS_DEFRAME_CODE" +tls_cert_stream_finish: + sta cert_data_len_lo + sta cert_buf_len + stx cert_data_len_hi + stx cert_buf_len+1 + lda #cert_buf + sta cert_data_ptr+1 + jmp x509_extract_pubkey ; carry propagates as our result +.endif + .segment "TLS_CODE" ; ============================================================================= diff --git a/src/tls_deframe.s b/src/tls_deframe.s index 588b9dca..0f6399ea 100644 --- a/src/tls_deframe.s +++ b/src/tls_deframe.s @@ -91,6 +91,8 @@ .import tls_handle_certificate .import tls_handle_cert_verify .import tls_verify_finished +.import tls_cert_stream_finish +.import cert_buf .import tls_recv_sub_progress ; Carry buffer: 4-byte handshake header + body. EncryptedExtensions, @@ -103,13 +105,29 @@ DF_CARRY_MAX_BODY = DF_CARRY_SIZE - 4 DF_ERR_HDR_LEN = $01 ; 24-bit length high byte non-zero DF_ERR_TOO_BIG = $02 ; spanning non-Certificate message > carry cap DF_ERR_DISPATCH = $03 ; message handler returned C=1 -DF_ERR_TYPE = $04 ; unknown handshake message type +DF_ERR_TYPE = $04 ; unknown handshake message type +DF_ERR_CERT_FMT = $05 ; streamed Certificate malformed / no usable key +DF_ERR_CERT_TOO_BIG = $06 ; leaf certificate exceeds cert_buf (1536 B) ; df_mode values DF_MODE_HDR = 0 ; collecting the 4-byte message header DF_MODE_CARRY = 1 ; assembling body into df_carry_buf DF_MODE_STREAM = 2 ; streaming Certificate body (W2) +; Streaming-Certificate sub-states (df_cs_state) +CS_CTX = 0 ; certificate_request_context length (1 B, must be 0) +CS_LIST_LEN = 1 ; certificate_list length (3 B) +CS_ENTRY_LEN = 2 ; CertificateEntry cert_data length (3 B) +CS_ENTRY_DATA = 3 ; cert_data bytes (leaf -> cert_buf, rest discarded) +CS_EXT_LEN = 4 ; per-entry extensions length (2 B) +CS_EXT_DATA = 5 ; extension bytes (discarded) + +; Leaf staging capacity — keep in step with cert_buf in src/der_decode.s +; (CERT_BUF_BSS, 1536 B). The cap check runs BEFORE any copy, so an +; oversize leaf is a clean DF_ERR_CERT_TOO_BIG, never a write past +; cert_buf+1536. +DF_CERT_BUF_SIZE = 1536 + .segment "TLS_DEFRAME_CODE" ; ============================================================================= @@ -331,6 +349,10 @@ df_carry_body: lda #>df_carry_buf adc df_carry_off+1 sta zp_tmp2 + lda df_chunk + sta df_copy_len + lda df_chunk+1 + sta df_copy_len+1 jsr df_copy_chunk ; df_carry_off += chunk @@ -410,12 +432,286 @@ df_need_data: rts ; ============================================================================= -; W2 streaming Certificate consumer — placeholder in W1: a Certificate -; that spans records is rejected until the streaming consumer lands. +; W2 streaming Certificate consumer +; +; Parses the Certificate message incrementally as its bytes arrive: +; certificate_request_context, certificate_list length, then per entry +; the cert_data length + data + extensions. Only the FIRST entry (the +; leaf) is copied — into cert_buf — and the moment it is complete the +; public key is extracted via tls_cert_stream_finish (same extraction +; as the in-place path). Every later entry is counted and discarded +; without buffering. Message bytes are folded into the running +; transcript in bulk at the top of each pump (fold-as-you-go — the +; bytes are not retained, so there is nothing to fold later). +; +; Reached only for a Certificate that is NOT wholly inside one record; +; the wholly-in-record case dispatches in place through +; tls_handle_certificate exactly as before. ; ============================================================================= df_stream_begin: + lda #$33 + sta tls_recv_sub_progress + lda #DF_MODE_STREAM + sta df_mode + lda #CS_CTX + sta df_cs_state + lda #0 + sta df_cs_idx + sta df_cs_entry + sta df_cs_off + sta df_cs_off+1 + ; fall through into the body consumer + df_stream_body: - lda #DF_ERR_TOO_BIG + ; anything to consume this pump? + lda df_avail + ora df_avail+1 + bne :+ + jmp df_need_data +: + ; df_take = min(df_avail, df_msg_rem); fold it all into the + ; running transcript up front — the parse loop below consumes + ; exactly df_take bytes before returning. + jsr df_chunk_min + lda df_chunk + sta df_take + lda df_chunk+1 + sta df_take+1 + jsr df_rec_cursor + lda df_take + sta zp_count + lda df_take+1 + sta zp_count+1 + jsr tls_transcript_update + +@parse: + lda df_take + ora df_take+1 + bne @have_bytes + ; out of bytes this pump — message complete? + lda df_msg_rem + ora df_msg_rem+1 + bne @more_later + jmp @msg_end +@more_later: + jmp df_need_data + +@have_bytes: + lda df_cs_state + cmp #CS_ENTRY_DATA + bne :+ + jmp @entry_data +: cmp #CS_EXT_DATA + bne :+ + jmp @ext_data +: cmp #CS_CTX + beq @ctx + cmp #CS_LIST_LEN + beq @list_len + cmp #CS_ENTRY_LEN + beq @entry_len + ; CS_EXT_LEN + jmp @ext_len + +@ctx: + ; certificate_request_context length — must be 0 for a server + ; Certificate (RFC 8446 §4.4.2) + jsr df_cs_take_byte + beq :+ + jmp @cert_fmt_err +: lda #CS_LIST_LEN + sta df_cs_state + lda #0 + sta df_cs_idx + jmp @parse + +@list_len: + ; certificate_list length, 3 bytes. High byte must be 0 (the + ; 16-bit message length already caps everything below 64 KB); + ; the low 16 bits are not tracked — df_msg_rem bounds the walk. + jsr df_cs_take_byte + ldx df_cs_idx + bne :+ + cmp #0 + beq :+ + jmp @cert_fmt_err +: inc df_cs_idx + lda df_cs_idx + cmp #3 + bne @parse + lda #CS_ENTRY_LEN + sta df_cs_state + lda #0 + sta df_cs_idx + jmp @parse + +@entry_len: + ; CertificateEntry cert_data length, 3 bytes big-endian + jsr df_cs_take_byte + ldx df_cs_idx + bne @el_not_first + cmp #0 + beq @el_next + jmp @cert_fmt_err ; entry >= 64 KB +@el_not_first: + cpx #1 + bne :+ + sta df_cs_tmp+1 ; length high byte + jmp @el_next +: sta df_cs_tmp ; length low byte +@el_next: + inc df_cs_idx + lda df_cs_idx + cmp #3 + beq :+ + jmp @parse +: ; length complete + lda df_cs_tmp + sta df_cs_need + lda df_cs_tmp+1 + sta df_cs_need+1 + lda #0 + sta df_cs_idx + lda df_cs_entry + bne @el_done ; not the leaf — just skip its bytes + ; leaf: must fit cert_buf. Checked BEFORE any copy. + lda df_cs_need+1 + cmp #>DF_CERT_BUF_SIZE + bcc @el_leaf_fits + bne @el_leaf_too_big + lda df_cs_need + beq @el_leaf_fits ; exactly 1536 ($0600) still fits +@el_leaf_too_big: + lda #DF_ERR_CERT_TOO_BIG + jmp df_fail +@el_leaf_fits: + lda #0 + sta df_cs_off + sta df_cs_off+1 +@el_done: + lda #CS_ENTRY_DATA + sta df_cs_state + jmp @parse + +@entry_data: + ; cert_data bytes: copy to cert_buf for the leaf, discard others + lda df_cs_need + ora df_cs_need+1 + beq @entry_data_done + jsr df_cs_min_take_need ; df_n = min(df_take, df_cs_need) + lda df_cs_entry + bne @ed_skip_copy + ; leaf: copy df_n bytes (record cursor) -> cert_buf + df_cs_off + jsr df_rec_cursor + clc + lda #cert_buf + adc df_cs_off+1 + sta zp_tmp2 + lda df_n + sta df_copy_len + lda df_n+1 + sta df_copy_len+1 + jsr df_copy_chunk + clc + lda df_cs_off + adc df_n + sta df_cs_off + lda df_cs_off+1 + adc df_n+1 + sta df_cs_off+1 +@ed_skip_copy: + jsr df_cs_advance_n ; consumes df_n, also df_cs_need -= df_n + lda df_cs_need + ora df_cs_need+1 + beq @entry_data_done + jmp @parse ; df_take exhausted — loop top handles it +@entry_data_done: + lda df_cs_entry + bne @edd_not_leaf + ; Leaf complete — extract the public key NOW (remaining chain + ; entries only get skipped; nothing else reads cert_buf). + lda df_cs_tmp ; leaf length (df_cs_tmp survives + ldx df_cs_tmp+1 ; CS_ENTRY_DATA untouched) + jsr tls_cert_stream_finish + bcc @edd_not_leaf + jmp @cert_fmt_err ; no usable EC public key in the leaf +@edd_not_leaf: + lda #CS_EXT_LEN + sta df_cs_state + lda #0 + sta df_cs_idx + jmp @parse + +@ext_len: + ; per-entry extensions length, 2 bytes big-endian + jsr df_cs_take_byte + ldx df_cs_idx + bne :+ + sta df_cs_tmp+1 + jmp @xl_next +: sta df_cs_tmp +@xl_next: + inc df_cs_idx + lda df_cs_idx + cmp #2 + beq :+ + jmp @parse +: lda df_cs_tmp + sta df_cs_need + lda df_cs_tmp+1 + sta df_cs_need+1 + lda #0 + sta df_cs_idx + lda df_cs_need + ora df_cs_need+1 + beq @entry_done ; no extensions + lda #CS_EXT_DATA + sta df_cs_state + jmp @parse + +@ext_data: + jsr df_cs_min_take_need + jsr df_cs_advance_n + lda df_cs_need + ora df_cs_need+1 + beq @entry_done + jmp @parse +@entry_done: + inc df_cs_entry + lda #CS_ENTRY_LEN + sta df_cs_state + lda #0 + sta df_cs_idx + jmp @parse + +@msg_end: + ; The declared message length ran out. It must have done so + ; cleanly between entries, with at least one entry consumed + ; (the leaf, whose extraction already succeeded above). + lda df_cs_state + cmp #CS_ENTRY_LEN + bne @cert_fmt_err + lda df_cs_idx + bne @cert_fmt_err + lda df_cs_entry + beq @cert_fmt_err + lda #$34 + sta tls_recv_sub_progress + ; message done — reset per-message state, report success. + ; (No df_dispatch here: the streamed Certificate was consumed + ; incrementally; there is nothing left to hand to a handler.) + lda #0 + sta df_mode + sta df_hdr_have + sta df_hdr_split + clc + rts + +@cert_fmt_err: + lda #DF_ERR_CERT_FMT jmp df_fail ; ============================================================================= @@ -481,10 +777,10 @@ df_consume_chunk: sta df_msg_rem+1 rts -; Copy df_chunk bytes from (zp_ptr) to (zp_tmp1). Clobbers A,X,Y and +; Copy df_copy_len bytes from (zp_ptr) to (zp_tmp1). Clobbers A,X,Y and ; advances both pointers' high bytes across pages. df_copy_chunk: - ldx df_chunk+1 + ldx df_copy_len+1 beq @tail @page: ldy #0 : lda (zp_ptr),y @@ -495,16 +791,94 @@ df_copy_chunk: inc zp_tmp2 dex bne @page -@tail: lda df_chunk +@tail: lda df_copy_len beq @done ldy #0 : lda (zp_ptr),y sta (zp_tmp1),y iny - cpy df_chunk + cpy df_copy_len bne :- @done: rts +; --- streaming-Certificate helpers ------------------------------------------ + +; Take one message byte from the record cursor. Returns it in A (flags +; reflect A). Advances df_rec_off; decrements df_take and df_msg_rem. +df_cs_take_byte: + jsr df_rec_cursor + ldy #0 + lda (zp_ptr),y + pha + inc df_rec_off + bne :+ + inc df_rec_off+1 +: lda df_take + bne :+ + dec df_take+1 +: dec df_take + lda df_msg_rem + bne :+ + dec df_msg_rem+1 +: dec df_msg_rem + pla + rts + +; df_n = min(df_take, df_cs_need). Clobbers A. +df_cs_min_take_need: + lda df_take+1 + cmp df_cs_need+1 + bcc @use_take + bne @use_need + lda df_take + cmp df_cs_need + bcc @use_take +@use_need: + lda df_cs_need + sta df_n + lda df_cs_need+1 + sta df_n+1 + rts +@use_take: + lda df_take + sta df_n + lda df_take+1 + sta df_n+1 + rts + +; Consume df_n message bytes in bulk: +; df_rec_off += n; df_take -= n; df_msg_rem -= n; df_cs_need -= n. +df_cs_advance_n: + clc + lda df_rec_off + adc df_n + sta df_rec_off + lda df_rec_off+1 + adc df_n+1 + sta df_rec_off+1 + sec + lda df_take + sbc df_n + sta df_take + lda df_take+1 + sbc df_n+1 + sta df_take+1 + sec + lda df_msg_rem + sbc df_n + sta df_msg_rem + lda df_msg_rem+1 + sbc df_n+1 + sta df_msg_rem+1 + sec + lda df_cs_need + sbc df_n + sta df_cs_need + lda df_cs_need+1 + sbc df_n+1 + sta df_cs_need+1 + rts + ; ============================================================================= ; Deframer state — NET_BSS_TAIL region (UCI: ~378 B free under comb) ; ============================================================================= @@ -521,6 +895,18 @@ df_avail: .res 2 ; unconsumed bytes in current record df_chunk: .res 2 ; current chunk length df_carry_off: .res 2 ; write offset into df_carry_buf df_last_err: .res 1 ; DF_ERR_* of the last failure +df_copy_len: .res 2 ; df_copy_chunk length parameter + +; streaming-Certificate consumer state +df_take: .res 2 ; message bytes left to parse this pump +df_n: .res 2 ; current bulk sub-operation length +df_cs_state: .res 1 ; CS_* +df_cs_idx: .res 1 ; byte index within a multi-byte field +df_cs_tmp: .res 2 ; field accumulator (hi/lo) +df_cs_need: .res 2 ; bytes remaining of current data field +df_cs_entry: .res 1 ; CertificateEntry counter (0 = leaf) +df_cs_off: .res 2 ; leaf write offset into cert_buf + df_carry_buf: .res DF_CARRY_SIZE .endif ; TLS_STREAM_DEFRAME diff --git a/tools/test_tls_deframe.py b/tools/test_tls_deframe.py index 875c7d4f..c3affa5f 100644 --- a/tools/test_tls_deframe.py +++ b/tools/test_tls_deframe.py @@ -38,6 +38,7 @@ from __future__ import annotations +import datetime import hashlib import hmac import os @@ -45,6 +46,11 @@ import subprocess import sys +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import NameOID + from c64_test_harness import ( Labels, ViceInstanceManager, @@ -68,6 +74,8 @@ "tls_rec_buf", "tls_rec_len", "tls_transcript_init", "tls_transcript_hash", "tls_transcript", "tls_s_hs_secret", + "cert_buf", "cert_buf_len", + "ecdsa_pubkey_x", "ecdsa_pubkey_y", "ecdsa_curve_id", ] # Handshake message types @@ -116,6 +124,40 @@ def hs_msg(msg_type: int, body: bytes) -> bytes: return bytes([msg_type]) + len(body).to_bytes(3, "big") + body +def cert_msg(entries, ctx: bytes = b"") -> bytes: + """RFC 8446 §4.4.2 Certificate message. entries = [(der, ext_bytes)].""" + lst = b"" + for der, ext in entries: + lst += len(der).to_bytes(3, "big") + der + lst += len(ext).to_bytes(2, "big") + ext + body = bytes([len(ctx)]) + ctx + len(lst).to_bytes(3, "big") + lst + return hs_msg(HS_CERT, body) + + +def chunks(data: bytes, n: int): + return [data[i:i + n] for i in range(0, len(data), n)] + + +def generate_p256_cert(): + """Self-signed ECDSA P-256 cert (same recipe as test_x509.py).""" + key = ec.generate_private_key(ec.SECP256R1()) + subject = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, "test.example.com"), + ]) + cert = (x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(subject) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.now(datetime.UTC)) + .not_valid_after(datetime.datetime.now(datetime.UTC) + + datetime.timedelta(days=365)) + .sign(key, hashes.SHA256())) + der = cert.public_bytes(serialization.Encoding.DER) + nums = key.public_key().public_numbers() + return der, nums.x.to_bytes(32, "big"), nums.y.to_bytes(32, "big") + + SECRET = hashlib.sha256(b"c64-https lane A deframe secret").digest() @@ -299,6 +341,113 @@ def check_transcript(name, *msgs): check("oversize spanning non-Certificate rejected", ev == [("err", ERR_TOO_BIG)], f"events={ev}") + # ======================================================================= + # W2: streaming Certificate consumer + # ======================================================================= + leaf_der, pub_x, pub_y = generate_p256_cert() + + def check_pubkey(name): + got_x = read_bytes(transport, labels["ecdsa_pubkey_x"], 32) + got_y = read_bytes(transport, labels["ecdsa_pubkey_y"], 32) + curve = read_bytes(transport, labels["ecdsa_curve_id"], 1)[0] + check(name + " (pubkey X)", got_x == pub_x, + f"got {got_x.hex()[:16]}.. want {pub_x.hex()[:16]}..") + check(name + " (pubkey Y)", got_y == pub_y, "") + check(name + " (curve id P-256)", curve == 0, f"curve={curve}") + + def clear_pubkey(): + write_bytes(transport, labels["ecdsa_pubkey_x"], b"\xa5" * 32) + write_bytes(transport, labels["ecdsa_pubkey_y"], b"\xa5" * 32) + write_bytes(transport, labels["ecdsa_curve_id"], b"\xff") + + # --- 10. Certificate wholly in one record: in-place path unchanged --- + cm = cert_msg([(leaf_der, b"")]) + if len(cm) <= 548: + rig.reset() + clear_pubkey() + ev = rig.drive([cm]) + check("in-place Certificate", ev == ["msg"], f"events={ev}") + check("in-place Certificate pointer = tls_rec_buf", + rig.hs_ptr() == labels["tls_rec_buf"], + f"ptr=${rig.hs_ptr():04X}") + check_pubkey("in-place Certificate") + check_transcript("in-place Certificate", cm) + else: + check("in-place Certificate", False, + f"generated cert msg too large for one record ({len(cm)} B)") + + # --- 11. Certificate chain spanning records (streamed leaf) --- + # Leaf with real extensions bytes, plus a second (discarded) chain + # entry — split into 529-byte-ish record chunks like a real MFL=512 + # server, but at a deliberately awkward 150 B to hit more edges. + chain = cert_msg([(leaf_der, b"\x00\x0b\x00\x01\x04"), + (b"\xaa" * 600, b"\x01\x02\x03")]) + rig.reset() + clear_pubkey() + write_bytes(transport, labels["cert_buf"], b"\x5a" * 16) # stale marker + ev = rig.drive(chunks(chain, 150)) + check("streamed Certificate chain", ev == ["msg"], f"events={ev}") + check_pubkey("streamed Certificate chain") + got_leaf = read_bytes(transport, labels["cert_buf"], len(leaf_der)) + check("streamed Certificate leaf staged in cert_buf", + got_leaf == leaf_der, "cert_buf contents differ") + got_len = read_bytes(transport, labels["cert_buf_len"], 2) + check("cert_buf_len set to leaf length", + struct.unpack(" Date: Fri, 21 Aug 2026 17:58:18 -0400 Subject: [PATCH 12/33] fix(deframer-test): collision-free overflow oracle + UCI default build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The overflow sentinel at cert_buf+1536 (=$A600) is tls_rec_buf[0] under both cfgs — the record layer legitimately writes it on every flight, so no implementation could pass that check (found by the deframer lane's first run against the gate). Replace it with a strictly stronger, collision-free oracle: fill cert_buf itself with the sentinel and require it untouched — the CertificateEntry length field precedes the data, so a correct guard rejects an oversized leaf before copying anything. Also: build BACKEND=uci by default (the deframer is TLS_STREAM_DEFRAME, UCI-only; ip65 xfails by design — DEFRAMER_BUILD_BACKEND=ip65 overrides), and early-exit the pump loop once the ring is drained and all expected messages dispatched, so surplus calls no longer spin the 65,536-poll timeout loop. Co-Authored-By: Claude Fable 5 --- tools/test_tls_deframer.py | 88 ++++++++++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 17 deletions(-) diff --git a/tools/test_tls_deframer.py b/tools/test_tls_deframer.py index 11cb76c2..95cb2a43 100644 --- a/tools/test_tls_deframer.py +++ b/tools/test_tls_deframer.py @@ -59,9 +59,17 @@ * the running transcript (read via ``tls_transcript_hash``) == SHA-256(concatenated handshake messages). Proves every message's bytes were folded regardless of record framing. - * for the oversized-leaf case, a sentinel at ``cert_buf + 1536`` survives and - no valid pubkey is extracted — proves the size guard fired instead of - overflowing. + * for the oversized-leaf case, ``cert_buf`` itself is pre-filled with a + sentinel pattern that must survive the flight untouched, and no valid + pubkey may be extracted. The CertificateEntry's 24-bit length field + arrives before any cert_data byte, so a correct implementation can (and + must) reject an oversized leaf BEFORE copying anything — "reject on the + length field" is the only overflow oracle this memory map permits, because + the byte after ``cert_buf``'s cap is ``tls_rec_buf[0]`` ($A600 under both + cfgs), which the record layer legitimately writes on every flight. (An + earlier revision put a sentinel at ``cert_buf+1536`` and was therefore + unpassable by ANY implementation — caught by the deframer lane's first run + against it.) Both crypto-verified messages (CertificateVerify, Finished) are intentionally absent from this suite: their transcript discipline needs a self-consistent @@ -74,11 +82,19 @@ Vectors are fully deterministic (fixed key/iv/seq, repo test cert), so a failure reproduces byte-for-byte. +BACKEND: the deframer ships UCI-only (``TLS_STREAM_DEFRAME`` defaults ON for +``BACKEND=uci``, OFF for ip65, which has no code/BSS headroom). This suite +therefore builds ``make BACKEND=uci`` by default; on an ip65 build the nine +``[deframer]`` scenarios xfail BY DESIGN, not by defect. The suite itself is +backend-agnostic — every address comes from ``build/labels.txt``. + Usage: python3 tools/test_tls_deframer.py [--verbose] Env: - C64_SKIP_BUILD=1 reuse the already-built PRG + C64_SKIP_BUILD=1 reuse the already-built PRG + DEFRAMER_BUILD_BACKEND=ip65 build the ip65 image instead (deframer + scenarios then xfail by design) Requires: Python 3.10+, c64_test_harness, VICE x64sc, cryptography """ @@ -233,15 +249,28 @@ def reset_recv_state(transport, labels) -> None: write_bytes(transport, labels["tls_state"], [TLS_STATE_CERTIFICATE]) -def drive_flight(transport, labels, record_bytes, n_records): +def ring_is_empty(transport, labels) -> bool: + head = read_bytes(transport, labels["tcp_recv_head"], 2) + tail = read_bytes(transport, labels["tcp_recv_tail"], 2) + return head == tail + + +def drive_flight(transport, labels, record_bytes, n_records, n_msgs): """Prime the ring and pump tls_recv_encrypted until the flight drains. Returns the list of carry flags observed (diagnostic only; the oracle is - end-state, so a build whose call granularity differs is still judged fairly). + end-state, so a build whose call granularity differs is still judged + fairly — one call may consume one message or the whole flight). + + Early exit: once the ring is empty and *n_msgs* calls have succeeded, + nothing more can arrive (net_poll is an RTS), so stop rather than let a + surplus call spin tls_recv_encrypted's 65,536-poll timeout loop. A C=1 + return also ends the flight (error, or drained + timed out). """ reset_recv_state(transport, labels) prime_ring(transport, labels, record_bytes) carries = [] + successes = 0 # An upper bound on calls under any sane design: never more than one call # per record, plus slack for a drain call. Capped so a wedged build cannot # spin the suite forever. @@ -255,6 +284,12 @@ def drive_flight(transport, labels, record_bytes, n_records): carry = regs[k] & 0x01 break carries.append(carry) + if carry == 0: + successes += 1 + if carry == 1: + break + if successes >= n_msgs and ring_is_empty(transport, labels): + break return carries @@ -276,13 +311,14 @@ def read_transcript(transport, labels): class Scenario: def __init__(self, name, kind, chunks, expect_pubkey, expect_transcript, - expect_overflow_safe=False): + expect_overflow_safe=False, n_msgs=1): self.name = name self.kind = kind # "plumbing" or "deframer" self.chunks = chunks self.expect_pubkey = expect_pubkey # (x, y) or None self.expect_transcript = expect_transcript # bytes or None self.expect_overflow_safe = expect_overflow_safe + self.n_msgs = n_msgs # handshake messages in flight def build_scenarios(cert_der, pubkey_xy): @@ -300,6 +336,7 @@ def build_scenarios(cert_der, pubkey_xy): [ee, cert], (px, py), hashlib.sha256(stream).digest(), + n_msgs=2, )) # A lone Certificate in a single record — the simplest reassembly-free @@ -319,6 +356,7 @@ def build_scenarios(cert_der, pubkey_xy): [ee + cert], (px, py), hashlib.sha256(ee + cert).digest(), + n_msgs=2, )) # --- deframer (b): one message split across records --- @@ -359,6 +397,7 @@ def build_scenarios(cert_der, pubkey_xy): [ee + cert[:half], cert[half:]], (px, py), hashlib.sha256(ee + cert).digest(), + n_msgs=2, )) # certificate spanning many small records (the real ~6-record pattern). @@ -398,8 +437,14 @@ def run_scenario(transport, labels, sc): write_bytes(transport, labels["ecdsa_pubkey_x"], bytes([POISON]) * 32) write_bytes(transport, labels["ecdsa_pubkey_y"], bytes([POISON]) * 32) write_bytes(transport, labels["tls_transcript"], bytes([POISON]) * 32) - # Overflow sentinel just past cert_buf's capacity. - write_bytes(transport, labels["cert_buf"] + CERT_BUF_MAX, bytes([SENTINEL])) + if sc.expect_overflow_safe: + # Fill cert_buf ITSELF with the sentinel. The CertificateEntry length + # field precedes the cert bytes, so a correct implementation rejects an + # oversized leaf before copying anything — cert_buf must come back + # untouched. A sentinel *past* cert_buf would be useless here: the next + # byte is tls_rec_buf[0], which the record layer legitimately writes. + write_bytes(transport, labels["cert_buf"], + bytes([SENTINEL]) * CERT_BUF_MAX) # Fresh running transcript so the fold accumulates exactly this flight. jsr(transport, labels["tls_transcript_init"], timeout=60.0) @@ -407,7 +452,8 @@ def run_scenario(transport, labels, sc): write_bytes(transport, labels["tls_hs_read_iv"], HS_READ_IV) record_bytes, n_records = records_from_chunks(sc.chunks) - carries = drive_flight(transport, labels, record_bytes, n_records) + carries = drive_flight(transport, labels, record_bytes, n_records, + sc.n_msgs) problems = [] @@ -431,12 +477,14 @@ def run_scenario(transport, labels, sc): ) if sc.expect_overflow_safe: - sentinel = read_bytes(transport, labels["cert_buf"] + CERT_BUF_MAX, 1)[0] - if sentinel != SENTINEL: + buf = read_bytes(transport, labels["cert_buf"], CERT_BUF_MAX) + if buf != bytes([SENTINEL]) * CERT_BUF_MAX: + first = next(i for i, b in enumerate(buf) if b != SENTINEL) problems.append( - f"BUFFER OVERFLOW: sentinel at cert_buf+{CERT_BUF_MAX} was " - f"clobbered (${sentinel:02X}, expected ${SENTINEL:02X}) — the " - f"oversized leaf was copied past cert_buf" + f"cert_buf written during an oversized-leaf flight (first " + f"clobber at cert_buf+{first}, ${buf[first]:02X}) — the size " + f"guard must fire on the CertificateEntry length field, " + f"BEFORE any cert_data byte is copied" ) gx, gy = read_pubkey(transport, labels) if gx != bytes([POISON]) * 32 or gy != bytes([POISON]) * 32: @@ -517,9 +565,15 @@ def main() -> int: if os.environ.get("C64_SKIP_BUILD"): print("\n=== Building (skipped: C64_SKIP_BUILD set) ===") else: - print("\n=== Building ===") + # UCI by default: the deframer is TLS_STREAM_DEFRAME, ON only under + # BACKEND=uci (ip65 has no headroom — its deframer scenarios xfail by + # design). make clean first: make tracks timestamps, not the command + # line, so a backend switch without clean produces a mixed link. + backend = os.environ.get("DEFRAMER_BUILD_BACKEND", "uci") + print(f"\n=== Building (BACKEND={backend}) ===") subprocess.run(["make", "clean"], capture_output=True, cwd=PROJECT_ROOT) - result = subprocess.run(["make"], capture_output=True, text=True, + result = subprocess.run(["make", f"BACKEND={backend}"], + capture_output=True, text=True, cwd=PROJECT_ROOT) if result.returncode != 0: print(f"Build failed:\n{result.stderr}") From 2757776f1de4c1fb1c2ead57e19bc25efc6736ac Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:59:00 -0400 Subject: [PATCH 13/33] fix(test): deframer-gate overflow sentinel collided with tls_rec_buf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cert_buf+1536 is $A600 = tls_rec_buf[0] under both cfgs, so the record layer clobbers the past-the-end sentinel on any build — the gate's oversized-leaf case failed against a correct deframer. Replace it with a full-buffer sentinel fill: W2's guard rejects before any copy, so the whole buffer staying intact is the stronger, collision-free oracle (it also catches partial copies). Analysis credit: Lane A's report. Co-Authored-By: Claude Fable 5 --- tools/test_tls_deframer.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tools/test_tls_deframer.py b/tools/test_tls_deframer.py index 11cb76c2..2636dbaa 100644 --- a/tools/test_tls_deframer.py +++ b/tools/test_tls_deframer.py @@ -59,9 +59,10 @@ * the running transcript (read via ``tls_transcript_hash``) == SHA-256(concatenated handshake messages). Proves every message's bytes were folded regardless of record framing. - * for the oversized-leaf case, a sentinel at ``cert_buf + 1536`` survives and - no valid pubkey is extracted — proves the size guard fired instead of - overflowing. + * for the oversized-leaf case, a sentinel fill of the whole ``cert_buf`` + survives and no valid pubkey is extracted — proves the size guard fired + before any copy. (Not one byte past the end: ``cert_buf + 1536`` is + $A600 = ``tls_rec_buf[0]``, which the record layer writes on any build.) Both crypto-verified messages (CertificateVerify, Finished) are intentionally absent from this suite: their transcript discipline needs a self-consistent @@ -398,8 +399,13 @@ def run_scenario(transport, labels, sc): write_bytes(transport, labels["ecdsa_pubkey_x"], bytes([POISON]) * 32) write_bytes(transport, labels["ecdsa_pubkey_y"], bytes([POISON]) * 32) write_bytes(transport, labels["tls_transcript"], bytes([POISON]) * 32) - # Overflow sentinel just past cert_buf's capacity. - write_bytes(transport, labels["cert_buf"] + CERT_BUF_MAX, bytes([SENTINEL])) + # Overflow sentinel: fill ALL of cert_buf. cert_buf+1536 cannot host a + # sentinel — it is $A600 = tls_rec_buf[0] under both cfgs, which the + # record layer writes on every build (found the hard way: the check + # failed against a correct deframer). The W2 guard rejects an oversized + # leaf BEFORE any copy, so the whole buffer staying intact is the + # stronger, collision-free oracle (it also catches a partial copy). + write_bytes(transport, labels["cert_buf"], bytes([SENTINEL]) * CERT_BUF_MAX) # Fresh running transcript so the fold accumulates exactly this flight. jsr(transport, labels["tls_transcript_init"], timeout=60.0) @@ -431,12 +437,13 @@ def run_scenario(transport, labels, sc): ) if sc.expect_overflow_safe: - sentinel = read_bytes(transport, labels["cert_buf"] + CERT_BUF_MAX, 1)[0] - if sentinel != SENTINEL: + got_buf = read_bytes(transport, labels["cert_buf"], CERT_BUF_MAX) + if got_buf != bytes([SENTINEL]) * CERT_BUF_MAX: + bad = next(i for i, b in enumerate(got_buf) if b != SENTINEL) problems.append( - f"BUFFER OVERFLOW: sentinel at cert_buf+{CERT_BUF_MAX} was " - f"clobbered (${sentinel:02X}, expected ${SENTINEL:02X}) — the " - f"oversized leaf was copied past cert_buf" + f"BUFFER WRITTEN ON OVERSIZED LEAF: cert_buf+{bad} was " + f"clobbered (${got_buf[bad]:02X}, expected ${SENTINEL:02X}) — " + f"the size guard must reject before any copy" ) gx, gy = read_pubkey(transport, labels) if gx != bytes([POISON]) * 32 or gy != bytes([POISON]) * 32: From 6c862b7bef5f65f33fa531706cbefa89fa339303 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:28:52 -0400 Subject: [PATCH 14/33] fix(record): inner content type misread on 512-content records (MFL real-server shape) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inner-type extraction in tls_record_decrypt indexed the decrypted plaintext with a two-page dispatch: high byte 0 -> tls_rec_buf,y, any other -> tls_rec_buf+256,y. A record carrying 512 content bytes puts the inner type at index 512 (page 2), so the code returned tls_rec_buf[256] — a mid-payload byte — and tls_recv_encrypted aborted on a bogus content type. Exactly-512-content records are what max_fragment_length-honoring servers send; every local fixture topped out one byte short, so this latent (pre-sprint) bug was reachable only live. Found on the first github.com attempt: abort in CERTIFICATE state, tls_rec_type=$08 = plaintext[256] of the first Certificate record (verified by offline decrypt of the captured ring against the DMA-dumped handshake key). Add the missing page-2 arm (legal indices reach $0213) and a gate vector shaped like the real flight — a 512-byte zero-padded EE message so the misread byte is $00 by construction, never $16 by luck (the first draft of the vector passed on the buggy build exactly that way). Mutation-proven in VICE: buggy build 9/10 failing only this scenario, fixed build 10/10. Co-Authored-By: Claude Fable 5 --- src/tls_record.s | 17 +++++++++++++---- tools/test_tls_deframer.py | 25 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/tls_record.s b/src/tls_record.s index 3e60d58c..36952cdd 100644 --- a/src/tls_record.s +++ b/src/tls_record.s @@ -459,11 +459,20 @@ tls_record_decrypt: tay ; Y = (ciphertext_len - 1) low byte lda tls_enc_aead_len+1 sbc #0 - beq @inner_lo_page ; if high byte = 0, in first page - ; High page: read from tls_rec_buf+256,y - lda tls_rec_buf+256,y + beq @inner_page0 ; index $0000-$00FF + cmp #2 + beq @inner_page2 ; index $0200-$02FF: 512-content records — + ; the max_fragment_length=512 real-server + ; shape (content 512 + type = index 512). + ; The old two-page dispatch read + ; tls_rec_buf+256 here and got a mid-DER + ; byte; found live against github.com. + lda tls_rec_buf+256,y ; index $0100-$01FF jmp @got_inner_type -@inner_lo_page: +@inner_page2: + lda tls_rec_buf+512,y + jmp @got_inner_type +@inner_page0: lda tls_rec_buf,y @got_inner_type: sta tls_rec_type diff --git a/tools/test_tls_deframer.py b/tools/test_tls_deframer.py index 95cb2a43..4ecb661a 100644 --- a/tools/test_tls_deframer.py +++ b/tools/test_tls_deframer.py @@ -409,6 +409,31 @@ def build_scenarios(cert_der, pubkey_xy): hashlib.sha256(cert).digest(), )) + # --- deframer: MFL-512 real-server record shape --- + # Records carrying EXACTLY 512 content bytes (inner plaintext 513: the + # inner content-type byte sits at index 512, i.e. page 2 of tls_rec_buf). + # This is what a max_fragment_length-honoring server (github.com) sends, + # and it is one byte past every local fixture: the record layer's + # inner-type extraction had a two-page dispatch that misread index 512 + # as tls_rec_buf[256], aborting the handshake with a bogus content type. + # Found live against github.com 2026-08-21; keep this vector so the + # class stays covered. + # The vector must be adversarial BY CONSTRUCTION: the buggy dispatch read + # tls_rec_buf[256] as the inner type, so a record whose content byte 256 + # happens to be $16 passes by luck (the first draft of this vector did). + # A 512-byte EE message padded with zeros pins content[256] to $00. + ee_big = hs_message(TLS_HS_ENCRYPTED_EXT, b"\x01\xfa" + b"\x00" * 506) + assert len(ee_big) == 512 + assert ee_big[256] != TLS_CT_HANDSHAKE + scenarios.append(Scenario( + "deframer/mfl512_full_records", + "deframer", + chunk_bytes(ee_big + cert, [512]), + (px, py), + hashlib.sha256(ee_big + cert).digest(), + n_msgs=2, + )) + # --- deframer (d): oversized leaf must error, not overflow --- # A single-entry Certificate whose cert_data exceeds cert_buf (1536 B). big_leaf = b"\x30\x82" + (CERT_BUF_MAX + 200).to_bytes(2, "big") + \ From dbf3c96d677a5c4ba20c19a89311eb023f7724b7 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:51:54 -0400 Subject: [PATCH 15/33] docs: record the real-server HTTPS milestone (github/browserleaks 200) + live-run findings Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 4a85e99c..5f5f469d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -647,6 +647,43 @@ The scripts: ### End-to-end HTTPS status +**REAL-SERVER MILESTONE (2026-08-21, U64E @ 48 MHz, comb):** the first +HTTPS GETs from the C64 to real public internet servers. + + target CFIN http_status note + github.com 32.5 s **200** full PASS, homepage HTML prefix + browserleaks.com 33.4 s **200** TLS + status PASS; rig sentinel + timed out on body drain (below) + +The W1 streaming deframer handled the real 9-11-record flights +(Certificate ~2.7 KB over ~6 records; CV+Finished sharing the tail), the +W2 streaming consumer staged the leaf, and the sibling library verified +real CA-issued chains. Local padded-chain bench: 11-record flight costs +only ~0.7 s over the single-cert control (31.5 vs 30.8 s CFIN). + +Two findings from the live runs: + - **512-content records** (what MFL-honoring servers actually send) hit + a latent page-dispatch bug in the record layer's inner-type read — + fixed in `src/tls_record.s` (page-2 arm), mutation-proven by + `tools/test_tls_deframer.py::mfl512_full_records`. Every local + fixture had been one byte short of the trigger. + - **W4 gap (open):** a large NON-chunked body cannot terminate by + Content-Length once `body_append` truncates at 512 B + (`http_resp_len` freezes, the equality never fires), leaving only + the connection-close fallback — browserleaks holds the connection + and the rig sentinel timed out at 300 s with status/body already + correct. Fix direction: count consumed-not-stored body bytes + (needs >16-bit for real pages) or cap the post-truncation drain. + +**U64E "writemem exhaustion wedge"** (user-named): fw 3.14d misses +garbage collection on a temp location filled by REST `writemem`; a long +hardware session (~15 PRG loads that day) fills it CUMULATIVELY and then +REST *and* the UCI bridge wedge together — ping alive, REST refuses +instantly, C64 parks mid-transfer. Power cycle fixes. Budget PRG loads +per session and power-cycle proactively; `readmem` is not implicated. + + + The TLS 1.3 handshake now completes end-to-end against the local test listener (ECDSA-P256 cert, `tools/https_e2e/certs/`) on **both** backends: UCI/U64E at 48 MHz turbo and stock 1 MHz, and ip65/VICE at stock 1 MHz From fc6effed0e9363492db9a71c929a9707ce17fe9a Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:59:51 -0400 Subject: [PATCH 16/33] =?UTF-8?q?feat(uci):=20userspace=20/Temp=20GC=20?= =?UTF-8?q?=E2=80=94=20defuses=20the=20writemem=20exhaustion=20wedge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fw <= 3.14d leaks one /Temp attachment per REST body (run_prg's ~63 KB PRG + a 3 B trigger POST per rig run, verified by FTP listing) and never collects them; ~15 runs fill /Temp and wedge REST + the UCI bridge together, recoverable only by power cycle. Upstream fixed it in GideonZ/1541ultimate#686 (auto-cleanup, keep youngest 10) but no released firmware carries it yet. _temp_gc.py mirrors that policy host-side over FTP: delete managed temp files oldest-first, keep the youngest 2, touch nothing else. Wired into rig_https_live.py right after enable_uci (under the DeviceLock); best-effort, C64_SKIP_TEMP_GC=1 bypasses. Verified live on the U64E. Co-Authored-By: Claude Fable 5 --- tools/uci/_temp_gc.py | 77 +++++++++++++++++++++++++++++++++++++ tools/uci/rig_https_live.py | 7 ++++ 2 files changed, 84 insertions(+) create mode 100644 tools/uci/_temp_gc.py diff --git a/tools/uci/_temp_gc.py b/tools/uci/_temp_gc.py new file mode 100644 index 00000000..e72a93a8 --- /dev/null +++ b/tools/uci/_temp_gc.py @@ -0,0 +1,77 @@ +"""Userspace GC for the U64E's /Temp attachment leak (fw <= 3.14d). + +Every REST call that carries a body — run_prg's PRG upload, the keystroke +trigger POST, machine:writemem — lands as a file in /Temp on the device, +and released firmware never deletes them: the "writemem exhaustion wedge". +When /Temp fills (~15 rig runs at ~63 KB PRG + 1 small attachment per +run), the REST API *and* the C64-facing UCI bridge wedge together and +only a power cycle recovers. Upstream fixed it in +GideonZ/1541ultimate#686 (auto-cleanup keeping the youngest 10 managed +files), but no released firmware carries it yet. + +This helper mirrors that policy from the host side over FTP: it deletes +/Temp files matching ``temp`` — the firmware's managed-attachment +naming — oldest-first, keeping the youngest ``keep``. It never touches +any other /Temp entry (user files, mounted images). + +Call ``gc_temp(host)`` at rig start (rigs run it right after enable_uci, +under the DeviceLock) or run standalone: + + python3 tools/uci/_temp_gc.py [host] + +Environment: ``U64_HOST`` (default 192.168.1.81), ``C64_SKIP_TEMP_GC=1`` +to bypass, ``TEMP_GC_KEEP`` to override the keep count (default 2). +""" + +from __future__ import annotations + +import os +import re +import sys +from ftplib import FTP, error_perm + +_MANAGED = re.compile(r"^temp\d+$") + + +def gc_temp(host: str, keep: int | None = None, timeout: float = 10.0) -> int: + """Delete managed /Temp attachments, keeping the youngest *keep*. + + Returns the number of files deleted. Any FTP failure is reported and + swallowed — GC is best-effort hygiene and must never fail a rig run. + """ + if os.environ.get("C64_SKIP_TEMP_GC") == "1": + return 0 + if keep is None: + keep = int(os.environ.get("TEMP_GC_KEEP", "2")) + deleted = 0 + try: + with FTP(host, timeout=timeout) as ftp: + ftp.login() + ftp.cwd("/Temp") + # NLST order is the firmware's directory order, which is + # creation order for the monotonically numbered tempNNNN + # files; the numeric sort below makes oldest-first explicit + # (and survives a counter that skips). + names = [n for n in ftp.nlst() if _MANAGED.match(n)] + names.sort(key=lambda n: int(n[4:])) + victims = names[: max(0, len(names) - keep)] if keep else names + for name in victims: + try: + ftp.delete(name) + deleted += 1 + except error_perm as exc: + # e.g. still referenced; skip — the next GC gets it + print(f" temp-gc: skip {name}: {exc}", file=sys.stderr) + except OSError as exc: + print(f" temp-gc: FTP unavailable on {host}: {exc}", file=sys.stderr) + return deleted + if deleted: + print(f" temp-gc: deleted {deleted} /Temp attachment(s) " + f"(kept youngest {keep}) — writemem-wedge budget restored") + return deleted + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else \ + os.environ.get("U64_HOST", "192.168.1.81") + gc_temp(target) diff --git a/tools/uci/rig_https_live.py b/tools/uci/rig_https_live.py index 9f6fa4df..b49bc5f9 100644 --- a/tools/uci/rig_https_live.py +++ b/tools/uci/rig_https_live.py @@ -95,6 +95,7 @@ from _memory_policy import build_policy_and_arbiter_with_overlay_carveout from _reu_preflight import ReuPreflightError, preflight_reu +from _temp_gc import gc_temp # Pure helpers shared with the local rig (all side-effect free; importing # the module only evaluates env defaults, it drives no hardware). @@ -411,6 +412,12 @@ def main() -> int: enable_uci(client) uci_enabled = True + # Userspace /Temp GC (fw <= 3.14d leaks one attachment per REST + # body; ~15 runs wedge REST + the UCI bridge together). Mirrors + # the GideonZ/1541ultimate#686 cleanup until a released firmware + # carries it. Best-effort; C64_SKIP_TEMP_GC=1 bypasses. + gc_temp(HOST) + # Wedged-runner pre-detect (see rig_https_local for the rationale). try: runner_health_check(client) From b0185acd175f779f81819e7a55904eeb443b0bd4 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:22:39 -0400 Subject: [PATCH 17/33] docs: HTTPS_HOST/TLS_STREAM_DEFRAME variables + rig_https_live in the reference sections Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 5f5f469d..c3af72a7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -191,6 +191,18 @@ Variables: ~7 MHz; boot costs ~50 s at 64 MHz (test scripts: set C64_INIT_WAIT). Uses cfg/c64-https-$(BACKEND)-onchip.cfg. + - `HTTPS_HOST=` — build-time HTTPS target (default `www.foo.bar`; + feeds the GET, SNI and DNS lookup). Travels via + a generated `build/https_host.inc` because + ca65's `-D` is numeric-only; the generator is + content-compared and deletes `boot.o` on a + change, so no `make clean` is needed for THIS + flag (the link step keeps the repo-wide + same-second caveat). Hosts >63 chars are a + build error (SNI copy guard). + - `TLS_STREAM_DEFRAME` — streaming handshake deframer gate; default ON + under `BACKEND=uci`, OFF (compiled out) under + ip65. See the real-server milestone entry. - `CA65`, `LD65` — toolchain overrides - `VICE` — override the `make run` emulator @@ -606,6 +618,15 @@ The scripts: stock `ssl`. `FINISHED_MODE=good` is the control and must be run first. See "Negative-path coverage — the server Finished" under Smoke tests. + - `rig_https_live.py` — HTTPS e2e against a REAL public server + (`HTTPS_TARGET=github.com`, DMA-fed to the + PRG so any build works). Pass criteria are + C64-side only (sentinel + `http_status=200`). + Runs the `/Temp` GC (`_temp_gc.py`) after + enable_uci — see the writemem-wedge note in + "End-to-end HTTPS status". Comb + `C64_INIT_WAIT=75` is auto-detected from + labels.txt. Keep `DEBUG_CAPTURE=0` here. - `rig_https_local.py` — HTTPS e2e scaffolding against a local TLS 1.3 listener (ECDSA-P256 cert from `tools/https_e2e/certs/`). DMAs a 6502 stub From 88972984352d28c85778025fb142874919c24df5 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:24:39 -0400 Subject: [PATCH 18/33] feat(w3b): HTTPS_PATH build knob alongside HTTPS_HOST Same generated-include mechanism; default '/' behaves identically. Needed for the Wikipedia-article stretch goal (action=raw endpoint). Build-time assert caps the path at 100 chars for the request builder. Co-Authored-By: Claude Fable 5 --- Makefile | 5 ++++- src/boot.s | 14 +++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index d3466bae..9b1eaa7a 100644 --- a/Makefile +++ b/Makefile @@ -86,6 +86,9 @@ endif # The generator rule is content-compared: changing HTTPS_HOST= rebuilds # boot.o with no `make clean`, and an unchanged value touches nothing. HTTPS_HOST ?= www.foo.bar +# Companion request-path override (default "/"), same mechanism/caveats. +# e.g. make HTTPS_PATH='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/w/index.php?title=Commodore_64&action=raw' +HTTPS_PATH ?= / # VIC-II blanking around the CPU-bound crypto primitives (src/vic.s). # ON by default and that is what ships; `make VIC_BLANK=0` degrades # vic_blank/vic_unblank to bare RTS so the badline tax can be measured @@ -440,7 +443,7 @@ build/net/ip65/ip65_blob.o: $(IP65_BIN) # it "up to date" and the file simply vanishes (measured too). build/https_host.inc: FORCE @mkdir -p build - @printf '.define HTTPS_HOST_STR "%s"\n' '$(HTTPS_HOST)' > $@.tmp + @printf '.define HTTPS_HOST_STR "%s"\n.define HTTPS_PATH_STR "%s"\n' '$(HTTPS_HOST)' '$(HTTPS_PATH)' > $@.tmp @if cmp -s $@.tmp $@; then rm -f $@.tmp; else mv $@.tmp $@ && rm -f build/boot.o; fi FORCE: diff --git a/src/boot.s b/src/boot.s index e355755d..c5fa5e8e 100644 --- a/src/boot.s +++ b/src/boot.s @@ -532,11 +532,11 @@ do_https_get: lda #http_host_target_len sta http_host_len - lda #http_path_root + lda #>https_path_target sta http_path_ptr+1 - lda #1 + lda #https_path_target_len sta http_path_len lda # Date: Fri, 21 Aug 2026 20:36:37 -0400 Subject: [PATCH 19/33] =?UTF-8?q?feat(uci):=20rig=5Fhttps=5Fwiki=20?= =?UTF-8?q?=E2=80=94=20Wikipedia-article=20stretch-goal=20rig=20(Lane=20H)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Host-side rig for the ~122 KB Commodore_64 wikitext fetch into the REU at $10:0000 with the on-screen viewer left live for the human. Based on rig_https_live.py; written against the 6502 lanes' contract exports (http_body_total 3B LE / http_body_sink / http_resp_buf 512 B prefix / viewer_enter) before those lanes land — main() refuses a PRG missing them and names the build line. Differences from the live rig: - REU forced to (Enabled, 16 MB) under the DeviceLock before reset, via the harness's device-generation-aware set_reu(); read first, write only on mismatch, settle after a real write, revert-on-power- cycle printed. preflight_reu() kept (presence, not size). - 600 s sentinel budget (WIKI_TIMEOUT), http_body_total polled at 1 Hz as the progress signal, printed at most every 10 s. - Verification: http_status==200 AND a host-side stdlib-ssl reference fetch of the same URL with C64-faithful headers + a User-Agent; exact size match + byte-exact 512 B prefix = PASS, size drift with matching prefix = WARN (both sizes printed), prefix mismatch or an unavailable reference = FAIL. - On PASS the machine is NOT reset and UCI stays enabled; the lock- release caveat (next queued job resets the device) is printed. - Trampoline JSRs viewer_enter carry-gated, after the sentinel write. Recon baked into the header (2026-08-21): wikipedia still MFL-honoring (529 B max record, 14-rec/4,659 B flight, ~120 s tolerance), ECDSA leaf still 1636 B DER; action=raw is Transfer-Encoding: chunked (NO Content-Length), 4 chunks <=32 KB, body 125,235 B, byte-stable; and a UA-less request — what src/http.s sends today — gets HTTP 403. Selfcheck: 26/26 offline+live; tools/test_pytest_boundary.py green. Co-Authored-By: Claude Fable 5 --- tools/uci/rig_https_wiki.py | 947 ++++++++++++++++++++++++++++++++++++ 1 file changed, 947 insertions(+) create mode 100644 tools/uci/rig_https_wiki.py diff --git a/tools/uci/rig_https_wiki.py b/tools/uci/rig_https_wiki.py new file mode 100644 index 00000000..753d95b6 --- /dev/null +++ b/tools/uci/rig_https_wiki.py @@ -0,0 +1,947 @@ +#!/usr/bin/env python3 +"""Wikipedia-article stretch goal: GET ~122 KB of wikitext from +en.wikipedia.org into the REU at $10:0000, then hand the machine to the +human with the on-screen viewer live. + +Build the PRG this rig expects with EXACTLY: + + make clean && make BACKEND=uci USE_NISTCURVES_ONCHIP_COMB=1 \\ + HTTPS_HOST=en.wikipedia.org \\ + 'HTTPS_PATH=/w/index.php?title=Commodore_64&action=raw' \\ + HTTPS_BODY_TO_REU=1 + +Contract with the 6502 lanes (exports expected in build/labels.txt): + http_body_total 3 bytes LE — de-chunked body bytes sunk so far; the + rig's progress signal AND the size it verifies + http_body_sink body-sink entry (presence proves the right build) + http_status 16-bit LE HTTP status + http_resp_buf holds the FIRST 512 body bytes after completion + viewer_enter on-screen viewer; the trampoline JSRs it on success + +Recon (2026-08-21, tools/probe_server_tls.py + header-faithful host +fetch — re-run both before trusting): + * en.wikipedia.org still honors max_fragment_length: 529 B max record + (fits TLS_REC_BUF_MAX 548), 14-record / 4,659 B server flight, + ~120 s silence tolerance, ECDSA-P256 leaf 1636 B DER (<= the 2048 B + cert_buf the certbuf lane sized for). + * action=raw responds **Transfer-Encoding: chunked** (NO + Content-Length): 4 data chunks of 26,931-32,768 B, body 125,235 B, + `text/x-wiki`, byte-stable across back-to-back requests. The + body-sink must ride the W4 chunked path; chunk sizes fit the + 16-bit http_chunk_rem. + * **A request without a User-Agent header gets HTTP 403** ("Please + set a user-agent...m robot policy"). src/http.s's http_build_get + sends only Host + Connection — the HTTP lane must add a UA header + or every run ends 403. + +Environment variables: + U64_HOST — device IP (default 192.168.1.81) + WIKI_HOST — hostname the trampoline dials (default + en.wikipedia.org; must match the build's + HTTPS_HOST for the banner/menu path but the + trampoline programs it either way) + WIKI_PATH — path (default /w/index.php?title=Commodore_64& + action=raw). Tip: pin a revision with &oldid=N + for a byte-exact compare that cannot drift. + WIKI_TIMEOUT — sentinel budget seconds (default 600; the fetch + is ~250+ TLS records after a ~35 s handshake) + WIKI_REF_UA — User-Agent for the HOST-side reference fetch + (Wikipedia requires one; default identifies this + project per their robot policy) + EXPECT_STATUS — expected http_status (default 200) + TURBO_MHZ — C64 CPU MHz (default 48); budgets scale by 48/MHz + C64_INIT_WAIT — boot wait; auto-detected 75 s for a comb build + REU_SETTLE / TURBO_SETTLE — post-config-write settle (default 3.0 s; + config writes glitch the next UCI command) + PHASE_TIMING — screen-scrape phase table (default 0 here: the + run is long and body progress is the signal) + UCI_DEBUG_DIR — artifact dir (default /tmp/uci_https_wiki_debug) + C64_SKIP_TEMP_GC / C64_SKIP_REU_PREFLIGHT — bypass those steps + WIKI_SELFCHECK_OFFLINE — set 1 to skip the live reference fetch in + --selfcheck + +Hardware conventions honored (CLAUDE.md "UCI rig scripts"): + * REU forced to (Enabled, 16 MB) BEFORE reset — the article lands at + REU bank 16, past the flash-saved 512 KB default. Read first, + write only on mismatch (any config write can drop the next UCI + command), settle after a real write. The write is runtime-only and + REVERTS ON POWER CYCLE — the rig prints exactly what it changed. + * preflight_reu() kept as well: it checks presence/enabled against + the build profile, not size. + * turbo set BEFORE reset with the read-skip-write pattern; + * /Temp GC after enable_uci (fw <= 3.14d writemem leak); + * every DMA address from build_policy_and_arbiter_with_overlay_ + carveout() — nothing hardcoded; + * DEBUG_CAPTURE (6510 bus stream) is NOT wired here at all — OFF; + * progress is read-only DMA at ~1-2 Hz (reads are wedge-safe; + writemem is the leaky side), progress PRINTS at most every 10 s; + * rig_*.py name keeps pytest collection out. + +On PASS the machine is NOT reset and UCI is left enabled: the viewer +is live on the real screen for the human. CAVEAT (DeviceLock module +docs): the lock is a queued flock — the moment this rig releases it, +any waiting harness job acquires the device and will typically reset +it. The viewer session lasts only until the next queued job; hold off +launching other device work while playing with it. + +Offline structural check (no hardware, no DeviceLock): + ./rig_https_wiki.py --selfcheck +""" +from __future__ import annotations + +import os +import socket +import ssl +import sys +import time +from pathlib import Path + +from c64_test_harness.backends.device_lock import DeviceLock, DeviceLockTimeout +from c64_test_harness.backends.ultimate64 import Ultimate64Transport +from c64_test_harness.backends.ultimate64_client import Ultimate64Client +from c64_test_harness.backends.ultimate64_helpers import ( + CAT_U64_SPECIFIC, + Ultimate64RunnerStuckError, + cpu_speed_enum, + get_reu_config, + runner_health_check, + set_reu, + set_turbo_mhz, +) +from c64_test_harness.keyboard import send_text +from c64_test_harness.labels import Labels +from c64_test_harness.uci_network import disable_uci, enable_uci + +from _memory_policy import build_policy_and_arbiter_with_overlay_carveout +from _reu_preflight import ReuPreflightError, preflight_reu +from _temp_gc import gc_temp + +# Pure helpers shared with the local/live rigs (side-effect free imports). +from rig_https_local import ( + _create_run_dir, + _current_phase, + _decode_screen_ram, + _dump_diag, + _dump_ring, + _dump_tls_state_snapshot, + _format_phase_log, + _prune_old_run_dirs, + _write_run_info, +) +from rig_https_live import SENTINEL_VALUE, default_init_wait + +HOST = os.environ.get("U64_HOST", "192.168.1.81") +REPO_ROOT = Path(__file__).resolve().parents[2] +PRG_PATH = REPO_ROOT / "build" / "c64-https.prg" +LABELS_PATH = REPO_ROOT / "build" / "labels.txt" + +WIKI_HOST = os.environ.get("WIKI_HOST", "en.wikipedia.org") +WIKI_PATH = os.environ.get( + "WIKI_PATH", "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/w/index.php?title=Commodore_64&action=raw") +WIKI_PORT = int(os.environ.get("WIKI_PORT", "443")) +EXPECT_STATUS = int(os.environ.get("EXPECT_STATUS", "200")) +# Wikipedia's robot policy 403s UA-less clients; the reference fetch +# must send one (the C64 side's UA is baked by the HTTP lane). +WIKI_REF_UA = os.environ.get( + "WIKI_REF_UA", + "c64-https-rig/0.1 (host-side reference fetch; " + "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/JC-000/c64-https)") + +TURBO_MHZ = int(os.environ.get("TURBO_MHZ", "48")) +_TIMEOUT_SCALE = max(1.0, 48.0 / float(TURBO_MHZ)) +WIKI_TIMEOUT = float(os.environ.get("WIKI_TIMEOUT", "600")) * _TIMEOUT_SCALE + +REQUIRED_REU_SIZE = "16 MB" # article sink base is REU bank $10 +RESP_PREFIX_BYTES = 512 # contract: first 512 body bytes +PROGRESS_PRINT_INTERVAL = 10.0 # print at most every 10 s +POLL_INTERVAL = 1.0 # readmem cadence (reads are safe) + +UCI_DEBUG_BASE_DIR = Path( + os.environ.get("UCI_DEBUG_DIR", "/tmp/uci_https_wiki_debug")) +UCI_DEBUG_KEEP = 5 + +# Labels the wiki-contract build must export, on top of the live-rig set. +CONTRACT_LABELS = [ + "http_body_total", "http_body_sink", "viewer_enter", +] +BASE_LABELS = [ + "http_get", "http_host_ptr", "http_host_len", + "http_path_ptr", "http_path_len", "http_port", + "net_init", "net_initialized", + "net_last_error", "net_tcp_state", + "tcp_recv_head", "tcp_recv_tail", + "http_resp_buf", "http_resp_len", "http_status", + "tls_state", "tls_last_state", +] + + +def _load_labels() -> dict[str, int]: + return dict(Labels.from_file(LABELS_PATH)) + + +# --------------------------------------------------------------------------- # +# 6502 trampoline # +# --------------------------------------------------------------------------- # + +def build_wiki_routine(labels: dict[str, int], *, + routine_addr: int, + host_str_addr: int, + path_str_addr: int, + sentinel_addr: int, + progress_addr: int, + carry_flag_addr: int, + host_len: int, + path_len: int, + port: int) -> bytes: + """Trampoline: http_get for the wiki target, then viewer on success. + + Same shape as rig_https_live's routine with two differences: the + path length is parameterised (the article path is 42 chars, not + "/"), and after latching carry + writing the sentinel it JSRs + ``viewer_enter`` when http_get returned C=0 — leaving the viewer + live on screen — and parks otherwise. + """ + if not (1 <= host_len <= 63): + raise ValueError(f"host_len {host_len} out of range 1..63 " + "(tls_hostname copy guards at 63)") + if not (1 <= path_len <= 128): + # http_req_buf is 256 B total for "GET HTTP/1.1" + Host + + # UA + Connection; 128 keeps comfortable margin. + raise ValueError(f"path_len {path_len} out of range 1..128 " + "(http_req_buf is 256 B for the whole request)") + code = bytearray() + + def emit(*bs: int) -> None: + code.extend(bs) + + def emit_lda_imm(v: int) -> None: + emit(0xA9, v & 0xFF) + + def emit_sta_abs(addr: int) -> None: + emit(0x8D, addr & 0xFF, (addr >> 8) & 0xFF) + + def emit_lda_abs(addr: int) -> None: + emit(0xAD, addr & 0xFF, (addr >> 8) & 0xFF) + + def emit_jsr(addr: int) -> None: + emit(0x20, addr & 0xFF, (addr >> 8) & 0xFF) + + def emit_progress(step: int) -> None: + emit_lda_imm(step) + emit_sta_abs(progress_addr) + + http_get = labels["http_get"] + http_host_ptr = labels["http_host_ptr"] + http_host_len = labels["http_host_len"] + http_path_ptr = labels["http_path_ptr"] + http_path_len = labels["http_path_len"] + http_port = labels["http_port"] + net_init = labels["net_init"] + tcp_recv_head = labels["tcp_recv_head"] + tcp_recv_tail = labels["tcp_recv_tail"] + viewer_enter = labels["viewer_enter"] + + # 0) Bank BASIC ROM OUT so $A000-$BFFF is RAM + emit_lda_abs(0x0001) + emit(0x29, 0xFE) # AND #$FE + emit_sta_abs(0x0001) + + # Clear markers + emit_lda_imm(0x00) + emit_sta_abs(sentinel_addr) + emit_sta_abs(progress_addr) + emit_sta_abs(carry_flag_addr) + + emit_progress(0x01) + + # Re-init UCI (ensure idle state after auto-init) + emit_jsr(net_init) + + # Zero the TCP ring head/tail to flush stale boot-poll data + emit_lda_imm(0x00) + emit_sta_abs(tcp_recv_head) + emit_sta_abs(tcp_recv_head + 1) + emit_sta_abs(tcp_recv_tail) + emit_sta_abs(tcp_recv_tail + 1) + + emit_progress(0x02) + + # http_host_ptr/len + emit_lda_imm(host_str_addr & 0xFF) + emit_sta_abs(http_host_ptr) + emit_lda_imm((host_str_addr >> 8) & 0xFF) + emit_sta_abs(http_host_ptr + 1) + emit_lda_imm(host_len) + emit_sta_abs(http_host_len) + + # http_path_ptr/len — real article path, length baked in + emit_lda_imm(path_str_addr & 0xFF) + emit_sta_abs(http_path_ptr) + emit_lda_imm((path_str_addr >> 8) & 0xFF) + emit_sta_abs(http_path_ptr + 1) + emit_lda_imm(path_len) + emit_sta_abs(http_path_len) + + # http_port (16-bit LE) + emit_lda_imm(port & 0xFF) + emit_sta_abs(http_port) + emit_lda_imm((port >> 8) & 0xFF) + emit_sta_abs(http_port + 1) + + emit_progress(0x03) + + # The REAL TLS 1.3 + HTTP + body-sink code path. + emit_jsr(http_get) + + # Latch carry (0=success, 1=failure) via PHP/PLA + emit(0x08) # PHP + emit(0x68) # PLA + emit_sta_abs(carry_flag_addr) + + emit_progress(0x04) + + # Sentinel FIRST so the rig sees completion promptly, then hand the + # screen to the human iff http_get succeeded (carry bit 0 clear). + emit_lda_imm(SENTINEL_VALUE) + emit_sta_abs(sentinel_addr) + + emit_lda_abs(carry_flag_addr) + emit(0x29, 0x01) # AND #$01 — carry is bit 0 of pushed P + emit(0xD0, 0x03) # BNE +3 (skip the JSR on failure) + emit_jsr(viewer_enter) # normally never returns while human plays + + # Park CPU (also the BNE target) + park = routine_addr + len(code) + emit(0x4C, park & 0xFF, (park >> 8) & 0xFF) # JMP park + + return bytes(code) + + +# --------------------------------------------------------------------------- # +# Host-side reference fetch (stdlib ssl only) # +# --------------------------------------------------------------------------- # + +def fetch_reference(host: str, path: str, port: int = 443, *, + user_agent: str = WIKI_REF_UA, + timeout: float = 30.0) -> dict: + """GET the same URL the C64 fetched, with C64-faithful headers plus a + User-Agent (Wikipedia 403s without one). Returns + {status, framing, body, content_length} — body is de-chunked. + """ + ctx = ssl.create_default_context() + ctx.minimum_version = ssl.TLSVersion.TLSv1_3 + raw = socket.create_connection((host, port), timeout=timeout) + s = ctx.wrap_socket(raw, server_hostname=host) + try: + req = (f"GET {path} HTTP/1.1\r\nHost: {host}\r\n" + f"User-Agent: {user_agent}\r\nConnection: close\r\n\r\n") + s.sendall(req.encode("ascii")) + buf = b"" + while True: + try: + chunk = s.recv(65536) + except (ssl.SSLError, OSError): + break + if not chunk: + break + buf += chunk + finally: + s.close() + + hdr_end = buf.find(b"\r\n\r\n") + if hdr_end < 0: + raise RuntimeError("reference fetch: no header terminator in reply") + header_lines = buf[:hdr_end].decode("latin-1").split("\r\n") + status = int(header_lines[0].split(" ", 2)[1]) + headers = {} + for line in header_lines[1:]: + if ":" in line: + k, v = line.split(":", 1) + headers[k.strip().lower()] = v.strip() + body = buf[hdr_end + 4:] + + framing = "close-delimited" + if "chunked" in headers.get("transfer-encoding", "").lower(): + framing = "chunked" + body = _dechunk(body) + elif "content-length" in headers: + framing = "content-length" + body = body[:int(headers["content-length"])] + return { + "status": status, + "framing": framing, + "content_length": headers.get("content-length"), + "content_type": headers.get("content-type"), + "body": body, + } + + +def _dechunk(raw: bytes) -> bytes: + out, off = bytearray(), 0 + while True: + i = raw.find(b"\r\n", off) + if i < 0: + break # truncated tail — return what we have + try: + n = int(raw[off:i].split(b";")[0], 16) + except ValueError: + break + if n == 0: + break + out += raw[i + 2:i + 2 + n] + off = i + 2 + n + 2 + return bytes(out) + + +# --------------------------------------------------------------------------- # +# Verification # +# --------------------------------------------------------------------------- # + +def verify_against_reference(c64_status: int, body_total: int, + prefix: bytes, ref: dict | None, + expect_status: int = EXPECT_STATUS, + ) -> tuple[str, list[str], list[str]]: + """Pass criteria. Returns (verdict, problems, warnings); verdict is + PASS / WARN (pass with size drift) / FAIL. + + * http_status must equal expect_status — else FAIL. + * reference unavailable — FAIL (an unverified pass is not a pass); + the viewer is still left live, rerun the rig to verify. + * body_total == reference size AND 512-byte prefix matches — PASS. + * sizes differ but prefix matches byte-exactly — WARN, not FAIL + (action=raw revisions change between fetches); both sizes are + reported. + * prefix mismatch — FAIL regardless of sizes. + """ + problems: list[str] = [] + warnings: list[str] = [] + if c64_status != expect_status: + problems.append(f"http_status is {c64_status}, expected " + f"{expect_status}" + + (" — note: Wikipedia 403s UA-less requests; is " + "the HTTP lane's User-Agent header in the build?" + if c64_status == 403 else "")) + return ("FAIL", problems, warnings) + if ref is None: + problems.append("host-side reference fetch failed — body cannot be " + "verified (C64-side status was correct; viewer left " + "live; re-run to verify)") + return ("FAIL", problems, warnings) + + ref_body = ref["body"] + n = min(RESP_PREFIX_BYTES, body_total, len(ref_body), len(prefix)) + prefix_ok = n > 0 and prefix[:n] == ref_body[:n] + size_ok = body_total == len(ref_body) + + if size_ok and prefix_ok: + return ("PASS", problems, warnings) + if prefix_ok: + warnings.append( + f"size drift: C64 http_body_total={body_total} vs reference " + f"{len(ref_body)} bytes, but first {n} body bytes match " + "byte-exactly (action=raw revision may have changed between " + "fetches) — WARN, not FAIL") + return ("WARN", problems, warnings) + problems.append( + f"body prefix mismatch in first {n} bytes " + f"(C64 total {body_total}, reference {len(ref_body)}); " + "the sink prefix in http_resp_buf does not match the article") + return ("FAIL", problems, warnings) + + +# --------------------------------------------------------------------------- # +# Device REU config (16 MB, before reset, read-skip-write) # +# --------------------------------------------------------------------------- # + +def ensure_reu_16mb(client: Ultimate64Client) -> None: + """Force (Enabled, 16 MB) — the sink writes REU bank $10, beyond the + flash-saved 512 KB. Runtime-only; reverts on power cycle.""" + try: + enabled, size = get_reu_config(client) + except Exception as exc: # probe best-effort, then write + print(f" (REU state probe failed: {exc}; writing anyway)") + enabled, size = False, "?" + if enabled and size == REQUIRED_REU_SIZE: + print(f"REU already (Enabled, {REQUIRED_REU_SIZE}) — " + "skipping config write") + return + print(f"REU config was (enabled={enabled}, size={size!r}); writing " + f"(Enabled, {REQUIRED_REU_SIZE}) — runtime-only, REVERTS ON " + "POWER CYCLE") + set_reu(client, True, size=REQUIRED_REU_SIZE) + time.sleep(float(os.environ.get("REU_SETTLE", "3.0"))) + + +# --------------------------------------------------------------------------- # +# Main hardware run # +# --------------------------------------------------------------------------- # + +def main() -> int: + if not PRG_PATH.is_file() or not LABELS_PATH.is_file(): + print(f"ERROR: build artifacts missing under {PRG_PATH.parent}", + file=sys.stderr) + print("Build with:\n make clean && make BACKEND=uci " + "USE_NISTCURVES_ONCHIP_COMB=1 HTTPS_HOST=en.wikipedia.org " + "'HTTPS_PATH=/w/index.php?title=Commodore_64&action=raw' " + "HTTPS_BODY_TO_REU=1", file=sys.stderr) + return 2 + + labels = _load_labels() + required = BASE_LABELS + CONTRACT_LABELS + missing = [n for n in required if n not in labels] + if missing: + print(f"ERROR: missing labels: {missing}", file=sys.stderr) + if set(missing) & set(CONTRACT_LABELS): + print("This PRG predates the wiki contract (HTTPS_BODY_TO_REU " + "body sink + viewer). Rebuild with the line in this " + "file's header.", file=sys.stderr) + return 2 + for n in sorted(required): + print(f" {n:22s} = ${labels[n]:04X}") + + init_wait = float(os.environ.get("C64_INIT_WAIT", + str(default_init_wait(labels)))) + + memory_policy, arbiter = build_policy_and_arbiter_with_overlay_carveout( + LABELS_PATH, PRG_PATH, + ) + routine_addr = arbiter.alloc(256, name="trampoline") + host_str_addr = arbiter.alloc(64, name="host_str") + path_str_addr = arbiter.alloc(128, name="path_str") + marker_base = arbiter.alloc(16, name="markers") + sentinel_addr, progress_addr, carry_flag_addr = ( + marker_base, marker_base + 1, marker_base + 2) + print(f"\nMemoryPolicy reserved {len(memory_policy.reserved_regions)}" + f" region(s); arbiter allocations:") + for base, last, note in arbiter.allocations: + print(f" ${base:04X}-${last:04X} {note}") + + host_bytes = WIKI_HOST.encode("ascii") + path_bytes = WIKI_PATH.encode("ascii") + routine_bytes = build_wiki_routine( + labels, + routine_addr=routine_addr, + host_str_addr=host_str_addr, + path_str_addr=path_str_addr, + sentinel_addr=sentinel_addr, + progress_addr=progress_addr, + carry_flag_addr=carry_flag_addr, + host_len=len(host_bytes), + path_len=len(path_bytes), + port=WIKI_PORT, + ) + print(f"\nTarget : https://{WIKI_HOST}{WIKI_PATH}") + print(f"Routine size : {len(routine_bytes)} bytes @ ${routine_addr:04X}") + print(f"C64_INIT_WAIT : {init_wait:.0f} s; sentinel budget " + f"{WIKI_TIMEOUT:.0f} s (x{_TIMEOUT_SCALE:.0f} at {TURBO_MHZ} MHz)") + + prg = PRG_PATH.read_bytes() + + lock = DeviceLock(HOST) + try: + lock.acquire_or_raise(timeout=120.0) + except DeviceLockTimeout as exc: + print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) + return 2 + print(f"Acquired DeviceLock({HOST}); holder info: {lock.read_info()!r}") + + removed = _prune_old_run_dirs(UCI_DEBUG_BASE_DIR, UCI_DEBUG_KEEP) + for d in removed: + print(f"Pruning old debug artifacts: {d}") + run_dir = _create_run_dir(UCI_DEBUG_BASE_DIR) + print(f"Debug artifacts dir: {run_dir}") + + client: Ultimate64Client | None = None + uci_enabled = False + outcome = "UNKNOWN" + exit_code = 1 + viewer_live = False + run_start = time.time() + try: + client = Ultimate64Client(host=HOST, timeout=15.0) + transport = Ultimate64Transport(host=HOST, timeout=15.0, client=client) + transport.memory_policy = memory_policy + + print("Enabling UCI...") + enable_uci(client) + uci_enabled = True + + # Userspace /Temp GC (fw <= 3.14d writemem attachment leak). + gc_temp(HOST) + + try: + runner_health_check(client) + except Ultimate64RunnerStuckError as exc: + print(f"[fatal] runner wedged at {HOST}: {exc}", file=sys.stderr) + print("[fatal] supervisor: investigate / authorize recover()", + file=sys.stderr) + return 3 + + # REU to (Enabled, 16 MB) — required, see header. Before reset, + # like every config write in this rig. + ensure_reu_16mb(client) + + # Presence/profile preflight kept too (issue #97 pattern). + try: + preflight_reu(client, LABELS_PATH) + except ReuPreflightError as exc: + print(str(exc), file=sys.stderr) + return 4 + + # Turbo BEFORE reset, read-skip-write (rig_https_live pattern). + try: + cat = client.get_config_category(CAT_U64_SPECIFIC) + inner = cat.get(CAT_U64_SPECIFIC, cat) + cur_speed = inner.get("CPU Speed") + cur_turbo = inner.get("Turbo Control") + except Exception as exc: + print(f" (turbo state probe failed: {exc}; writing anyway)") + cur_speed = cur_turbo = None + want_speed = str(cpu_speed_enum(TURBO_MHZ)) + if str(cur_speed) == want_speed and cur_turbo == "Manual": + print(f"Turbo already {TURBO_MHZ} MHz (Manual) — skipping " + "config write") + else: + print(f"Setting turbo to {TURBO_MHZ} MHz " + f"(from {cur_turbo}/{cur_speed})...") + set_turbo_mhz(client, TURBO_MHZ) + time.sleep(float(os.environ.get("TURBO_SETTLE", "3.0"))) + + print("Resetting machine...") + client.reset() + time.sleep(2.5) + + print("run_prg(PRG)...") + client.run_prg(prg) + print(f"Waiting {init_wait * _TIMEOUT_SCALE:.0f} s for auto-init " + "(entropy, REU init, comb precompute, DHCP)...") + time.sleep(init_wait * _TIMEOUT_SCALE) + + init_flag = transport.read_memory(labels["net_initialized"], 1)[0] + print(f"net_initialized = ${init_flag:02X}") + if init_flag == 0: + print("WARNING: net_initialized is 0 — auto-init may have failed") + + print("Sending 'Q' to exit PRG main_loop...") + send_text(transport, "q\r") + time.sleep(2.0 * _TIMEOUT_SCALE) + + # DMA-write routine + data + CHUNK = 64 + for i in range(0, len(routine_bytes), CHUNK): + transport.write_memory(routine_addr + i, + routine_bytes[i:i + CHUNK]) + transport.write_memory(host_str_addr, + (host_bytes + b"\x00").ljust(64, b"\x00")) + transport.write_memory(path_str_addr, + (path_bytes + b"\x00").ljust(128, b"\x00")) + transport.write_memory(marker_base, bytes(16)) + + sys_line = f"sys{routine_addr}\r" + print(f"Triggering: {sys_line.strip()}") + send_text(transport, sys_line) + + # --- Poll sentinel + body progress ------------------------------- # + body_total_addr = labels["http_body_total"] + deadline = time.time() + WIKI_TIMEOUT + start = time.time() + last_progress = -1 + last_print = 0.0 + last_body_total = 0 + sentinel_seen = False + phase_timing = os.environ.get("PHASE_TIMING", "0") == "1" + phase_log: list[tuple[str, float]] = [] + cur_phase = None + while time.time() < deadline: + time.sleep(POLL_INTERVAL) + blob = transport.read_memory(sentinel_addr, 2) + sentinel, progress = blob[0], blob[1] + bt = transport.read_memory(body_total_addr, 3) + body_total = bt[0] | (bt[1] << 8) | (bt[2] << 16) + now = time.time() + if progress != last_progress: + print(f" [{now - start:6.1f}s] progress=0x{progress:02X}") + last_progress = progress + last_print = now + elif now - last_print >= PROGRESS_PRINT_INTERVAL: + rate = ((body_total - last_body_total) + / max(now - last_print, 1e-9)) + print(f" [{now - start:6.1f}s] http_body_total=" + f"{body_total:,} B ({rate:,.0f} B/s)") + last_print = now + last_body_total = body_total + if phase_timing: + scr = _decode_screen_ram( + bytes(transport.read_memory(0x0400, 1000))) + phase = _current_phase(scr) + if cur_phase is None: + cur_phase = phase + elif phase != cur_phase: + cur_phase = phase + phase_log.append((phase, now - start)) + print(f" phase +{phase_log[-1][1]:7.1f}s {phase}") + if sentinel == SENTINEL_VALUE: + print(f" sentinel set after {now - start:.1f}s — " + "routine complete") + sentinel_seen = True + break + + if phase_timing and phase_log: + table = _format_phase_log(phase_log) + print("\n=== Phase timing ===") + print(table) + try: + (run_dir / "phase_timing.txt").write_text(table + "\n") + except Exception as exc: + print(f" (could not write phase_timing.txt: {exc})") + + # --- Read results (all DMA reads — safe even with viewer live) --- # + carry = transport.read_memory(carry_flag_addr, 1)[0] & 0x01 + status_raw = transport.read_memory(labels["http_status"], 2) + http_status = status_raw[0] | (status_raw[1] << 8) + bt = transport.read_memory(body_total_addr, 3) + body_total = bt[0] | (bt[1] << 8) | (bt[2] << 16) + prefix = bytes(transport.read_memory(labels["http_resp_buf"], + RESP_PREFIX_BYTES)) + print(f"\nhttp_get carry = {carry} (0=success, 1=failure)") + print(f"http_status = {http_status}") + print(f"http_body_total = {body_total:,} bytes") + print(f"prefix = {prefix[:80]!r}") + + try: + _dump_diag(transport, labels) + _dump_ring(transport, labels, run_dir) + _dump_tls_state_snapshot(transport, labels, run_dir) + (run_dir / "resp_prefix.bin").write_bytes(prefix) + print(f" snapshots -> {run_dir}") + except Exception as exc: + print(f"WARNING: snapshot dump failed: {exc}") + + if not sentinel_seen: + print(f"\nFAIL: sentinel not set within {WIKI_TIMEOUT:.0f}s " + f"(progress=0x{last_progress:02X}, " + f"http_body_total={body_total:,})", file=sys.stderr) + outcome = "TIMEOUT" + return 1 + if carry != 0: + print("\nFAIL: http_get returned C=1 — TLS handshake or HTTP " + "exchange failed (see tls_state_dump.json)", + file=sys.stderr) + outcome = "FAIL" + return 1 + + # --- Host-side reference fetch + comparison ---------------------- # + ref = None + for attempt in (1, 2): + try: + print(f"\nReference fetch (attempt {attempt}): " + f"https://{WIKI_HOST}{WIKI_PATH}") + ref = fetch_reference(WIKI_HOST, WIKI_PATH, WIKI_PORT) + print(f" status={ref['status']} framing={ref['framing']} " + f"body={len(ref['body']):,} B " + f"type={ref['content_type']}") + (run_dir / "reference_body.bin").write_bytes(ref["body"]) + break + except Exception as exc: + print(f" reference fetch failed: {exc}") + ref = None + if ref is not None and ref["status"] != EXPECT_STATUS: + print(f" WARNING: reference status {ref['status']} != " + f"{EXPECT_STATUS}; treating reference as unusable") + ref = None + + verdict, problems, warnings = verify_against_reference( + http_status, body_total, prefix, ref) + for w in warnings: + print(f"\nWARN: {w}") + if verdict == "FAIL": + print(f"\nFAIL against {WIKI_HOST}:", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + outcome = "FAIL" + return 1 + + outcome = verdict # PASS or WARN + exit_code = 0 + viewer_live = True + print(f"\n{verdict}: {body_total:,} article bytes sunk to REU " + f"$10:0000, http_status={http_status}, prefix verified " + f"against the live article.") + print("\nviewer live on device; scroll with CRSR/SPACE/F1/HOME, " + "Q to quit") + print("NOTE: the machine is NOT reset and UCI stays enabled. The " + "DeviceLock is released on exit — any queued harness job " + "will grab the device and reset it, ending the viewer " + "session. Hold other device work while playing.") + return 0 + + finally: + try: + _write_run_info( + run_dir / "run_info.txt", + outcome=outcome, + duration=time.time() - run_start, + exit_code=exit_code, + extra={ + "target": f"{WIKI_HOST}{WIKI_PATH}", + "turbo_mhz": TURBO_MHZ, + "host": HOST, + "viewer_live": viewer_live, + }, + ) + except Exception as exc: + print(f"WARNING: run_info write failed: {exc}") + print(f"\nDebug artifacts: {run_dir}") + + if viewer_live: + # Leave the machine exactly as-is for the human. + pass + elif uci_enabled and client is not None: + print("\nDisabling UCI...") + try: + disable_uci(client) + except Exception as exc: + print(f"WARNING: disable_uci failed: {exc}") + lock.release() + print(f"Released DeviceLock({HOST})") + + +# --------------------------------------------------------------------------- # +# Offline structural self-check — no hardware, no DeviceLock # +# --------------------------------------------------------------------------- # + +def _selfcheck() -> int: + failures: list[str] = [] + + def check(name: str, cond: bool, detail: str = "") -> None: + status = "ok" if cond else "FAIL" + print(f" [{status}] {name}" + (f" — {detail}" if detail else "")) + if not cond: + failures.append(name) + + print("rig_https_wiki selfcheck (offline unless noted)") + + # 1) env handling + check("WIKI_HOST default/env", + WIKI_HOST == os.environ.get("WIKI_HOST", "en.wikipedia.org")) + check("WIKI_PATH default has action=raw", + "action=raw" in os.environ.get("WIKI_PATH", WIKI_PATH)) + check("timeout scale >= 1", _TIMEOUT_SCALE >= 1.0, + f"TURBO_MHZ={TURBO_MHZ} -> x{_TIMEOUT_SCALE:g}") + check("sentinel budget scaled", WIKI_TIMEOUT >= 600.0 or + "WIKI_TIMEOUT" in os.environ, f"{WIKI_TIMEOUT:.0f}s") + + # 2) trampoline structure against synthetic labels + fake_labels = { + "http_get": 0x6123, "http_host_ptr": 0xA801, "http_host_len": 0xA803, + "http_path_ptr": 0xA804, "http_path_len": 0xA806, + "http_port": 0xA807, "net_init": 0x2001, + "tcp_recv_head": 0xA810, "tcp_recv_tail": 0xA812, + "viewer_enter": 0x7C40, + } + addrs = dict(routine_addr=0x3C00, host_str_addr=0x3D00, + path_str_addr=0x3D40, sentinel_addr=0x3DC0, + progress_addr=0x3DC1, carry_flag_addr=0x3DC2) + path = "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/w/index.php?title=Commodore_64&action=raw" + code = build_wiki_routine(fake_labels, **addrs, + host_len=len("en.wikipedia.org"), + path_len=len(path), port=443) + check("banks ROM out first", + code[:8] == bytes([0xAD, 0x01, 0x00, 0x29, 0xFE, 0x8D, 0x01, 0x00])) + jsr_http_get = bytes([0x20, 0x23, 0x61]) + check("JSR http_get present", jsr_http_get in code) + check("PHP/PLA carry latch after http_get", + code[code.index(jsr_http_get) + 3: + code.index(jsr_http_get) + 5] == b"\x08\x68") + check("path_len baked in", + bytes([0xA9, len(path), 0x8D, 0x06, 0xA8]) in code) + check("port 443 stored LE", + bytes([0xA9, 443 & 0xFF, 0x8D, 0x07, 0xA8, 0xA9, 443 >> 8]) in code) + check("sentinel value written", + bytes([0xA9, SENTINEL_VALUE]) in code) + # Conditional viewer entry: LDA carry / AND #1 / BNE +3 / JSR viewer + cond = bytes([0xAD, 0xC2, 0x3D, 0x29, 0x01, 0xD0, 0x03, + 0x20, 0x40, 0x7C]) + check("carry-gated JSR viewer_enter", cond in code) + check("viewer JSR after sentinel write", + code.index(cond) > code.index(bytes([0xA9, SENTINEL_VALUE]))) + park = addrs["routine_addr"] + len(code) - 3 + check("parks with JMP self (also the BNE target)", + code[-3:] == bytes([0x4C, park & 0xFF, (park >> 8) & 0xFF])) + check("BNE target is the park JMP", + code.index(cond) + len(cond) == len(code) - 3) + check("routine fits its 256 B slot", len(code) <= 256, + f"{len(code)} bytes") + + # 3) guards + for kwargs, name in ( + (dict(host_len=64, path_len=1), "host_len > 63 rejected"), + (dict(host_len=10, path_len=129), "path_len > 128 rejected"), + (dict(host_len=10, path_len=0), "path_len 0 rejected"), + ): + try: + build_wiki_routine(fake_labels, **addrs, port=443, **kwargs) + check(name, False) + except ValueError: + check(name, True) + + # 4) verification logic (synthetic) + ref = {"status": 200, "framing": "chunked", "content_length": None, + "content_type": "text/x-wiki", "body": b"A" * 1000} + v, p, w = verify_against_reference(200, 1000, b"A" * 512, ref, 200) + check("exact match -> PASS", v == "PASS" and not p and not w) + v, p, w = verify_against_reference(200, 998, b"A" * 512, ref, 200) + check("size drift + prefix match -> WARN", v == "WARN" and w and not p) + v, p, w = verify_against_reference(200, 1000, b"B" * 512, ref, 200) + check("prefix mismatch -> FAIL", v == "FAIL") + v, p, w = verify_against_reference(403, 126, b"A" * 126, ref, 200) + check("403 -> FAIL with UA hint", v == "FAIL" and "User-Agent" in p[0]) + v, p, w = verify_against_reference(200, 1000, b"A" * 512, None, 200) + check("no reference -> FAIL (unverified is not a pass)", v == "FAIL") + + # 5) de-chunker + raw = b"5\r\nHELLO\r\n6;ext=1\r\n WORLD\r\n0\r\n\r\n" + check("de-chunker strips framing", _dechunk(raw) == b"HELLO WORLD") + + # 6) live reference fetch (real Wikipedia; skippable) + if os.environ.get("WIKI_SELFCHECK_OFFLINE", "0") == "1": + print(" (WIKI_SELFCHECK_OFFLINE=1 — live reference fetch skipped)") + else: + try: + ref = fetch_reference(WIKI_HOST, WIKI_PATH, WIKI_PORT) + check("live reference fetch returns 200", + ref["status"] == 200, + f"framing={ref['framing']} body={len(ref['body']):,} B") + check("live body non-trivial", len(ref["body"]) > 50_000, + "article should be ~125 KB of wikitext") + except Exception as exc: + check("live reference fetch", False, f"{exc}") + + # 7) local build, if present (informational for contract labels) + if LABELS_PATH.is_file(): + labels = _load_labels() + base_missing = [n for n in BASE_LABELS if n not in labels] + check("base labels present in local build", not base_missing, + f"missing: {base_missing}" if base_missing else str(LABELS_PATH)) + contract_missing = [n for n in CONTRACT_LABELS if n not in labels] + if contract_missing: + print(f" (info) contract labels not in local build yet " + f"(6502 lanes pending): {contract_missing}") + else: + print(" (info) local build carries the full wiki contract") + print(f" (info) init wait would default to " + f"{default_init_wait(labels):.0f} s") + else: + print(f" (no {LABELS_PATH} — local-build check skipped)") + + if failures: + print(f"\nSELFCHECK FAIL: {len(failures)} check(s): {failures}") + return 1 + print("\nSELFCHECK PASS") + return 0 + + +if __name__ == "__main__": + if "--selfcheck" in sys.argv[1:]: + raise SystemExit(_selfcheck()) + raise SystemExit(main()) From beb1ed23d735d811dddad01aee7f1436d73a1fe6 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:40:50 -0400 Subject: [PATCH 20/33] fix(http): terminate identity bodies on bytes CONSUMED, 24-bit Content-Length MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The W4 body-termination gap found live against browserleaks.com: once the stored copy truncated at the 512 B http_resp_buf cap, http_resp_len froze, the Content-Length match could never fire, and only the connection-close / poll-timeout fallback ended the read — a server that holds the connection open left http_get never returning. - New 24-bit consumed-byte counter http_body_total (exported, LE), incremented in the new shared http_body_append for every body byte, stored or not; bytes past the 512 B cap are drained and discarded instead of ending the parse. - http_content_length widened to 24 bits (real bodies exceed 64 KB); the $FFFF magic 'absent' sentinel is replaced by a separate http_cl_valid flag byte that cannot collide with a real length. - Termination compares consumed count against Content-Length; with the header absent the legacy 512 B cap behaviour is preserved (chunked/streaming fallback). - http_resp_len keeps its contract: min(512, total), first 512 bytes. - The body machinery moved to a new HTTP_AUX_CODE segment: LOADER slack under UCI, CRYPTO_OVERLAY under ip65 (LOADER is 8 B from full there; the grown 24-bit parser no longer fits the NET_CODE tail on either backend). All five build profiles link. - tools/test_http.py: three new span-driven vectors (700 B body with CL — the regression; 24-bit CL parse; CL-absent legacy path); 38/38 PASS in VICE. Co-Authored-By: Claude Fable 5 --- cfg/c64-https-ip65.cfg | 5 + cfg/c64-https-uci-onchip.cfg | 3 + cfg/c64-https-uci.cfg | 5 + src/data.s | 12 +- src/http.s | 232 +++++++++++++++++++++++++---------- tools/test_http.py | 189 ++++++++++++++++++++++++++++ 6 files changed, 383 insertions(+), 63 deletions(-) diff --git a/cfg/c64-https-ip65.cfg b/cfg/c64-https-ip65.cfg index 06b0fde8..7afa9bcd 100644 --- a/cfg/c64-https-ip65.cfg +++ b/cfg/c64-https-ip65.cfg @@ -188,6 +188,11 @@ SEGMENTS { # below $A000. TLS_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; CRYPTO_AUX_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; + # W4 HTTP body machinery (24-bit Content-Length termination, and the + # buffer side of the REU body sink, src/http.s). LOADER is 8 B from + # full under ip65 and NET_CODE ~57 B, so the W4 growth rides the + # CRYPTO_OVERLAY slack (~450 B free pre-W4) alongside TLS_CODE. + HTTP_AUX_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; CRYPTO_CODE: load = CRYPTO_RESIDENT, type = ro; CRYPTO_RODATA: load = CRYPTO_RESIDENT, type = ro; # libs/nistcurves segment names (c64-lib-contract / cfa9085+). diff --git a/cfg/c64-https-uci-onchip.cfg b/cfg/c64-https-uci-onchip.cfg index f258fbba..311669f5 100644 --- a/cfg/c64-https-uci-onchip.cfg +++ b/cfg/c64-https-uci-onchip.cfg @@ -182,6 +182,9 @@ SEGMENTS { CODE: load = LOADER, type = ro; RODATA: load = CRYPTO_OVERLAY, type = ro; # onchip-comb variant: relieve CRYPTO_HOT INIT: load = LOADER, type = ro, optional = yes; + # W4 HTTP body machinery (24-bit Content-Length termination + REU + # body sink, src/http.s) — same placement as c64-https-uci.cfg. + HTTP_AUX_CODE: load = LOADER, type = ro, optional = yes; # NET_CODE hosts the UCI adapter + the LOADER_OVERFLOW tail. # Phase C.2 (UCI only): TLS_CODE and CRYPTO_AUX_CODE (SHA-256, diff --git a/cfg/c64-https-uci.cfg b/cfg/c64-https-uci.cfg index d5110a8e..2f635f5c 100644 --- a/cfg/c64-https-uci.cfg +++ b/cfg/c64-https-uci.cfg @@ -168,6 +168,11 @@ SEGMENTS { CODE: load = LOADER, type = ro; RODATA: load = CRYPTO_HOT, type = ro; INIT: load = LOADER, type = ro, optional = yes; + # W4 HTTP body machinery (24-bit Content-Length termination + REU + # body sink, src/http.s). Rides the LOADER slack under UCI (~590 B + # free pre-W4); NET_CODE is 8 B from full so LOADER_OVERFLOW could + # not absorb the growth. + HTTP_AUX_CODE: load = LOADER, type = ro, optional = yes; # NET_CODE hosts the UCI adapter + the LOADER_OVERFLOW tail. # Phase C.2 (UCI only): TLS_CODE and CRYPTO_AUX_CODE (SHA-256, diff --git a/src/data.s b/src/data.s index a4dffd5e..7f218806 100644 --- a/src/data.s +++ b/src/data.s @@ -358,11 +358,21 @@ http_resp_len: .res 2 .export http_line_idx .export http_line_buf .export http_content_length +.export http_cl_valid +.export http_body_total http_parse_state: .res 1 http_hdr_match: .res 1 http_line_idx: .res 1 http_line_buf: .res 32 -http_content_length: .res 2 ; 16-bit; $FFFF when header absent +; W4 body-termination fix: Content-Length is 24-bit (real bodies exceed +; 64 KB) and its "absent" state is a separate valid-flag byte instead of +; the historical $FFFF magic (a 24-bit magic could collide with a real +; length). http_body_total counts body bytes CONSUMED — not stored — so +; the Content-Length termination fires even after the 512 B stored copy +; truncates (the browserleaks.com hang; see src/http.s @state_body). +http_content_length: .res 3 ; 24-bit LE; meaningful iff http_cl_valid=1 +http_cl_valid: .res 1 ; 1 = Content-Length header seen +http_body_total: .res 3 ; 24-bit LE count of body bytes consumed ; ----------------------------------------------------------------------------- ; Application data pointers (for tls_send) diff --git a/src/http.s b/src/http.s index 8e842d12..b9d7448a 100644 --- a/src/http.s +++ b/src/http.s @@ -22,6 +22,11 @@ .export http_crlf .export http_bg_idx .export http_bg_src + ; W4 body-termination fix (see @state_body / HTTP_AUX_CODE): + .export http_body_append + .export http_in_mode + .export http_in_ptr + .export http_in_len ; ---- imports: data.asm BSS (HTTP I/O + parser state) ---- .import http_host_ptr @@ -39,6 +44,8 @@ .import http_line_idx .import http_line_buf .import http_content_length + .import http_cl_valid + .import http_body_total ; ---- imports: data.asm BSS (TLS app data) ---- ; (tcp_recv_head/tail imports dropped in the issue #72 redesign — @@ -523,11 +530,11 @@ http_recv_response: lda #0 sta http_hdr_match sta http_line_idx - ; Reset Content-Length sentinel to $FFFF ("unknown") — will be - ; overwritten by check_content_length if the header is seen. - lda #$ff - sta http_content_length - sta http_content_length+1 + ; Content-Length is "absent until seen": http_cl_valid = 0. The + ; historical $FFFF magic sentinel went away with the 24-bit + ; Content-Length extension — a separate valid-flag byte cannot + ; collide with any real length value. + sta http_cl_valid jmp @state_headers ; ----- state 1: read header lines into http_line_buf until an empty line ----- @@ -567,9 +574,7 @@ http_recv_response: ; fits entirely in one record no further record ever arrives. lda #2 sta http_parse_state - lda #0 - sta http_resp_len - sta http_resp_len+1 + jsr http_body_begin ; zero resp_len + consumed counter jmp @state_body ; ----- state 2: read body bytes ----- @@ -587,43 +592,25 @@ http_recv_response: ; first post-handshake TLS record the parser sees contains only the ; status line + headers and the body lands in the next record. @state_body: - ; Content-Length termination: http_content_length defaults to $FFFF - ; ("unknown"), overwritten by check_content_length if the header - ; was seen. $FFFF will never match because the buffer-full check - ; below bails at 512 bytes. For a real Content-Length value we - ; finish the instant resp_len matches — handles Content-Length: 0 - ; on entry and the post-append case below. - lda http_resp_len - cmp http_content_length - bne @body_read - lda http_resp_len+1 - cmp http_content_length+1 - beq @done -@body_read: + ; W4 termination fix: terminate on body bytes CONSUMED, not + ; stored. The old check compared http_resp_len against + ; Content-Length, but resp_len freezes at the 512 B buffer cap, + ; so for any body > 512 B the Content-Length match could never + ; fire and only the connection-close / poll-timeout fallback + ; ended the read — against a server that holds the connection + ; open (observed live: browserleaks.com), http_get never + ; returned. http_body_total (24-bit, HTTP_AUX_CODE routines) + ; now counts every consumed byte; bytes past the 512 B cap are + ; consumed and discarded instead of ending the parse. + jsr http_body_done_check + bcc @body_done ; C=0 -> body complete jsr http_in_byte bcs @not_done ; no byte right now -> poll more - ; store byte using zp_ptr = http_resp_buf + http_resp_len - pha ; save received byte - clc - lda #http_resp_buf - adc http_resp_len+1 - sta zp_ptr+1 - pla ; restore byte - ldy #0 - sta (zp_ptr),y - ; increment 16-bit length - inc http_resp_len - bne @body_check_full - inc http_resp_len+1 -@body_check_full: - ; check if buffer full (512 bytes = $0200) - lda http_resp_len+1 - cmp #$02 - bcs @done ; high byte >= 2 means >= 512 - jmp @state_body ; keep reading (also re-checks Content-Length) + jsr http_body_append ; count + store/sink the byte + jmp @state_body ; keep reading (re-checks termination) + +@body_done: + jmp @done @not_done: sec @@ -636,19 +623,25 @@ http_recv_response: ; check_content_length - inspect http_line_buf[0..http_line_idx) for the ; header "Content-Length: " (case-insensitive on the name, a ; single SP after the colon). On match, the decimal value is stored in -; http_content_length (16-bit) — which the caller pre-loaded with the -; $FFFF "unknown" sentinel. No-op otherwise. -; Clobbers: A, X, Y +; http_content_length (24-bit little-endian — real-server bodies exceed +; 64 KB) and http_cl_valid is set to 1. No-op otherwise (http_cl_valid +; stays 0 from the status-line reset). +; Clobbers: A, X, Y, zp_ptr, zp_temp ; -; Lives in the LOADER_OVERFLOW segment (reachable via JSR from CODE in -; LOADER) because the LOADER region is packed; see c64-https-ip65.cfg. +; Lives in the HTTP_AUX_CODE segment (reachable via JSR from CODE in +; LOADER): the W4 24-bit extension no longer fits the packed +; LOADER_OVERFLOW/NET_CODE tail on either backend, so the whole W4 HTTP +; body machinery rides a dedicated segment — LOADER slack under UCI, +; CRYPTO_OVERLAY under ip65 (see the cfgs). ; ============================================================================= - .segment "LOADER_OVERFLOW" + .segment "HTTP_AUX_CODE" check_content_length: ; Need at least len("content-length: ") = 16 bytes in the line. lda http_line_idx cmp #16 - bcc @ccl_rts ; line too short -> no-op + bcs :+ ; line too short -> no-op + rts +: ; Case-insensitive compare against lowercase pattern. Each line ; byte is OR'd with $20, which folds A..Z to a..z and leaves ':' ; and ' ' unchanged. @@ -657,37 +650,52 @@ check_content_length: lda http_line_buf,x ora #$20 cmp cl_pattern,x - bne @ccl_rts - dex + beq :+ + rts ; name mismatch -> no-op +: dex bpl @ccl_cmp - ; Matched. Zero the accumulator and parse digits starting at idx 16. + ; Matched. Zero the accumulator, mark the header seen, and parse + ; digits starting at idx 16. lda #0 sta http_content_length sta http_content_length+1 + sta http_content_length+2 + lda #1 + sta http_cl_valid ldx #16 @ccl_digit_loop: + ; (branch-inverted exits: the 24-bit loop body outgrew the + ; 127-byte relative-branch reach to @ccl_rts) cpx http_line_idx - bcs @ccl_rts - lda http_line_buf,x + bcc :+ + jmp @ccl_rts +: lda http_line_buf,x cmp #$30 ; '0' - bcc @ccl_rts - cmp #$3a ; '9'+1 - bcs @ccl_rts - sec + bcs :+ + jmp @ccl_rts +: cmp #$3a ; '9'+1 + bcc :+ + jmp @ccl_rts +: sec sbc #$30 ; A = digit 0..9 pha ; old *= 10 = old*8 + old*2 (need THREE ASLs to reach *8; tmp - ; holds *2 after the first shift). + ; holds *2 after the first shift). 24-bit: tmp = zp_ptr(2)+zp_temp. asl http_content_length rol http_content_length+1 + rol http_content_length+2 lda http_content_length - sta zp_ptr ; tmp_lo = old*2 + sta zp_ptr ; tmp = old*2 lda http_content_length+1 - sta zp_ptr+1 ; tmp_hi = old*2 + sta zp_ptr+1 + lda http_content_length+2 + sta zp_temp asl http_content_length rol http_content_length+1 + rol http_content_length+2 asl http_content_length rol http_content_length+1 + rol http_content_length+2 clc lda http_content_length adc zp_ptr @@ -695,12 +703,17 @@ check_content_length: lda http_content_length+1 adc zp_ptr+1 sta http_content_length+1 + lda http_content_length+2 + adc zp_temp + sta http_content_length+2 pla clc adc http_content_length sta http_content_length bcc @ccl_nc inc http_content_length+1 + bne @ccl_nc + inc http_content_length+2 @ccl_nc: inx jmp @ccl_digit_loop @@ -714,6 +727,101 @@ check_content_length: cl_pattern: .byte "content-length: " +; ============================================================================= +; http_body_begin - reset per-response body state at the headers->body +; transition (state 1 -> 2). Living here (not in the recv-loop inits) +; guarantees the reset runs on every parse walk regardless of entry +; path (http_recv_body, http_get_plain, or a test driving +; http_recv_response directly). +; Clobbers: A +; ============================================================================= +http_body_begin: + lda #0 + sta http_resp_len + sta http_resp_len+1 + sta http_body_total + sta http_body_total+1 + sta http_body_total+2 + rts + +; ============================================================================= +; http_body_done_check - is the body complete? +; C=0 -> complete; C=1 -> keep reading. +; +; Content-Length seen (http_cl_valid=1): complete exactly when the +; 24-bit consumed count http_body_total equals http_content_length. +; Works whether or not the stored copy was truncated at 512 B — +; that is the W4 fix. +; Content-Length absent: legacy behaviour — the 512 B buffer cap ends +; the read (chunked/streaming fallback preserved). +; Clobbers: A +; ============================================================================= +http_body_done_check: + lda http_cl_valid + beq @no_cl + lda http_body_total + cmp http_content_length + bne @more + lda http_body_total+1 + cmp http_content_length+1 + bne @more + lda http_body_total+2 + cmp http_content_length+2 + bne @more + clc + rts +@no_cl: + lda http_resp_len+1 + cmp #$02 + bcs @full ; high byte >= 2 means >= 512 +@more: + sec + rts +@full: + clc + rts + +; ============================================================================= +; http_body_append - consume one body byte (in A). +; +; Always increments the 24-bit http_body_total (bytes CONSUMED). +; Stores into http_resp_buf while resp_len < 512; bytes past the cap +; are counted and discarded. +; Clobbers: A, Y, zp_ptr. Preserves X. +; ============================================================================= +http_body_append: + pha + ; http_body_total++ (24-bit) + inc http_body_total + bne @stored_count + inc http_body_total+1 + bne @stored_count + inc http_body_total+2 +@stored_count: + ; room in http_resp_buf? (resp_len < 512) + lda http_resp_len+1 + cmp #$02 + bcs @discard ; buffer-mode overflow: count only + ; store byte at http_resp_buf + resp_len + clc + lda #http_resp_buf + adc http_resp_len+1 + sta zp_ptr+1 + pla + ldy #0 + sta (zp_ptr),y + inc http_resp_len + bne @rts + inc http_resp_len+1 +@rts: + rts +@discard: + pla ; byte counted, not stored + rts + ; ============================================================================= ; http_get_plain - perform a plain HTTP (no TLS) GET request ; Input: http_host_ptr/http_host_len = hostname diff --git a/tools/test_http.py b/tools/test_http.py index e9f9207d..d5b6bd65 100755 --- a/tools/test_http.py +++ b/tools/test_http.py @@ -357,6 +357,180 @@ def test_recv_response_with_body(transport, labels): return passed, failed +def _load_span(transport, labels, data, addr=0xC400): + """Feed canned data to the parser as a linear span (http_in_mode=1). + + This is the TLS-path input mode (issue #72): the parser reads from + http_in_ptr/http_in_len instead of the TCP ring, which also lifts the + ring helper's 256-byte limit — needed for the >512 B body vectors. + """ + write_bytes(transport, addr, data) + write_bytes(transport, labels.address("http_in_mode"), [1]) + write_bytes(transport, labels.address("http_in_ptr"), + [addr & 0xFF, (addr >> 8) & 0xFF]) + write_bytes(transport, labels.address("http_in_len"), + [len(data) & 0xFF, (len(data) >> 8) & 0xFF]) + + +def _read_u16(transport, labels, name): + b = read_bytes(transport, labels.address(name), 2) + return b[0] | (b[1] << 8) + + +def _read_u24(transport, labels, name): + b = read_bytes(transport, labels.address(name), 3) + return b[0] | (b[1] << 8) | (b[2] << 16) + + +def test_body_larger_than_buffer(transport, labels): + """W4 regression (browserleaks.com): identity body > 512 B with + Content-Length must terminate by CONSUMED count, not stored count. + + Before the fix, http_resp_len froze at the 512 B cap, the + Content-Length match could never fire, and only connection-close / + poll-timeout ended the read.""" + passed = 0 + failed = 0 + + body = bytes((i * 7 + (i >> 8)) & 0xFF for i in range(700)) + response = b"HTTP/1.1 200 OK\r\nContent-Length: 700\r\n\r\n" + body + + _load_span(transport, labels, response) + _reset_parser(transport, labels) + + complete = _run_parser_loop(transport, labels, timeout=60.0) + if not complete: + print(" FAIL: parser did not complete (Content-Length termination " + "did not fire on truncated body — the W4 regression)") + return 0, 1 + print(" PASS: parser terminated on consumed-count match") + passed += 1 + + total = _read_u24(transport, labels, "http_body_total") + if total == 700: + print(f" PASS: http_body_total = {total}") + passed += 1 + else: + print(f" FAIL: http_body_total = {total}, expected 700") + failed += 1 + + resp_len = _read_u16(transport, labels, "http_resp_len") + if resp_len == 512: + print(f" PASS: http_resp_len = {resp_len} (min(512, total))") + passed += 1 + else: + print(f" FAIL: http_resp_len = {resp_len}, expected 512") + failed += 1 + + stored = read_bytes(transport, labels.address("http_resp_buf"), 512) + if stored == body[:512]: + print(" PASS: http_resp_buf holds the first 512 body bytes") + passed += 1 + else: + print(" FAIL: http_resp_buf does not match first 512 body bytes") + failed += 1 + + in_len = _read_u16(transport, labels, "http_in_len") + if in_len == 0: + print(" PASS: span fully consumed (bytes past the cap were drained)") + passed += 1 + else: + print(f" FAIL: http_in_len = {in_len}, expected 0 " + "(bytes past the 512 cap were not consumed)") + failed += 1 + + return passed, failed + + +def test_content_length_24bit(transport, labels): + """Content-Length parses to 24 bits (real bodies exceed 64 KB) with + the valid-flag replacing the old $FFFF magic sentinel.""" + passed = 0 + failed = 0 + + # Headers + a token 5 bytes of a (claimed) 100000-byte body. A single + # dispatch consumes the span and returns not-done; we then inspect the + # parsed header state. + response = b"HTTP/1.1 200 OK\r\nContent-Length: 100000\r\n\r\nABCDE" + _load_span(transport, labels, response) + _reset_parser(transport, labels) + + jsr(transport, labels.address("http_recv_response")) + + cl_valid = read_bytes(transport, labels.address("http_cl_valid"), 1)[0] + if cl_valid == 1: + print(" PASS: http_cl_valid = 1") + passed += 1 + else: + print(f" FAIL: http_cl_valid = {cl_valid}, expected 1") + failed += 1 + + cl = _read_u24(transport, labels, "http_content_length") + if cl == 100000: + print(f" PASS: http_content_length = {cl} (24-bit)") + passed += 1 + else: + print(f" FAIL: http_content_length = {cl}, expected 100000") + failed += 1 + + total = _read_u24(transport, labels, "http_body_total") + state = read_bytes(transport, labels.address("http_parse_state"), 1)[0] + if total == 5 and state == 2: + print(f" PASS: body_total = {total}, still in body state " + "(not terminated early)") + passed += 1 + else: + print(f" FAIL: body_total = {total} (exp 5), parse_state = {state} " + "(exp 2)") + failed += 1 + + return passed, failed + + +def test_body_no_content_length(transport, labels): + """Content-Length absent: legacy behaviour preserved — bytes store into + http_resp_buf, no premature termination, body_total still counts.""" + passed = 0 + failed = 0 + + body = b"NO-CL BODY BYTES HERE" # 21 bytes + response = b"HTTP/1.1 200 OK\r\nX-Info: none\r\n\r\n" + body + _load_span(transport, labels, response) + _reset_parser(transport, labels) + + # Single dispatch: consumes the whole span, returns not-done (C=1) + # because with no Content-Length the parser keeps polling. + jsr(transport, labels.address("http_recv_response")) + + resp_len = _read_u16(transport, labels, "http_resp_len") + if resp_len == len(body): + print(f" PASS: http_resp_len = {resp_len}") + passed += 1 + else: + print(f" FAIL: http_resp_len = {resp_len}, expected {len(body)}") + failed += 1 + + stored = read_bytes(transport, labels.address("http_resp_buf"), len(body)) + if stored == body: + print(" PASS: body stored intact") + passed += 1 + else: + print(f" FAIL: body mismatch: {stored!r}") + failed += 1 + + total = _read_u24(transport, labels, "http_body_total") + if total == len(body): + print(f" PASS: http_body_total = {total}") + passed += 1 + else: + print(f" FAIL: http_body_total = {total}, expected {len(body)}") + failed += 1 + + # Restore ring input mode for any later ring-driven vectors. + write_bytes(transport, labels.address("http_in_mode"), [0]) + return passed, failed + + def run_tests(transport, labels, verbose=False): total_passed = 0 total_failed = 0 @@ -408,6 +582,21 @@ def run_tests(transport, labels, verbose=False): total_passed += p total_failed += f + print("\n--- W4: body > 512 with Content-Length (browserleaks regression) ---") + p, f = test_body_larger_than_buffer(transport, labels) + total_passed += p + total_failed += f + + print("\n--- W4: 24-bit Content-Length ---") + p, f = test_content_length_24bit(transport, labels) + total_passed += p + total_failed += f + + print("\n--- W4: Content-Length absent (legacy path) ---") + p, f = test_body_no_content_length(transport, labels) + total_passed += p + total_failed += f + return total_passed, total_failed From 28e673f77c16bf13efcf255f0b6547262a3f2857 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:41:36 -0400 Subject: [PATCH 21/33] feat(http): send a User-Agent header (Wikipedia 403s UA-less requests) Lane H's live recon (2026-08-21): en.wikipedia.org returns HTTP 403 to the exact request c64-https sends today because it carries no User-Agent; with one it returns 200. Emit a short honest UA line, 'User-Agent: c64-https/0.1', between Host and Connection. Request is now 87 B for the test vector (256 B http_req_buf; the 42-char article path leaves ample headroom). String rides RODATA; only the ~20 B copy loop lands in LOADER CODE. tools/test_http.py asserts the exact UA line and the updated full request bytes; 39/39 PASS in VICE. Co-Authored-By: Claude Fable 5 --- src/http.s | 18 ++++++++++++++++++ tools/test_http.py | 14 +++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/http.s b/src/http.s index b9d7448a..5abe2b22 100644 --- a/src/http.s +++ b/src/http.s @@ -19,6 +19,7 @@ .export http_version .export http_host_hdr .export http_conn_hdr + .export http_ua_hdr .export http_crlf .export http_bg_idx .export http_bg_src @@ -366,6 +367,21 @@ http_build_get: cpx #2 bne @cv_crlf1 + ; --- "User-Agent: c64-https/0.1\r\n" (27 bytes) --- + ; Wikipedia's robot policy 403s a UA-less request (measured live, + ; lane H recon 2026-08-21); a short honest UA gets a 200. + ldx #0 +@cv_ua: + lda http_ua_hdr,x + stx http_bg_src + ldx http_bg_idx + sta http_req_buf,x + inc http_bg_idx + ldx http_bg_src + inx + cpx #27 + bne @cv_ua + ; --- "Connection: close\r\n" (19 bytes) --- ldx #0 @cv_conn: @@ -910,6 +926,8 @@ http_host_hdr: .byte "Host: " http_conn_hdr: .byte "Connection: close", $0d, $0a +http_ua_hdr: + .byte "User-Agent: c64-https/0.1", $0d, $0a http_crlf: .byte $0d, $0a diff --git a/tools/test_http.py b/tools/test_http.py index d5b6bd65..7f9bb15e 100755 --- a/tools/test_http.py +++ b/tools/test_http.py @@ -90,6 +90,7 @@ def test_build_get_basic(transport, labels): version_addr = labels.address("http_version") host_hdr_addr = labels.address("http_host_hdr") conn_hdr_addr = labels.address("http_conn_hdr") + ua_hdr_addr = labels.address("http_ua_hdr") crlf_addr = labels.address("http_crlf") # Build expected output by reading the actual constant bytes from VICE @@ -99,6 +100,7 @@ def test_build_get_basic(transport, labels): ver_bytes = read_bytes(transport, version_addr, 11) # " HTTP/1.1\r\n" hosthdr_bytes = read_bytes(transport, host_hdr_addr, 6) # "Host: " conn_bytes = read_bytes(transport, conn_hdr_addr, 19) # "Connection: close\r\n" + ua_bytes = read_bytes(transport, ua_hdr_addr, 27) # "User-Agent: ...\r\n" crlf_bytes = read_bytes(transport, crlf_addr, 2) # \r\n # The hostname and path are written by us in raw bytes, so they stay as-is. @@ -109,11 +111,21 @@ def test_build_get_basic(transport, labels): hosthdr_bytes + # "Host: " (6) hostname + # "example.com" (11) crlf_bytes + # "\r\n" (2) + ua_bytes + # "User-Agent: c64-https/0.1\r\n" (27) conn_bytes + # "Connection: close\r\n" (19) crlf_bytes # "\r\n" (2) ) - expected_len = len(expected) # should be 60 + expected_len = len(expected) # should be 87 + + # The UA header exists because Wikipedia 403s UA-less requests: assert + # the emitted line is exactly the honest short UA (raw ASCII). + if ua_bytes == b"User-Agent: c64-https/0.1\r\n": + print(" PASS: User-Agent header line emitted as expected") + passed += 1 + else: + print(f" FAIL: User-Agent line = {bytes(ua_bytes)!r}") + failed += 1 # Check length if actual_len == expected_len: From 51ac79d4db8651106c0df165816ed065b0c3c572 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:45:36 -0400 Subject: [PATCH 22/33] feat(viewer): REU-backed scrolling text viewer for the body-sink stretch goal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New src/viewer.s (UCI-only, .ifdef BACKEND_UCI — the ip65 PRGs are byte-identical with and without this change, verified by hash on both ip65 profiles): renders a large ASCII document stored in the REU at HTTP_REU_BODY_BASE (default $10:0000, -D-overridable; length in the 24-bit http_body_total) on the 40x25 screen, writing screen RAM directly with the ascii_chrout strategy-B character mapping. - 24 content rows + status row 0 (hex offset/total + Q=QUIT) - hard wrap at 40 cols, LF newline (one LF after a wrap is consumed), CR ignored, TAB -> space; clamped at both ends; EMPTY screen for total = 0 - keys: CRSR-DOWN/UP one row, SPACE/F1 page, HOME top, Q quit - window model: 3,584 B REU->C64 DMA into the dead tcp_recv_buf ring (1 KB back-coverage); row-start table + REU shadow-register save slots live in the ring tail, so viewer BSS is 20 B in NET_BSS_TAIL - scroll-up derives the previous display-row start from a backward LF scan (P + 40*floor((n-1)/40)), matching forward rendering exactly - REU shadow registers ($DF02-$DF0A) are saved/restored around every DMA-using entry so reu_fetch_mul_row's latched state survives VIEWER_CODE segment: CRYPTO_HOT in the comb cfg (265 B still free), CRYPTO_OVERLAY in the plain UCI cfg (CRYPTO_HOT there has only ~69 B of slack at the v0.11.2 pin). Code+rodata 1,189 B. boot.s hook (BACKEND_UCI only): after http_recv_body, a non-zero http_body_sink routes to viewer_enter instead of print_resp_body. viewer.s carries .ifndef-guarded provisional definitions of http_body_sink / http_body_total until the body-sink lane's land (-D VIEWER_EXTERNAL_SINK=1 flips them to imports). tools/test_viewer.py (VICE, default_vice_config): 7/7 PASS — exact 25x40 screen-code assertions for LF handling, wrap + wrap-LF consume, lowercase folding, TAB/CR, odd bytes, mid-document and arbitrary offsets, EOF clamp, offset==total, EMPTY, plus 9 scroll-up vectors against a forward-rendered row-start list. Test image builds with make BACKEND=uci HTTP_REU_BODY_BASE=196608 VIEWER_TEST_HELPERS=1. The interactive key loop is a thin dispatch over these tested primitives and is deliberately not driven from VICE. Co-Authored-By: Claude Fable 5 --- Makefile | 13 + cfg/c64-https-uci-onchip.cfg | 6 + cfg/c64-https-uci.cfg | 9 + src/boot.s | 16 + src/viewer.s | 823 +++++++++++++++++++++++++++++++++++ tools/test_viewer.py | 390 +++++++++++++++++ 6 files changed, 1257 insertions(+) create mode 100644 src/viewer.s create mode 100644 tools/test_viewer.py diff --git a/Makefile b/Makefile index 9e49935e..c5595d9a 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,19 @@ endif ifdef HTTPS_PORT CA65FLAGS += -D HTTPS_PORT=$(HTTPS_PORT) endif +# Optional REU document-base override for the Lane G viewer +# (src/viewer.s; defaults to $10:0000 = 1048576). VICE tests use a +# 512 KB REU, so they build with a base inside it: +# `make BACKEND=uci HTTP_REU_BODY_BASE=196608` ($03:0000; decimal — +# ca65 -D does not take 0x, and $ needs shell quoting). +ifdef HTTP_REU_BODY_BASE +CA65FLAGS += -D HTTP_REU_BODY_BASE=$(HTTP_REU_BODY_BASE) +endif +# Viewer test helpers (viewer_test_blit RAM->REU blit + viewer_scroll_up). +# Test-image only; never set for a shipping build. +ifdef VIEWER_TEST_HELPERS +CA65FLAGS += -D VIEWER_TEST_HELPERS=1 +endif # VIC-II blanking around the CPU-bound crypto primitives (src/vic.s). # ON by default and that is what ships; `make VIC_BLANK=0` degrades # vic_blank/vic_unblank to bare RTS so the badline tax can be measured diff --git a/cfg/c64-https-uci-onchip.cfg b/cfg/c64-https-uci-onchip.cfg index f258fbba..88908e1e 100644 --- a/cfg/c64-https-uci-onchip.cfg +++ b/cfg/c64-https-uci-onchip.cfg @@ -241,6 +241,12 @@ SEGMENTS { X25519_RODATA: load = CRYPTO_OVERLAY, type = ro, optional = yes, align = $100; X25519_BSS: load = CRYPTO_OVERLAY, type = bss, optional = yes, align = $100; + # Lane G stretch goal: REU-backed text viewer (src/viewer.s). This + # comb variant keeps VIEWER_CODE in CRYPTO_HOT (the RODATA moves + # above leave it ~1.4 KB of slack); the plain uci cfg routes it to + # CRYPTO_OVERLAY instead because its CRYPTO_HOT is nearly full. + VIEWER_CODE: load = CRYPTO_HOT, type = ro, optional = yes; + # --- Resident crypto + TLS code / rodata. --- CRYPTO_CODE: load = CRYPTO_HOT, type = ro; CRYPTO_RODATA: load = CRYPTO_OVERLAY, type = ro; # onchip-comb variant: relieve CRYPTO_HOT diff --git a/cfg/c64-https-uci.cfg b/cfg/c64-https-uci.cfg index d5110a8e..3ebbf4f1 100644 --- a/cfg/c64-https-uci.cfg +++ b/cfg/c64-https-uci.cfg @@ -227,6 +227,15 @@ SEGMENTS { X25519_RODATA: load = CRYPTO_OVERLAY, type = ro, optional = yes, align = $100; X25519_BSS: load = CRYPTO_OVERLAY, type = bss, optional = yes, align = $100; + # Lane G stretch goal: REU-backed text viewer (src/viewer.s). Rides + # CRYPTO_OVERLAY here because CRYPTO_HOT has only ~69 B of slack at + # the v0.11.2 nistcurves pin under this cfg (the -onchip.cfg comb + # variant, whose CRYPTO_HOT has room, places it there instead). The + # overlay slot is free in the shipped non-comb UCI profiles; under + # the (broken, non-shipping) P-384/P-256 overlay-embed flags this + # placement would contend for the slot and fail loudly at link time. + VIEWER_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; + # --- Resident crypto + TLS code / rodata. --- CRYPTO_CODE: load = CRYPTO_HOT, type = ro; CRYPTO_RODATA: load = CRYPTO_HOT, type = ro; diff --git a/src/boot.s b/src/boot.s index 40e5e736..3945f99a 100644 --- a/src/boot.s +++ b/src/boot.s @@ -18,6 +18,12 @@ .export print_null_terminated .export print_resp_body + ; ---- Lane G viewer hook (UCI only; see do_https_get) ---- +.ifdef BACKEND_UCI + .import viewer_enter + .import http_body_sink +.endif + ; ---- exports: REU multiply table routines ---- ; Phase C.5: under USE_X25519_SIBLING=1 the sibling's ; libs/x25519/src/x25519_init.s owns reu_mul_init + @@ -633,6 +639,16 @@ do_https_get: ; body whenever it arrived as a second TLS record. jsr http_recv_body +.ifdef BACKEND_UCI + ; Lane G stretch-goal hook: when the body sink routed the + ; response into the REU (http_body_sink != 0), drop into the + ; scrollable viewer instead of printing the bounce buffer. + lda http_body_sink + beq @print_body + jsr viewer_enter + jmp @close +@print_body: +.endif ; display response body jsr print_resp_body diff --git a/src/viewer.s b/src/viewer.s new file mode 100644 index 00000000..9c27d01c --- /dev/null +++ b/src/viewer.s @@ -0,0 +1,823 @@ +; ============================================================================= +; viewer.s — REU-backed scrolling text viewer (Lane G stretch goal) +; +; Lets a human scroll through a large ASCII document stored in the REU +; (sunk there by the HTTP body-sink lane) on the 40x25 screen. +; +; UCI-only: the whole file is gated on BACKEND_UCI so the ip65 build +; stays byte-identical (this TU assembles to an empty object there). +; +; Document contract (agreed with the body-sink lane): +; - raw ASCII bytes, LF line endings, arbitrary line length, in the +; REU at HTTP_REU_BODY_BASE (default $10:0000 = bank 16) +; - 24-bit little-endian length in `http_body_total` +; +; Display model: +; - row 0 : status line "OOOOOO/TTTTTT Q=QUIT" (hex offset/total) +; - rows 1-24 : 24 content rows, hard-wrapped at 40 columns +; - LF starts a new row; a single LF immediately after a 40-column +; hard wrap is consumed (no phantom blank row); CR is ignored; +; TAB renders as a space; ASCII $20-$5F map to screen codes +; directly, a-z fold to the uppercase glyphs, everything else is +; a space. Mirrors boot.s::ascii_chrout strategy B, but writes +; screen RAM ($0400) directly for speed. +; +; Keys (KERNAL GETIN): +; CRSR-DOWN ($11) scroll down one row CRSR-UP ($91) up one row +; SPACE ($20) page down (24 rows) F1 ($85) page up +; HOME ($13) top Q ($51/$71) quit (rts) +; +; Scrolling model: a 24-bit document offset of the top displayed row +; (`viewer_offset`). Every render DMAs a 3,584 B window from the REU +; into tcp_recv_buf ($C000 — dead ring after the connection closes) +; positioned 1,024 B before the top offset, and re-renders the full +; screen. Scroll-down / page-down take the next row start from a +; 25-entry row-start table recorded during the render (at $CE00, +; inside the reclaimed ring — zero BSS). Scroll-up scans backward in +; the window for the previous LF and derives the previous display-row +; start as P + 40*floor((n-1)/40) (P = previous logical line start, +; n = its length), which reproduces exactly the row starts forward +; rendering would produce. If no LF exists within the ~1 KB of back +; window (a logical line longer than 1 KB), it falls back to top-40, +; which is still forward-consistent for any line that long unless the +; line *ends* inside that kilobyte (rare; display re-syncs at the +; next LF). +; +; REU discipline: the viewer only FETCHes (REU->C64); document banks +; and the crypto banks 0-2 are never written. The REU address/length +; shadow registers ARE clobbered by our DMAs, and reu_fetch_mul_row +; relies on the latched values reu_mul_init leaves behind — so entry +; saves $DF02-$DF08/$DF0A and exit restores them (slot at $CEF0/$CEF8, +; inside the reclaimed ring page). +; +; Zero-page: reuses zp_ptr ($FB-$FC). The viewer only runs while no +; TLS/HTTP/crypto is in flight, so the time-share is safe. +; ============================================================================= + +.ifdef BACKEND_UCI + + .include "constants.inc" + .macpack cbm + + .export viewer_enter + .export viewer_render_at + .export viewer_offset + + ; --- body-sink lane contract ------------------------------------- + ; The REU base of the document. -D-overridable for tests (VICE + ; uses a 512 KB REU, so tests build with a base inside it, e.g. + ; make HTTP_REU_BODY_BASE=196608 = $03:0000). The body-sink lane + ; exports the canonical equate; this .ifndef merges cleanly with + ; a -D from the command line either way. + .ifndef HTTP_REU_BODY_BASE + HTTP_REU_BODY_BASE = $100000 + .endif + + ; PROVISIONAL fallback definitions — laneE-bodysink owns + ; http_body_total (24-bit LE body length) and http_body_sink + ; (flag: body was sunk to REU). Until its definitions land this + ; block makes the branch link standalone. Merge resolution: + ; build with -D VIEWER_EXTERNAL_SINK=1 (or delete this block); + ; a 'Duplicate external identifier' from ld65 IS the reminder. + .ifdef VIEWER_EXTERNAL_SINK + .import http_body_total + .else + .export http_body_total + .export http_body_sink + .endif + +; ----------------------------------------------------------------------------- +; Constants +; ----------------------------------------------------------------------------- +SCREEN = $0400 +STATUS_ROW = SCREEN ; row 0 +CONTENT_ROW0 = SCREEN + 40 ; row 1 +COLOR_RAM = $d800 +VW_ROWS = 24 ; content rows +VW_COLS = 40 + +VW_WIN_BUF = tcp_recv_buf ; $C000 bounce buffer +VW_WIN_SIZE = $0e00 ; 3,584 B window +VW_WIN_BACK = $0400 ; keep 1 KB before top for back-scans + +VW_TBL = $ce00 ; 25 row starts x 3 B (in dead ring) +VW_REU_SAVE1 = $cef0 ; 9 B REU shadow save (viewer_enter) +VW_REU_SAVE2 = $cefa ; 9 B REU shadow save (viewer_render_at) + +KEY_CRSR_DOWN = $11 +KEY_CRSR_UP = $91 +KEY_SPACE = $20 +KEY_F1 = $85 +KEY_HOME = $13 +KEY_Q = $51 +KEY_Q_LC = $71 + +; ----------------------------------------------------------------------------- +; BSS (NET_BSS_TAIL — 20 bytes production) +; ----------------------------------------------------------------------------- + .segment "NET_BSS_TAIL" + +viewer_offset: ; 24-bit doc offset of top row +vw_top: .res 3 +vw_win: .res 3 ; doc offset of window start +vw_wok: .res 1 ; window-valid flag +vw_cur: .res 3 ; render cursor (doc offset) +vw_req: .res 3 ; ensure_window request offset +vw_tmp: .res 3 ; scratch (24-bit / scan counter) +vw_cnt: .res 2 ; scratch (16-bit index) +vw_row: .res 1 ; current content row +vw_tbi: .res 1 ; row-table byte index + + .ifndef VIEWER_EXTERNAL_SINK +http_body_total: .res 3 ; PROVISIONAL — see header +http_body_sink: .res 1 ; PROVISIONAL — see header + .endif + + .ifdef VIEWER_TEST_HELPERS + .export viewer_blit_off +viewer_blit_off: .res 3 ; test blit: doc offset of chunk + .endif + +; ----------------------------------------------------------------------------- + .segment "VIEWER_CODE" + +; ============================================================================= +; viewer_enter — interactive viewer. Returns (rts) on Q. +; ============================================================================= +viewer_enter: + jsr reu_save1 + lda #0 + sta vw_wok ; document may have changed + jsr top_zero + jsr fill_color + jsr render_core +@key: + jsr getin + cmp #0 + beq @key + cmp #KEY_Q + beq @quit + cmp #KEY_Q_LC + beq @quit + cmp #KEY_CRSR_DOWN + beq @down + cmp #KEY_CRSR_UP + beq @up + cmp #KEY_SPACE + beq @pgdn + cmp #KEY_F1 + beq @pgup + cmp #KEY_HOME + bne @key + jsr top_zero ; HOME -> top + jmp @redraw +@down: + ldx #(3*1) ; next row start = table entry 1 + jsr take_tbl_entry + jmp @redraw +@up: + jsr scroll_up_one + jmp @redraw +@pgdn: + ldx #(3*VW_ROWS) ; entry 24 = next screen top + jsr take_tbl_entry + jmp @redraw +@pgup: + lda #VW_ROWS + sta vw_row +@pgup_l: + jsr scroll_up_one + dec vw_row + bne @pgup_l +@redraw: + jsr render_core + jmp @key +@quit: + jsr reu_restore1 + rts + +; ============================================================================= +; viewer_render_at — non-interactive: render one screen at viewer_offset. +; Preserves the REU shadow registers around its own DMAs. +; ============================================================================= +viewer_render_at: + ldx #8 ; save shadows $DF02..$DF0A to slot 2 +@sv: lda reu_c64_lo,x + sta VW_REU_SAVE2,x + dex + bpl @sv + lda #0 + sta vw_wok ; test may have re-blitted the doc + jsr render_core + ldx #8 +@rs: lda VW_REU_SAVE2,x + sta reu_c64_lo,x + dex + bpl @rs + rts + +; ----------------------------------------------------------------------------- +; reu_save1 / reu_restore1 — shadow-register save around a viewer session. +; reu_fetch_mul_row (REU profile) relies on the latched C64 addr/len that +; reu_mul_init leaves behind; our FETCHes clobber them. The whole +; contiguous $DF02-$DF0A span is saved (the $DF09 interrupt-mask +; readback/writeback is a no-op, and including it makes the copy a +; plain indexed loop). +; ----------------------------------------------------------------------------- +reu_save1: + ldx #8 +@l: lda reu_c64_lo,x + sta VW_REU_SAVE1,x + dex + bpl @l + rts + +reu_restore1: + ldx #8 +@l: lda VW_REU_SAVE1,x + sta reu_c64_lo,x + dex + bpl @l + rts + +; ----------------------------------------------------------------------------- +top_zero: + lda #0 + sta vw_top + sta vw_top+1 + sta vw_top+2 + rts + +; ----------------------------------------------------------------------------- +; take_tbl_entry — X = byte index of a row-start table entry. Copies it to +; vw_top unless it would land at/after http_body_total (clamp at bottom). +; ----------------------------------------------------------------------------- +take_tbl_entry: + lda VW_TBL,x ; entry >= total ? -> no move + cmp http_body_total + lda VW_TBL+1,x + sbc http_body_total+1 + lda VW_TBL+2,x + sbc http_body_total+2 + bcs @no + lda VW_TBL,x + sta vw_top + lda VW_TBL+1,x + sta vw_top+1 + lda VW_TBL+2,x + sta vw_top+2 +@no: rts + +; ============================================================================= +; render_core — render status + 24 content rows from vw_top. +; Records the 24 row-start offsets plus the next-screen top in VW_TBL. +; ============================================================================= +render_core: + lda http_body_total + ora http_body_total+1 + ora http_body_total+2 + bne @go + jmp @empty +@go: + lda vw_top + sta vw_cur + sta vw_req + lda vw_top+1 + sta vw_cur+1 + sta vw_req+1 + lda vw_top+2 + sta vw_cur+2 + sta vw_req+2 + jsr ensure_window + jsr draw_status + + lda #CONTENT_ROW0 + sta row_store+2 + lda #0 + sta vw_row + sta vw_tbi +@rowloop: + ldx vw_tbi ; record row start + lda vw_cur + sta VW_TBL,x + lda vw_cur+1 + sta VW_TBL+1,x + lda vw_cur+2 + sta VW_TBL+2,x + inx + inx + inx + stx vw_tbi + jsr render_row + clc ; store base += 40 + lda row_store+1 + adc #VW_COLS + sta row_store+1 + lda row_store+2 + adc #0 + sta row_store+2 + inc vw_row + lda vw_row + cmp #VW_ROWS + bne @rowloop + ldx vw_tbi ; entry 24 = next-screen top + lda vw_cur + sta VW_TBL,x + lda vw_cur+1 + sta VW_TBL+1,x + lda vw_cur+2 + sta VW_TBL+2,x + rts +@empty: + jsr clear_screen + ldx #(empty_txt_end - empty_txt - 1) +@el: lda empty_txt,x + sta STATUS_ROW,x + dex + bpl @el + rts + +; ----------------------------------------------------------------------------- +; render_row — render one 40-column row at the SMC base in row_store, +; advancing vw_cur. Blank-fills after LF / EOF. Consumes a single LF +; that immediately follows a 40-column hard wrap. +; ----------------------------------------------------------------------------- +render_row: + ldx #0 +@loop: + jsr at_eof + bcs @fill + jsr getbyte + cmp #$0a + beq @lf + cmp #$0d + beq @cr + cmp #$09 + bne @conv + lda #$20 +@conv: + jsr conv_ascii + jsr row_store + jsr inc_cur + inx + cpx #VW_COLS + bne @loop + jsr at_eof ; wrapped: eat one trailing LF + bcs @done + jsr getbyte + cmp #$0a + bne @done + jsr inc_cur +@done: rts +@cr: + jsr inc_cur ; CR: skip, no column advance + jmp @loop +@lf: + jsr inc_cur ; LF: consume, blank rest of row +@fill: + lda #$20 +@fl: cpx #VW_COLS + beq @done + jsr row_store + inx + bne @fl + +row_store: + sta CONTENT_ROW0,x ; operand patched by render_core + rts + +; ----------------------------------------------------------------------------- +; at_eof — C=1 when vw_cur >= http_body_total. +; ----------------------------------------------------------------------------- +at_eof: + lda vw_cur + cmp http_body_total + lda vw_cur+1 + sbc http_body_total+1 + lda vw_cur+2 + sbc http_body_total+2 + rts + +inc_cur: + inc vw_cur + bne @r + inc vw_cur+1 + bne @r + inc vw_cur+2 +@r: rts + +; ----------------------------------------------------------------------------- +; conv_ascii — ASCII in A -> screen code in A (uppercase/graphics set). +; $20-$3F direct; $40-$5F -> $00-$1F; a-z -> $01-$1A (uppercase glyphs); +; everything else -> space. Preserves X/Y. +; ----------------------------------------------------------------------------- +conv_ascii: + cmp #$20 + bcc @sp + cmp #$40 + bcc @ok ; $20-$3F + cmp #$60 + bcc @upper ; $40-$5F + cmp #$61 + bcc @sp ; $60 backtick + cmp #$7b + bcs @sp ; $7B-$FF + sec + sbc #$60 ; a-z -> $01-$1A + rts +@upper: + sec + sbc #$40 ; @A-Z[\]^_ -> $00-$1F + rts +@sp: lda #$20 +@ok: rts + +; ----------------------------------------------------------------------------- +; getbyte — byte at doc offset vw_cur from the window, refreshing the +; window if vw_cur is outside it. Preserves X. Clobbers Y, zp_ptr. +; ----------------------------------------------------------------------------- +getbyte: + sec ; t = vw_cur - vw_win + lda vw_cur + sbc vw_win + sta zp_ptr + lda vw_cur+1 + sbc vw_win+1 + sta zp_ptr+1 + lda vw_cur+2 + sbc vw_win+2 + bne @refresh ; t >= 64 KB + lda vw_wok + beq @refresh + lda zp_ptr+1 + cmp #>VW_WIN_SIZE + bcs @refresh ; t >= window size + clc + adc #>VW_WIN_BUF + sta zp_ptr+1 + ldy #0 + lda (zp_ptr),y + rts +@refresh: + lda vw_cur + sta vw_req + lda vw_cur+1 + sta vw_req+1 + lda vw_cur+2 + sta vw_req+2 + jsr ensure_window + jmp getbyte + +; ----------------------------------------------------------------------------- +; ensure_window — make the bounce window cover vw_req with ~1 KB of +; back-coverage. Skips the DMA when the desired window is already +; loaded. Window start = max(vw_req - VW_WIN_BACK, 0). +; Clobbers A. Preserves X/Y. +; ----------------------------------------------------------------------------- +ensure_window: + sec + lda vw_req + sbc #VW_WIN_BACK + sta vw_tmp+1 + lda vw_req+2 + sbc #0 + sta vw_tmp+2 + bcs @have_ws + lda #0 ; clamp to document start + sta vw_tmp + sta vw_tmp+1 + sta vw_tmp+2 +@have_ws: + lda vw_wok + beq @load + lda vw_tmp ; same window already loaded? + cmp vw_win + bne @load + lda vw_tmp+1 + cmp vw_win+1 + bne @load + lda vw_tmp+2 + cmp vw_win+2 + beq @done +@load: + lda vw_tmp + sta vw_win + lda vw_tmp+1 + sta vw_win+1 + lda vw_tmp+2 + sta vw_win+2 + lda #1 + sta vw_wok + clc ; REU addr = base + window start + lda vw_win + adc #HTTP_REU_BODY_BASE + sta reu_reu_hi + lda vw_win+2 + adc #^HTTP_REU_BODY_BASE + sta reu_reu_bank + lda #VW_WIN_BUF + sta reu_c64_hi + lda #VW_WIN_SIZE + sta reu_len_hi + lda #0 + sta reu_addr_ctrl + lda #%10110001 ; execute + autoload + FETCH (REU->C64) + sta reu_command +@done: rts + +; ============================================================================= +; scroll_up_one — move vw_top to the previous display-row start. +; See the header for the derivation. No-op at offset 0. +; ============================================================================= +scroll_up_one: + lda vw_top + ora vw_top+1 + ora vw_top+2 + beq @done + lda vw_top + sta vw_req + lda vw_top+1 + sta vw_req+1 + lda vw_top+2 + sta vw_req+2 + jsr ensure_window + sec ; idx_top = vw_top - vw_win (16-bit) + lda vw_top + sbc vw_win + sta vw_cnt + lda vw_top+1 + sbc vw_win+1 + sta vw_cnt+1 + sec ; zp_ptr -> window byte idx_top-1 + lda vw_cnt + sbc #1 + sta zp_ptr + lda vw_cnt+1 + sbc #0 + clc + adc #>VW_WIN_BUF + sta zp_ptr+1 + ldy #0 + lda (zp_ptr),y + cmp #$0a + beq @at_line_start +@sub40: + sec ; mid logical line: top -= 40 + lda vw_top + sbc #40 + sta vw_top + lda vw_top+1 + sbc #0 + sta vw_top+1 + lda vw_top+2 + sbc #0 + sta vw_top+2 + bcs @done + jmp top_zero ; underflow guard -> 0 +@done: rts + +@at_line_start: + lda vw_cnt+1 ; idx_top < 2 -> top = 0 + bne @scan + lda vw_cnt + cmp #2 + bcs @scan + jmp top_zero +@scan: + sec ; zp_ptr -> idx_top-2, scan back for LF + lda vw_cnt + sbc #2 + sta zp_ptr + lda vw_cnt+1 + sbc #0 + clc + adc #>VW_WIN_BUF + sta zp_ptr+1 + lda #0 ; c = 0 (16-bit in vw_tmp) + sta vw_tmp + sta vw_tmp+1 +@sloop: + lda (zp_ptr),y ; Y still 0 + cmp #$0a + beq @found + lda zp_ptr ; at window index 0? + bne @dec + lda zp_ptr+1 + cmp #>VW_WIN_BUF + beq @floor +@dec: + lda zp_ptr + bne @d1 + dec zp_ptr+1 +@d1: dec zp_ptr + inc vw_tmp + bne @sloop + inc vw_tmp+1 + jmp @sloop +@floor: + lda vw_win ; window start = doc start? + ora vw_win+1 + ora vw_win+2 + beq @mod ; doc start: m = c + jmp @sub40 ; >1 KB logical line: fallback top-40 +@found: + lda vw_tmp ; c == 0 -> empty prev line: top -= 1 + ora vw_tmp+1 + bne @have_m + sec + lda vw_top + sbc #1 + sta vw_top + lda vw_top+1 + sbc #0 + sta vw_top+1 + lda vw_top+2 + sbc #0 + sta vw_top+2 + rts +@have_m: + lda vw_tmp ; m = c - 1 + bne @m1 + dec vw_tmp+1 +@m1: dec vw_tmp +@mod: + lda vw_tmp+1 ; r = m mod 40 + bne @msub + lda vw_tmp + cmp #40 + bcc @modok +@msub: + sec + lda vw_tmp + sbc #40 + sta vw_tmp + lda vw_tmp+1 + sbc #0 + sta vw_tmp+1 + jmp @mod +@modok: + lda vw_tmp ; top -= (r + 2) + clc + adc #2 + sta vw_tmp + sec + lda vw_top + sbc vw_tmp + sta vw_top + lda vw_top+1 + sbc #0 + sta vw_top+1 + lda vw_top+2 + sbc #0 + sta vw_top+2 + rts + +; ----------------------------------------------------------------------------- +; draw_status — row 0: "OOOOOO/TTTTTT Q=QUIT" + space padding. +; ----------------------------------------------------------------------------- +draw_status: + ldy #0 + lda vw_top+2 + jsr hex_byte + lda vw_top+1 + jsr hex_byte + lda vw_top + jsr hex_byte + lda #$2f ; '/' + sta STATUS_ROW,y + iny + lda http_body_total+2 + jsr hex_byte + lda http_body_total+1 + jsr hex_byte + lda http_body_total + jsr hex_byte + ldx #0 +@txt: + cpx #(status_txt_end - status_txt) + bcs @pad + lda status_txt,x + inx + bne @put +@pad: + lda #$20 +@put: + sta STATUS_ROW,y + iny + cpy #VW_COLS + bne @txt + rts + +hex_byte: + pha + lsr + lsr + lsr + lsr + jsr hex_dig + pla + and #$0f +hex_dig: + cmp #10 + bcc @num + sbc #9 ; 10..15 -> $01-$06 (screen A-F) + bne @sta +@num: + ora #$30 ; 0..9 -> $30-$39 +@sta: + sta STATUS_ROW,y + iny + rts + +; ----------------------------------------------------------------------------- +; clear_screen / fill_color +; ----------------------------------------------------------------------------- +clear_screen: + ldx #0 + lda #$20 +@l: sta SCREEN,x + sta SCREEN+250,x + sta SCREEN+500,x + sta SCREEN+750,x + inx + cpx #250 + bne @l + rts + +fill_color: + ldx #0 + lda $0286 ; current text colour +@l: sta COLOR_RAM,x + sta COLOR_RAM+250,x + sta COLOR_RAM+500,x + sta COLOR_RAM+750,x + inx + cpx #250 + bne @l + rts + +; ----------------------------------------------------------------------------- +; rodata (rides VIEWER_CODE) +; ----------------------------------------------------------------------------- +status_txt: + scrcode " Q=QUIT" +status_txt_end: +empty_txt: + scrcode "EMPTY" +empty_txt_end: + +; ============================================================================= +; Test helpers — assembled only with -D VIEWER_TEST_HELPERS=1. +; ============================================================================= + .ifdef VIEWER_TEST_HELPERS + .export viewer_test_blit + .export viewer_scroll_up + +; viewer_test_blit — STASH 4,096 B from tcp_recv_buf ($C000) to the REU +; at HTTP_REU_BODY_BASE + viewer_blit_off. Lets the VICE harness build +; a multi-KB document in the REU 4 KB at a time. +viewer_test_blit: + clc + lda viewer_blit_off + adc #HTTP_REU_BODY_BASE + sta reu_reu_hi + lda viewer_blit_off+2 + adc #^HTTP_REU_BODY_BASE + sta reu_reu_bank + lda #VW_WIN_BUF + sta reu_c64_hi + lda #0 + sta reu_len_lo + sta reu_addr_ctrl + lda #$10 + sta reu_len_hi ; 4,096 B + lda #%10110000 ; execute + autoload + STASH (C64->REU) + sta reu_command + lda #0 + sta vw_wok ; document changed under the window + rts + +; viewer_scroll_up — one scroll-up step from viewer_offset (updates it, +; no render). Exposes the backward-scan logic to the VICE harness. +viewer_scroll_up: + jmp scroll_up_one + .endif + +.endif ; BACKEND_UCI diff --git a/tools/test_viewer.py b/tools/test_viewer.py new file mode 100644 index 00000000..655bbe38 --- /dev/null +++ b/tools/test_viewer.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +"""test_viewer.py - REU text viewer render tests (Lane G stretch goal). + +Exercises the non-interactive `viewer_render_at` entry of src/viewer.s +against a known multi-KB ASCII document blitted into the REU, asserting +the exact 40x25 screen-code content for vectors covering LF handling, +long-line hard wrap (including the wrap-then-LF consume), lowercase +folding, TAB/CR handling, EOF clamping, mid-document offsets, the +total=0 EMPTY screen, and the status line. Also drives the exported +`viewer_scroll_up` helper to verify the backward row-start scan against +a forward-rendered row-start list. + +The interactive key loop of `viewer_enter` is a thin dispatch over these +tested primitives and is deliberately NOT driven from VICE. + +Build (done automatically unless C64_SKIP_BUILD=1): + + make clean && make BACKEND=uci HTTP_REU_BODY_BASE=196608 \ + VIEWER_TEST_HELPERS=1 + +(196608 = $03:0000 — inside the 512 KB VICE REU; the production default +base is $10:0000, outside it.) + +Usage: + python3 tools/test_viewer.py [--verbose] + +Requires: Python 3.10+, c64_test_harness, VICE x64sc. +""" + +import os +import subprocess +import sys + +from c64_test_harness import ( + Labels, + ViceInstanceManager, + read_bytes, + write_bytes, + jsr, + wait_for_text, +) + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from _vice_helpers import default_vice_config + +PROJECT_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +PRG_PATH = os.path.join(PROJECT_ROOT, "build", "c64-https.prg") +LABELS_PATH = os.path.join(PROJECT_ROOT, "build", "labels.txt") + +REU_BASE = 196608 # $03:0000 — must match the make line above + +SCREEN = 0x0400 +COLS = 40 +CONTENT_ROWS = 24 +BLIT_BUF = 0xC000 +BLIT_CHUNK = 4096 + +REQUIRED_LABELS = [ + "viewer_render_at", + "viewer_offset", + "viewer_test_blit", + "viewer_blit_off", + "viewer_scroll_up", + "http_body_total", +] + +VERBOSE = False + + +# --------------------------------------------------------------------------- +# Python model of the renderer (must mirror src/viewer.s exactly) +# --------------------------------------------------------------------------- + +def conv(b): + """ASCII byte -> screen code, mirroring conv_ascii.""" + if b < 0x20: + return 0x20 + if b < 0x40: + return b + if b < 0x60: + return b - 0x40 + if b == 0x60: + return 0x20 + if 0x61 <= b <= 0x7A: + return b - 0x60 + return 0x20 + + +def txt_codes(s): + """Uppercase-glyph screen codes for an ASCII string (A-Z -> 1-26).""" + out = [] + for ch in s: + o = ord(ch) + if 0x41 <= o <= 0x5A: + out.append(o - 0x40) + else: + out.append(o) + return out + + +def render_model(doc, total, off): + """Render 24 content rows from offset. Returns (rows, row_starts). + + row_starts has 25 entries: the 24 row starts plus the next-screen top. + """ + rows = [] + starts = [] + cur = off + for _ in range(CONTENT_ROWS): + starts.append(cur) + row = [] + while True: + if cur >= total: + row += [0x20] * (COLS - len(row)) + break + b = doc[cur] + if b == 0x0A: + cur += 1 + row += [0x20] * (COLS - len(row)) + break + if b == 0x0D: + cur += 1 + continue + row.append(0x20 if b == 0x09 else conv(b)) + cur += 1 + if len(row) == COLS: + # hard wrap: consume a single immediately-following LF + if cur < total and doc[cur] == 0x0A: + cur += 1 + break + rows.append(row) + starts.append(cur) + return rows, starts + + +def hexdig(d): + return 0x30 + d if d < 10 else d - 9 # screen '0'-'9' / A-F ($01-$06) + + +def hex24(v): + out = [] + for byte in ((v >> 16) & 0xFF, (v >> 8) & 0xFF, v & 0xFF): + out.append(hexdig(byte >> 4)) + out.append(hexdig(byte & 0x0F)) + return out + + +def status_model(off, total): + row = hex24(off) + [0x2F] + hex24(total) + txt_codes(" Q=QUIT") + row += [0x20] * (COLS - len(row)) + return row + + +def expected_screen(doc, total, off): + """Full 25x40 expected screen-code grid.""" + if total == 0: + grid = [txt_codes("EMPTY") + [0x20] * (COLS - 5)] + grid += [[0x20] * COLS for _ in range(CONTENT_ROWS)] + return grid + rows, _ = render_model(doc, total, off) + return [status_model(off, total)] + rows + + +def forward_row_starts(doc, total): + """Every display-row start offset from document start to EOF.""" + starts = [] + cur = 0 + while cur < total: + starts.append(cur) + col = 0 + while True: + if cur >= total: + break + b = doc[cur] + if b == 0x0A: + cur += 1 + break + if b == 0x0D: + cur += 1 + continue + cur += 1 + col += 1 + if col == COLS: + if cur < total and doc[cur] == 0x0A: + cur += 1 + break + return starts + + +# --------------------------------------------------------------------------- +# Test document +# --------------------------------------------------------------------------- + +def build_doc(): + parts = [] + parts.append(b"Hello, C64 viewer!\n") # folding + punct + parts.append(b"lowercase line with words\n") # folding + parts.append(b"\n") # empty line + parts.append(b"A" * 40 + b"\n") # exactly 40 + LF (consumed) + parts.append(b"B" * 41 + b"\n") # wraps to 2 rows + parts.append(b"tab\there and CR\r ignored\n") # TAB, CR + parts.append(b"odd bytes: `{|}~ \x7f\x80\xff end\n") # -> spaces + parts.append(b"digits 0123456789 and SYMBOLS @[]^_\n") + # a long run of numbered lines to give mid-document offsets substance + for i in range(120): + parts.append(("line %04d: the quick brown fox jumps over the lazy dog\n" % i).encode()) + parts.append(b"C" * 100 + b"\n") # 3-row wrap + parts.append(b"last line without trailing newline") + return b"".join(parts) + + +# --------------------------------------------------------------------------- +# Harness plumbing +# --------------------------------------------------------------------------- + +def write24(transport, addr, value): + write_bytes(transport, addr, bytes([value & 0xFF, (value >> 8) & 0xFF, + (value >> 16) & 0xFF])) + + +def upload_doc(transport, labels, doc): + for off in range(0, len(doc), BLIT_CHUNK): + chunk = doc[off:off + BLIT_CHUNK] + chunk += b"\x00" * (BLIT_CHUNK - len(chunk)) + write_bytes(transport, BLIT_BUF, chunk) + write24(transport, labels["viewer_blit_off"], off) + jsr(transport, labels["viewer_test_blit"], timeout=30.0) + + +def read_screen(transport): + data = read_bytes(transport, SCREEN, 1000) + return [list(data[r * COLS:(r + 1) * COLS]) for r in range(25)] + + +def dump_row(row): + return " ".join("%02x" % b for b in row) + + +def check_screen(name, transport, labels, doc, total, off): + print(f"\n--- {name} (offset {off}, total {total}) ---") + write24(transport, labels["http_body_total"], total) + write24(transport, labels["viewer_offset"], off) + jsr(transport, labels["viewer_render_at"], timeout=60.0) + got = read_screen(transport) + want = expected_screen(doc, total, off) + for r in range(25): + if got[r] != want[r]: + print(f" FAIL: row {r} mismatch") + print(f" want: {dump_row(want[r])}") + print(f" got: {dump_row(got[r])}") + return False + print(" PASS: all 25 rows match") + return True + + +def check_scroll_up(name, transport, labels, expect_from_to): + """expect_from_to: list of (top, expected_new_top).""" + print(f"\n--- {name} ---") + ok = True + for top, want in expect_from_to: + write24(transport, labels["viewer_offset"], top) + jsr(transport, labels["viewer_scroll_up"], timeout=30.0) + got_b = read_bytes(transport, labels["viewer_offset"], 3) + got = got_b[0] | (got_b[1] << 8) | (got_b[2] << 16) + if got == want: + print(f" PASS: scroll_up({top}) -> {got}") + else: + print(f" FAIL: scroll_up({top}) -> {got}, expected {want}") + ok = False + return ok + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + global VERBOSE + VERBOSE = "--verbose" in sys.argv + os.chdir(PROJECT_ROOT) + + if os.environ.get("C64_SKIP_BUILD"): + print("=== Building (skipped: C64_SKIP_BUILD set) ===") + else: + print("=== Building (BACKEND=uci, test REU base, viewer helpers) ===") + subprocess.run(["make", "clean"], capture_output=True) + result = subprocess.run( + ["make", "BACKEND=uci", f"HTTP_REU_BODY_BASE={REU_BASE}", + "VIEWER_TEST_HELPERS=1"], + capture_output=True, text=True) + if result.returncode != 0: + print(f" Build failed:\n{result.stderr[-2000:]}") + sys.exit(1) + print(" Build OK") + + if not os.path.exists(PRG_PATH): + print(f"FATAL: {PRG_PATH} not found") + sys.exit(1) + + labels = Labels.from_file(LABELS_PATH) + for name in REQUIRED_LABELS: + if labels.address(name) is None: + print(f"FATAL: required label '{name}' not found " + f"(build with VIEWER_TEST_HELPERS=1)") + sys.exit(1) + + doc = build_doc() + total = len(doc) + print(f" Document: {total} bytes") + starts = forward_row_starts(doc, total) + + config = default_vice_config( + prg_path=PRG_PATH, warp=True, ntsc=True, sound=False) + + passed = failed = 0 + + def tally(ok): + nonlocal passed, failed + if ok: + passed += 1 + else: + failed += 1 + + with ViceInstanceManager(config=config) as mgr: + inst = mgr.acquire() + transport = inst.transport + print(f" VICE PID={inst.pid}, port={inst.port}") + + print(" Waiting for main menu...") + if wait_for_text(transport, "Q=QUIT", timeout=120.0, verbose=False) is None: + print("FATAL: main menu did not appear") + sys.exit(1) + + print(" Uploading document to REU...") + upload_doc(transport, labels, doc) + + # 1: document start — LF handling, empty line, lowercase folding, + # TAB, CR, odd-byte squashing, exact-40 wrap + LF consume + tally(check_screen("Test 1: top of document", transport, labels, + doc, total, 0)) + + # 2: mid-document offset (a genuine forward row start) + mid = starts[len(starts) // 2] + tally(check_screen("Test 2: mid-document row start", transport, + labels, doc, total, mid)) + + # 3: arbitrary (non-row-start) offset — render_at takes any offset + tally(check_screen("Test 3: arbitrary offset", transport, labels, + doc, total, 1234)) + + # 4: EOF clamp — fewer than 24 rows remain, rest blank-filled + tally(check_screen("Test 4: EOF clamp", transport, labels, + doc, total, starts[-3])) + + # 5: offset exactly at total — fully blank content + tally(check_screen("Test 5: offset == total", transport, labels, + doc, total, total)) + + # 6: empty document + tally(check_screen("Test 6: EMPTY document", transport, labels, + doc, 0, 0)) + + # restore total for the scroll tests + write24(transport, labels["http_body_total"], total) + + # 7: scroll-up — for a sample of forward row starts, one step up + # must land on the preceding forward row start; top 0 stays. + sample_idx = [1, 2, 3, 4, 5, 8, len(starts) // 2, len(starts) - 1] + vectors = [(0, 0)] + for i in sample_idx: + vectors.append((starts[i], starts[i - 1])) + tally(check_scroll_up("Test 7: scroll-up row-start scan", + transport, labels, vectors)) + + mgr.release(inst) + + total_t = passed + failed + print("\n" + "=" * 60) + print(f" Passed: {passed}/{total_t}") + print(f" Failed: {failed}/{total_t}") + if failed == 0: + print(f"\n [+] viewer: ALL {total_t} TESTS PASSED") + else: + print(f"\n [-] viewer: {failed} TEST(S) FAILED") + print("=" * 60) + sys.exit(0 if failed == 0 else 1) + + +if __name__ == "__main__": + main() From 6fc4c90cc218a6a6cd2ed4240699db00137591f0 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:47:53 -0400 Subject: [PATCH 23/33] feat(http): chunked transfer-decoding (RFC 7230 4.1) with 0-chunk termination MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wikipedia — the stretch-goal target — serves the article CHUNKED with NO Content-Length (lane H live recon 2026-08-21: 4 data chunks of 26,931-32,768 B, de-chunked body 125,235 B), so on that path the terminal 0-chunk, not a Content-Length match, is what must end the body read. - 'Transfer-Encoding: chunked' matched case-insensitively per header line (check_header_line now runs both matchers); http_te_chunked routes the end-of-headers transition to the new chunk machine. - parse_state 3..6: chunk-size line (hex, case-insensitive, chunk extensions skipped), chunk payload, closing CRLF, trailer section; the empty trailer line after the 0-chunk completes the body (C=0). - Only payload bytes reach http_body_append, so http_body_total counts de-chunked payload — never framing — and the first-512/http_resp_len contracts hold unchanged on the chunked path. - http_chunk_rem is 16-bit; chunks >= 64 KB would mis-parse (largest observed anywhere is Wikipedia's 32,768 B) — documented at the declaration. - New HTTP_CHUNK_CODE segment: CRYPTO_RESIDENT tail under ip65 (the onchip profile now has 2 B free there — the region is at capacity), CRYPTO_OVERLAY under UCI. chunk_hex_digit rides HTTP_AUX_CODE for exactly that fit. All five profiles link. - tools/test_http.py: two chunked vectors (700 B across 3 chunks with an extension + uppercase hex + trailer; small 2-chunk body with case-folded TE header); 47/47 PASS in VICE. Co-Authored-By: Claude Fable 5 --- cfg/c64-https-ip65.cfg | 4 + cfg/c64-https-uci-onchip.cfg | 4 + cfg/c64-https-uci.cfg | 6 + src/http.s | 259 +++++++++++++++++++++++++++++++++-- tools/test_http.py | 136 ++++++++++++++++++ 5 files changed, 398 insertions(+), 11 deletions(-) diff --git a/cfg/c64-https-ip65.cfg b/cfg/c64-https-ip65.cfg index 7afa9bcd..40f8d95b 100644 --- a/cfg/c64-https-ip65.cfg +++ b/cfg/c64-https-ip65.cfg @@ -223,6 +223,10 @@ SEGMENTS { LIB_NISTCURVES_SHA384_TABLES: load = CRYPTO_OVERLAY, type = ro, optional = yes, align = $100; LIB_NISTCURVES_SHA384_BSS: load = CRYPTO_COLD_SHADOW, type = bss, optional = yes; RESIDENT_RODATA: load = CRYPTO_RESIDENT, type = ro, optional = yes; + # W4 chunked transfer-decoding (src/http.s http_chunk_dispatch). + # CRYPTO_OVERLAY is full under ip65 once HTTP_AUX_CODE lands, so the + # chunk machine rides the CRYPTO_RESIDENT tail slack instead. + HTTP_CHUNK_CODE: load = CRYPTO_RESIDENT, type = ro, optional = yes; # Phase C.5: sibling c64-x25519 rodata + bss segments. Under ip65 # CRYPTO_OVERLAY is now real ($4F8C-$5FFF, 4,212 B); the segments diff --git a/cfg/c64-https-uci-onchip.cfg b/cfg/c64-https-uci-onchip.cfg index 311669f5..57b91640 100644 --- a/cfg/c64-https-uci-onchip.cfg +++ b/cfg/c64-https-uci-onchip.cfg @@ -185,6 +185,10 @@ SEGMENTS { # W4 HTTP body machinery (24-bit Content-Length termination + REU # body sink, src/http.s) — same placement as c64-https-uci.cfg. HTTP_AUX_CODE: load = LOADER, type = ro, optional = yes; + # W4 chunked transfer-decoding — same placement rationale as + # c64-https-uci.cfg (CRYPTO_OVERLAY shared with LIB_NISTCURVES_ + # MUL_CODE here; both are resident, the slot is 7.5 KB). + HTTP_CHUNK_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; # NET_CODE hosts the UCI adapter + the LOADER_OVERFLOW tail. # Phase C.2 (UCI only): TLS_CODE and CRYPTO_AUX_CODE (SHA-256, diff --git a/cfg/c64-https-uci.cfg b/cfg/c64-https-uci.cfg index 2f635f5c..dc95d72d 100644 --- a/cfg/c64-https-uci.cfg +++ b/cfg/c64-https-uci.cfg @@ -173,6 +173,12 @@ SEGMENTS { # free pre-W4); NET_CODE is 8 B from full so LOADER_OVERFLOW could # not absorb the growth. HTTP_AUX_CODE: load = LOADER, type = ro, optional = yes; + # W4 chunked transfer-decoding. LOADER cannot absorb it on top of + # HTTP_AUX_CODE, so it rides CRYPTO_OVERLAY, which the production + # UCI build leaves free (P-384 swap-ins would clobber it, but every + # P-384 build target is broken and TLS-level P-384 is stubbed — the + # same trade ip65 already makes by keeping TLS_CODE there). + HTTP_CHUNK_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; # NET_CODE hosts the UCI adapter + the LOADER_OVERFLOW tail. # Phase C.2 (UCI only): TLS_CODE and CRYPTO_AUX_CODE (SHA-256, diff --git a/src/http.s b/src/http.s index 5abe2b22..ab380af8 100644 --- a/src/http.s +++ b/src/http.s @@ -28,6 +28,9 @@ .export http_in_mode .export http_in_ptr .export http_in_len + ; W4 chunked transfer-decoding (see http_chunk_dispatch): + .export http_te_chunked + .export http_chunk_rem ; ---- imports: data.asm BSS (HTTP I/O + parser state) ---- .import http_host_ptr @@ -457,6 +460,10 @@ http_recv_response: beq @state_status cmp #1 beq @jmp_headers + cmp #2 + beq @jmp_body + jmp http_chunk_dispatch ; states 3..6: chunked transfer-decoding +@jmp_body: jmp @state_body @jmp_headers: jmp @state_headers @@ -549,8 +556,10 @@ http_recv_response: ; Content-Length is "absent until seen": http_cl_valid = 0. The ; historical $FFFF magic sentinel went away with the 24-bit ; Content-Length extension — a separate valid-flag byte cannot - ; collide with any real length value. + ; collide with any real length value. Transfer-Encoding likewise + ; resets to "not chunked". sta http_cl_valid + sta http_te_chunked jmp @state_headers ; ----- state 1: read header lines into http_line_buf until an empty line ----- @@ -578,7 +587,7 @@ http_recv_response: @hl_eol: lda http_line_idx beq @hdr_end ; empty line -> end of headers - jsr check_content_length + jsr check_header_line ; Content-Length + Transfer-Encoding lda #0 sta http_line_idx jmp @state_headers @@ -588,10 +597,9 @@ http_recv_response: ; fallthrough below — returning @not_done here would strand a ; short body until the next record, and for a response that ; fits entirely in one record no further record ever arrives. - lda #2 - sta http_parse_state - jsr http_body_begin ; zero resp_len + consumed counter - jmp @state_body + jsr http_body_begin ; zero counters + pick body state: + ; 2 = identity, 3 = chunked + jmp http_recv_response ; re-dispatch into the selected state ; ----- state 2: read body bytes ----- ; RFC 7230 behaviour: we treat an empty ring as "body still arriving" @@ -744,11 +752,12 @@ cl_pattern: .byte "content-length: " ; ============================================================================= -; http_body_begin - reset per-response body state at the headers->body -; transition (state 1 -> 2). Living here (not in the recv-loop inits) -; guarantees the reset runs on every parse walk regardless of entry -; path (http_recv_body, http_get_plain, or a test driving -; http_recv_response directly). +; http_body_begin - reset per-response body state at the end-of-headers +; transition and select the body parse state: 2 (identity) normally, +; 3 (chunk-size line) when a Transfer-Encoding: chunked header was +; seen. Living here (not in the recv-loop inits) guarantees the reset +; runs on every parse walk regardless of entry path (http_recv_body, +; http_get_plain, or a test driving http_recv_response directly). ; Clobbers: A ; ============================================================================= http_body_begin: @@ -758,7 +767,60 @@ http_body_begin: sta http_body_total sta http_body_total+1 sta http_body_total+2 + sta http_chunk_rem + sta http_chunk_rem+1 + sta http_chunk_skip + lda http_te_chunked + beq @identity + lda #3 ; chunked: start at the chunk-size line + sta http_parse_state rts +@identity: + lda #2 + sta http_parse_state + rts + +; ============================================================================= +; check_header_line - run all per-header-line matchers on the line +; accumulated in http_line_buf. Called once per non-empty header +; line from @hl_eol. +; Clobbers: A, X, Y, zp_ptr, zp_temp +; ============================================================================= +check_header_line: + jsr check_content_length + ; fall through to check_transfer_encoding + +; ============================================================================= +; check_transfer_encoding - match "Transfer-Encoding: chunked" (case- +; insensitive name and value, single SP after the colon — the same +; shape check_content_length matches). Sets http_te_chunked = 1 on +; match. A prefix match suffices: "chunked" is by spec the final +; coding, and Wikipedia (the stretch-goal target) sends exactly this +; line. +; Clobbers: A, X +; ============================================================================= +check_transfer_encoding: + lda http_line_idx + cmp #26 ; len("transfer-encoding: chunked") + bcs :+ + rts +: ldx #25 +@cte_cmp: + lda http_line_buf,x + ora #$20 + cmp te_pattern,x + beq :+ + rts +: dex + bpl @cte_cmp + lda #1 + sta http_te_chunked + rts + +; Lowercase pattern; incoming bytes are OR'd with $20 (folds A..Z, +; leaves '-', ':', ' ' unchanged). +te_pattern: + .byte "transfer-encoding: chunked" ; ============================================================================= ; http_body_done_check - is the body complete? @@ -838,6 +900,177 @@ http_body_append: pla ; byte counted, not stored rts +; ============================================================================= +; Chunked transfer-decoding (RFC 7230 §4.1). Wikipedia — the stretch- +; goal target — serves the article chunked with NO Content-Length +; (measured live 2026-08-21: 4 data chunks of 26,931-32,768 B, body +; 125,235 B), so on that path the terminal 0-chunk is the termination +; signal, not a Content-Length match. +; +; parse_state values (3..6 dispatch here from http_recv_response): +; 3 - chunk-size line: hex digits, optional ";extension" skipped, +; terminated by LF +; 4 - chunk payload: http_chunk_rem bytes handed to http_body_append +; 5 - the CRLF that closes a chunk's payload +; 6 - trailer section after the 0-chunk: lines until an empty one, +; then the body is complete (C=0) +; +; Only payload bytes (state 4) reach http_body_append, so +; http_body_total counts de-chunked payload — never framing — and the +; http_resp_buf first-512 / REU-sink contracts hold unchanged. +; +; http_chunk_rem is 16-bit: chunks >= 64 KB would mis-parse, but no +; observed server sends them (Wikipedia's max is 32,768 B) and RFC 7230 +; sets no minimum chunking granularity a client may rely on. +; +; Lives in its own HTTP_CHUNK_CODE segment: CRYPTO_RESIDENT slack under +; ip65 (CRYPTO_OVERLAY is full there once HTTP_AUX_CODE lands), +; CRYPTO_OVERLAY under UCI. +; ============================================================================= + .segment "HTTP_CHUNK_CODE" +http_chunk_dispatch: + lda http_parse_state + cmp #4 + bcc @st_size ; state 3 + beq @jd_data ; state 4 + cmp #5 + bne @jd_trailer ; state 6+ + jmp @st_crlf ; state 5 +@jd_data: + jmp @st_data +@jd_trailer: + jmp @st_trailer + +@not_done: ; near exit for the early states (the + sec ; machine outgrew one branch's reach) + rts + +; ----- state 3: chunk-size line ----- +@st_size: + jsr http_in_byte + bcs @not_done + cmp #$0a ; LF ends the size line + beq @size_eol + ldy http_chunk_skip + bne @st_size ; already skipping extension/junk + cmp #$0d ; CR discarded + beq @st_size + jsr chunk_hex_digit + bcs @size_ext ; not a hex digit: ";ext" etc. + ; http_chunk_rem = rem*16 + digit + asl http_chunk_rem + rol http_chunk_rem+1 + asl http_chunk_rem + rol http_chunk_rem+1 + asl http_chunk_rem + rol http_chunk_rem+1 + asl http_chunk_rem + rol http_chunk_rem+1 + ora http_chunk_rem + sta http_chunk_rem + jmp @st_size +@size_ext: + inc http_chunk_skip ; ignore the rest of the line (reached at + jmp @st_size ; most once per line: the skip check above + ; short-circuits every later byte) +@size_eol: + lda #0 + sta http_chunk_skip + lda http_chunk_rem + ora http_chunk_rem+1 + beq @last_chunk + lda #4 + sta http_parse_state + jmp @st_data +@last_chunk: + lda #6 ; 0-chunk -> trailer section + sta http_parse_state + lda #0 + sta http_line_idx ; trailer-line char count + jmp @st_trailer + +; ----- state 4: chunk payload ----- +@st_data: + lda http_chunk_rem + ora http_chunk_rem+1 + beq @data_done + jsr http_in_byte + bcs @not_done2 + jsr http_body_append ; count + store/sink (payload bytes only) + lda http_chunk_rem + bne :+ + dec http_chunk_rem+1 +: dec http_chunk_rem + jmp @st_data +@data_done: + lda #5 + sta http_parse_state + ; fall through into the CRLF eater + +; ----- state 5: CRLF closing the chunk payload ----- +@st_crlf: + jsr http_in_byte + bcs @not_done2 + cmp #$0a + bne @st_crlf ; eat the CR (and tolerate junk) until LF + lda #3 ; next chunk-size line (rem is 0 — state 4 + sta http_parse_state ; counted it down — and skip is 0 too, so + jmp @st_size ; the size parser starts fresh) + +; ----- state 6: trailer lines until an empty one ----- +@st_trailer: + jsr http_in_byte + bcs @not_done2 + cmp #$0d + beq @st_trailer ; CR never counts toward the line + cmp #$0a + beq @tr_eol + inc http_line_idx + jmp @st_trailer +@tr_eol: + lda http_line_idx + beq @tr_done ; empty line -> body complete + lda #0 + sta http_line_idx + jmp @st_trailer +@tr_done: + clc + rts + +@not_done2: ; far exit for the late states + sec + rts + +; ----------------------------------------------------------------------------- +; chunk_hex_digit - ASCII hex digit in A -> C=0 + A=0..15, else C=1 +; (case-insensitive). Clobbers: A. +; Rides HTTP_AUX_CODE (jsr'd from the chunk machine): the ip65-onchip +; profile's CRYPTO_RESIDENT is too tight for it to sit with +; HTTP_CHUNK_CODE. +; ----------------------------------------------------------------------------- + .segment "HTTP_AUX_CODE" +chunk_hex_digit: + cmp #$30 ; '0' + bcc @nothex + cmp #$3a ; '9'+1 + bcs @alpha + and #$0f + clc + rts +@alpha: + ora #$20 ; fold A-F to a-f + cmp #$61 ; 'a' + bcc @nothex + cmp #$67 ; 'f'+1 + bcs @nothex + sec + sbc #$57 ; 'a' ($61) -> 10 + clc + rts +@nothex: + sec + rts + ; ============================================================================= ; http_get_plain - perform a plain HTTP (no TLS) GET request ; Input: http_host_ptr/http_host_len = hostname @@ -945,3 +1178,7 @@ http_bg_src: .res 1 ; build_get source index temp http_in_mode: .res 1 ; 0 = TCP ring (plain HTTP), 1 = span (TLS) http_in_ptr: .res 2 ; span read cursor http_in_len: .res 2 ; span bytes remaining (16-bit) +; Chunked transfer-decoding state (see http_chunk_dispatch) +http_te_chunked: .res 1 ; 1 = "Transfer-Encoding: chunked" seen +http_chunk_rem: .res 2 ; 16-bit payload bytes left in this chunk +http_chunk_skip: .res 1 ; 1 = skipping rest of the chunk-size line diff --git a/tools/test_http.py b/tools/test_http.py index 7f9bb15e..195974e4 100755 --- a/tools/test_http.py +++ b/tools/test_http.py @@ -543,6 +543,132 @@ def test_body_no_content_length(transport, labels): return passed, failed +def _chunk(data, ext=b""): + """Encode one HTTP/1.1 chunk (RFC 7230 §4.1).""" + return (b"%x" % len(data)) + ext + b"\r\n" + data + b"\r\n" + + +def test_chunked_body(transport, labels): + """Chunked transfer-decoding: Wikipedia serves the stretch-goal article + chunked with NO Content-Length, so the terminal 0-chunk is the + termination signal and http_body_total must count de-chunked payload + bytes only (never framing).""" + passed = 0 + failed = 0 + + part1 = bytes((i * 3 + 1) & 0xFF for i in range(300)) + part2 = bytes((i * 5 + 2) & 0xFF for i in range(300)) + part3 = bytes((i * 11 + 3) & 0xFF for i in range(100)) + body = part1 + part2 + part3 # 700 B de-chunked + + stream = ( + b"HTTP/1.1 200 OK\r\n" + b"Transfer-Encoding: chunked\r\n" + b"\r\n" + + _chunk(part1) # "12c\r\n"... + + _chunk(part2, ext=b";name=val") # chunk extension + + (b"%X" % len(part3)) + b"\r\n" + part3 + b"\r\n" # UPPERCASE hex + + b"0\r\n" + + b"X-Trailer: yes\r\n" # trailer section + + b"\r\n" + ) + + _load_span(transport, labels, stream) + _reset_parser(transport, labels) + + complete = _run_parser_loop(transport, labels, timeout=90.0) + if not complete: + print(" FAIL: parser did not complete on the terminal 0-chunk") + return 0, 1 + print(" PASS: parser terminated on the 0-chunk (+ trailer)") + passed += 1 + + te = read_bytes(transport, labels.address("http_te_chunked"), 1)[0] + if te == 1: + print(" PASS: http_te_chunked = 1") + passed += 1 + else: + print(f" FAIL: http_te_chunked = {te}, expected 1") + failed += 1 + + total = _read_u24(transport, labels, "http_body_total") + if total == 700: + print(f" PASS: http_body_total = {total} (payload only, no framing)") + passed += 1 + else: + print(f" FAIL: http_body_total = {total}, expected 700") + failed += 1 + + resp_len = _read_u16(transport, labels, "http_resp_len") + if resp_len == 512: + print(f" PASS: http_resp_len = {resp_len}") + passed += 1 + else: + print(f" FAIL: http_resp_len = {resp_len}, expected 512") + failed += 1 + + stored = read_bytes(transport, labels.address("http_resp_buf"), 512) + if stored == body[:512]: + print(" PASS: http_resp_buf holds the first 512 de-chunked bytes") + passed += 1 + else: + print(" FAIL: http_resp_buf does not match de-chunked body prefix") + failed += 1 + + in_len = _read_u16(transport, labels, "http_in_len") + if in_len == 0: + print(" PASS: span fully consumed") + passed += 1 + else: + print(f" FAIL: http_in_len = {in_len}, expected 0") + failed += 1 + + return passed, failed + + +def test_chunked_small_body(transport, labels): + """Chunked body smaller than the buffer: stored intact, exact length.""" + passed = 0 + failed = 0 + + body = b"CHUNKED-SMALL-BODY-0123456789" # 29 B + stream = ( + b"HTTP/1.1 200 OK\r\n" + b"transfer-encoding: CHUNKED\r\n" # case-folded match + b"\r\n" + + _chunk(body[:10]) + _chunk(body[10:]) + b"0\r\n\r\n" + ) + + _load_span(transport, labels, stream) + _reset_parser(transport, labels) + + complete = _run_parser_loop(transport, labels, timeout=30.0) + if not complete: + print(" FAIL: parser did not complete") + return 0, 1 + + total = _read_u24(transport, labels, "http_body_total") + resp_len = _read_u16(transport, labels, "http_resp_len") + stored = read_bytes(transport, labels.address("http_resp_buf"), len(body)) + if total == len(body) and resp_len == len(body): + print(f" PASS: body_total = resp_len = {total}") + passed += 1 + else: + print(f" FAIL: body_total = {total}, resp_len = {resp_len}, " + f"expected {len(body)}") + failed += 1 + if stored == body: + print(" PASS: de-chunked body stored intact") + passed += 1 + else: + print(f" FAIL: body mismatch: {stored!r}") + failed += 1 + + # Restore ring input mode for any later ring-driven vectors. + write_bytes(transport, labels.address("http_in_mode"), [0]) + return passed, failed + + def run_tests(transport, labels, verbose=False): total_passed = 0 total_failed = 0 @@ -609,6 +735,16 @@ def run_tests(transport, labels, verbose=False): total_passed += p total_failed += f + print("\n--- W4: chunked body > 512 (Wikipedia shape) ---") + p, f = test_chunked_body(transport, labels) + total_passed += p + total_failed += f + + print("\n--- W4: chunked small body ---") + p, f = test_chunked_small_body(transport, labels) + total_passed += p + total_failed += f + return total_passed, total_failed From 7d3ef2b50eb4b5b395f8fb8aa755e5e3715b5506 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:48:00 -0400 Subject: [PATCH 24/33] =?UTF-8?q?feat(cert):=20grow=20cert=5Fbuf=20to=2020?= =?UTF-8?q?48=20B=20under=20UCI=20=E2=80=94=20Wikipedia's=201636=20B=20lea?= =?UTF-8?q?f=20fits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CERT_BUF_SIZE is now a per-backend equate in net_tuning.inc (2048 UCI / 1536 ip65); der_decode.s, the deframer's leaf cap (DF_CERT_BUF_SIZE) and the tests all derive from it instead of a literal. src/exports.s exports it as the absolute symbol `cert_buf_size` so tests/rigs read the real capacity from build/labels.txt. Under UCI, CERT_BUF_BSS moves from CRYPTO_COLD_SHADOW into CRYPTO_OVERLAY (after TLS_DEFRAME_CODE; plain/onchip end at $4F62 with 4,253 B free, comb through ~$5BD2 with ~1.1 KB free — still room for the rigs' DMA scratch, proven via the _memory_policy arbiter + rig_https_live --selfcheck). Moving it removed the packing that landed the shared sqtab at its baked $BC00 anchor, so both UCI cfgs now pin TABLES_BSS with start=$BA00; the Makefile's post-link check passes on all UCI profiles. Under ip65 nothing moves: cert_buf stays 1536 B at $A000 in the #68 SCRATCH_UNION, and both ip65 PRGs are byte-identical across this change. Casualty, accepted: USE_X25519_SIBLING=1 no longer links under UCI (X25519_BSS overflows CRYPTO_OVERLAY by 1,280 B; it linked pre-growth). EMBED_P256_OVERLAY was already broken before this change (labels.txt ordering defect) and is unchanged. Documented in CLAUDE.md. Tests: tools/test_tls_deframe.py gains cap-relative boundary vectors (cap accepted, cap+64 rejected, 1636 B accepted on UCI) — 42/42; tools/test_tls_deframer.py gains a wikipedia_sized_leaf_accepted gate (real minted 1636 B P-256 cert, pubkey extraction as oracle) — 13/13. The acceptance rig now calls tls_deframe_init per scenario, mirroring tls_connect's per-connection reset (the oversize-abort scenario used to leak DF_MODE_STREAM state into whatever ran after it). ip65 regression: run_all_tests 287/287, test_x509 11/11, test_tls_handshake 21/21. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 65 ++++++++++++++++++---- cfg/c64-https-uci-onchip.cfg | 28 ++++++++-- cfg/c64-https-uci.cfg | 29 ++++++++-- src/der_decode.s | 6 +- src/exports.s | 8 +++ src/net/ip65/net_tuning.inc | 9 +++ src/net/uci/net_tuning.inc | 12 ++++ src/tls_deframe.s | 18 ++++-- tools/test_tls_deframe.py | 32 ++++++++--- tools/test_tls_deframer.py | 104 +++++++++++++++++++++++++++++++---- tools/uci/rig_https_live.py | 29 ++++++---- 11 files changed, 285 insertions(+), 55 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c3af72a7..413e08e8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -162,8 +162,11 @@ Variables: (`cfg/c64-https-$(BACKEND).cfg`; default ip65). Changing it requires `make clean` — see above. - `USE_X25519_SIBLING=1` — swap the in-tree X25519 for the - `libs/x25519@v0.11.2` sibling. **UCI links; - ip65 overflows CRYPTO_OVERLAY** — see + `libs/x25519@v0.11.2` sibling. **Currently + links under NEITHER backend: ip65 overflows + CRYPTO_OVERLAY (long-standing), and UCI + overflows it by 1,280 B since the 2048 B + cert_buf moved in (Wikipedia growth)** — see "Known issues". Off by default either way. - `EMBED_P256_OVERLAY=1` — stage the P-256 verify image into the CRYPTO_OVERLAY slot at PRG-load (UCI). Its @@ -903,12 +906,20 @@ Five latent bugs and three new ones were cleared to get here: overflow and is fixed (see the CRYPTO_COLD_SHADOW entry under "Memory layout"). - **`USE_X25519_SIBLING=1` LINKS UNDER UCI from the v0.10.1 / - v0.11.0 wave pins** (issue #112), and still does not under ip65. - Both backends previously died on the same symbol collision; that - half is closed, and what remains on ip65 is a fit problem: - - make BACKEND=uci USE_X25519_SIBLING=1 # 62,977 B PRG, links + **`USE_X25519_SIBLING=1` no longer links under EITHER backend.** + It linked under UCI from the v0.10.1 / v0.11.0 wave pins (issue + #112, symbol collision resolved) until the Wikipedia cert_buf + growth moved the resident 2,048 B `CERT_BUF_BSS` into + CRYPTO_OVERLAY; the sibling's page-aligned tables no longer fit + alongside it + TLS_DEFRAME_CODE (measured 2026-08-21, both + baselines confirmed against the pre-growth tree). This is an + accepted casualty: the flag ships in nothing, and the resident + deframer + Wikipedia-capable cert_buf are worth more than an + opt-in nobody ships. ip65 remains the older, structural overflow: + + make BACKEND=uci USE_X25519_SIBLING=1 + Segment 'X25519_BSS' overflows memory area + 'CRYPTO_OVERLAY' by 1280 bytes # post-cert_buf-growth make USE_X25519_SIBLING=1 # ip65 Segment 'X25519_RODATA' overflows memory area 'CRYPTO_OVERLAY' by 3584 bytes @@ -1908,9 +1919,18 @@ UCI layout (W1 reference — `cfg/c64-https-uci.cfg`): $4000 (size 0) UCI_BSS_REGION Zero-size alias post-W1 (UCI_BSS moved into NET_BSS_TAIL above, which is why NET_BSS_TAIL now runs through $41FF) - $4200-$5FFF CRYPTO_OVERLAY 7.5 KB swappable overlay slot + $4200-$5FFF CRYPTO_OVERLAY 7.5 KB slot. Resident in EVERY + default UCI build since the sprint: + TLS_DEFRAME_CODE (~1.4 KB) + + CERT_BUF_BSS (2,048 B — Wikipedia + growth); comb adds RODATA/ + CRYPTO_RODATA/LIB rodata/LIMLEE_BSS + (used through ~$5BD2, ~1.1 KB tail + free). The overlay-embed flags (X25519 sibling / P-384 SHA-384 / - P-384 curve / W3 P-256 verify embed) + P-384 curve / W3 P-256 embed) contend + for what remains — see the + CRYPTO_OVERLAY bullet below. $6000-$9FFF CRYPTO_HOT 16 KB file-backed; resident code + rodata + small BSS (UCI_BSS, most of libs/nistcurves P-256). No segment @@ -1986,7 +2006,26 @@ Tight regions (post-W1): - **CRYPTO_OVERLAY** under UCI doubles as P-384 SHA-384/curve overlay paging slot, the W3 P-256 overlay embed slot, AND the USE_X25519_SIBLING=1 X25519 sibling rodata + BSS slot. Mutually - exclusive at link time across the three flags. + exclusive at link time across the three flags — and since the + real-server sprint the region also carries two RESIDENT tenants in + every default UCI build: `TLS_DEFRAME_CODE` (~1.4 KB, W1 deframer) + and `CERT_BUF_BSS` (2,048 B — `CERT_BUF_SIZE` in + `src/net/uci/net_tuning.inc`, grown from 1536 so + en.wikipedia.org's 1636 B leaf fits; under ip65 cert_buf stays + 1536 B at $A000 in the SCRATCH_UNION, untouched). Measured + occupancy 2026-08-21: plain-uci and uci-onchip use $4200-$4F62 + (4,253 B free); uci-comb through ~$5BD2 (~1.1 KB free, still + enough for the rigs' ~390 B of DMA scratch — verified via + `_memory_policy` arbiter + `rig_https_live.py --selfcheck`). + Consequences for the flags: `USE_X25519_SIBLING` now overflows + under UCI too (see Known issues); `EMBED_P256_OVERLAY` / + `USE_OVERLAY_P384_EMBED` were already broken before the growth + (the `build/labels.txt` ordering defect) and remain so. Moving + cert_buf out of CRYPTO_COLD_SHADOW also removed the packing that + used to land sqtab at $BC00, so both UCI cfgs now pin + `TABLES_BSS` with `start = $BA00` — the sibling reads the shared + sqtab through equates baked to LIB_SHARED_SQTAB_BASE=$BC00 and + the Makefile's post-link check still asserts it. There is a known TODO to restructure the MEMORY map so that all file-backed regions are physically contiguous in a single ROM-like @@ -2014,7 +2053,9 @@ ld65 and ca65 edge cases; they are intentional and should stay: doing it inside the `.inc` header would duplicate on every include. Only **backend-agnostic** symbols live here: `tcp_recv_buf`, `fe_src1`, `fe_src2`, `fe_dst`, `cc20_data_ptr`, `cc20_remain`, - `zp_ptr`. The ip65-specific ones (`ip65_init`, `ip65_process`) moved + `zp_ptr`, and `cert_buf_size` (the CERT_BUF_SIZE equate as an + absolute export — 2048 UCI / 1536 ip65 — which tests/rigs MUST + read from labels.txt instead of hardcoding either number). The ip65-specific ones (`ip65_init`, `ip65_process`) moved to `src/net/ip65/exports.s`, which is linked only under `BACKEND=ip65` and also carries the `cert_buf = $A000` SCRATCH_UNION link-time assert. diff --git a/cfg/c64-https-uci-onchip.cfg b/cfg/c64-https-uci-onchip.cfg index 3d0ddfbf..568d3158 100644 --- a/cfg/c64-https-uci-onchip.cfg +++ b/cfg/c64-https-uci-onchip.cfg @@ -218,9 +218,10 @@ SEGMENTS { # to CRYPTO_COLD_SHADOW ($A000-$BFFF) under the hot/cold split. # ip65 refit: cert_buf moved from BSS_TAIL into its own segment so # the ip65 cfg can pin + union it with the lib verify scratch (see - # cfg/c64-https-ip65.cfg SCRATCH_UNION). Under UCI it simply packs - # here, first in the cold shadow — same total occupancy as before. - CERT_BUF_BSS: load = CRYPTO_COLD_SHADOW, type = bss, optional = yes; + # cfg/c64-https-ip65.cfg SCRATCH_UNION). Under UCI it used to pack + # first in the cold shadow; the Wikipedia growth (CERT_BUF_SIZE + # 1536 -> 2048, src/net/uci/net_tuning.inc) moved it to + # CRYPTO_OVERLAY — see the overlay-slot block below. BSS_TAIL: load = CRYPTO_COLD_SHADOW, type = bss, optional = yes; # --- Overlay slot. --- @@ -240,6 +241,16 @@ SEGMENTS { # combination with the onchip profiles by the Makefile). TLS_DEFRAME_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; + # cert_buf (src/der_decode.s), grown to 2048 B under UCI so + # en.wikipedia.org's 1636 B leaf fits (CERT_BUF_SIZE in + # src/net/uci/net_tuning.inc). Moved out of CRYPTO_COLD_SHADOW, + # which had no room for the extra 512 B; CRYPTO_OVERLAY does. + # Tightens the overlay contention for the overlay-embed flags — + # already forbidden with the onchip profiles by the Makefile — + # and shares the region with CRYPTO_RODATA (routed here by this + # cfg variant) + TLS_DEFRAME_CODE: check the .map after changes. + CERT_BUF_BSS: load = CRYPTO_OVERLAY, type = bss, optional = yes; + # Phase C.5: sibling c64-x25519 rodata tables (mul38, sqr_lo/hi, # a24_b0..b3 — ~2 KB) AND the sibling's page-aligned BSS buffers # (fe25519_tmp1..4, x25_*, mul_dma_lo/hi/carry — ~1.5 KB) ride @@ -324,7 +335,16 @@ SEGMENTS { # TABLES_BSS pins the 1.5 KB sqtab/mul_dma tables on a page # boundary. Page alignment is load-bearing for the no-page-penalty # `lda abs,Y` addressing in the hot-path multiply routines. - TABLES_BSS: load = CRYPTO_COLD_SHADOW, type = bss, align = $100; + # start = $BA00 (page-aligned, so the .align 256 requirement is + # still met): the sibling nistcurves archive reads the shared + # quarter-square table through equates BAKED to + # LIB_SHARED_SQTAB_BASE = $BC00 (sqtab_lo at TABLES_BSS+$200), and + # the Makefile's post-link check asserts it. That address used to + # fall out of cert_buf packing first in the cold shadow; with + # CERT_BUF_BSS moved to CRYPTO_OVERLAY (Wikipedia growth) the + # anchor must be explicit. $BA00 + $600 = $C000 — flush to the + # region end, exactly the historical placement. + TABLES_BSS: load = CRYPTO_COLD_SHADOW, type = bss, start = $BA00; # Phase 3: P-384 split overlay blobs embedded in the PRG. Boot # DMAs them out to REU banks 6/7 then the staging RAM is free. diff --git a/cfg/c64-https-uci.cfg b/cfg/c64-https-uci.cfg index 151d7649..eddf7ca2 100644 --- a/cfg/c64-https-uci.cfg +++ b/cfg/c64-https-uci.cfg @@ -204,9 +204,10 @@ SEGMENTS { # to CRYPTO_COLD_SHADOW ($A000-$BFFF) under the hot/cold split. # ip65 refit: cert_buf moved from BSS_TAIL into its own segment so # the ip65 cfg can pin + union it with the lib verify scratch (see - # cfg/c64-https-ip65.cfg SCRATCH_UNION). Under UCI it simply packs - # here, first in the cold shadow — same total occupancy as before. - CERT_BUF_BSS: load = CRYPTO_COLD_SHADOW, type = bss, optional = yes; + # cfg/c64-https-ip65.cfg SCRATCH_UNION). Under UCI it used to pack + # first in the cold shadow; the Wikipedia growth (CERT_BUF_SIZE + # 1536 -> 2048, src/net/uci/net_tuning.inc) moved it to + # CRYPTO_OVERLAY — see the overlay-slot block below. BSS_TAIL: load = CRYPTO_COLD_SHADOW, type = bss, optional = yes; # --- Overlay slot. --- @@ -228,6 +229,17 @@ SEGMENTS { # LIB_NISTCURVES_MUL_CODE already is under the onchip profile. TLS_DEFRAME_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; + # cert_buf (src/der_decode.s), grown to 2048 B under UCI so + # en.wikipedia.org's 1636 B leaf fits (CERT_BUF_SIZE in + # src/net/uci/net_tuning.inc). Moved out of CRYPTO_COLD_SHADOW, + # which had no room for the extra 512 B; CRYPTO_OVERLAY does. + # This tightens the overlay contention: TLS_DEFRAME_CODE + + # CERT_BUF_BSS are resident in every default UCI build, so the + # mutually-exclusive overlay-embed flags (EMBED_P256_OVERLAY / + # USE_OVERLAY_P384_EMBED / USE_X25519_SIBLING) now have ~2 KB less + # to work with — check the .map before re-enabling any of them. + CERT_BUF_BSS: load = CRYPTO_OVERLAY, type = bss, optional = yes; + # Phase C.5: sibling c64-x25519 rodata tables (mul38, sqr_lo/hi, # a24_b0..b3 — ~2 KB) AND the sibling's page-aligned BSS buffers # (fe25519_tmp1..4, x25_*, mul_dma_lo/hi/carry — ~1.5 KB) ride @@ -307,7 +319,16 @@ SEGMENTS { # TABLES_BSS pins the 1.5 KB sqtab/mul_dma tables on a page # boundary. Page alignment is load-bearing for the no-page-penalty # `lda abs,Y` addressing in the hot-path multiply routines. - TABLES_BSS: load = CRYPTO_COLD_SHADOW, type = bss, align = $100; + # start = $BA00 (page-aligned, so the .align 256 requirement is + # still met): the sibling nistcurves archive reads the shared + # quarter-square table through equates BAKED to + # LIB_SHARED_SQTAB_BASE = $BC00 (sqtab_lo at TABLES_BSS+$200), and + # the Makefile's post-link check asserts it. That address used to + # fall out of cert_buf packing first in the cold shadow; with + # CERT_BUF_BSS moved to CRYPTO_OVERLAY (Wikipedia growth) the + # anchor must be explicit. $BA00 + $600 = $C000 — flush to the + # region end, exactly the historical placement. + TABLES_BSS: load = CRYPTO_COLD_SHADOW, type = bss, start = $BA00; # Phase 3: P-384 split overlay blobs embedded in the PRG. Boot # DMAs them out to REU banks 6/7 then the staging RAM is free. diff --git a/src/der_decode.s b/src/der_decode.s index 20719e57..c431c3fc 100644 --- a/src/der_decode.s +++ b/src/der_decode.s @@ -12,6 +12,7 @@ ; zp_count ($FE) - temporary .include "constants.inc" +.include "net_tuning.inc" ; CERT_BUF_SIZE (2048 UCI / 1536 ip65) ; --- Public exports: code --- .export der_read_tag @@ -584,7 +585,10 @@ cert_curve_id: .res 1 ; 0=P-256, 1=P-384 ; cert_buf freely. Any future cert-chain validation that re-reads ; cert_buf (or hashes TBS via cert_tbs_ptr) after Certificate ; processing MUST first break the union in cfg/c64-https-ip65.cfg. +; Size is backend-conditional (net_tuning.inc): 2048 under UCI (fits +; en.wikipedia.org's 1636 B leaf; segment routed to CRYPTO_OVERLAY), +; 1536 under ip65 (SCRATCH_UNION cap — see cfg/c64-https-ip65.cfg). .segment "CERT_BUF_BSS" -cert_buf: .res 1536 ; certificate DER buffer +cert_buf: .res CERT_BUF_SIZE ; certificate DER buffer .segment "BSS" cert_buf_len: .res 2 ; certificate length diff --git a/src/exports.s b/src/exports.s index 71e59a60..e229e67d 100644 --- a/src/exports.s +++ b/src/exports.s @@ -14,9 +14,17 @@ ; Add symbols here as the harness needs them. .include "constants.inc" +.include "net_tuning.inc" ; CERT_BUF_SIZE (backend-conditional) .export tcp_recv_buf +; cert_buf capacity as a linker-visible absolute symbol (2048 UCI / +; 1536 ip65 — src/net//net_tuning.inc). Tests and rigs MUST +; read this from build/labels.txt instead of hardcoding either number: +; tools/test_tls_deframer.py, tools/test_tls_deframe.py. +cert_buf_size = CERT_BUF_SIZE +.export cert_buf_size + ; Promote the fe25519 ZP equates so tools/test_x25519.py can resolve ; them via labels.txt. .export fe_src1 diff --git a/src/net/ip65/net_tuning.inc b/src/net/ip65/net_tuning.inc index 364cbe60..e358e3a8 100644 --- a/src/net/ip65/net_tuning.inc +++ b/src/net/ip65/net_tuning.inc @@ -19,3 +19,12 @@ ; do not shrink it without re-running tests/test_vice_https_macos.py. NET_SH_DRAIN_OUTER = 8 NET_SH_DRAIN_INNER = 250 + +; CERT_BUF_SIZE: capacity of cert_buf (src/der_decode.s, CERT_BUF_BSS). +; +; 1536 under ip65 — cert_buf is pinned at $A000 and time-shared with +; LIB_NISTCURVES_P256_BSS via the #68 SCRATCH_UNION, whose $600 cap is +; a link-time assert (src/net/ip65/exports.s + the cfg). Growing this +; would break the union; the UCI backend carries the 2048 B variant +; (Wikipedia's 1636 B leaf) in CRYPTO_OVERLAY instead. +CERT_BUF_SIZE = 1536 diff --git a/src/net/uci/net_tuning.inc b/src/net/uci/net_tuning.inc index 8ea107a1..66acbbc2 100644 --- a/src/net/uci/net_tuning.inc +++ b/src/net/uci/net_tuning.inc @@ -30,3 +30,15 @@ ; an INNER of 0 into 256 iterations. NET_SH_DRAIN_OUTER = 1 NET_SH_DRAIN_INNER = 16 + +; CERT_BUF_SIZE: capacity of cert_buf (src/der_decode.s, CERT_BUF_BSS). +; +; 2048 under UCI so en.wikipedia.org's 1636 B leaf certificate fits — +; the last blocker for Wikipedia as a real-server HTTPS target. The +; segment rides CRYPTO_OVERLAY under both UCI cfgs (it no longer fits +; CRYPTO_COLD_SHADOW at this size); ip65 keeps 1536, pinned at $A000 +; and unioned with LIB_NISTCURVES_P256_BSS (see cfg/c64-https-ip65.cfg +; SCRATCH_UNION — its $600 cap is a link-time assert and MUST NOT +; grow). The deframer's leaf cap check requires the low byte to be +; zero (asserted in src/tls_deframe.s). +CERT_BUF_SIZE = 2048 diff --git a/src/tls_deframe.s b/src/tls_deframe.s index 0f6399ea..3699c183 100644 --- a/src/tls_deframe.s +++ b/src/tls_deframe.s @@ -68,6 +68,7 @@ ; ip65 has neither the code nor the BSS headroom — see the Makefile). .include "constants.inc" +.include "net_tuning.inc" ; CERT_BUF_SIZE (2048 UCI / 1536 ip65) .ifdef TLS_STREAM_DEFRAME @@ -107,7 +108,7 @@ DF_ERR_TOO_BIG = $02 ; spanning non-Certificate message > carry cap DF_ERR_DISPATCH = $03 ; message handler returned C=1 DF_ERR_TYPE = $04 ; unknown handshake message type DF_ERR_CERT_FMT = $05 ; streamed Certificate malformed / no usable key -DF_ERR_CERT_TOO_BIG = $06 ; leaf certificate exceeds cert_buf (1536 B) +DF_ERR_CERT_TOO_BIG = $06 ; leaf certificate exceeds cert_buf (CERT_BUF_SIZE) ; df_mode values DF_MODE_HDR = 0 ; collecting the 4-byte message header @@ -122,11 +123,16 @@ CS_ENTRY_DATA = 3 ; cert_data bytes (leaf -> cert_buf, rest discar CS_EXT_LEN = 4 ; per-entry extensions length (2 B) CS_EXT_DATA = 5 ; extension bytes (discarded) -; Leaf staging capacity — keep in step with cert_buf in src/der_decode.s -; (CERT_BUF_BSS, 1536 B). The cap check runs BEFORE any copy, so an +; Leaf staging capacity — cert_buf in src/der_decode.s (CERT_BUF_BSS), +; sized per-backend by net_tuning.inc (2048 UCI / 1536 ip65), so the +; two can never drift. The cap check runs BEFORE any copy, so an ; oversize leaf is a clean DF_ERR_CERT_TOO_BIG, never a write past -; cert_buf+1536. -DF_CERT_BUF_SIZE = 1536 +; cert_buf+CERT_BUF_SIZE. +DF_CERT_BUF_SIZE = CERT_BUF_SIZE +; The @el_leaf_fits comparison below tests only the high byte plus a +; low-byte-of-need == 0 check, which is exact ONLY while the cap's own +; low byte is zero. Keep it a whole number of pages or rewrite the check. +.assert (DF_CERT_BUF_SIZE .MOD 256) = 0, error, "DF_CERT_BUF_SIZE must be page-aligned (leaf cap check assumes low byte 0)" .segment "TLS_DEFRAME_CODE" @@ -580,7 +586,7 @@ df_stream_body: bcc @el_leaf_fits bne @el_leaf_too_big lda df_cs_need - beq @el_leaf_fits ; exactly 1536 ($0600) still fits + beq @el_leaf_fits ; exactly DF_CERT_BUF_SIZE still fits @el_leaf_too_big: lda #DF_ERR_CERT_TOO_BIG jmp df_fail diff --git a/tools/test_tls_deframe.py b/tools/test_tls_deframe.py index c3affa5f..ca51e522 100644 --- a/tools/test_tls_deframe.py +++ b/tools/test_tls_deframe.py @@ -74,7 +74,7 @@ "tls_rec_buf", "tls_rec_len", "tls_transcript_init", "tls_transcript_hash", "tls_transcript", "tls_s_hs_secret", - "cert_buf", "cert_buf_len", + "cert_buf", "cert_buf_len", "cert_buf_size", "ecdsa_pubkey_x", "ecdsa_pubkey_y", "ecdsa_curve_id", ] @@ -412,10 +412,12 @@ def clear_pubkey(): # --- 13. leaf larger than cert_buf: explicit error, before any copy --- # The cap check fires when the entry length completes, i.e. BEFORE a # single cert_data byte is copied — cert_buf must remain untouched. - # (A sentinel at cert_buf+1536 cannot test this: that address is - # tls_rec_buf[0] under both cfgs, which the record layer writes - # legitimately. Watching cert_buf itself is collision-free.) - big_cert = cert_msg([(b"\xbb" * 1600, b"")]) # wikipedia-sized leaf + # (A sentinel at cert_buf+cap cannot test this: the byte after the + # cap can be legitimately-written memory. Watching cert_buf itself + # is collision-free.) + # cap comes from labels.txt (`cert_buf_size`): 2048 UCI / 1536 ip65. + cap = labels.address("cert_buf_size") + big_cert = cert_msg([(b"\xbb" * (cap + 64), b"")]) rig.reset() write_bytes(transport, labels["cert_buf"], b"\x5a" * 64) ev = rig.drive(chunks(big_cert, 300)) @@ -424,15 +426,29 @@ def clear_pubkey(): check("oversize leaf: cert_buf untouched (guard before copy)", read_bytes(transport, labels["cert_buf"], 64) == b"\x5a" * 64, "") - # --- 13b. leaf of exactly 1536 B is still accepted by the cap --- + # --- 13b. leaf of exactly cap bytes is still accepted by the cap --- # (no EC key inside, so it must fail LATER with CERT_FMT — proving # the boundary itself is not rejected) - edge_cert = cert_msg([(b"\xcc" * 1536, b"")]) + edge_cert = cert_msg([(b"\xcc" * cap, b"")]) rig.reset() ev = rig.drive(chunks(edge_cert, 300)) - check("1536 B leaf passes the cap (fails later on extraction)", + check(f"{cap} B leaf passes the cap (fails later on extraction)", ev == [("err", ERR_CERT_FMT)], f"events={ev}") + # --- 13c. Wikipedia-sized 1636 B leaf: the gate the 2048 B UCI cap + # exists for. Above the historical 1536 B cap; must pass the size + # check on a 2048 B build (garbage DER, so CERT_FMT later) and be + # the clean TOO_BIG reject on a 1536 B (ip65) build. + wiki_cert = cert_msg([(b"\xdd" * 1636, b"")]) + rig.reset() + ev = rig.drive(chunks(wiki_cert, 300)) + if 1636 <= cap: + check("1636 B (wikipedia) leaf passes the cap", + ev == [("err", ERR_CERT_FMT)], f"events={ev}") + else: + check("1636 B (wikipedia) leaf rejected on a 1536 B build", + ev == [("err", ERR_CERT_TOO_BIG)], f"events={ev}") + # --- 14. non-zero certificate_request_context --- bad_ctx = bytearray(cert_msg([(leaf_der, b"")])) bad_ctx[4] = 0x05 # ctx length byte inside the body diff --git a/tools/test_tls_deframer.py b/tools/test_tls_deframer.py index 4ecb661a..ab48e786 100644 --- a/tools/test_tls_deframer.py +++ b/tools/test_tls_deframer.py @@ -23,8 +23,10 @@ inside the 4-byte message header, (c) the Certificate message (~2.7 KB, multi-cert chain) spans ~6 records; only the first (leaf) cert must be captured, the rest counted+discarded, - (d) a leaf certificate larger than ``cert_buf`` (1536 B) must produce a clean - "certificate too large" error, never a buffer overflow. + (d) a leaf certificate larger than ``cert_buf`` (2048 B UCI / 1536 B ip65, + read from labels.txt ``cert_buf_size``) must produce a clean + "certificate too large" error, never a buffer overflow — while a + Wikipedia-sized ~1636 B leaf must be ACCEPTED on the UCI build. The current code (``tls13.s`` ~555-660) assumes exactly one message per record. Every scenario below that is marked ``[deframer]`` therefore FAILS on the @@ -134,8 +136,13 @@ TLS_HS_CERTIFICATE = 11 TLS_STATE_CERTIFICATE = 4 # >= ENCRYPTED_EXT(3): decrypt with hs read keys -# cert_buf capacity from the sprint plan / src/der_decode.s:588. -CERT_BUF_MAX = 1536 +# cert_buf capacity — resolved in main() from build/labels.txt +# (`cert_buf_size`, an absolute export in src/exports.s: 2048 under UCI, +# 1536 under ip65 — src/net//net_tuning.inc). Never hardcode +# either number here: the Wikipedia growth made the size +# backend-conditional, and the oversize/boundary vectors below scale +# with it. +CERT_BUF_MAX = None # Fixed handshake read key/iv/seq so every record is reproducible. HS_READ_KEY = bytes((i * 7 + 3) & 0xFF for i in range(32)) @@ -155,6 +162,7 @@ "tls_transcript", "tls_transcript_init", "tls_transcript_hash", "ecdsa_pubkey_x", "ecdsa_pubkey_y", "cert_buf", + "cert_buf_size", "sqtab_init", ] @@ -247,6 +255,16 @@ def reset_recv_state(transport, labels) -> None: write_bytes(transport, labels["tls_recv_count"], [0, 0]) write_bytes(transport, labels["tls_read_seq"], b"\x00" * 8) write_bytes(transport, labels["tls_state"], [TLS_STATE_CERTIFICATE]) + # Fresh-connection deframer reset, exactly as tls_connect does after + # ServerHello (src/tls13.s @ `jsr tls_deframe_init`). Each scenario + # is a fresh simulated connection; without this, a scenario that + # aborts MID-STREAM (the oversized-leaf reject) leaks DF_MODE_STREAM + # state into the next scenario's flight — invisible while the + # error vector ran last, real once anything follows it. Conditional + # because pre-deframer / ip65 builds export no such symbol (their + # deframer scenarios xfail by design). + if labels.address("tls_deframe_init") is not None: + jsr(transport, labels["tls_deframe_init"], timeout=30.0) def ring_is_empty(transport, labels) -> bool: @@ -321,7 +339,7 @@ def __init__(self, name, kind, chunks, expect_pubkey, expect_transcript, self.n_msgs = n_msgs # handshake messages in flight -def build_scenarios(cert_der, pubkey_xy): +def build_scenarios(cert_der, pubkey_xy, wiki_leaf=None): ee = encrypted_extensions() cert = certificate_message(cert_der) px, py = pubkey_xy @@ -435,19 +453,47 @@ def build_scenarios(cert_der, pubkey_xy): )) # --- deframer (d): oversized leaf must error, not overflow --- - # A single-entry Certificate whose cert_data exceeds cert_buf (1536 B). + # A single-entry Certificate whose cert_data exceeds cert_buf + # (CERT_BUF_MAX, read from labels.txt — 2048 UCI / 1536 ip65). big_leaf = b"\x30\x82" + (CERT_BUF_MAX + 200).to_bytes(2, "big") + \ bytes((i * 3) & 0xFF for i in range(CERT_BUF_MAX + 160)) big_cert = certificate_message(big_leaf) scenarios.append(Scenario( "deframer/oversized_leaf_errors", "deframer", - chunk_bytes(big_cert, [480, 480, 480, 480]), + chunk_bytes(big_cert, [480, 480, 480, 480, 480]), None, # must NOT extract a pubkey None, # transcript not asserted expect_overflow_safe=True, )) + # --- wiki gate: a ~1636 B leaf (en.wikipedia.org's leaf size) --- + # The whole point of the 2048 B UCI cert_buf: a leaf bigger than the + # historical 1536 B cap but under 2048 must be ACCEPTED and fully + # parsed (real minted P-256 cert, pubkey extraction is the oracle). + # On a 1536 B build (ip65 cap) the same leaf must instead be the + # clean TOO_BIG reject, cert_buf untouched. + if wiki_leaf is not None: + wleaf_der, (wx, wy) = wiki_leaf + wcert = certificate_message(wleaf_der) + if len(wleaf_der) <= CERT_BUF_MAX: + scenarios.append(Scenario( + "deframer/wikipedia_sized_leaf_accepted", + "deframer", + chunk_bytes(wcert, [480, 480, 480]), + (wx, wy), + hashlib.sha256(wcert).digest(), + )) + else: + scenarios.append(Scenario( + "deframer/wikipedia_sized_leaf_errors", + "deframer", + chunk_bytes(wcert, [480, 480, 480]), + None, + None, + expect_overflow_safe=True, + )) + return scenarios @@ -526,7 +572,7 @@ def run_scenario(transport, labels, sc): return (not problems), detail -def run_tests(transport, labels, cert_der, pubkey_xy): +def run_tests(transport, labels, cert_der, pubkey_xy, wiki_leaf=None): print("\n Initializing sqtab (Poly1305 multiply table)...") jsr(transport, labels["sqtab_init"], timeout=60.0) @@ -534,7 +580,7 @@ def run_tests(transport, labels, cert_der, pubkey_xy): write_bytes(transport, labels["net_poll"], [RTS]) print(" net_poll patched to RTS (ring-driven, no live NIC)") - scenarios = build_scenarios(cert_der, pubkey_xy) + scenarios = build_scenarios(cert_der, pubkey_xy, wiki_leaf) plumb_pass = plumb_fail = defr_pass = defr_fail = 0 for sc in scenarios: @@ -576,6 +622,36 @@ def load_cert_fixture(): return der, (nums.x.to_bytes(32, "big"), nums.y.to_bytes(32, "big")) +def build_wiki_leaf(target: int = 1636): + """Mint a real, parseable ~1636 B P-256 leaf (en.wikipedia.org's size). + + Uses the padded-cert generator from tools/https_e2e/chain_certs.py + (standard v3 shape: SPKI before the padding extension, so the C64's + skip-and-seek parser extracts the key the same way it does for the + repo cert). The key is minted fresh per run, so the expected pubkey + comes from the cert itself rather than a fixed vector; everything + else about the scenario stays deterministic. Sizing converges in a + couple of passes (the ECDSA signature length wobbles by ±2 B, which + is irrelevant to the gate — anything in (1536, 2048] exercises it). + """ + from cryptography.x509 import load_der_x509_certificate + + sys.path.insert(0, os.path.join(PROJECT_ROOT, "tools", "https_e2e")) + from chain_certs import build_padded_intermediate # noqa: PLC0415 + + pad = 1200 + der = build_padded_intermediate("C64 Wiki-Sized Leaf", pad) + for _ in range(4): + if len(der) == target: + break + pad += target - len(der) + der = build_padded_intermediate("C64 Wiki-Sized Leaf", pad) + if not (1536 < len(der) <= 2048): + raise RuntimeError(f"wiki leaf sizing failed: {len(der)} B") + nums = load_der_x509_certificate(der).public_key().public_numbers() + return der, (nums.x.to_bytes(32, "big"), nums.y.to_bytes(32, "big")) + + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -615,9 +691,16 @@ def main() -> int: print(f"FATAL: required label(s) not found: {', '.join(missing)}") return 1 + global CERT_BUF_MAX + CERT_BUF_MAX = labels.address("cert_buf_size") + print(f" cert_buf capacity: {CERT_BUF_MAX} B (labels.txt cert_buf_size)") + cert_der, pubkey_xy = load_cert_fixture() print(f" Test cert: {len(cert_der)} B DER, " f"pubkey x={pubkey_xy[0][:6].hex()}...") + wiki_leaf = build_wiki_leaf() + print(f" Wiki-sized leaf: {len(wiki_leaf[0])} B DER " + f"({'fits' if len(wiki_leaf[0]) <= CERT_BUF_MAX else 'over cap'})") config = default_vice_config(prg_path=PRG_PATH, warp=True, ntsc=True, sound=False) @@ -636,7 +719,8 @@ def main() -> int: print("\n=== Deframer scenarios ===") try: - pp, pf, dp, df = run_tests(transport, labels, cert_der, pubkey_xy) + pp, pf, dp, df = run_tests(transport, labels, cert_der, + pubkey_xy, wiki_leaf) finally: mgr.release(inst) diff --git a/tools/uci/rig_https_live.py b/tools/uci/rig_https_live.py index b49bc5f9..dca7a39d 100644 --- a/tools/uci/rig_https_live.py +++ b/tools/uci/rig_https_live.py @@ -16,9 +16,11 @@ browserleaks.com 529 B 45 s (2/2) 952 B 12 recs en.wikipedia.org 529 B ~120 s (2/2) 1636 B 14 recs -en.wikipedia.org's leaf (1636 B) exceeds the 1536 B ``cert_buf``; the rig -warns loudly when it is selected but does not refuse — the run is how the -"certificate too large" error path gets exercised once W2 lands. +en.wikipedia.org's leaf (1636 B) fits the UCI ``cert_buf`` since it grew +to 2048 B (``cert_buf_size`` in labels.txt — the rig reads it from there); +on an old 1536 B build the rig warns loudly when wikipedia is selected but +does not refuse, since that run exercises the W2 "certificate too large" +error path. How the target reaches the C64: this rig programs ``http_host_ptr`` / ``http_host_len`` over DMA and calls ``http_get``, whose prologue copies the @@ -143,7 +145,11 @@ "browserleaks.com": {"tolerance_s": 45, "leaf_b": 952, "records": 12}, "en.wikipedia.org": {"tolerance_s": 120, "leaf_b": 1636, "records": 14}, } -CERT_BUF_BYTES = 1536 # src/der_decode.s cert_buf capacity +# cert_buf capacity fallback, used only when labels.txt predates the +# `cert_buf_size` export (src/exports.s). Current builds carry the real +# value in labels.txt: 2048 UCI / 1536 ip65 (Wikipedia growth) — always +# prefer the label over this constant. +CERT_BUF_FALLBACK_BYTES = 1536 # Comb marker: manifest equate bit for REU bank 2 (Lim-Lee anchor table). _BANKS_EQUATE = "LIB_NISTCURVES_REU_BANKS_USED" @@ -292,7 +298,8 @@ def emit_progress(step: int) -> None: return bytes(code) -def _print_target_context(target: str, port: int) -> None: +def _print_target_context(target: str, port: int, + cert_buf_b: int = CERT_BUF_FALLBACK_BYTES) -> None: info = _KNOWN_TARGETS.get(target) print(f"\nLive target : {target}:{port}") if info is None: @@ -303,13 +310,13 @@ def _print_target_context(target: str, port: int) -> None: f"(2026-08-21 probe; live servers change — re-probe before " f"trusting)") print(f" leaf cert : {info['leaf_b']} B " - f"(cert_buf = {CERT_BUF_BYTES} B)") + f"(cert_buf = {cert_buf_b} B)") print(f" server flight : ~{info['records']} records, all <= 529 B") - if info["leaf_b"] > CERT_BUF_BYTES: + if info["leaf_b"] > cert_buf_b: print(f" *** WARNING: leaf exceeds cert_buf by " - f"{info['leaf_b'] - CERT_BUF_BYTES} B — expect the client to " + f"{info['leaf_b'] - cert_buf_b} B — expect the client to " f"abort at Certificate (this exercises the W2 'certificate " - f"too large' path; it cannot PASS until cert_buf is raised)") + f"too large' path; it cannot PASS on this build)") def main() -> int: @@ -361,7 +368,9 @@ def main() -> int: for base, last, note in arbiter.allocations: print(f" ${base:04X}-${last:04X} {note}") - _print_target_context(HTTPS_TARGET, HTTPS_TARGET_PORT) + _print_target_context(HTTPS_TARGET, HTTPS_TARGET_PORT, + labels.get("cert_buf_size", + CERT_BUF_FALLBACK_BYTES)) host_bytes = HTTPS_TARGET.encode("ascii") routine_bytes = build_live_routine( From 6492c0fe243f1907a5f2d6d1e4fb2e5105da299c Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:49:54 -0400 Subject: [PATCH 25/33] docs(cfg): overlay embed-flag exclusion is now a data-integrity constraint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With CERT_BUF_BSS resident in CRYPTO_OVERLAY, a runtime overlay swap-in would clobber the staged leaf certificate mid-handshake — live data, not just deframer code. Note it in both UCI cfgs (raised by the deframer lane's review). Co-Authored-By: Claude Fable 5 --- cfg/c64-https-uci-onchip.cfg | 3 +++ cfg/c64-https-uci.cfg | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/cfg/c64-https-uci-onchip.cfg b/cfg/c64-https-uci-onchip.cfg index 568d3158..2240917b 100644 --- a/cfg/c64-https-uci-onchip.cfg +++ b/cfg/c64-https-uci-onchip.cfg @@ -249,6 +249,9 @@ SEGMENTS { # already forbidden with the onchip profiles by the Makefile — # and shares the region with CRYPTO_RODATA (routed here by this # cfg variant) + TLS_DEFRAME_CODE: check the .map after changes. + # A runtime overlay swap-in would now clobber the staged leaf + # certificate mid-handshake (live data, not just code) — the + # embed-flag exclusion is a data-integrity constraint here. CERT_BUF_BSS: load = CRYPTO_OVERLAY, type = bss, optional = yes; # Phase C.5: sibling c64-x25519 rodata tables (mul38, sqr_lo/hi, diff --git a/cfg/c64-https-uci.cfg b/cfg/c64-https-uci.cfg index eddf7ca2..3d29398e 100644 --- a/cfg/c64-https-uci.cfg +++ b/cfg/c64-https-uci.cfg @@ -238,6 +238,11 @@ SEGMENTS { # mutually-exclusive overlay-embed flags (EMBED_P256_OVERLAY / # USE_OVERLAY_P384_EMBED / USE_X25519_SIBLING) now have ~2 KB less # to work with — check the .map before re-enabling any of them. + # It also HARDENS the incompatibility from "would evict code" to + # "would corrupt live data": a runtime overlay swap-in would now + # clobber the staged leaf certificate mid-handshake, not just the + # deframer's code. Embed flags and the streaming build stay + # mutually exclusive for data-integrity reasons, not merely fit. CERT_BUF_BSS: load = CRYPTO_OVERLAY, type = bss, optional = yes; # Phase C.5: sibling c64-x25519 rodata tables (mul38, sqr_lo/hi, From ae97db9c930134cc07797138cbcb9f9905b3a86d Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:56:32 -0400 Subject: [PATCH 26/33] =?UTF-8?q?feat(http):=20REU=20body=20sink=20?= =?UTF-8?q?=E2=80=94=20stream=20large=20response=20bodies=20to=20the=20REU?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With http_body_sink=1 the 512 B http_resp_buf becomes a bounce buffer: http_body_append fills it, and each fill is REU-DMA-STASHed (HTTP_SINK_CODE, register programming mirrors boot.s::reu_mul_init) to http_reu_body_base + a 24-bit http_reu_cursor. At completion http_body_finish (idempotent via a latch, also invoked from the poll-timeout fallbacks) blits the final partial bounce, then FETCHes the FIRST min(512, total) body bytes back into http_resp_buf — the host-side verification contract: first-512 prefix + http_body_total + REU contents. Works under both identity and chunked bodies (the sink sits under the shared http_body_append). - Base: bank 16 ($10:0000, 16 MB REU — the stretch-goal config), clear of mul-table banks 0-1, comb's Lim-Lee bank 2, reserved banks 6-7. HTTP_REU_BODY_BASE equate in constants.inc, overridable with `make HTTP_REU_BODY_BASE=`; the runtime copy http_reu_body_base (3 B LE, RODATA = RAM) can also be retargeted by DMA — VICE's 512 KB REU cannot hold bank 16, so the harness pokes bank 3 ($03:0000) at runtime. - New knob HTTPS_BODY_TO_REU=1: boot.s's do_https_get sets http_body_sink=1 before http_recv_body (the only boot.s change). - UCI-ONLY: every ip65 code pool is within ~30 B of full after the W4 termination + chunked work, so ip65 builds get 3-byte inert stubs (sanctioned by the sprint brief; the knob is UCI-only anyway) and http_body_sink is a dead flag there. All five profiles link; the knob+base-override build links too. - http_body_finish re-latches the reu_fetch_mul_row register state (mul_dma_lo target, 512 B length) that the sink clobbers, so a second handshake on a REU-profile build still fetches multiply rows correctly (guard mirrors boot.s's). - tools/test_http.py: sink vectors (chunked 700 B wiki-shape + identity/CL 600 B) verify body_total, the first-512 restore, and the REU contents via a C64-side DMA FETCH stub; the suite now uses default_vice_config (mandatory -reu). On an ip65 build the sink vectors report loudly that they did NOT run (C64_EXPECT_SINK=1 makes absence a failure). 47/47 PASS on ip65, 52/52 on uci. Co-Authored-By: Claude Fable 5 --- Makefile | 12 ++ cfg/c64-https-ip65.cfg | 4 + cfg/c64-https-uci-onchip.cfg | 3 + cfg/c64-https-uci.cfg | 3 + src/boot.s | 7 + src/constants.inc | 14 ++ src/data.s | 10 ++ src/http.s | 239 +++++++++++++++++++++++++++++++++-- tools/test_http.py | 171 ++++++++++++++++++++++++- 9 files changed, 450 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 9e49935e..ba9629af 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,18 @@ endif ifdef HTTPS_PORT CA65FLAGS += -D HTTPS_PORT=$(HTTPS_PORT) endif +# W4 REU body sink: HTTPS_BODY_TO_REU=1 makes boot.s's do_https_get set +# http_body_sink=1, streaming the HTTP response body into the REU at +# HTTP_REU_BODY_BASE (default $10:0000 = bank 16 — needs a 16 MB REU; +# see src/constants.inc). HTTP_REU_BODY_BASE= overrides the +# base (ca65 -D takes no 0x and $ needs shell quoting): e.g. +# `make HTTP_REU_BODY_BASE=196608` targets bank 3 for a 512 KB VICE REU. +ifdef HTTPS_BODY_TO_REU +CA65FLAGS += -D HTTPS_BODY_TO_REU=1 +endif +ifdef HTTP_REU_BODY_BASE +CA65FLAGS += -D HTTP_REU_BODY_BASE=$(HTTP_REU_BODY_BASE) +endif # VIC-II blanking around the CPU-bound crypto primitives (src/vic.s). # ON by default and that is what ships; `make VIC_BLANK=0` degrades # vic_blank/vic_unblank to bare RTS so the badline tax can be measured diff --git a/cfg/c64-https-ip65.cfg b/cfg/c64-https-ip65.cfg index 40f8d95b..e1ee3c55 100644 --- a/cfg/c64-https-ip65.cfg +++ b/cfg/c64-https-ip65.cfg @@ -153,6 +153,10 @@ SEGMENTS { # CRYPTO_AUX_CODE2 (hmac_drbg alone) also rides the NET_CODE tail to # spread crypto code across NET_CODE / CRYPTO_RESIDENT / CRYPTO_OVERLAY. LOADER_OVERFLOW: load = NET_CODE, type = ro, optional = yes; + # W4 REU body sink (src/http.s HTTP_SINK_CODE) — rides the NET_CODE + # tail slack under ip65 (CRYPTO_OVERLAY and CRYPTO_RESIDENT are both + # at capacity once HTTP_AUX_CODE / HTTP_CHUNK_CODE land). + HTTP_SINK_CODE: load = NET_CODE, type = ro, optional = yes; CRYPTO_AUX_CODE2: load = NET_CODE, type = ro, optional = yes; NET_BSS: load = NET_BSS, type = bss, optional = yes; diff --git a/cfg/c64-https-uci-onchip.cfg b/cfg/c64-https-uci-onchip.cfg index 57b91640..26a508b0 100644 --- a/cfg/c64-https-uci-onchip.cfg +++ b/cfg/c64-https-uci-onchip.cfg @@ -189,6 +189,9 @@ SEGMENTS { # c64-https-uci.cfg (CRYPTO_OVERLAY shared with LIB_NISTCURVES_ # MUL_CODE here; both are resident, the slot is 7.5 KB). HTTP_CHUNK_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; + # W4 REU body sink — same CRYPTO_OVERLAY rationale as + # HTTP_CHUNK_CODE above. + HTTP_SINK_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; # NET_CODE hosts the UCI adapter + the LOADER_OVERFLOW tail. # Phase C.2 (UCI only): TLS_CODE and CRYPTO_AUX_CODE (SHA-256, diff --git a/cfg/c64-https-uci.cfg b/cfg/c64-https-uci.cfg index dc95d72d..4b6ab661 100644 --- a/cfg/c64-https-uci.cfg +++ b/cfg/c64-https-uci.cfg @@ -179,6 +179,9 @@ SEGMENTS { # P-384 build target is broken and TLS-level P-384 is stubbed — the # same trade ip65 already makes by keeping TLS_CODE there). HTTP_CHUNK_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; + # W4 REU body sink — same CRYPTO_OVERLAY rationale as + # HTTP_CHUNK_CODE above. + HTTP_SINK_CODE: load = CRYPTO_OVERLAY, type = ro, optional = yes; # NET_CODE hosts the UCI adapter + the LOADER_OVERFLOW tail. # Phase C.2 (UCI only): TLS_CODE and CRYPTO_AUX_CODE (SHA-256, diff --git a/src/boot.s b/src/boot.s index 40e5e736..e733abd3 100644 --- a/src/boot.s +++ b/src/boot.s @@ -126,6 +126,9 @@ .import http_get_plain .import http_build_get .import http_recv_body +.ifdef HTTPS_BODY_TO_REU + .import http_body_sink +.endif ; ---- imports: HTTP I/O state (data.asm) ---- .import http_host_ptr @@ -631,6 +634,10 @@ do_https_get: ; the BODY in http_resp_buf / http_resp_len. Replaces the old ; first-record-only copy loop that showed headers and lost the ; body whenever it arrived as a second TLS record. +.ifdef HTTPS_BODY_TO_REU + lda #1 ; W4: stream the body into the REU at + sta http_body_sink ; http_reu_body_base (see src/http.s) +.endif jsr http_recv_body ; display response body diff --git a/src/constants.inc b/src/constants.inc index 21672667..73fe2ff5 100644 --- a/src/constants.inc +++ b/src/constants.inc @@ -130,6 +130,20 @@ reu_len_lo = $df07 ; transfer length low reu_len_hi = $df08 ; transfer length high reu_addr_ctrl = $df0a ; address control +; --- W4 REU body sink --- +; Default REU destination for a streamed HTTP response body: bank 16 +; ($10:0000) — clear of the mul-table banks 0-1, the comb profile's +; Lim-Lee bank 2, and the reserved P-384 overlay banks 6-7. The +; shipped default therefore needs a 16 MB REU (the stretch-goal +; config). Build-time override: `make HTTP_REU_BODY_BASE=196608` +; (decimal; ca65 -D takes no 0x and $ needs shell quoting) puts it at +; bank 3 for a 512 KB VICE REU. The runtime copy lives in src/http.s +; (`http_reu_body_base`, 3 bytes LE in RODATA — RAM at runtime), so +; tests and rigs can also retarget it by DMA without a rebuild. +.ifndef HTTP_REU_BODY_BASE +HTTP_REU_BODY_BASE = $100000 +.endif + ; --- SID voice 3 setup for noise (entropy collection) --- sid_base = $d400 sid_v3_freq_lo = $d40e diff --git a/src/data.s b/src/data.s index 7f218806..8b2dbbfb 100644 --- a/src/data.s +++ b/src/data.s @@ -374,6 +374,16 @@ http_content_length: .res 3 ; 24-bit LE; meaningful iff http_cl_vali http_cl_valid: .res 1 ; 1 = Content-Length header seen http_body_total: .res 3 ; 24-bit LE count of body bytes consumed +; W4 REU body sink (see src/http.s HTTP_SINK_CODE). http_body_sink is +; a MODE flag owned by the caller (boot.s under HTTPS_BODY_TO_REU=1, or +; a test/rig via DMA): 0 = classic 512 B buffer capture (default), +; 1 = stream the body to the REU at http_reu_body_base, bouncing +; through http_resp_buf. It is deliberately NOT reset by the parser. +.export http_body_sink +.export http_reu_cursor +http_body_sink: .res 1 ; 0 = buffer (default), 1 = REU sink +http_reu_cursor: .res 3 ; 24-bit LE REU write offset + ; ----------------------------------------------------------------------------- ; Application data pointers (for tls_send) ; ----------------------------------------------------------------------------- diff --git a/src/http.s b/src/http.s index ab380af8..a80b7603 100644 --- a/src/http.s +++ b/src/http.s @@ -31,6 +31,14 @@ ; W4 chunked transfer-decoding (see http_chunk_dispatch): .export http_te_chunked .export http_chunk_rem + ; W4 REU body sink (see HTTP_SINK_CODE; UCI-only — ip65 gets + ; inert stubs, the backend cannot fit the sink and the knob is + ; UCI-only anyway): + .export http_body_finish +.ifdef BACKEND_UCI + .export http_reu_body_base + .export http_sink_blit +.endif ; ---- imports: data.asm BSS (HTTP I/O + parser state) ---- .import http_host_ptr @@ -50,6 +58,11 @@ .import http_content_length .import http_cl_valid .import http_body_total + .import http_body_sink + .import http_reu_cursor +.ifndef USE_X25519_SIBLING + .import mul_dma_lo ; REU-latch restore in http_body_finish +.endif ; ---- imports: data.asm BSS (TLS app data) ---- ; (tcp_recv_head/tail imports dropped in the issue #72 redesign — @@ -240,6 +253,11 @@ http_recv_body: ; Timeout — accept whatever we have @recv_complete: + ; Sink finalize is idempotent (http_sink_flushed latch): on the + ; parse-complete path it already ran inside http_recv_response; + ; this call covers the poll-timeout fallback so a sink body cut + ; short still gets its final blit + first-512 restore. + jsr http_body_finish clc rts @@ -625,7 +643,8 @@ http_recv_response: ; open (observed live: browserleaks.com), http_get never ; returned. http_body_total (24-bit, HTTP_AUX_CODE routines) ; now counts every consumed byte; bytes past the 512 B cap are - ; consumed and discarded instead of ending the parse. + ; consumed and discarded (or streamed to the REU under + ; http_body_sink=1) instead of ending the parse. jsr http_body_done_check bcc @body_done ; C=0 -> body complete jsr http_in_byte @@ -634,7 +653,7 @@ http_recv_response: jmp @state_body ; keep reading (re-checks termination) @body_done: - jmp @done + jmp sink_finish_done ; sink finalize (no-op in buffer mode), C=0 @not_done: sec @@ -770,6 +789,10 @@ http_body_begin: sta http_chunk_rem sta http_chunk_rem+1 sta http_chunk_skip + sta http_reu_cursor + sta http_reu_cursor+1 + sta http_reu_cursor+2 + sta http_sink_flushed lda http_te_chunked beq @identity lda #3 ; chunked: start at the chunk-size line @@ -831,7 +854,8 @@ te_pattern: ; Works whether or not the stored copy was truncated at 512 B — ; that is the W4 fix. ; Content-Length absent: legacy behaviour — the 512 B buffer cap ends -; the read (chunked/streaming fallback preserved). +; the read (chunked/streaming fallback preserved). In sink mode +; there is no cap: the caller's poll-timeout is the bound. ; Clobbers: A ; ============================================================================= http_body_done_check: @@ -849,6 +873,10 @@ http_body_done_check: clc rts @no_cl: +.ifdef BACKEND_UCI + lda http_body_sink + bne @more ; sink mode: no 512 B cap +.endif lda http_resp_len+1 cmp #$02 bcs @full ; high byte >= 2 means >= 512 @@ -863,8 +891,11 @@ http_body_done_check: ; http_body_append - consume one body byte (in A). ; ; Always increments the 24-bit http_body_total (bytes CONSUMED). -; Stores into http_resp_buf while resp_len < 512; bytes past the cap -; are counted and discarded. +; Buffer mode (http_body_sink=0): stores into http_resp_buf while +; resp_len < 512; bytes past the cap are counted and discarded. +; Sink mode (http_body_sink=1): http_resp_buf is a 512 B bounce +; buffer — on fill it is REU-DMA-blitted to http_reu_body_base + +; http_reu_cursor and reused (see HTTP_SINK_CODE). ; Clobbers: A, Y, zp_ptr. Preserves X. ; ============================================================================= http_body_append: @@ -892,8 +923,20 @@ http_body_append: ldy #0 sta (zp_ptr),y inc http_resp_len - bne @rts + bne @check_sink inc http_resp_len+1 +@check_sink: +.ifdef BACKEND_UCI + lda http_body_sink + beq @rts + lda http_resp_len+1 ; sink mode: bounce full at 512? + cmp #$02 + bcc @rts + jsr http_sink_blit ; blit the bounce, advance the cursor + lda #0 ; reuse the bounce buffer + sta http_resp_len + sta http_resp_len+1 +.endif @rts: rts @discard: @@ -1034,8 +1077,7 @@ http_chunk_dispatch: sta http_line_idx jmp @st_trailer @tr_done: - clc - rts + jmp sink_finish_done ; sink finalize (no-op in buffer mode), C=0 @not_done2: ; far exit for the late states sec @@ -1071,6 +1113,172 @@ chunk_hex_digit: sec rts +; ============================================================================= +; W4 REU body sink (HTTP_SINK_CODE segment: NET_CODE tail under ip65, +; CRYPTO_OVERLAY under UCI). +; +; With http_body_sink = 1, http_resp_buf acts as a 512 B bounce buffer: +; http_body_append fills it, and each time it fills http_sink_blit +; REU-DMA-STASHes the whole bounce to http_reu_body_base + +; http_reu_cursor and resets it. At completion http_body_finish blits +; the final partial bounce, then re-FETCHes the FIRST min(512, total) +; body bytes from the REU back into http_resp_buf — the host-side +; verification contract: after a sunk body, http_resp_buf holds the +; first 512 body bytes, http_resp_len = min(512, total), and +; http_body_total the full de-chunked size. +; +; The default base is bank 16 ($10:0000 — 16 MB REU, the stretch-goal +; config); see HTTP_REU_BODY_BASE in constants.inc for the override +; story. Register programming mirrors src/boot.s::reu_mul_init. +; +; UCI-ONLY: the sink is a turbo/UCI stretch feature (the build knob is +; UCI-only) and ip65's memory map cannot fit it — every ip65 code pool +; is within ~30 B of full after the W4 termination + chunked work. The +; ip65 arm below provides 3-byte inert stubs so the shared jmp/jsr +; sites link; http_body_sink is then a dead flag on ip65 (nothing +; tests it — the .ifdef BACKEND_UCI guards in http_body_append / +; http_body_done_check compile the sink branches out). +; ============================================================================= + .segment "HTTP_SINK_CODE" +.ifndef BACKEND_UCI +sink_finish_done: ; ip65: no sink — complete the parse +http_body_finish: ; and finalize nothing + clc + rts +.else + +; ----------------------------------------------------------------------------- +; sink_reu_setup - common $DF00 programming for blit + fetch-back: +; C64 addr = http_resp_buf, len = http_resp_len, both addrs increment. +; Clobbers: A. +; ----------------------------------------------------------------------------- +sink_reu_setup: + lda #http_resp_buf + sta reu_c64_hi + lda http_resp_len + sta reu_len_lo + lda http_resp_len+1 + sta reu_len_hi + lda #0 + sta reu_addr_ctrl + rts + +; ----------------------------------------------------------------------------- +; http_sink_blit - STASH http_resp_len bytes of http_resp_buf to +; http_reu_body_base + http_reu_cursor; cursor += resp_len. +; No-op when resp_len = 0. Clobbers: A. +; ----------------------------------------------------------------------------- +http_sink_blit: + lda http_resp_len + ora http_resp_len+1 + beq @rts + jsr sink_reu_setup + clc ; REU addr = 24-bit base + cursor + lda http_reu_body_base + adc http_reu_cursor + sta reu_reu_lo + lda http_reu_body_base+1 + adc http_reu_cursor+1 + sta reu_reu_hi + lda http_reu_body_base+2 + adc http_reu_cursor+2 + sta reu_reu_bank + lda #%10110000 ; execute + autoload + STASH (C64->REU) + sta reu_command + clc ; cursor += resp_len + lda http_reu_cursor + adc http_resp_len + sta http_reu_cursor + lda http_reu_cursor+1 + adc http_resp_len+1 + sta http_reu_cursor+1 + bcc @rts + inc http_reu_cursor+2 +@rts: + rts + +; ----------------------------------------------------------------------------- +; sink_finish_done - shared completion tail for both body paths +; (identity @body_done, chunked @tr_done): finalize the sink and +; return C=0 (parse complete). +; ----------------------------------------------------------------------------- +sink_finish_done: + jsr http_body_finish + clc + rts + +; ----------------------------------------------------------------------------- +; http_body_finish - finalize the REU sink at body completion (or the +; poll-timeout fallback). Idempotent via the http_sink_flushed +; latch; a no-op in buffer mode. Steps: final partial blit; +; http_resp_len = min(512, http_body_total); FETCH the first +; resp_len body bytes from the REU back into http_resp_buf; restore +; the reu_fetch_mul_row register latch this sink clobbered. +; Clobbers: A. +; ----------------------------------------------------------------------------- +http_body_finish: + lda http_body_sink + beq @rts + lda http_sink_flushed + bne @rts + inc http_sink_flushed + jsr http_sink_blit ; final partial bounce (may be 0 bytes) + ; http_resp_len = min(512, http_body_total) + lda http_body_total+2 + bne @cap + lda http_body_total+1 + cmp #$02 + bcs @cap + lda http_body_total + sta http_resp_len + lda http_body_total+1 + sta http_resp_len+1 + jmp @fetch +@cap: + lda #$00 + sta http_resp_len + lda #$02 + sta http_resp_len+1 +@fetch: + lda http_resp_len + ora http_resp_len+1 + beq @restore + jsr sink_reu_setup + lda http_reu_body_base + sta reu_reu_lo + lda http_reu_body_base+1 + sta reu_reu_hi + lda http_reu_body_base+2 + sta reu_reu_bank + lda #%10110001 ; execute + autoload + FETCH (REU->C64) + sta reu_command +@restore: +.ifndef USE_NISTCURVES_ONCHIP +.ifndef USE_X25519_SIBLING + ; Re-latch the reu_fetch_mul_row register state that + ; reu_mul_init pre-set at boot (this sink clobbered it). A + ; REU-profile build doing a SECOND handshake in the same + ; session would otherwise DMA multiply rows to the wrong C64 + ; address with no diagnostic. Onchip/comb profiles never use + ; the latch (guard mirrors boot.s's reu_fetch_mul_row guard). + lda #mul_dma_lo + sta reu_c64_hi + lda #0 + sta reu_reu_lo + sta reu_len_lo + sta reu_addr_ctrl + lda #2 + sta reu_len_hi ; 512 B row fetches +.endif +.endif +@rts: + rts +.endif ; BACKEND_UCI (REU sink) + ; ============================================================================= ; http_get_plain - perform a plain HTTP (no TLS) GET request ; Input: http_host_ptr/http_host_len = hostname @@ -1132,6 +1340,8 @@ http_get_plain: bne @plain_poll ; timeout: accept whatever we got so far @plain_done: + jsr http_body_finish ; sink finalize (idempotent; no-op unless + ; http_body_sink=1 — see @recv_complete) ; --- 8. Close TCP --- jsr net_tcp_close @@ -1164,6 +1374,17 @@ http_ua_hdr: http_crlf: .byte $0d, $0a +.ifdef BACKEND_UCI +; W4 REU body sink: runtime copy of the REU destination base (24-bit +; LE). Initialised from the HTTP_REU_BODY_BASE equate (constants.inc; +; default bank 16 = $10:0000, -D-overridable at build). Lives in +; RODATA — which is RAM at runtime — so tests and rigs can retarget it +; by DMA without a rebuild (e.g. VICE's 512 KB REU cannot hold bank 16; +; the harness pokes bank 3 here). +http_reu_body_base: + .byte HTTP_REU_BODY_BASE, ^HTTP_REU_BODY_BASE +.endif + ; ============================================================================= ; Module-local scratch (build_get temporaries only; parser state is in @@ -1182,3 +1403,5 @@ http_in_len: .res 2 ; span bytes remaining (16-bit) http_te_chunked: .res 1 ; 1 = "Transfer-Encoding: chunked" seen http_chunk_rem: .res 2 ; 16-bit payload bytes left in this chunk http_chunk_skip: .res 1 ; 1 = skipping rest of the chunk-size line +; REU sink finalize-once latch (see http_body_finish) +http_sink_flushed: .res 1 ; 1 = finish already ran for this body diff --git a/tools/test_http.py b/tools/test_http.py index 195974e4..cd168f58 100755 --- a/tools/test_http.py +++ b/tools/test_http.py @@ -17,6 +17,7 @@ Labels, ViceConfig, ViceInstanceManager, read_bytes, write_bytes, jsr, wait_for_text, ) +from _vice_helpers import default_vice_config PROJECT_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") PRG_PATH = os.path.join(PROJECT_ROOT, "build", "c64-https.prg") @@ -669,6 +670,152 @@ def test_chunked_small_body(transport, labels): return passed, failed +# --- W4 REU body sink helpers --------------------------------------------- +# VICE runs with a 512 KB REU (-reusize 512), which cannot hold the +# shipped default base of bank 16 ($10:0000, 16 MB REU). The sink reads +# its base from the runtime variable http_reu_body_base (3 bytes LE in +# RODATA — RAM at runtime), so the test retargets it to bank 3 +# ($03:0000) by DMA, no rebuild needed. Hardware rigs with a 16 MB REU +# use the shipped default (or `make HTTP_REU_BODY_BASE=`). +SINK_TEST_BASE = (0x00, 0x00, 0x03) # $03:0000 — inside a 512 KB REU + +REU_STATUS = 0xDF00 + + +def _reu_fetch_to_ram(transport, labels, length, c64_addr=0xC800, + reu_base=SINK_TEST_BASE, stub_addr=0xC300): + """Drive a C64-side REU FETCH so the host can read REU contents back. + + Writes a small 6502 stub that programs the $DF00 REC registers for a + REU->C64 transfer of `length` bytes from `reu_base` to `c64_addr`, + executes it via jsr(), and returns the fetched bytes. + """ + lo, hi = length & 0xFF, (length >> 8) & 0xFF + stub = bytes([ + 0xA9, c64_addr & 0xFF, 0x8D, 0x02, 0xDF, # c64 addr lo + 0xA9, (c64_addr >> 8) & 0xFF, 0x8D, 0x03, 0xDF, # c64 addr hi + 0xA9, reu_base[0], 0x8D, 0x04, 0xDF, # reu addr lo + 0xA9, reu_base[1], 0x8D, 0x05, 0xDF, # reu addr hi + 0xA9, reu_base[2], 0x8D, 0x06, 0xDF, # reu bank + 0xA9, lo, 0x8D, 0x07, 0xDF, # len lo + 0xA9, hi, 0x8D, 0x08, 0xDF, # len hi + 0xA9, 0x00, 0x8D, 0x0A, 0xDF, # addr ctrl + 0xA9, 0xB1, 0x8D, 0x01, 0xDF, # execute FETCH + 0x60, # rts + ]) + write_bytes(transport, stub_addr, stub) + jsr(transport, stub_addr) + return read_bytes(transport, c64_addr, length) + + +def test_reu_body_sink(transport, labels): + """W4 REU body sink: >512 B chunked body streamed to the REU through + the 512 B bounce buffer; verifies http_body_total, the first-512 + http_resp_buf contract, and the REU contents read back by DMA.""" + passed = 0 + failed = 0 + + sink_blit = labels.address("http_sink_blit") + base_var = labels.address("http_reu_body_base") + sink_flag = labels.address("http_body_sink") + if sink_blit is None or base_var is None: + # The REU sink is BACKEND=uci-only (ip65 cannot fit it). Do not + # let this read as coverage: say loudly what was not run. + print(" !! SINK VECTORS NOT RUN: this PRG has no REU sink " + "(BACKEND=uci-only).") + print(" !! To cover them: make clean && make BACKEND=uci && " + "C64_SKIP_BUILD=1 python3 tools/test_http.py") + if os.environ.get("C64_EXPECT_SINK"): + print(" FAIL: C64_EXPECT_SINK=1 but the sink is absent") + return 0, 1 + return 0, 0 + + # Retarget the sink base into the 512 KB VICE REU (bank 3) and + # enable sink mode. + write_bytes(transport, base_var, list(SINK_TEST_BASE)) + write_bytes(transport, sink_flag, [1]) + + # --- vector 1: chunked 700 B (the Wikipedia shape) --- + part1 = bytes((i * 13 + 5) & 0xFF for i in range(400)) + part2 = bytes((i * 17 + 9) & 0xFF for i in range(300)) + body = part1 + part2 + stream = ( + b"HTTP/1.1 200 OK\r\n" + b"Transfer-Encoding: chunked\r\n" + b"\r\n" + + _chunk(part1) + _chunk(part2) + b"0\r\n\r\n" + ) + _load_span(transport, labels, stream) + _reset_parser(transport, labels) + + complete = _run_parser_loop(transport, labels, timeout=90.0) + if not complete: + print(" FAIL: sink parse (chunked) did not complete") + write_bytes(transport, sink_flag, [0]) + return passed, failed + 1 + print(" PASS: chunked sink parse complete") + passed += 1 + + total = _read_u24(transport, labels, "http_body_total") + resp_len = _read_u16(transport, labels, "http_resp_len") + if total == 700 and resp_len == 512: + print(f" PASS: body_total = {total}, resp_len = {resp_len}") + passed += 1 + else: + print(f" FAIL: body_total = {total} (exp 700), resp_len = " + f"{resp_len} (exp 512)") + failed += 1 + + stored = read_bytes(transport, labels.address("http_resp_buf"), 512) + if stored == body[:512]: + print(" PASS: http_resp_buf restored to the FIRST 512 body bytes") + passed += 1 + else: + print(" FAIL: http_resp_buf does not hold the first 512 body bytes") + failed += 1 + + reu = _reu_fetch_to_ram(transport, labels, 700) + if bytes(reu) == body: + print(" PASS: REU holds all 700 de-chunked body bytes") + passed += 1 + else: + diff = next((i for i in range(700) if reu[i] != body[i]), None) + print(f" FAIL: REU contents mismatch (first diff at {diff})") + failed += 1 + + # --- vector 2: identity 600 B with Content-Length --- + body2 = bytes((i * 31 + 7) & 0xFF for i in range(600)) + stream2 = (b"HTTP/1.1 200 OK\r\nContent-Length: 600\r\n\r\n" + body2) + _load_span(transport, labels, stream2) + _reset_parser(transport, labels) + + complete = _run_parser_loop(transport, labels, timeout=90.0) + if not complete: + print(" FAIL: sink parse (identity/CL) did not complete") + write_bytes(transport, sink_flag, [0]) + return passed, failed + 1 + + total = _read_u24(transport, labels, "http_body_total") + resp_len = _read_u16(transport, labels, "http_resp_len") + stored = read_bytes(transport, labels.address("http_resp_buf"), 512) + reu = _reu_fetch_to_ram(transport, labels, 600) + if (total == 600 and resp_len == 512 and stored == body2[:512] + and bytes(reu) == body2): + print(" PASS: identity/CL sink: body_total = 600, first-512 " + "restored, REU contents match") + passed += 1 + else: + print(f" FAIL: identity/CL sink: total={total}, resp_len={resp_len}, " + f"prefix_ok={stored == body2[:512]}, reu_ok={bytes(reu) == body2}") + failed += 1 + + # Cleanup: buffer mode, shipped default base (bank 16), ring input. + write_bytes(transport, sink_flag, [0]) + write_bytes(transport, base_var, [0x00, 0x00, 0x10]) + write_bytes(transport, labels.address("http_in_mode"), [0]) + return passed, failed + + def run_tests(transport, labels, verbose=False): total_passed = 0 total_failed = 0 @@ -688,12 +835,17 @@ def run_tests(transport, labels, verbose=False): print(f" FATAL: VICE connection lost: {e}") return total_passed, total_failed + 1 - # jsr() smoke test + # jsr() smoke test. net_save_zp is ip65-only; http_body_finish + # exists under both backends and is a no-op while http_body_sink=0. print("\n--- jsr() Smoke Test ---") try: - save_zp = labels.address("net_save_zp") - jsr(transport, save_zp) - print(" PASS: jsr(net_save_zp) returned OK") + target = labels.address("net_save_zp") + name = "net_save_zp" + if target is None: + target = labels.address("http_body_finish") + name = "http_body_finish" + jsr(transport, target) + print(f" PASS: jsr({name}) returned OK") total_passed += 1 except Exception as e: print(f" FAIL: jsr() crashed VICE: {e}") @@ -745,6 +897,11 @@ def run_tests(transport, labels, verbose=False): total_passed += p total_failed += f + print("\n--- W4: REU body sink (BACKEND=uci builds only) ---") + p, f = test_reu_body_sink(transport, labels) + total_passed += p + total_failed += f + return total_passed, total_failed @@ -784,7 +941,11 @@ def main(): print(f" FATAL: required label '{name}' not found") sys.exit(1) - config = ViceConfig(prg_path=PRG_PATH, warp=True, ntsc=True, sound=False) + # default_vice_config pre-applies the mandatory -reu -reusize 512 + # (see "VICE harness gotcha" in CLAUDE.md) — the REU body-sink + # vectors DMA through the emulated REU. + config = default_vice_config(prg_path=PRG_PATH, warp=True, ntsc=True, + sound=False) print("\n=== Starting VICE ===") with ViceInstanceManager(config=config) as mgr: From b5d31c846870f44f2d060a891a069456a1e640c8 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:57:54 -0400 Subject: [PATCH 27/33] fix(uci): rig_https_wiki arms http_body_sink by DMA before http_get MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lane E's landed contract sets http_body_sink=1 inside do_https_get (the menu 'G' path), but the rig's trampoline JSRs http_get directly and never crosses it — without the poke the body would not be sunk. Poking the exported flag by DMA before http_get is the supported alternative per lane E. Co-Authored-By: Claude Fable 5 --- tools/uci/rig_https_wiki.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/uci/rig_https_wiki.py b/tools/uci/rig_https_wiki.py index 753d95b6..828582da 100644 --- a/tools/uci/rig_https_wiki.py +++ b/tools/uci/rig_https_wiki.py @@ -637,6 +637,14 @@ def main() -> int: (path_bytes + b"\x00").ljust(128, b"\x00")) transport.write_memory(marker_base, bytes(16)) + # Arm the REU body sink explicitly. The HTTPS_BODY_TO_REU build + # sets http_body_sink=1 in do_https_get (the menu 'G' path), but + # this trampoline JSRs http_get directly and never crosses it — + # without this poke the body would not be sunk (lane E contract: + # poking the exported flag by DMA before http_get is supported). + transport.write_memory(labels["http_body_sink"], b"\x01") + print("Armed http_body_sink=1 (trampoline bypasses do_https_get)") + sys_line = f"sys{routine_addr}\r" print(f"Triggering: {sys_line.strip()}") send_text(transport, sys_line) From 2087fdf08c75d2f3f26b2c5f92842592f1e6f0b6 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 21:19:58 -0400 Subject: [PATCH 28/33] fix(uci): wiki rig sink-arm poke needs a MemoryPolicy override (CRYPTO_COLD_SHADOW) Co-Authored-By: Claude Fable 5 --- tools/uci/rig_https_wiki.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/uci/rig_https_wiki.py b/tools/uci/rig_https_wiki.py index 828582da..76bf3b22 100644 --- a/tools/uci/rig_https_wiki.py +++ b/tools/uci/rig_https_wiki.py @@ -642,7 +642,12 @@ def main() -> int: # this trampoline JSRs http_get directly and never crosses it — # without this poke the body would not be sunk (lane E contract: # poking the exported flag by DMA before http_get is supported). - transport.write_memory(labels["http_body_sink"], b"\x01") + # http_body_sink lives in CRYPTO_COLD_SHADOW ($A000-$BFFF), which + # the MemoryPolicy reserves — this single-byte poke is the lane E + # supported contract, so bypass with a named override rather than + # widening the policy. + transport.write_memory(labels["http_body_sink"], b"\x01", + override="arm http_body_sink (lane E DMA contract)") print("Armed http_body_sink=1 (trampoline bypasses do_https_get)") sys_line = f"sys{routine_addr}\r" From a4f4c46f9693c51a2f1dc5f9e29e5d2cacf7e7f7 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 21:36:25 -0400 Subject: [PATCH 29/33] =?UTF-8?q?fix(uci):=20/Temp=20GC=20=E2=80=94=20the?= =?UTF-8?q?=20firmware=20attachment=20counter=20is=20hex?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit temp0009 is followed by temp000A (observed live on the U64E); the \d+ pattern silently skipped lettered names forever. Match and sort hex. Co-Authored-By: Claude Fable 5 --- tools/uci/_temp_gc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/uci/_temp_gc.py b/tools/uci/_temp_gc.py index e72a93a8..c85a2d0f 100644 --- a/tools/uci/_temp_gc.py +++ b/tools/uci/_temp_gc.py @@ -30,7 +30,9 @@ import sys from ftplib import FTP, error_perm -_MANAGED = re.compile(r"^temp\d+$") +# The firmware's attachment counter is HEX: temp0009 is followed by +# temp000A (observed live). A \d+ pattern silently skips lettered names. +_MANAGED = re.compile(r"^temp[0-9a-fA-F]+$") def gc_temp(host: str, keep: int | None = None, timeout: float = 10.0) -> int: @@ -53,7 +55,7 @@ def gc_temp(host: str, keep: int | None = None, timeout: float = 10.0) -> int: # files; the numeric sort below makes oldest-first explicit # (and survives a counter that skips). names = [n for n in ftp.nlst() if _MANAGED.match(n)] - names.sort(key=lambda n: int(n[4:])) + names.sort(key=lambda n: int(n[4:], 16)) victims = names[: max(0, len(names) - keep)] if keep else names for name in victims: try: From 54153049db8b92fba80b28021a78756cf87d5de8 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Fri, 21 Aug 2026 23:02:18 -0400 Subject: [PATCH 30/33] fix(uci): drain the flight before the crypto stall; close orphaned sockets on rig abort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings from the U64E wikipedia attempts (2026-08-21): 1. UCI firmware ACK autonomy ends at its ~4.3 KB socket buffer — bytes beyond are silently DROPPED, not flow-controlled. wikipedia's 4,659 B flight is the first to exceed it: with 16 drain polls the ~8 s X25519 stall let the buffer overflow, leaving a permanent hole at record 9 (tls_read_seq=8, mid-header — measured twice). github's 3.2 KB flight squeaked under. Raise the post-SH drain budget to 8x16=128 polls (~5.2 s), which moves ~4 KB firmware->ring before the stall. Client silence grows ~30 s vs 45/90/120 s tolerances. 2. Abandoning an ACTIVE firmware socket (rig timeout with the C64 parked in http_get) is the leading suspect for the recurring DHCP-0.0.0.0 poisoning that survives machine:reboot. The wiki rig's timeout path now saves the socket id, resets the C64 (PRG stays in RAM), and SYS-trampolines net_tcp_close so the firmware sees a real SOCKET_CLOSE. Co-Authored-By: Claude Fable 5 --- src/net/uci/net_tuning.inc | 22 ++++++++++++++++------ tools/uci/rig_https_wiki.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/src/net/uci/net_tuning.inc b/src/net/uci/net_tuning.inc index 66acbbc2..8535bb89 100644 --- a/src/net/uci/net_tuning.inc +++ b/src/net/uci/net_tuning.inc @@ -23,12 +23,22 @@ ; cost ~80 s and regressed the shipped handshake from 51.0 s to 125.4 s ; (issue #73). ; -; 16 polls (~0.6 s at 48 MHz) is a deliberate small hedge rather than 0: -; it still pulls anything already queued into the ring before the long -; stalls, at negligible cost, without relying on firmware autonomy being -; absolute. Keep it non-zero — the drain loop's `dex`/`bne` shape turns -; an INNER of 0 into 256 iterations. -NET_SH_DRAIN_OUTER = 1 +; REVISED 2026-08-21 (the wikipedia stall): firmware ACK autonomy has a +; LIMIT — its socket buffer holds ~4.3 KB, and bytes beyond that are +; silently dropped, not flow-controlled. en.wikipedia.org's 4,659 B +; flight is the first target to exceed it: with only 16 drain polls the +; C64 entered the ~8 s X25519 stall pulling nothing, the firmware +; buffer overflowed, and the stream had a permanent hole at record 9 +; (tls_read_seq=8, mid-header, ~4.3 KB consumed — measured, U64E +; 48 MHz). github's 3.2 KB flight squeaked under, which is why every +; earlier live run passed. The drain therefore DOES have something to +; buy on UCI for large flights: moving ~4 KB firmware->ring before the +; crypto stall frees the firmware buffer for the flight tail. 128 +; polls ≈ 5.2 s at the ~40 ms clock-invariant poll cost — client +; silence grows to ~30 s against 45 s (browserleaks) / 90 s (github) / +; 120 s (wikipedia) tolerances. Keep INNER non-zero — the drain loop's +; `dex`/`bne` shape turns an INNER of 0 into 256 iterations. +NET_SH_DRAIN_OUTER = 8 NET_SH_DRAIN_INNER = 16 ; CERT_BUF_SIZE: capacity of cert_buf (src/der_decode.s, CERT_BUF_BSS). diff --git a/tools/uci/rig_https_wiki.py b/tools/uci/rig_https_wiki.py index 76bf3b22..1ebeb421 100644 --- a/tools/uci/rig_https_wiki.py +++ b/tools/uci/rig_https_wiki.py @@ -476,6 +476,40 @@ def ensure_reu_16mb(client: Ultimate64Client) -> None: # Main hardware run # # --------------------------------------------------------------------------- # +def _close_orphan_socket(transport, client, labels) -> None: + """Best-effort SOCKET_CLOSE for a connection abandoned mid-run. + + Abandoning an ACTIVE firmware TCP socket (rig timeout while the C64 + is parked inside http_get's recv loop) is the leading suspect for + the U64E's recurring DHCP-0.0.0.0 poisoning (2026-08-21; survives + machine:reboot, cleared only by cold power). The stuck C64 cannot be + jsr'd into, so: save the live socket id, reset the C64 (the PRG + stays in RAM), let BASIC come up, restore the id, and SYS a 4-byte + trampoline in the cassette buffer straight at net_tcp_close so the + firmware sees a proper SOCKET_CLOSE. Never raises. + """ + try: + sock = transport.read_memory(labels["uci_socket_id"], 1)[0] + if sock == 0: + return + close_addr = labels["net_tcp_close"] + print(f" orphan-socket hygiene: closing firmware socket " + f"{sock} via net_tcp_close (${close_addr:04X})") + client.reset() + time.sleep(6.0) # BASIC READY (PRG stays in RAM) + transport.write_memory(labels["uci_socket_id"], bytes([sock]), + override="restore socket id for orphan close") + tramp = bytes([0x20, close_addr & 0xFF, close_addr >> 8, 0x60]) + transport.write_memory(0x0334, tramp, + override="orphan-close trampoline (cassette buffer)") + send_text(transport, "sys820\r") + time.sleep(2.0) + print(" orphan-socket hygiene: SOCKET_CLOSE issued") + except Exception as exc: # noqa: BLE001 — best-effort + print(f" orphan-socket hygiene failed (non-fatal): {exc}", + file=sys.stderr) + + def main() -> int: if not PRG_PATH.is_file() or not LABELS_PATH.is_file(): print(f"ERROR: build artifacts missing under {PRG_PATH.parent}", @@ -734,6 +768,7 @@ def main() -> int: print(f"\nFAIL: sentinel not set within {WIKI_TIMEOUT:.0f}s " f"(progress=0x{last_progress:02X}, " f"http_body_total={body_total:,})", file=sys.stderr) + _close_orphan_socket(transport, client, labels) outcome = "TIMEOUT" return 1 if carry != 0: From 8bff7d31616d9e7a353ab9d1aa768732656e635c Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Sat, 22 Aug 2026 06:57:49 -0400 Subject: [PATCH 31/33] =?UTF-8?q?feat(debug):=20HTTPS=5FSNI=20override=20?= =?UTF-8?q?=E2=80=94=20present=20a=20different=20SNI=20than=20the=20connec?= =?UTF-8?q?t=20host?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets the C64 dial a local TCP relay while the ClientHello names the real origin (wikipedia rejects absent/mismatched SNI — verified). Built for the wikipedia-stall forensics: the instrumented relay proved the full 4,658 B flight is delivered into the firmware connection with zero backpressure and the client stalls anyway, killing the delivery/ window theories. Off unless HTTPS_SNI= is set; ip65 unaffected. Co-Authored-By: Claude Fable 5 --- Makefile | 11 ++++++++++- src/boot.s | 11 +++++++++++ src/http.s | 20 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2c91a3cb..592e0859 100644 --- a/Makefile +++ b/Makefile @@ -99,6 +99,15 @@ HTTPS_HOST ?= www.foo.bar # Companion request-path override (default "/"), same mechanism/caveats. # e.g. make HTTPS_PATH='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/w/index.php?title=Commodore_64&action=raw' HTTPS_PATH ?= / +# Debug knob: present a DIFFERENT SNI than the connect host (default: +# SNI = host, unchanged). Lets the C64 dial a local TCP relay while the +# ClientHello names the real origin — e.g. tcpdump forensics via +# make HTTPS_HOST=10.43.23.99 HTTPS_PORT=8443 HTTPS_SNI=en.wikipedia.org +# UCI-only, off unless set; empty means no override. +HTTPS_SNI ?= +ifneq ($(strip $(HTTPS_SNI)),) +CA65FLAGS += -D HTTPS_SNI_OVERRIDE=1 +endif # Optional REU document-base override, shared by the http.s body sink # and the Lane G viewer (both read the HTTP_REU_BODY_BASE equate / # runtime var; defaults to $10:0000 = 1048576). VICE tests use a @@ -467,7 +476,7 @@ build/net/ip65/ip65_blob.o: $(IP65_BIN) # it "up to date" and the file simply vanishes (measured too). build/https_host.inc: FORCE @mkdir -p build - @printf '.define HTTPS_HOST_STR "%s"\n.define HTTPS_PATH_STR "%s"\n' '$(HTTPS_HOST)' '$(HTTPS_PATH)' > $@.tmp + @printf '.define HTTPS_HOST_STR "%s"\n.define HTTPS_PATH_STR "%s"\n.define HTTPS_SNI_STR "%s"\n' '$(HTTPS_HOST)' '$(HTTPS_PATH)' '$(HTTPS_SNI)' > $@.tmp @if cmp -s $@.tmp $@; then rm -f $@.tmp; else mv $@.tmp $@ && rm -f build/boot.o; fi FORCE: diff --git a/src/boot.s b/src/boot.s index b8be129c..3c8aa887 100644 --- a/src/boot.s +++ b/src/boot.s @@ -1243,6 +1243,17 @@ https_path_target: https_path_target_len = * - https_path_target - 1 .assert https_path_target_len <= 100, error, "HTTPS_PATH too long for the request builder" +.ifdef HTTPS_SNI_OVERRIDE + ; `make HTTPS_SNI=` — SNI presented instead of the connect + ; host (relay/forensics debug knob; see Makefile). Consumed by + ; http_get's SNI copy in src/http.s. + .export https_sni_override +https_sni_override: + .byte HTTPS_SNI_STR + .byte 0 + .assert (* - https_sni_override - 1) <= 63, error, "HTTPS_SNI exceeds the 63-char SNI guard" +.endif + ; ============================================================================= ; Local BSS ; ============================================================================= diff --git a/src/http.s b/src/http.s index 97b4774c..f7be310c 100644 --- a/src/http.s +++ b/src/http.s @@ -35,6 +35,9 @@ ; ---- imports: data.asm BSS (HTTP I/O + parser state) ---- .import http_host_ptr .import http_host_len +.ifdef HTTPS_SNI_OVERRIDE + .import https_sni_override +.endif .import http_path_ptr .import http_path_len .import http_port @@ -110,6 +113,22 @@ http_get: @tcp_ok: ; --- 4. Copy hostname to tls_hostname for SNI --- +.ifdef HTTPS_SNI_OVERRIDE + ; Debug knob (make HTTPS_SNI=...): present a fixed SNI that + ; differs from the connect host, so the C64 can dial a local + ; TCP relay while the ClientHello names the real origin. + ldy #0 +@copy_sni: + lda https_sni_override,y + beq @copy_sni_done + sta tls_hostname,y + iny + bne @copy_sni ; always branches (SNI < 256) +@copy_sni_done: + lda #0 + sta tls_hostname,y ; null-terminate + sty tls_hostname_len +.else lda http_host_ptr sta zp_ptr lda http_host_ptr+1 @@ -126,6 +145,7 @@ http_get: lda #0 sta tls_hostname,y ; null-terminate sty tls_hostname_len +.endif ; --- 5. TLS handshake --- jsr tls_connect From d9cd021c4b0fd9a13c48190dbf16cfdf0032cce5 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Sat, 22 Aug 2026 07:13:47 -0400 Subject: [PATCH 32/33] =?UTF-8?q?fix(uci):=20clamp=20SOCKET=5FREAD=20to=20?= =?UTF-8?q?ring=20free=20space=20=E2=80=94=20THE=20wikipedia=20stall?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit net_poll always requested a fixed 512 B from firmware, but the fill loop drops any byte past the ring's current free space and the response drain discards the remainder — bytes the firmware counts as delivered. Once the ~4 KB ring wraps (any TLS flight over ring capacity, i.e. en.wikipedia.org's 4.7 KB cert flight), the stream gets a permanent hole and the client parks forever at that record. Found by decrypting the stalled run's captured ring (wrap-aware, per- record seq brute-force): the hole sits exactly one ring-capacity past the wrap origin. The relay experiment had already excluded delivery/ window causes (full flight handed to the firmware, zero backpressure, client stalled anyway) — this is the read side. Fix: request min(ring_free - 1, 512) and skip the read entirely when nothing fits; unread bytes wait in the firmware for the next poll. Adds a wikipedia-shaped gate scenario (multi-cert chain, real 1636 B leaf) that proves the CONSUMER handles the shape — but note in-code: the gate primes the ring directly with net_poll stubbed, so it CANNOT exercise this adapter fill path. The fix is validated on hardware, not in VICE. Gate 12/12, viewer 7/7, http 61/61. Co-Authored-By: Claude Fable 5 --- src/net/uci/net.s | 54 +++++++++++++++++++++++++++++++++++--- tools/test_tls_deframer.py | 35 ++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 3 deletions(-) diff --git a/src/net/uci/net.s b/src/net/uci/net.s index 4296d689..be8715cf 100644 --- a/src/net/uci/net.s +++ b/src/net/uci/net.s @@ -138,6 +138,52 @@ net_poll: beq @do_poll rts @do_poll: + ; --- Clamp the SOCKET_READ request to ring free space ----------- + ; THE WIKIPEDIA-STALL BUG (found 2026-08-22 by ring forensics): + ; requesting a fixed 512 while the ring has less free space makes + ; the fill loop below hit "ring full" mid-response, and the + ; response drain then DISCARDS the remainder — bytes the firmware + ; considers delivered. The TLS stream acquires a permanent hole + ; exactly one ring-capacity past the wrap origin and the client + ; parks forever (en.wikipedia.org's 4.7 KB flight; the local + ; 4.68 KB control missed the razor-edge by phasing). Request + ; min(free-1, UCI_READ_CHUNK_MAX) instead, and skip the read + ; entirely when nothing fits — unread bytes stay in the firmware + ; for a later poll, which is exactly what TCP is for. + ; free-1 = (head - tail - 1) & TCP_RECV_MASK (head==tail: empty) + lda tcp_recv_head+0 + sec + sbc tcp_recv_tail+0 + sta uci_req_len+0 + lda tcp_recv_head+1 + sbc tcp_recv_tail+1 + and #>TCP_RECV_MASK + sta uci_req_len+1 + lda uci_req_len+0 + sec + sbc #$01 + sta uci_req_len+0 + lda uci_req_len+1 + sbc #$00 + and #>TCP_RECV_MASK + sta uci_req_len+1 + ora uci_req_len+0 + bne :+ + rts ; ring effectively full — poll again later +: + ; clamp to UCI_READ_CHUNK_MAX (512 = $0200) + lda uci_req_len+1 + cmp #>UCI_READ_CHUNK_MAX + bcc @len_clamped ; high < 2 -> under 512, keep + bne @len_use_max ; high > 2 -> over, clamp + lda uci_req_len+0 + beq @len_clamped ; exactly 512, keep +@len_use_max: + lda #UCI_READ_CHUNK_MAX + sta uci_req_len+1 +@len_clamped: jsr uci_wait_not_busy bcc :+ ; FPGA wedged before we could push SOCKET_READ — net_last_error is @@ -157,10 +203,11 @@ net_poll: lda uci_socket_id jsr uci_put_byte - ; maxlen — fixed UCI_READ_CHUNK_MAX (512), LE. - lda #UCI_READ_CHUNK_MAX + lda uci_req_len+1 jsr uci_put_byte jsr uci_push_wait @@ -1053,5 +1100,6 @@ uci_chunk_len: .res 2 ; 16-bit bytes remaining in current chunk uci_write_resp: .res 2 ; written_lo/hi from SOCKET_WRITE uci_read_hdr: .res 2 ; actual_len_lo/hi from SOCKET_READ uci_poll_rem: .res 2 ; net_poll per-cycle remaining +uci_req_len: .res 2 ; net_poll: clamped SOCKET_READ maxlen uci_next_lo: .res 1 ; scratch: (tail+1) & mask, low uci_next_hi: .res 1 ; scratch: (tail+1) & mask, high diff --git a/tools/test_tls_deframer.py b/tools/test_tls_deframer.py index ab48e786..b1fab2f4 100644 --- a/tools/test_tls_deframer.py +++ b/tools/test_tls_deframer.py @@ -452,6 +452,41 @@ def build_scenarios(cert_der, pubkey_xy, wiki_leaf=None): n_msgs=2, )) + # --- deframer: wikipedia-SHAPED flight (2026-08-22 stall forensics) --- + # Real en.wikipedia.org shape: EE in a small record, then a multi-cert + # Certificate whose FIRST entry is the 1636 B leaf (> the historical + # 1536 B cap — the reason for the 2048 UCI cert_buf), spanning many + # 512-content records. This pins the CONSUMER side: deframer + record + # layer + multi-cert framing keep the big leaf and discard the rest. + # + # NOTE — SCOPE: this harness primes the whole flight into the ring up + # front with net_poll stubbed to RTS, so it CANNOT reproduce the + # actual hardware stall, which is in the ADAPTER fill loop + # (src/net/uci/net.s net_poll requesting >ring-free and the response + # drain discarding the overflow — fixed there by clamping the + # SOCKET_READ maxlen to ring free space). That fix is validated on + # hardware / a poll-driven rig, NOT here. Kept under the 4 KB ring + # (leaf 1636 + one 656 intermediate) so prime_ring can hold it. + if wiki_leaf is not None and len(wiki_leaf[0]) <= CERT_BUF_MAX: + shaped_leaf, shaped_xy = wiki_leaf + else: + shaped_leaf, shaped_xy = cert_der, (px, py) + shaped_inter = bytes([0x30, 0x82]) + (656 - 4).to_bytes(2, "big") + \ + bytes((i * 7 + 656) & 0xFF for i in range(656 - 4)) + entries = b"" + for c in (shaped_leaf, shaped_inter): + entries += len(c).to_bytes(3, "big") + c + b"\x00\x00" + body = b"\x00" + len(entries).to_bytes(3, "big") + entries + shaped_cert = hs_message(TLS_HS_CERTIFICATE, body) + scenarios.append(Scenario( + "deframer/wikipedia_shaped_flight", + "deframer", + [ee] + chunk_bytes(shaped_cert, [512] * 6), + shaped_xy, + hashlib.sha256(ee + shaped_cert).digest(), + n_msgs=2, + )) + # --- deframer (d): oversized leaf must error, not overflow --- # A single-entry Certificate whose cert_data exceeds cert_buf # (CERT_BUF_MAX, read from labels.txt — 2048 UCI / 1536 ip65). From 4e063c6b235b08114bd0fd89cb98e8351f3b5e85 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Sat, 22 Aug 2026 07:54:18 -0400 Subject: [PATCH 33/33] docs: record the real-server + wikipedia milestone in README and CLAUDE.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README: milestone callout in the intro, two Project Status entries (real public-internet HTTPS on UCI/comb; wikipedia article into REU + viewer), and refreshed the two stalest Known Issues (real-server reach/limits; the now-fixed wikipedia ring-drop). CLAUDE.md: rewrote the End-to-end HTTPS status milestone block for the final state — 4 live sites at HTTP 200 incl. wikipedia's 125,235 B article to REU + viewer; the three live-only bugs fixed (512-content inner-type, large-body consumed-count termination, the net_poll ring-drop stall d9cd021); and the two distinct U64E device gotchas (lease-poisoning via reset-with-live-socket vs the writemem/temp wedge). Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 103 +++++++++++++++++++++++++++++++++++++----------------- README.md | 13 ++++++- 2 files changed, 82 insertions(+), 34 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 413e08e8..0ab1dc59 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -671,40 +671,77 @@ The scripts: ### End-to-end HTTPS status -**REAL-SERVER MILESTONE (2026-08-21, U64E @ 48 MHz, comb):** the first -HTTPS GETs from the C64 to real public internet servers. - - target CFIN http_status note - github.com 32.5 s **200** full PASS, homepage HTML prefix - browserleaks.com 33.4 s **200** TLS + status PASS; rig sentinel - timed out on body drain (below) - -The W1 streaming deframer handled the real 9-11-record flights -(Certificate ~2.7 KB over ~6 records; CV+Finished sharing the tail), the -W2 streaming consumer staged the leaf, and the sibling library verified -real CA-issued chains. Local padded-chain bench: 11-record flight costs -only ~0.7 s over the single-cert control (31.5 vs 30.8 s CFIN). - -Two findings from the live runs: - - **512-content records** (what MFL-honoring servers actually send) hit - a latent page-dispatch bug in the record layer's inner-type read — - fixed in `src/tls_record.s` (page-2 arm), mutation-proven by +**REAL-SERVER MILESTONE (2026-08-21/22, U64E @ 48 MHz, comb):** HTTPS GETs +from the C64 to real public internet servers, and the stretch goal — the +Commodore 64's own Wikipedia article, over TLS, into the REU, scrollable +on screen. + + target result + github.com **HTTP 200**, homepage HTML (CFIN 32.5 s) + browserleaks.com **HTTP 200** (CFIN 33.4 s) + lwn.net **HTTP 200** (CFIN 38.5 s) + en.wikipedia.org **HTTP 200**, 125,235 B article -> REU $10:0000, + body byte-verified vs a host reference fetch, + on-screen scroll viewer entered (~76 s to full body) + +The W1 streaming deframer (`src/tls_deframe.s`) handles the real 11-14 +record flights (Certificate spanning ~6 records; CV+Finished sharing the +tail); the W2 streaming consumer stages the leaf into a UCI-only 2048 B +`cert_buf` (`CERT_BUF_SIZE`, so wikipedia's 1636 B leaf fits); the +sibling library verifies real CA-issued chains. Target is a build knob: +`make HTTPS_HOST= HTTPS_PATH=` (+ `HTTPS_BODY_TO_REU=1` and +the `src/viewer.s` viewer for the wikipedia flow). Rigs: +`tools/uci/rig_https_live.py`, `rig_https_wiki.py`. + +Three bugs the local-listener path never exposed, all fixed: + - **512-content records** (what MFL-honoring servers send) hit a latent + page-dispatch bug in the record layer's inner-type read — fixed in + `src/tls_record.s` (page-2 arm), mutation-proven by `tools/test_tls_deframer.py::mfl512_full_records`. Every local - fixture had been one byte short of the trigger. - - **W4 gap (open):** a large NON-chunked body cannot terminate by - Content-Length once `body_append` truncates at 512 B - (`http_resp_len` freezes, the equality never fires), leaving only - the connection-close fallback — browserleaks holds the connection - and the rig sentinel timed out at 300 s with status/body already - correct. Fix direction: count consumed-not-stored body bytes - (needs >16-bit for real pages) or cap the post-truncation drain. - -**U64E "writemem exhaustion wedge"** (user-named): fw 3.14d misses -garbage collection on a temp location filled by REST `writemem`; a long -hardware session (~15 PRG loads that day) fills it CUMULATIVELY and then -REST *and* the UCI bridge wedge together — ping alive, REST refuses -instantly, C64 parks mid-transfer. Power cycle fixes. Budget PRG loads -per session and power-cycle proactively; `readmem` is not implicated. + fixture was one byte short of the trigger. + - **Large-body termination**: identity/chunked bodies past the 512 B + `http_resp_buf` cap now terminate on a 24-bit CONSUMED-byte count + (`http_body_total`), not on the frozen stored length (Lane E W4 + rework in `src/http.s`; wikipedia serves the article chunked with no + Content-Length, so this is the primary path). + - **THE wikipedia stall (`net_poll` ring-drop, commit d9cd021)**: the + UCI adapter requested a fixed 512 B per `SOCKET_READ` while its ring + fill loop silently dropped any byte past the ring's CURRENT free + space — and the response drain discarded them, bytes the firmware + counts as delivered. Once the 4 KB ring wrapped (any flight over ring + capacity: wikipedia's 4.7 KB cert flight is the first target to + cross it), the stream got a permanent hole one ring-capacity past + the wrap and the deframer parked forever at that record. github/ + browserleaks/lwn (3.2-4.0 KB flights) squeaked under. Fixed by + clamping the request to `min(ring_free - 1, 512)` and skipping the + read when nothing fits (`src/net/uci/net.s`). **Not a firmware bug.** + Found by decrypting the stalled run's captured ring wrap-aware, after + a TCP-relay experiment excluded delivery/window causes (full flight + handed to firmware with zero backpressure, client stalled anyway — + see `HTTPS_SNI` override knob, added for exactly that experiment). + +**U64E lease-poisoning (open device gotcha, separate from the writemem +wedge below).** Resetting the C64 while a firmware TCP socket is still +live poisons the UCI lease path: `GET_IPADDR` then returns 0.0.0.0 on +all interfaces with no error while REST/FTP/menu stay healthy, so the +program loops forever at "REQUESTING DHCP". Survives C64 resets, +`machine:reboot`, and UCI disable/enable — **only a wall power cycle +clears it.** Any live rig that times out leaves the C64 parked in +`http_get` with an open socket, so its cleanup reset triggers this; the +completing runs above do not. Diagnostic trap: probe DHCP with a +FAST-BOOT image (onchip), never the comb image, which does not reach +DHCP inside `phase2_check`/`boot_check` windows at stock clock and reads +as a false 0.0.0.0. + +**U64E "writemem exhaustion wedge"** (user-named, DIFFERENT failure): fw +3.14d misses garbage collection on a temp location filled by REST +`writemem`; a long hardware session (~15 PRG loads) fills it CUMULATIVELY +and then REST *and* the UCI bridge wedge together — ping alive, REST +refuses instantly, C64 parks mid-transfer. Power cycle fixes; upstream +GideonZ/1541ultimate#686 (merged, unreleased). Mitigated by +`tools/uci/_temp_gc.py` (FTP-deletes managed `temp` files, keep +youngest 2; wired into the live/wiki rigs). Budget PRG loads per session; +`readmem` is not implicated. diff --git a/README.md b/README.md index da4aaae9..065e1046 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,14 @@ An HTTPS client for the Commodore 64 in 6502 assembly. Implements TLS 1.3 over T **For demonstration and educational purposes only — not cryptographically secure.** +> **Real-server milestone (2026-08-21/22).** The client now completes TLS 1.3 +> handshakes and HTTP GETs against real public servers on the open internet — +> **github.com, browserleaks.com and lwn.net** all return HTTP 200 — and, as a +> demonstration, fetches the full 125 KB **Wikipedia article about the +> Commodore 64** over TLS into a 16 MB REU and scrolls it on the C64's own +> screen. Requires the UCI backend at turbo (comb profile). See the Project +> Status section and the "End-to-end HTTPS status" notes in `CLAUDE.md`. + ## I just want to run it Grab a release — latest is @@ -291,13 +299,16 @@ Progress: - [x] **End-to-end HTTPS GET demo (both backends)** — TLS 1.3 handshake + HTTP GET completes against a local Python TLS listener (ECDSA-P256 cert). Returns `http_status=200`, body `"HELLO FROM TLS SERVER"`. - UCI: real Ultimate 64 Elite hardware at both 48 MHz turbo and stock 1 MHz. See `tools/uci/rig_https_local.py` (supports `TURBO_MHZ` env var). - ip65: VICE + RR-Net at stock 1 MHz, no warp. The bridge-rig script is `tests/rig_phase3_https_1mhz.py`; the hardware-free macOS feth/pcap rig is `tests/rig_vice_https_macos.py`, and that is where the wall-clock below was taken. +- [x] **Real public-internet HTTPS (UCI/comb, turbo)** — github.com, browserleaks.com and lwn.net all return `http_status=200` on real U64E hardware at 48 MHz. Needed three pieces of work over the local-listener path: a streaming handshake-message deframer (`src/tls_deframe.s`) for flights where handshake messages don't align with TLS records; a 2048 B `cert_buf` under UCI so larger real leaves fit; and — the last blocker — clamping the UCI adapter's `SOCKET_READ` request to the receive ring's free space, without which any flight over ~4 KB lost its tail to a ring-wrap drop. Build-time target is `make HTTPS_HOST=` / `HTTPS_PATH=`; the rig is `tools/uci/rig_https_live.py`. +- [x] **Wikipedia article into REU + on-screen viewer (stretch goal, UCI/comb)** — `make HTTPS_HOST=en.wikipedia.org HTTPS_PATH='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/w/index.php?title=Commodore_64&action=raw' HTTPS_BODY_TO_REU=1` streams the 125,235 B article body into REU bank 16 (`$10:0000`) and drops into a scroll viewer (`src/viewer.s`, CRSR/SPACE/F1/HOME/Q). Verified on U64E @ 48 MHz, body byte-checked against a host-side reference fetch. Rig: `tools/uci/rig_https_wiki.py`. ### Known Issues - **The handshake is slow, and the ECDSA P-256 verify dominates it.** Every figure here is quoted from the measurement record in `CLAUDE.md`. Except where noted they were taken at the **`libs/nistcurves` v0.6.0 pin**, and the pin is now v0.11.2, so treat them as a baseline rather than as current. The one profile re-measured at the current pin is comb: 46.986 / 24.440 / 16.402 s verify at 16 / 32 / 48 MHz (U64E, n=3, VIC blanking active). End-to-end handshake + GET against the local listener, U64E, master 2ceb5b1: **80.8 s** (REU profile, 48 MHz), **45.5 s** (onchip profile, 48 MHz), **1,157.7 s** (REU, stock 1 MHz). One point of that sweep has been carried forward: 48 MHz REU measures **82.1 s** at v0.9.1 and **82.4 s** at v0.10.1 (n=1 each, so the 0.4% step between them is noise; the 1.6% from v0.6.0 is the FIPS 186-5 public-key validation gate v0.7.0 added). No other clock or profile has been re-measured. On the REU-less stock-C64 path (ip65 + onchip, no REU, honest 1 MHz in VICE) the whole run measured **2,159.7 s = 36.0 min**, of which the verify stretch alone was 1,416.7 s. That is fine for the local listener, which holds the connection open; it exceeds a typical 10-30 s real-world server handshake window. - **P-384** ECDSA is stubbed at the TLS layer. The dispatcher advertises `ecdsa_secp384r1_sha384` (0x0503) and routes to `src/crypto/ecdsa_verify_384.s`, but no P-384 build target completes. Measured 2026-08-14: `make p384-overlay` from a clean tree stops at `No rule to make target 'build/labels.txt'`, and after a main build has produced that file it stops at `Segment 'LIB_NISTCURVES_SHA384_TABLES' overflows memory area 'OVERLAY_REGION' by 1536 bytes`. Cert chains requiring P-384 will not verify. - **`USE_X25519_SIBLING=1` now links under UCI, and still does not under ip65.** The duplicate-symbol failure this entry used to record — `ld65: Error: Duplicate external identifier: 'reu_mul_tables_init'`, on **both** backends — was closed by the `libs/nistcurves` v0.10.1 / `libs/x25519` v0.11.0 bump plus one line of archive surgery: `tools/integration/build_nistcurves_p256.sh` now also drops `reu_mul_init.o`, the SPEC §8.2 `reu_mul` provider that `src/boot.s` supplies itself. Re-measured at the current v0.11.2 pins, unchanged: `make clean && make BACKEND=uci USE_X25519_SIBLING=1` produces a 62,977 B PRG, and ip65 stops instead at `Segment 'X25519_RODATA' overflows memory area 'CRYPTO_OVERLAY' by 3584 bytes` — a placement problem (ip65's overlay slot is 4,212 B against UCI's 7,680 B), not a symbol collision. The flag remains **off by default** and no shipped artifact contains the sibling; the in-tree X25519 in `src/crypto/{x25519,fe25519}.s` is what every release PRG is built from. Flipping the default is a separate decision that wants a hardware handshake behind it. -- **Live internet HTTP GET (UCI backend)** has not been re-verified since the FPGA-fence rework; only the local multi-segment listener is exercised regularly. +- **Real-server reach is UCI/comb + turbo only, and has size limits.** The public-internet HTTPS above works on the comb profile at turbo; the stock-C64 ip65 path is far too slow for a real server's connection window (~36 min/handshake). Among real leaves, en.wikipedia.org's 1636 B leaf needs the 2048 B UCI `cert_buf` (fits); anything larger, or a server that ignores `max_fragment_length` and sends >548 B records (e.g. Cloudflare), is out of scope. Cloudflare additionally enforces a ~15 s connect-to-first-request deadline the C64 cannot meet and is deliberately unsupported. +- **The wikipedia stall was a client bug, now fixed.** Historical note for anyone bisecting: TLS flights larger than the ~4 KB UCI receive ring used to stall permanently, because `net_poll` requested a fixed 512 B and its fill loop dropped bytes past the ring's current free space (discarded as "delivered"). Fixed by clamping the `SOCKET_READ` request to ring free space (`src/net/uci/net.s`). It was never a firmware bug; github/browserleaks/lwn flights are under 4 KB and were unaffected. - **VICE 3.9** previously appeared to crash on chained HMAC-SHA256 calls (backend-independent — affects the crypto-only test suites), but this was caused by hardcoded port numbers bypassing the test harness port allocator. With proper `ViceInstanceManager` usage (no hardcoded ports), all N=1..10 chained calls succeed reliably. ## Test Automation