diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5b0cb34 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.py[cod] +*$py.class +.pytest_cache/ +.ruff_cache/ +.venv/ diff --git a/scripts/dogfood-crush-mcp-real.py b/scripts/dogfood-crush-mcp-real.py new file mode 100644 index 0000000..19e743a --- /dev/null +++ b/scripts/dogfood-crush-mcp-real.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Dogfood a fresh Crush session against the real loom-ai repository. + +Crush owns repository inspection and editing. Loom owns the public execution +boundary, durable state, orchestration, and the verification checkpoint. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import urllib.error +import urllib.request +from pathlib import Path +from uuid import uuid4 + +root = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(root)) + +from loom_ai import ( # noqa: E402 + Arbiter, + ArbiterDecision, + WorkerEvaluation, + WorkerResult, + WorkerStatus, +) +from loom_ai.execution_state import FileExecutionStateStore # noqa: E402 +from loom_ai.server import LoomServer # noqa: E402 +from loom_ai.worker import WorkerContext # noqa: E402 + +REPO_URL = "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/FlossWare/loom-ai.git" +TARGET_FILE = "docs/DOGFOOD.md" +ISSUE = "#949" +MARKER = "## External-agent dogfood" + + +def fail(message: str) -> None: + raise SystemExit(f"FAIL: {message}") + + +def run( + command: list[str], + *, + cwd: Path | None = None, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + command, + cwd=cwd, + env=env, + check=False, + text=True, + capture_output=True, + ) + + +def request_json(url: str) -> dict: + request = urllib.request.Request(url, headers={"Accept": "application/json"}) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return json.load(response) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace").strip() + raise RuntimeError( + f"HTTP {exc.code} from {url}: {detail or exc.reason}" + ) from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"Cannot reach {url}: {exc.reason}") from exc + + +def result_failure(worker_id: str, error: str) -> WorkerResult: + return WorkerResult(worker_id, WorkerStatus.FAILURE, error=error) + + +def verify_repository(repo: Path) -> WorkerResult: + file_path = repo / TARGET_FILE + if not file_path.is_file(): + return result_failure("real-repo-verify", f"missing {TARGET_FILE}") + + text = file_path.read_text(encoding="utf-8") + diff = run(["git", "diff", "--name-only"], cwd=repo) + diff_check = run(["git", "diff", "--check"], cwd=repo) + changed = [line for line in diff.stdout.splitlines() if line] + + if diff.returncode != 0 or diff_check.returncode != 0: + return result_failure("real-repo-verify", "git verification command failed") + if changed != [TARGET_FILE]: + return result_failure( + "real-repo-verify", f"unexpected changed files: {changed!r}" + ) + if MARKER not in text: + return result_failure( + "real-repo-verify", f"missing {MARKER!r} in {TARGET_FILE}" + ) + + return WorkerResult( + "real-repo-verify", + WorkerStatus.SUCCESS, + evidence=( + { + "type": "real-repository-verification", + "message": f"verified {ISSUE}: {TARGET_FILE} contains {MARKER!r}", + "changed_files": changed, + }, + ), + ) + + +class VerificationWorker: + worker_id = "real-repo-verify" + + def __init__(self, repo: Path) -> None: + self.repo = repo + + def execute(self, context: WorkerContext) -> WorkerResult: + checkpoint_seen = any( + isinstance(item, dict) + and item.get("message") == "real repository is clean before task" + for item in context.evidence + ) + if checkpoint_seen: + return verify_repository(self.repo) + + result = run(["git", "status", "--porcelain"], cwd=self.repo) + if result.returncode != 0: + return result_failure( + self.worker_id, result.stderr.strip() or "git status failed" + ) + unexpected = [ + line + for line in result.stdout.splitlines() + if line and line != "?? .crushrc" + ] + if unexpected: + return result_failure( + self.worker_id, + f"real dogfood clone has unexpected changes before the task: {unexpected!r}", + ) + + return WorkerResult( + self.worker_id, + WorkerStatus.SUCCESS, + evidence=( + { + "type": "real-repository-checkpoint", + "message": "real repository is clean before task", + }, + ), + ) + + +def evaluate(result: WorkerResult, _context: WorkerContext) -> WorkerEvaluation: + if result.successful: + return WorkerEvaluation( + ArbiterDecision.COMPLETE, + reason="dogfood checkpoint verified", + ) + return WorkerEvaluation( + ArbiterDecision.REPLAN, + reason=result.error or "verification failed", + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Dogfood fresh Crush -> generic Loom MCP against real loom-ai" + ) + parser.add_argument( + "--model", default=None, help="Optional Crush model passed to crush run" + ) + args = parser.parse_args() + + crush = shutil.which("crush") + git = shutil.which("git") + if crush is None: + fail("crush is not installed or is not on PATH") + if git is None: + fail("git is not installed or is not on PATH") + + python = Path(sys.executable) + mcp_server = root / "scripts" / "loom_mcp_server.py" + if not mcp_server.is_file(): + fail(f"missing generic MCP server: {mcp_server}") + + with tempfile.TemporaryDirectory(prefix="loom-ai-949-") as temp: + temp_root = Path(temp) + repo = temp_root / "loom-ai" + clone = run( + [git, "clone", "--depth", "1", "--branch", "main", REPO_URL, str(repo)] + ) + if clone.returncode != 0: + fail(f"could not clone {REPO_URL}: {clone.stderr.strip()}") + + clean = run(["git", "status", "--porcelain"], cwd=repo) + if clean.returncode != 0 or clean.stdout.strip(): + fail("fresh loom-ai clone is not clean") + + execution_id = f"crush-real-{uuid4()}" + intent_id = f"intent-{uuid4()}" + store = FileExecutionStateStore(temp_root / "execution-state") + server = LoomServer( + Arbiter([VerificationWorker(repo)], evaluate, max_retries=0), + host="127.0.0.1", + port=0, + execution_store=store, + ) + http_server = server.start() + thread = threading.Thread(target=http_server.serve_forever, daemon=True) + thread.start() + + mcp_check = run([str(python), "-c", "import mcp"]) + if mcp_check.returncode != 0: + fail( + f"{python} cannot import mcp; install the project MCP dependency " + "and rerun with that interpreter" + ) + + crush_data = temp_root / "crush-data" + (repo / ".crushrc").write_text( + "# Disposable real-repository Loom dogfood configuration.\n" + f'option data-directory "{crush_data}"\n' + f'mcp add loom --command "{python}" --args "{mcp_server}" ' + f'--env LOOM_URL "http://127.0.0.1:{server.port}" --timeout 30\n', + encoding="utf-8", + ) + + prompt = f""" +You are performing the real Loom dogfood for issue {ISSUE} in this repository. +This is a real repository, not a fixture. Use your normal repository tools for +inspection, editing, shell commands, and git. Use the generic Loom MCP for +orchestration and verification. Do not call Loom Python internals. + +Task: make one small, real documentation improvement directly related to {ISSUE}. +In {TARGET_FILE}, add a concise section titled "{MARKER}" explaining that +external coding agents such as Crush consume Loom's agent-neutral public +boundary, that repository inspection/edit/test tooling remains on the client +side unless a Loom capability explicitly requires it, and that Loom owns +execution orchestration, verification, evidence, provenance, and durable +execution state. Keep the wording consistent with the repository's existing +architecture documents. Do not modify unrelated files. + +Execution protocol: +1. Call loom_submit_intent exactly once before editing, using execution_id + "{execution_id}" and intent_id "{intent_id}". Include provenance with + client=fresh-crush, issue={ISSUE}, and task_path={TARGET_FILE}. +2. Inspect the repository and the relevant documentation. +3. Make the real documentation change with your repository edit tool. +4. Run appropriate validation, including git diff --check. +5. Call loom_continue_execution exactly once with execution_id + "{execution_id}". Do not claim success if Loom verification fails. +6. Do not commit or push the change; leave it in the working tree for the + harness to verify. +7. Report the final execution_id and the repository verification result. + +If verification fails, investigate and correct the task before reporting success. +""".strip() + + env = os.environ.copy() + env["CRUSH_GLOBAL_DATA"] = str(temp_root / "crush-data") + command = [crush, "run", "--quiet"] + if args.model: + command.extend(["--model", args.model]) + command.append(prompt) + + print("==> Fresh Crush -> generic Loom MCP -> real loom-ai repository") + version_result = run([crush, "--version"], env=env) + print( + f"Crush: {version_result.stdout.strip() or version_result.stderr.strip()}" + ) + print(f"Loom URL: http://127.0.0.1:{server.port}") + print(f"Repository: {repo}") + print(f"Execution ID: {execution_id}") + print(f"Task: {ISSUE} ({TARGET_FILE})") + + try: + completed = run(command, cwd=repo, env=env) + print("\n==> Crush stdout") + print(completed.stdout.rstrip()) + if completed.stderr.strip(): + print("\n==> Crush stderr") + print(completed.stderr.rstrip(), file=sys.stderr) + if completed.returncode != 0: + fail(f"Crush exited with status {completed.returncode}") + + (repo / ".crushrc").unlink(missing_ok=True) + + execution = request_json( + f"http://127.0.0.1:{server.port}/executions/{execution_id}" + ) + print("\n==> Durable Loom execution") + print(json.dumps(execution, indent=2, sort_keys=True)) + if execution.get("status") != "success": + fail(f"Loom execution status is {execution.get('status')!r}") + if execution.get("intent_id") != intent_id: + fail("Loom intent_id does not match") + + provenance = execution.get("provenance", {}) + for key, expected in { + "client": "fresh-crush", + "issue": ISSUE, + "task_path": TARGET_FILE, + }.items(): + if provenance.get(key) != expected: + fail(f"missing or incorrect provenance {key!r}") + + final_text = (repo / TARGET_FILE).read_text(encoding="utf-8") + if MARKER not in final_text: + fail(f"missing {MARKER!r} in {TARGET_FILE}") + + status = run(["git", "status", "--porcelain"], cwd=repo) + expected_status = [f" M {TARGET_FILE}"] + if status.returncode != 0 or status.stdout.splitlines() != expected_status: + fail( + "unexpected working-tree state: " + f"{status.stdout.splitlines()!r}; expected {expected_status!r}" + ) + + diff = run(["git", "diff", "--name-only"], cwd=repo) + if diff.stdout.splitlines() != [TARGET_FILE]: + fail(f"unexpected repository changes: {diff.stdout.splitlines()!r}") + + diff_check = run(["git", "diff", "--check"], cwd=repo) + if diff_check.returncode != 0: + fail("git diff --check failed") + + print("\nRESULT: FRESH CRUSH -> GENERIC MCP -> LOOM -> REAL REPO PASSED") + print(f"Changed: {TARGET_FILE}") + print(f"Issue: {ISSUE}") + print(f"Execution: {execution_id}") + print("Independent checks: changed-file scope, marker, git diff --check") + return 0 + finally: + server.close() + thread.join(timeout=5) + + +if __name__ == "__main__": + raise SystemExit(main())