Implement atomic run bundle retention

README.md

@@ -747,15 +747,34 @@ history writer to persist redacted command metadata, timing, exit status,
 project context, and a pointer to the raw log file. History writes should be
 best-effort and should not fail the user command when an index cannot be updated.
 
-High-frequency tools can set `base_cli.App(max_log_files=<count>)` to keep at
-most that many default persistent log files across the owner's run bundles.
+Durable applications use a bounded complete-bundle policy by default, so
+retention never splits a run's metadata, log, and temporary diagnostics.
+High-frequency tools can tune it explicitly:
+
+```python
+app = base_cli.App(
+    name="workspace-tools",
+    retention=base_cli.RetentionPolicy(
+        max_bundles=20,
+        max_age_seconds=30 * 24 * 60 * 60,
+        max_total_bytes=512 * 1024 * 1024,
+    ),
+)
+```
+
 Retention runs during startup after the current run's default log file is
-resolved, and the current run's log file is never pruned. The policy is skipped
-for `ctx.dry_run`,
-`log_to_file=False`, and explicit `--log-file` paths so no-durable-write modes
-and caller-selected log locations stay under caller control. Use this as a
-small guardrail for busy local tools; an application can provide broader
-maintenance commands for caches, logs, and retained temp files.
+resolved. The active invocation, inherited parent bundle, and bundles marked
+`preserve` (including `--keep-temp`) are never removed. A stale `running`
+bundle is eligible for crash recovery only when an age bound is configured;
+without one it is retained for diagnosis. The metadata and retention index are
+written with same-filesystem temporary files, flush/sync, and atomic
+replacement, and concurrent pruners serialize through a sidecar lock.
+
+The policy is skipped for `ctx.dry_run`, `log_to_file=False`, and explicit
+`--log-file` paths so no-durable-write modes and caller-selected log locations
+stay under caller control. The original `max_log_files=<count>` option remains
+available as a compatibility per-file policy; new applications should use
+`RetentionPolicy`.
 
 Logs use a stable, human-readable shape:
 

lib/python/base_cli/__init__.py

@@ -117,7 +117,7 @@ def _resolve_version() -> str:
     UserConfigLoader,
     WorkspaceRootResolver,
 )
-from .runtime import RuntimeLayout
+from .runtime import RetentionPolicy, RuntimeLayout
 from .typer import TyperAdapter, attach_typer, get_typer_command
 
 __all__ = [
@@ -195,6 +195,7 @@ def _resolve_version() -> str:
     "ProjectDiscovery",
     "RECORD_SCHEMAS",
     "RuntimeLayout",
+    "RetentionPolicy",
     "RuntimeResolver",
     "is_terminal",
     "output_format_choices",

lib/python/base_cli/_lifecycle.py

@@ -9,6 +9,7 @@
 from .context import Context
 from .exit_codes import ExitCode
 from .history import format_timestamp
+from ._runtime import refresh_run_bundle_index
 
 
 @dataclass(frozen=True)
@@ -57,6 +58,9 @@ def finish(
             }
         )
         write_private_json(self.context._run_metadata_path, metadata)
+        owner_root = self.context.owner_root
+        if owner_root is not None:
+            refresh_run_bundle_index(owner_root / "runs", logger=self.context.log)
 
     def _existing_metadata(self) -> dict[str, Any]:
         path = self.context._run_metadata_path
@@ -103,6 +107,10 @@ def _metadata(self, *, status: str) -> dict[str, Any]:
             "project_root": str(context.project_root) if context.project_root else None,
             "manifest": str(context.manifest_path) if context.manifest_path else None,
             "workspace_root": str(context.workspace_root) if context.workspace_root else None,
+            # ``--keep-temp`` is an explicit request to retain diagnostics;
+            # retention therefore protects the complete invocation bundle,
+            # not only its temporary directory.
+            "preserve": context.keep_temp,
         }
 
 

lib/python/base_cli/_private_files.py

@@ -4,6 +4,7 @@
 
 import json
 import os
+import secrets
 from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
@@ -32,20 +33,148 @@ def restrict_directory(path: Path) -> None:
 
 
 def write_private_json(path: Path, value: Mapping[str, Any]) -> None:
-    """Write a JSON mapping with owner-only permissions from the moment it is created."""
+    """Atomically write a private JSON mapping.
 
+    The temporary file is created beside the destination, written and synced
+    before it is replaced.  A failed serialization, write, or replacement
+    therefore leaves the previous snapshot intact instead of exposing a
+    truncated metadata/index file.  On POSIX, directory-relative operations
+    also prevent a swapped ancestor from redirecting the replacement through
+    a symlink.
+    """
+
+    path = Path(path)
     path.parent.mkdir(parents=True, exist_ok=True)
-    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, PRIVATE_FILE_MODE)
+    parent_fd: int | None = None
+    temporary_name: str | None = None
+    fd: int | None = None
     try:
+        parent_fd = _open_parent_directory(path.parent)
+        for _ in range(32):
+            candidate = f".{path.name}.{secrets.token_hex(12)}.tmp"
+            try:
+                if parent_fd is not None and _supports_dir_fd_open():
+                    fd = os.open(
+                        candidate,
+                        os.O_WRONLY | os.O_CREAT | os.O_EXCL,
+                        PRIVATE_FILE_MODE,
+                        dir_fd=parent_fd,
+                    )
+                else:
+                    fd = os.open(
+                        path.parent / candidate,
+                        os.O_WRONLY | os.O_CREAT | os.O_EXCL,
+                        PRIVATE_FILE_MODE,
+                    )
+                temporary_name = candidate
+                break
+            except FileExistsError:
+                continue
+        if fd is None or temporary_name is None:
+            raise OSError("unable to allocate a private JSON temporary file")
+
         fchmod = getattr(os, "fchmod", None)
         if os.name != "nt" and fchmod is not None:
             fchmod(fd, PRIVATE_FILE_MODE)
-        stream = os.fdopen(fd, "w", encoding="utf-8")
-        fd = -1
-        with stream:
+        with os.fdopen(fd, "w", encoding="utf-8") as stream:
+            fd = None
             json.dump(value, stream, sort_keys=True)
             stream.write("\n")
+            stream.flush()
+            os.fsync(stream.fileno())
+
+        # Refuse an existing symlink as the destination.  Replacing a regular
+        # file is safe and atomic; replacing a symlink entry would be safe from
+        # following it, but rejecting it makes an unexpected redirection
+        # explicit to callers.
+        if parent_fd is not None and _supports_dir_fd_stat():
+            try:
+                existing = os.stat(path.name, dir_fd=parent_fd, follow_symlinks=False)
+            except FileNotFoundError:
+                existing = None
+            if existing is not None and _is_symlink_mode(existing.st_mode):
+                raise OSError(f"refusing to replace symlink '{path}'")
+            _atomic_replace_relative(parent_fd, temporary_name, path.name)
+        else:
+            if path.is_symlink():
+                raise OSError(f"refusing to replace symlink '{path}'")
+            os.replace(path.parent / temporary_name, path)
+        temporary_name = None
+        if parent_fd is not None:
+            _sync_directory(parent_fd)
+        restrict_file(path)
     finally:
-        if fd != -1:
+        if fd is not None:
             os.close(fd)
-    restrict_file(path)
+        if temporary_name is not None:
+            try:
+                if parent_fd is not None and _supports_dir_fd_unlink():
+                    os.unlink(temporary_name, dir_fd=parent_fd)
+                else:
+                    (path.parent / temporary_name).unlink()
+            except OSError:
+                pass
+        if parent_fd is not None:
+            os.close(parent_fd)
+
+
+def _supports_dir_fd_open() -> bool:
+    return os.open in os.supports_dir_fd
+
+
+def _supports_dir_fd_stat() -> bool:
+    return os.stat in os.supports_dir_fd and os.stat in os.supports_follow_symlinks
+
+
+def _supports_dir_fd_unlink() -> bool:
+    return os.unlink in os.supports_dir_fd
+
+
+def _is_symlink_mode(mode: int) -> bool:
+    # Avoid importing stat in the hot path and keep this helper usable on
+    # platforms where the POSIX mode constants are still available.
+    return (mode & 0o170000) == 0o120000
+
+
+def _open_parent_directory(path: Path) -> int | None:
+    """Open a parent directory without following path components on POSIX."""
+
+    if os.name == "nt" or not hasattr(os, "O_NOFOLLOW") or not hasattr(os, "O_DIRECTORY"):
+        return None
+    # Resolve the platform's standard compatibility links (for example
+    # macOS's /var -> /private/var), then bind each real component with
+    # O_NOFOLLOW so a later ancestor swap cannot redirect the write.
+    absolute = path.resolve(strict=True)
+    descriptor = os.open(
+        absolute.anchor,
+        os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0),
+    )
+    try:
+        for component in absolute.relative_to(Path(absolute.anchor)).parts:
+            next_descriptor = os.open(
+                component,
+                os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0),
+                dir_fd=descriptor,
+            )
+            os.close(descriptor)
+            descriptor = next_descriptor
+        return descriptor
+    except BaseException:
+        os.close(descriptor)
+        raise
+
+
+def _atomic_replace_relative(parent_fd: int, source: str, destination: str) -> None:
+    if os.rename in os.supports_dir_fd:
+        os.rename(source, destination, src_dir_fd=parent_fd, dst_dir_fd=parent_fd)
+        return
+    os.replace(source, destination)
+
+
+def _sync_directory(parent_fd: int) -> None:
+    try:
+        os.fsync(parent_fd)
+    except OSError:
+        # Directory fsync is not available on all supported filesystems and
+        # platforms.  The file itself was still flushed before replacement.
+        pass