fix(server): reuse the checkpoint index so unchanged untracked files are not rehashed every turn

apps/server/src/vcs/GitVcsDriver.test.ts

@@ -14,7 +14,12 @@ import * as PlatformError from "effect/PlatformError";
 import { ChildProcessSpawner } from "effect/unstable/process";
 import { assert, it } from "@effect/vitest";
 
-import { CheckpointRef, GitCommandError, VcsProcessExitError } from "@t3tools/contracts";
+import {
+  CheckpointRef,
+  GitCommandError,
+  VcsProcessExitError,
+  VcsProcessTimeoutError,
+} from "@t3tools/contracts";
 import * as ServerConfig from "../config.ts";
 import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";
 import * as ProcessRunner from "../processRunner.ts";
@@ -941,6 +946,271 @@ for (const indexState of ["missing", "invalid"] as const) {
   );
 }
 
+const makeRecordingDriver = Effect.fn("makeRecordingDriver")(function* () {
+  const liveProcess = yield* VcsProcess.VcsProcess;
+  const commands: ReadonlyArray<string>[] = [];
+  const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
+    Effect.provideService(VcsProcess.VcsProcess, {
+      run: (input) => {
+        commands.push(input.args);
+        return liveProcess.run(input);
+      },
+    }),
+  );
+  return { driver, commands };
+});
+
+it.effect("checkpoint capture reuses its index so unchanged untracked files are not rehashed", () =>
+  Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const { driver, commands } = yield* makeRecordingDriver();
+    const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-reuse-" });
+    const { git } = yield* makeCheckpointFixture(driver, cwd);
+    // An old mtime keeps a file out of Git's racy window, where it would be hashed again.
+    const write = Effect.fn(function* (name: string, contents: string) {
+      yield* fs.makeDirectory(path.dirname(path.join(cwd, name)), { recursive: true });
+      yield* fs.writeFileString(path.join(cwd, name), contents);
+      yield* fs.utimes(path.join(cwd, name), 1_700_000_000, 1_700_000_000);
+    });
+    yield* fs.writeFileString(path.join(cwd, ".gitattributes"), "*.dat filter=probe\n");
+    yield* fs.writeFileString(
+      path.join(cwd, ".git/filter.cjs"),
+      'require("node:fs").appendFileSync(".git/reads", "read\\n"); process.stdin.pipe(process.stdout);',
+    );
+    yield* git(["config", "filter.probe.clean", "node .git/filter.cjs"]);
+    for (const name of ["cache/stable.dat", "cache/edited.dat", "deleted.dat"]) {
+      yield* write(name, `${name}\n`);
+    }
+    const originalIndex = yield* fs.readFile(path.join(cwd, ".git/index"));
+    const checkpointIndex = path.join(cwd, ".git/t3-checkpoint-index");
+    const first = CheckpointRef.make("refs/t3/checkpoints/reuse/first");
+    const second = CheckpointRef.make("refs/t3/checkpoints/reuse/second");
+    const cold = CheckpointRef.make("refs/t3/checkpoints/reuse/cold");
+    const captureCommands = Effect.fn(function* (checkpointRef: CheckpointRef) {
+      commands.length = 0;
+      yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
+      return commands.map((args) => args.join(" "));
+    });
+
+    yield* captureCommands(first);
+    assert.isTrue(yield* fs.exists(checkpointIndex));
+    yield* write("cache/edited.dat", "edited\n");
+    yield* fs.remove(path.join(cwd, "deleted.dat"));
+    yield* fs.writeFileString(path.join(cwd, "file.txt"), "second turn\n");
+    yield* write("added.txt", "added\n");
+    yield* fs.writeFileString(path.join(cwd, ".git/reads"), "");
+    const warmCommands = yield* captureCommands(second);
+
+    assert.strictEqual(yield* fs.readFileString(path.join(cwd, ".git/reads")), "read\n");
+    assert.isFalse(warmCommands.some((command) => command.includes("read-tree --reset")));
+    for (const [name, content] of [
+      ["cache/stable.dat", "cache/stable.dat\n"],
+      ["cache/edited.dat", "edited\n"],
+      ["file.txt", "second turn\n"],
+      ["added.txt", "added\n"],
+    ]) {
+      assert.strictEqual((yield* git(["show", `${second}:${name}`])).stdout, content);
+    }
+    assert.notInclude((yield* git(["ls-tree", "-r", "--name-only", second])).stdout, "deleted");
+
+    yield* fs.remove(checkpointIndex);
+    const coldCommands = yield* captureCommands(cold);
+    assert.isTrue(coldCommands.some((command) => command.includes("read-tree --reset")));
+    assert.strictEqual(
+      (yield* git(["rev-parse", `${second}^{tree}`])).stdout,
+      (yield* git(["rev-parse", `${cold}^{tree}`])).stdout,
+    );
+    assert.deepEqual(yield* fs.readFile(path.join(cwd, ".git/index")), originalIndex);
+  }).pipe(Effect.scoped, Effect.provide(GitContractLayer)),
+);
+
+it.effect("checkpoint index reuse follows ignore rules like a fresh capture", () =>
+  Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const driver = yield* GitVcsDriver.makeVcsDriverShape();
+    const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-reuse-ignored-" });
+    const { git } = yield* makeCheckpointFixture(driver, cwd);
+    const write = (name: string) =>
+      fs
+        .makeDirectory(path.dirname(path.join(cwd, name)), { recursive: true })
+        .pipe(Effect.andThen(fs.writeFileString(path.join(cwd, name), `${name}\n`)));
+    yield* write("tracked.log");
+    yield* write("unindexed.log");
+    yield* git(["add", "tracked.log", "unindexed.log"]);
+    yield* git(["commit", "-m", "track logs"]);
+    yield* write("later.log");
+    yield* write("build/out.o");
+    const first = CheckpointRef.make("refs/t3/checkpoints/ignored/first");
+    const second = CheckpointRef.make("refs/t3/checkpoints/ignored/second");
+    const cold = CheckpointRef.make("refs/t3/checkpoints/ignored/cold");
+    const files = (checkpointRef: CheckpointRef) =>
+      git(["ls-tree", "-r", "--name-only", checkpointRef]).pipe(
+        Effect.map((result) => result.stdout.trim().split("\n")),
+      );
+
+    yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef: first });
+    assert.includeMembers(yield* files(first), ["later.log", "build/out.o", "tracked.log"]);
+    yield* fs.writeFileString(path.join(cwd, ".gitignore"), "*.log\nbuild/\n");
+    yield* write("forced.log");
+    yield* git(["add", "--force", "forced.log"]);
+    yield* git(["commit", "-m", "track an ignored log"]);
+    // A fresh capture starts from HEAD, so staged changes to ignored files do not count.
+    yield* write("staged.log");
+    yield* git(["add", "--force", "staged.log"]);
+    yield* git(["rm", "--cached", "--quiet", "unindexed.log"]);
+    yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef: second });
+
+    assert.sameMembers(yield* files(second), [
+      ".gitignore",
+      "file.txt",
+      "forced.log",
+      "tracked.log",
+      "unindexed.log",
+    ]);
+    yield* fs.remove(path.join(cwd, ".git/t3-checkpoint-index"));
+    yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef: cold });
+    assert.strictEqual(
+      (yield* git(["rev-parse", `${second}^{tree}`])).stdout,
+      (yield* git(["rev-parse", `${cold}^{tree}`])).stdout,
+    );
+  }).pipe(Effect.scoped, Effect.provide(GitContractLayer)),
+);
+
+it.effect.each(["corrupt", "pruned"] as const)(
+  "checkpoint capture starts over when its reused index is %s",
+  (state) =>
+    Effect.gen(function* () {
+      const fs = yield* FileSystem.FileSystem;
+      const path = yield* Path.Path;
+      const { driver, commands } = yield* makeRecordingDriver();
+      const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-reuse-invalid-" });
+      const { git } = yield* makeCheckpointFixture(driver, cwd);
+      yield* fs.writeFileString(path.join(cwd, "untracked.txt"), "untracked\n");
+      const checkpointIndex = path.join(cwd, ".git/t3-checkpoint-index");
+      const first = CheckpointRef.make("refs/t3/checkpoints/invalid/first");
+      const second = CheckpointRef.make("refs/t3/checkpoints/invalid/second");
+      yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef: first });
+      if (state === "corrupt") {
+        yield* fs.writeFileString(checkpointIndex, "invalid index");
+      } else {
+        // Nothing else references the untracked blob, so Git prunes it from under the index.
+        yield* git(["update-ref", "-d", first]);
+        yield* git(["prune", "--expire=now"]);
+      }
+      commands.length = 0;
+
+      yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef: second });
+
+      assert.isTrue(commands.some((args) => args.join(" ").includes("read-tree --reset")));
+      assert.strictEqual((yield* git(["show", `${second}:untracked.txt`])).stdout, "untracked\n");
+      const replaced = yield* fs.readFile(checkpointIndex);
+      assert.strictEqual(new TextDecoder().decode(replaced.subarray(0, 4)), "DIRC");
+    }).pipe(Effect.scoped, Effect.provide(GitContractLayer)),
+);
+
+it.effect("a timed-out reused checkpoint capture keeps its index and does not retry cold", () =>
+  Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const liveProcess = yield* VcsProcess.VcsProcess;
+    const commands: ReadonlyArray<string>[] = [];
+    const isStaging = (args: ReadonlyArray<string>) => args.includes("add") && args.includes("-A");
+    let timeOutStaging = false;
+    const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
+      Effect.provideService(VcsProcess.VcsProcess, {
+        run: (input) => {
+          commands.push(input.args);
+          return timeOutStaging && isStaging(input.args)
+            ? Effect.fail(
+                new VcsProcessTimeoutError({
+                  operation: input.operation,
+                  command: input.command,
+                  cwd: input.cwd,
+                  argumentCount: input.args.length,
+                  timeoutMs: input.timeoutMs ?? 0,
+                }),
+              )
+            : liveProcess.run(input);
+        },
+      }),
+    );
+    const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-reuse-timeout-" });
+    yield* makeCheckpointFixture(driver, cwd);
+    yield* fs.writeFileString(path.join(cwd, "untracked.txt"), "untracked\n");
+    const checkpointIndex = path.join(cwd, ".git/t3-checkpoint-index");
+    const first = CheckpointRef.make("refs/t3/checkpoints/timeout/first");
+    const second = CheckpointRef.make("refs/t3/checkpoints/timeout/second");
+    yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef: first });
+    const savedIndex = yield* fs.readFile(checkpointIndex);
+    timeOutStaging = true;
+    commands.length = 0;
+
+    const error = yield* driver.checkpoints
+      .captureCheckpoint({ cwd, checkpointRef: second })
+      .pipe(Effect.flip);
+
+    assert.strictEqual(error._tag, "VcsProcessTimeoutError");
+    assert.lengthOf(commands.filter(isStaging), 1);
+    assert.isFalse(commands.some((args) => args.join(" ").includes("read-tree --reset")));
+    // A timeout is the working tree's cost, not a bad index, so the saved index stays as it was.
+    assert.deepEqual(yield* fs.readFile(checkpointIndex), savedIndex);
+    assert.isFalse(yield* driver.checkpoints.hasCheckpointRef({ cwd, checkpointRef: second }));
+  }).pipe(Effect.scoped, Effect.provide(GitContractLayer)),
+);
+
+it.effect("checkpoint capture stays cold when its ignored entries overflow the listing", () =>
+  Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const liveProcess = yield* VcsProcess.VcsProcess;
+    const commands: string[] = [];
+    const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
+      Effect.provideService(VcsProcess.VcsProcess, {
+        run: (input) => {
+          commands.push(input.args.join(" "));
+          // Any listing is over a one-byte cap, as a huge one is over the real cap.
+          return liveProcess.run(
+            input.args.includes("--ignored") ? { ...input, maxOutputBytes: 1 } : input,
+          );
+        },
+      }),
+    );
+    const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-reuse-overflow-" });
+    const { git } = yield* makeCheckpointFixture(driver, cwd);
+    yield* fs.writeFileString(path.join(cwd, ".gitignore"), "*.log\n");
+    yield* fs.writeFileString(path.join(cwd, "tracked.log"), "tracked\n");
+    yield* git(["add", "--force", ".gitignore", "tracked.log"]);
+    yield* git(["commit", "-m", "track an ignored log"]);
+    yield* fs.writeFileString(path.join(cwd, "untracked.txt"), "untracked\n");
+    const checkpointIndex = path.join(cwd, ".git/t3-checkpoint-index");
+    const tree = (checkpointRef: CheckpointRef) =>
+      git(["rev-parse", `${checkpointRef}^{tree}`]).pipe(Effect.map((result) => result.stdout));
+    const first = CheckpointRef.make("refs/t3/checkpoints/overflow/first");
+    yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef: first });
+    assert.isTrue(yield* fs.exists(checkpointIndex));
+
+    for (const turn of ["second", "third"]) {
+      const checkpointRef = CheckpointRef.make(`refs/t3/checkpoints/overflow/${turn}`);
+      commands.length = 0;
+      yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
+
+      assert.strictEqual(
+        commands.some((command) => command.includes("--ignored")),
+        turn === "second",
+      );
+      assert.isTrue(commands.some((command) => command.includes("read-tree --reset")));
+      assert.strictEqual(yield* tree(checkpointRef), yield* tree(first));
+      assert.strictEqual(
+        (yield* git(["show", `${checkpointRef}:tracked.log`])).stdout,
+        "tracked\n",
+      );
+      assert.isFalse(yield* fs.exists(checkpointIndex));
+    }
+  }).pipe(Effect.scoped, Effect.provide(GitContractLayer)),
+);
+
 it.effect("restores empty checkpoints without changing paths outside the workspace", () =>
   Effect.gen(function* () {
     const fileSystem = yield* FileSystem.FileSystem;