fix(server): back off failed git status refreshes

apps/server/src/git/Layers/GitCore.test.ts

@@ -3,7 +3,8 @@ import path from "node:path";
 
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import { it } from "@effect/vitest";
-import { Effect, FileSystem, Layer, PlatformError, Scope } from "effect";
+import { Duration, Effect, FileSystem, Layer, PlatformError, Scope } from "effect";
+import { TestClock } from "effect/testing";
 import { describe, expect, vi } from "vitest";
 
 import { GitCoreLive, makeGitCore } from "./GitCore.ts";
@@ -1713,6 +1714,51 @@ it.layer(TestLayer)("git integration", (it) => {
       }),
     );
 
+    it.effect("backs off upstream auto-refresh after a failed status fetch", () =>
+      Effect.gen(function* () {
+        const remote = yield* makeTmpDir();
+        const source = yield* makeTmpDir();
+        yield* git(remote, ["init", "--bare"]);
+
+        yield* initRepoWithCommit(source);
+        const initialBranch = (yield* (yield* GitCore).listBranches({
+          cwd: source,
+        })).branches.find((branch) => branch.current)!.name;
+        yield* git(source, ["remote", "add", "origin", remote]);
+        yield* git(source, ["push", "-u", "origin", initialBranch]);
+
+        const realGitCore = yield* GitCore;
+        let refreshFetchAttempts = 0;
+        const core = yield* makeIsolatedGitCore((input) => {
+          if (input.args[0] === "--git-dir" && input.args[2] === "fetch") {
+            refreshFetchAttempts += 1;
+            return Effect.succeed({
+              code: 128,
+              stdout: "",
+              stderr: "simulated fetch timeout",
+              stdoutTruncated: false,
+              stderrTruncated: false,
+            });
+          }
+          return realGitCore.execute(input);
+        });
+
+        const firstStatus = yield* core.statusDetails(source);
+        expect(firstStatus.branch).toBe(initialBranch);
+        expect(refreshFetchAttempts).toBe(1);
+
+        yield* TestClock.adjust(Duration.seconds(15));
+        const secondStatus = yield* core.statusDetails(source);
+        expect(secondStatus.branch).toBe(initialBranch);
+        expect(refreshFetchAttempts).toBe(1);
+
+        yield* TestClock.adjust(Duration.minutes(10));
+        const thirdStatus = yield* core.statusDetails(source);
+        expect(thirdStatus.branch).toBe(initialBranch);
+        expect(refreshFetchAttempts).toBe(2);
+      }),
+    );
+
     it.effect("computes ahead count against base branch when no upstream is configured", () =>
       Effect.gen(function* () {
         const tmp = yield* makeTmpDir();

apps/server/src/git/Layers/GitCore.ts

@@ -56,7 +56,7 @@ const WORKSPACE_GIT_HARDENED_CONFIG_ARGS = [
 ] as const;
 const STATUS_UPSTREAM_REFRESH_INTERVAL = Duration.seconds(15);
 const STATUS_UPSTREAM_REFRESH_TIMEOUT = Duration.seconds(5);
-const STATUS_UPSTREAM_REFRESH_FAILURE_COOLDOWN = Duration.seconds(5);
+const STATUS_UPSTREAM_REFRESH_FAILURE_COOLDOWN = Duration.minutes(10);
 const STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY = 2_048;
 const DEFAULT_BASE_BRANCH_CANDIDATES = ["main", "master"] as const;
 const GIT_LIST_BRANCHES_DEFAULT_LIMIT = 100;
@@ -931,7 +931,23 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {
         allowNonZeroExit: true,
         timeoutMs: Duration.toMillis(STATUS_UPSTREAM_REFRESH_TIMEOUT),
       },
-    ).pipe(Effect.asVoid);
+    ).pipe(
+      Effect.flatMap((result) => {
+        if (result.code === 0) {
+          return Effect.void;
+        }
+
+        const stderr = result.stderr.trim();
+        return Effect.fail(
+          createGitCommandError(
+            "GitCore.fetchRemoteForStatus",
+            fetchCwd,
+            ["--git-dir", gitCommonDir, "fetch", "--quiet", "--no-tags", remoteName],
+            stderr.length > 0 ? stderr : "git fetch failed",
+          ),
+        );
+      }),
+    );
   };
 
   const resolveGitCommonDir = Effect.fn("resolveGitCommonDir")(function* (cwd: string) {
@@ -951,7 +967,7 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {
 
   const statusRemoteRefreshCache = yield* Cache.makeWith(refreshStatusRemoteCacheEntry, {
     capacity: STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY,
-    // Keep successful refreshes warm and briefly back off failed refreshes to avoid retry storms.
+    // Keep successful refreshes warm and back off failed auto-refreshes to avoid retry storms.
     timeToLive: (exit) =>
       Exit.isSuccess(exit)
         ? STATUS_UPSTREAM_REFRESH_INTERVAL