diff --git a/packages/core/src/integration.ts b/packages/core/src/integration.ts index 055f90ca7046..5df2884f64b3 100644 --- a/packages/core/src/integration.ts +++ b/packages/core/src/integration.ts @@ -409,8 +409,12 @@ export const locationLayer = Layer.effect( pools.set(id, entryPool) } const selected = entryPool.pool.next(yield* Clock.currentTimeMillis) - if (!selected) return + if (!selected) { + console.log(`[credential-pool] ${id} owner=${owner}: no available credential (all on cooldown or reserved)`) + return + } const connection = connections.find((connection) => connection.type === "credential" && connection.id === selected.id) + console.log(`[credential-pool] ${id} owner=${owner}: selected ${selected.id.slice(0, 12)}...`) if (owner === "legacy") entryPool.pool.release(selected.id) else selection.set(ownerKey, Credential.ID.make(selected.id)) return connection @@ -418,6 +422,7 @@ export const locationLayer = Layer.effect( cooldown: Effect.fn("Integration.connection.cooldown")(function* (id, owner, retryAfterMs = 0) { const selected = selection.get(`${id}:${owner}`) if (!selected) return + console.log(`[credential-pool] ${id} owner=${owner}: penalizing ${selected.slice(0, 12)}... retryAfterMs=${retryAfterMs}`) pools.get(id)?.pool.penalize(selected, retryAfterMs, yield* Clock.currentTimeMillis) }), release: Effect.fn("Integration.connection.release")(function* (id, owner) { diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index b186396ed0bc..1205ecd85d14 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -131,10 +131,16 @@ const layer = Layer.effect( const switchProvider = Effect.fnUntraced(function* (session: SessionSchema.Info, model: Model, error: LLMError) { const destinations = CredentialFailover.fallbackRoutes(model.provider) - if (!CredentialFailover.eligible(error) || destinations.length === 0) return false + const isEligible = CredentialFailover.eligible(error) + console.log(`[failover] provider=${model.provider} eligible=${isEligible} destinations=[${destinations.join(",")}] error=${error.reason?.message?.slice(0, 100) ?? "unknown"}`) + if (!isEligible || destinations.length === 0) return false const attempted = failoverRoutes.get(session.id) ?? new Set([model.provider]) const destination = destinations.find((candidate) => !attempted.has(candidate)) - if (!destination) return false + if (!destination) { + console.log(`[failover] no untried destination remaining (attempted: ${[...attempted].join(",")})`) + return false + } + console.log(`[failover] switching from ${model.provider} to ${destination}`) attempted.add(destination) failoverRoutes.set(session.id, attempted) const available = yield* catalog.model.available() diff --git a/packages/opencode/src/auth/index.ts b/packages/opencode/src/auth/index.ts index b3221d445ed1..66c22c9f02a6 100644 --- a/packages/opencode/src/auth/index.ts +++ b/packages/opencode/src/auth/index.ts @@ -76,7 +76,9 @@ const layer = Layer.effect( if (keys.length === 0) return info const index = rotations.get(providerID) ?? 0 rotations.set(providerID, (index + 1) % keys.length) - return new Api({ ...info, key: keys[index % keys.length] }) + const selectedKey = keys[index % keys.length] + console.log(`[key-rotation] ${providerID} using key ${index % keys.length + 1}/${keys.length}: ${selectedKey.slice(0, 6)}...${selectedKey.slice(-4)}`) + return new Api({ ...info, key: selectedKey }) }) const set = Effect.fn("Auth.set")(function* (key: string, info: Info) { diff --git a/packages/opencode/src/session/retry.ts b/packages/opencode/src/session/retry.ts index 61cb294a4a89..af44daefe6f9 100644 --- a/packages/opencode/src/session/retry.ts +++ b/packages/opencode/src/session/retry.ts @@ -186,6 +186,7 @@ export function policy(opts: { const retry = retryable(error, opts.provider) const eligible = (opts.provider === "opencode" || opts.provider === "opencode-go") && CredentialFailover.eligible(error) + console.log(`[retry] provider=${opts.provider} attempt=${meta.attempt} retry=${!!retry} eligible=${eligible} statusCode=${(error as any)?.data?.statusCode} message=${(error as any)?.data?.message?.slice(0, 120) ?? "none"}`) if (!retry && !eligible) return Cause.done(meta.attempt) const before = opts.beforeRetry ? opts.beforeRetry(error) : Effect.succeed(false) return before.pipe(