From 0a67196a872404a8933fb1bfb2b31c7873880821 Mon Sep 17 00:00:00 2001
From: Muhannad Al-Khatib <mmalkhatib@hotmail.com>
Date: Fri, 28 Aug 2026 12:38:23 +0300
Subject: [PATCH] Bound pooled connection lifetime to stop stale-connection
 failures

AddApiClient's AddHttpClient<...>() calls never configured a
SocketsHttpHandler, so PooledConnectionLifetime defaulted to
Timeout.InfiniteTimeSpan - a pooled connection is only ever recycled
by HttpClientFactory's own handler rotation (every 2 minutes by
default), which doesn't help a connection that's mid-use or was
recently reused. A connection that goes stale on the network side
(NAT/conntrack entry expiring, a load balancer dropping an idle
socket) fails silently on its next reuse instead of erroring where the
network actually dropped it.

Confirmed live in production: Ship's calls to Lookups (traceType
lookups via ProcessAddedShipmentTrace) were failing with StatusCode=0
at high volume, while a fresh raw HTTP request to the exact same
endpoint from the exact same pod succeeded in 10-20ms every time -
ruling out the app, the query, and the network path itself, and
pointing squarely at a stale pooled connection.

Bounding PooledConnectionLifetime to 2 minutes forces a fresh
connection (and fresh DNS resolution) at least that often, so a
connection can never go stale for longer than this window before
being discarded and replaced.
---
 .../IServiceCollectionExtensions.cs           | 32 +++++++++++++++----
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/SW.HttpExtensions/IServiceCollectionExtensions.cs b/SW.HttpExtensions/IServiceCollectionExtensions.cs
index 7687254..b130209 100644
--- a/SW.HttpExtensions/IServiceCollectionExtensions.cs
+++ b/SW.HttpExtensions/IServiceCollectionExtensions.cs
@@ -2,6 +2,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using System;
 using System.Collections.Generic;
+using System.Net.Http;
 using System.Runtime.CompilerServices;
 using System.Text;
 
@@ -9,6 +10,23 @@ namespace SW.HttpExtensions
 {
     public static class IServiceCollectionExtensions
     {
+        // IHttpClientFactory pools connections inside each SocketsHttpHandler indefinitely by
+        // default (PooledConnectionLifetime is Timeout.InfiniteTimeSpan) - a connection is only
+        // ever recycled if the *handler* itself gets rotated (HttpClientFactory does that every 2
+        // minutes by default, but a handler with in-flight or recently-used connections can live
+        // far longer than that). A connection that goes stale server-side - the remote pod
+        // restarting, a NAT/conntrack entry expiring, a load balancer dropping an idle socket -
+        // fails silently on its next reuse instead of erroring at the point the underlying
+        // network dropped it. Bounding PooledConnectionLifetime forces a fresh connection (and
+        // fresh DNS resolution) at least this often, so a connection can never go stale for
+        // longer than this window before it is discarded and replaced.
+        private static readonly TimeSpan PooledConnectionLifetime = TimeSpan.FromMinutes(2);
+
+        private static void ConfigureHandler(IHttpClientBuilder builder) =>
+            builder.ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler
+            {
+                PooledConnectionLifetime = PooledConnectionLifetime
+            });
 
         public static IServiceCollection AddJwtTokenParameters(this IServiceCollection serviceCollection, Action<JwtTokenParameters> configure = null)
         {
@@ -37,10 +55,10 @@ public static IServiceCollection AddApiClient<TInterface, TImplementation, TImpl
                 serviceCollection.AddTransient<TInterface, TImplementationMock>();
 
             else
-                serviceCollection.AddHttpClient<TInterface, TImplementation>(httpClient =>
+                ConfigureHandler(serviceCollection.AddHttpClient<TInterface, TImplementation>(httpClient =>
                 {
                     httpClient.BaseAddress = new Uri(clientOptions.BaseUrl);
-                });
+                }));
 
             return serviceCollection;
         }
@@ -53,10 +71,10 @@ public static IServiceCollection AddApiClient<TInterface, TImplementation, TOpti
 
             var clientOptions = serviceCollection.AddApiClientInternal(configure);
 
-            serviceCollection.AddHttpClient<TInterface, TImplementation>(httpClient =>
+            ConfigureHandler(serviceCollection.AddHttpClient<TInterface, TImplementation>(httpClient =>
             {
                 httpClient.BaseAddress = new Uri(clientOptions.BaseUrl);
-            });
+            }));
 
             return serviceCollection;
         }
@@ -64,13 +82,13 @@ public static IServiceCollection AddApiClient<TInterface, TImplementation, TOpti
         public static IServiceCollection AddApiClient<TImplementation, TOptions>(this IServiceCollection serviceCollection, Action<TOptions> configure = null)
             where TOptions : ApiClientOptionsBase, new()
             where TImplementation : ApiClientBase<TOptions>
-            
+
         {
             var clientOptions = serviceCollection.AddApiClientInternal(configure);
-            serviceCollection.AddHttpClient<TImplementation>(httpClient =>
+            ConfigureHandler(serviceCollection.AddHttpClient<TImplementation>(httpClient =>
             {
                 httpClient.BaseAddress = new Uri(clientOptions.BaseUrl);
-            });
+            }));
 
             return serviceCollection;
         }