SSR RPC cache: no replacement fill after the lookup's deadline

dotnet/EcencyApi.Tests/SsrRpcTests.cs

@@ -87,6 +87,7 @@
         SsrRpc.FillGate = new SemaphoreSlim(maxFills, maxFills);
         SsrRpc.MaxQueuedFills = maxQueued;
         SsrRpc.SecretDigest = null;
+        SsrRpc.Now = () => Environment.TickCount64;
         SsrRpc.ResetForTests();
     }
 
@@ -101,7 +102,7 @@
         var tasks = Enumerable.Range(0, 12).Select(_ => SsrRpc.Resolve(Post, P("a", "b"))).ToArray();
         var results = await Task.WhenAll(tasks);
         Assert.Equal(1, stub.Hits);
         Assert.Single(results.Where(r => r.Outcome == SsrRpc.Outcome.Miss));
         Assert.Equal(11, results.Count(r => r.Outcome == SsrRpc.Outcome.Coalesced));
         var bodies = results.Select(r => Encoding.UTF8.GetString(r.Bytes)).Distinct().ToArray();
         Assert.Single(bodies);
@@ -314,8 +315,8 @@
         await Task.Delay(180);
         var c = SsrRpc.Resolve(Post, P("k", "2"));   // fresh reader at ~190ms, coalesces
         await Task.WhenAll(a, b, c);
         Assert.Equal(SsrRpc.Outcome.Timeout, a.Result.Outcome);
         Assert.Equal(SsrRpc.Outcome.Timeout, b.Result.Outcome);
         // Judged by the fresh attach, not by the original enqueue: the fill ran.
         await Task.Delay(500);
         Assert.Equal(2, stub.Hits);
@@ -323,6 +324,38 @@
         Assert.Equal(SsrRpc.Outcome.Hit, (await SsrRpc.Resolve(Post, P("k", "2"))).Outcome);
     }
 
+    [Fact]
+    public async Task A_coalesced_reader_whose_deadline_passed_before_its_fill_was_rejected_gets_timeout_and_no_replacement_fill()
+    {
+        await using var stub = new RpcStub { DelayMs = 300 };
+        Use(stub, budgetMs: 1000, maxFills: 1);
+        // A controllable clock: real time drives the stub and the waits, the
+        // clock drives the deadline and the attach/expiry bookkeeping.
+        long offset = 0;
+        SsrRpc.Now = () => Environment.TickCount64 + Interlocked.Read(ref offset);
+        var timeoutsBefore = Interlocked.Read(ref SsrRpc.CounterFor(Post.Key).Timeout);
+
+        var a = SsrRpc.Resolve(Post, P("d", "1"));   // holds the gate for ~300ms
+        await Task.Delay(30);
+        var creator = SsrRpc.Resolve(Post, P("d", "2")); // queued fill, waits for the gate
+        await Task.Delay(30);
+        var late = SsrRpc.Resolve(Post, P("d", "2"));    // coalesces onto it, deadline = now + 1000
+        await Task.Delay(30);
+        // Jump the clock past every deadline and past the attach window, while
+        // the readers' real waits (1000ms) are still running.
+        Interlocked.Exchange(ref offset, 1_500);
+        // The gate frees at ~300ms real; the queued fill is then judged expired.
+        var results = await Task.WhenAll(a, creator, late);
+
+        Assert.Equal(SsrRpc.Outcome.Miss, results[0].Outcome);
+        Assert.Equal(SsrRpc.Outcome.Unavailable, results[1].Outcome); // the creator is not coalesced
+        Assert.Equal(SsrRpc.Outcome.Timeout, results[2].Outcome);     // past its deadline: timeout, no retry
+        Assert.Equal(timeoutsBefore + 1, Interlocked.Read(ref SsrRpc.CounterFor(Post.Key).Timeout));
+        await Task.Delay(400);
+        Assert.Equal(1, stub.Hits); // no replacement fill went upstream
+        SsrRpc.Now = () => Environment.TickCount64;
+    }
+
     [Fact]
     public async Task With_the_secret_configured_both_routes_serve_the_matching_header_and_nothing_else()
     {

dotnet/EcencyApi/Handlers/SsrRpc.cs

@@ -77,6 +77,10 @@ internal sealed record MethodPolicy(string Api, string Method, int TtlMs)
 
     internal static int BudgetMs = Config.SsrBudgetMs;
 
+    // Clock behind the lookup deadline and the attach/expiry timestamps;
+    // replaceable so tests can drive the post-deadline paths deterministically.
+    internal static Func<long> Now = () => Environment.TickCount64;
+
     // Bounds detached fills: a fill outlives the request budget on purpose, so
     // a slow pool plus many distinct keys must not pile up unbounded calls.
     internal static SemaphoreSlim FillGate = new(Config.SsrMaxConcurrentFills, Config.SsrMaxConcurrentFills);
@@ -95,15 +99,15 @@ private sealed class Pending
         // the expiry decision both happen under `lock (this)`, so a reader can
         // never attach to a fill that has just decided to expire: it finds
         // Expired set and starts a fresh fill instead.
-        public long LastAttachMs = Environment.TickCount64;
+        public long LastAttachMs = Now();
         public bool Expired;
 
         public bool TryAttach()
         {
             lock (this)
             {
                 if (Expired) return false;
-                LastAttachMs = Environment.TickCount64;
+                LastAttachMs = Now();
                 return true;
             }
         }
@@ -112,7 +116,7 @@ public bool TryExpire(int budgetMs)
         {
             lock (this)
             {
-                if (Environment.TickCount64 - LastAttachMs <= budgetMs) return false;
+                if (Now() - LastAttachMs <= budgetMs) return false;
                 Expired = true;
                 return true;
             }
@@ -216,7 +220,7 @@ internal static async Task<Resolution> Resolve(MethodPolicy policy, JsonNode @pa
         }
 
         // One wall-clock budget for the whole lookup, retry included.
-        var deadline = Environment.TickCount64 + BudgetMs;
+        var deadline = Now() + BudgetMs;
         var retried = false;
     again:
         var coalesced = true;
@@ -255,7 +259,7 @@ internal static async Task<Resolution> Resolve(MethodPolicy policy, JsonNode @pa
         if (coalesced) Interlocked.Increment(ref counter.Coalesced);
         else Interlocked.Increment(ref counter.Miss);
 
-        var remaining = (int)Math.Max(0, deadline - Environment.TickCount64);
+        var remaining = (int)Math.Max(0, deadline - Now());
         var finished = await Task.WhenAny(pending.Task, Task.Delay(remaining));
         if (!ReferenceEquals(finished, pending.Task))
         {
@@ -268,15 +272,23 @@ internal static async Task<Resolution> Resolve(MethodPolicy policy, JsonNode @pa
             var bytes = await pending.Task;
             return new Resolution(coalesced ? Outcome.Coalesced : Outcome.Miss, bytes, null);
         }
-        catch (FillRejectedException) when (coalesced && !retried)
+        catch (FillRejectedException) when (coalesced && !retried && Now() < deadline)
         {
             // The fill this reader attached to was judged expired (or refused) before
             // the reader's own wait began, which can only happen if the reader was
             // descheduled for longer than the budget between attaching and waiting.
-            // Its budget has not been spent on anything yet, so start over once.
+            // While its deadline has not passed, start over once; past it, the
+            // lookup is a timeout and no replacement fill is started for it.
             retried = true;
             goto again;
         }
+        catch (FillRejectedException) when (coalesced && Now() >= deadline)
+        {
+            // Past the deadline the lookup is a timeout; a repeat rejection
+            // before it falls through to the generic unavailable path below.
+            Interlocked.Increment(ref counter.Timeout);
+            return new Resolution(Outcome.Timeout, Array.Empty<byte>(), "budget exceeded");
+        }
         catch (HiveRpcClient.RpcException e)
         {
             Interlocked.Increment(ref counter.Error);