RPC pool: park only a node that is not answering

dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs

@@ -345,6 +345,55 @@ public async Task ANodeOutOfItsPark_AdmitsOneProbeAtATime()
         Assert.Equal(4, client.HealthSnapshot()[0]!["calls"]!.GetValue<long>());
     }
 
+    [Fact]
+    public async Task AHealthyNodeWithOverlappingTimeouts_IsNotParked()
+    {
+        // Production shape: the node that answers ~98% of calls has three
+        // heavy queries time out at once. Three "consecutive" failures, yet it
+        // is answering everything else; parking it would push the load onto
+        // weaker nodes. Only a node that is not answering gets parked.
+        var hang = false;
+        await using var busy = new StubNode(() => hang ? -1 : 200);
+        await using var spare = new StubNode(() => 200);
+        long now = 0;
+        var client = new HiveRpcClient(new[] { busy.Url, spare.Url }, timeoutMs: 200, failoverThreshold: 1, clock: () => now);
+
+        for (var i = 0; i < 40; i++)
+        {
+            await client.Call("condenser_api", "get_accounts", new JsonArray());
+        }
+        // Three overlapping timeouts: the calls start together, each holding an
+        // ordering with the busy node first, and all three fail on it at the
+        // same time before failing over to the spare.
+        hang = true;
+        await Task.WhenAll(Enumerable.Range(0, 3)
+            .Select(_ => client.Call("condenser_api", "get_accounts", new JsonArray())));
+        hang = false;
+        var view = client.HealthSnapshot()[0]!;
+        Assert.Equal(3, view["consecutive_failures"]!.GetValue<int>());
+        Assert.Equal(0, view["parked_for_ms"]!.GetValue<long>());
+        Assert.True(view["failure_rate"]!.GetValue<double>() < 0.5);
+
+        // With an alternative available the ranking stops trying it, so its
+        // fraction can only climb where it keeps being tried: a node that is the
+        // only option and keeps failing crosses the fraction and is parked.
+        var dying = false;
+        await using var only = new StubNode(() => dying ? -1 : 200);
+        var lone = new HiveRpcClient(new[] { only.Url }, timeoutMs: 200, failoverThreshold: 1, clock: () => now);
+        for (var i = 0; i < 10; i++)
+        {
+            await lone.Call("condenser_api", "get_accounts", new JsonArray());
+        }
+        dying = true;
+        for (var i = 0; i < 8; i++)
+        {
+            await Assert.ThrowsAnyAsync<Exception>(() => lone.Call("condenser_api", "get_accounts", new JsonArray()));
+        }
+        view = lone.HealthSnapshot()[0]!;
+        Assert.True(view["failure_rate"]!.GetValue<double>() >= 0.5);
+        Assert.True(view["parked_for_ms"]!.GetValue<long>() > 0);
+    }
+
     [Fact]
     public async Task RateLimits_DoNotCountTowardFailureParking()
     {

dotnet/EcencyApi/Infrastructure/HiveRpcClient.cs

@@ -67,6 +67,7 @@ public JsonArray HealthSnapshot()
                 ["recent_failure"] = v.RecentFailure,
                 ["rate_limited_for_ms"] = v.RateLimitedForMs,
                 ["parked_for_ms"] = v.FailureParkedForMs,
+                ["failure_rate"] = Math.Round(v.FailureRate, 3),
             });
         }
         return arr;

dotnet/EcencyApi/Infrastructure/NodeHealthTracker.cs

@@ -40,6 +40,13 @@ public sealed class NodeHealthTracker
     private const int FailureParkThreshold = 3;
     private const int FailureParkBaseMs = 30_000;
     private const int FailureParkMaxMs = 120_000;
+    // Under concurrency "consecutive" is not "sequential": with hundreds of
+    // calls in flight, three overlapping timeouts of a heavy query satisfy the
+    // count while the node is answering everything else. Parking therefore
+    // also needs the node to have never answered, or to be failing most of its
+    // recent calls (an EWMA of the failure fraction, alpha 0.1).
+    private const double FailureRateAlpha = 0.1;
+    private const double FailureRateParkFloor = 0.5;
 
     private sealed class NodeHealth
     {
@@ -50,6 +57,8 @@ private sealed class NodeHealth
         public long LastRateLimitAtMs;
         public long FailureParkedUntilMs;
         public int FailureParkStreak;
+        // Recent failure fraction (1 = every recent call failed).
+        public double FailureRate;
         // Hard failures only (timeouts, refusals, bad answers), never 429s: a
         // throttled node is responsive and has its own parking.
         public int ConsecutiveHardFailures;
@@ -67,7 +76,7 @@ private sealed class NodeHealth
     public sealed record NodeView(
         int Index, long Calls, long Successes, long Failures, long Timeouts, long RateLimits,
         double? EwmaLatencyMs, int LatencySamples, int ConsecutiveFailures,
-        bool RecentFailure, long RateLimitedForMs, long FailureParkedForMs);
+        bool RecentFailure, long RateLimitedForMs, long FailureParkedForMs, double FailureRate);
 
     private readonly NodeHealth[] _health;
     private readonly object _lock = new();
@@ -94,6 +103,7 @@ public void RecordSuccess(int nodeIndex, double elapsedMs)
             var h = _health[nodeIndex];
             h.Calls++;
             h.Successes++;
+            h.FailureRate *= 1 - FailureRateAlpha;
             h.ConsecutiveFailures = 0;
             h.ConsecutiveHardFailures = 0;
             h.RateLimitStreak = 0;
@@ -122,6 +132,7 @@ public bool RecordFailure(int nodeIndex, double elapsedMs, bool timedOut = false
             h.Calls++;
             h.Failures++;
             if (timedOut) h.Timeouts++;
+            h.FailureRate = h.FailureRate * (1 - FailureRateAlpha) + FailureRateAlpha;
             h.ConsecutiveFailures++;
             h.ConsecutiveHardFailures++;
             h.LastFailureAtMs = now;
@@ -136,7 +147,8 @@ public bool RecordFailure(int nodeIndex, double elapsedMs, bool timedOut = false
             {
                 RecordLatency(h, elapsedMs);
             }
-            if (h.ConsecutiveHardFailures >= FailureParkThreshold)
+            var notAnswering = h.Successes == 0 || h.FailureRate >= FailureRateParkFloor;
+            if (h.ConsecutiveHardFailures >= FailureParkThreshold && notAnswering)
             {
                 var parkMs = Math.Min(FailureParkBaseMs << Math.Min(h.FailureParkStreak, 2), FailureParkMaxMs);
                 h.FailureParkedUntilMs = now + parkMs;
@@ -284,7 +296,8 @@ public List<NodeView> Snapshot()
                 return new NodeView(i, h.Calls, h.Successes, h.Failures, h.Timeouts, h.RateLimits,
                     h.EwmaLatencyMs, h.LatencySampleCount, h.ConsecutiveFailures,
                     h.ConsecutiveFailures > 0 && now - h.LastFailureAtMs < RecentFailureWindowMs,
-                    Math.Max(0, h.RateLimitedUntilMs - now), Math.Max(0, h.FailureParkedUntilMs - now));
+                    Math.Max(0, h.RateLimitedUntilMs - now), Math.Max(0, h.FailureParkedUntilMs - now),
+                    h.FailureRate);
             }).ToList();
         }
     }