From 1c865b8b577d9d3992f43448b74ee64c969c3f32 Mon Sep 17 00:00:00 2001 From: feruzm Date: Fri, 21 Aug 2026 16:55:27 +0000 Subject: [PATCH 1/2] rpc pool: park only a node that is not answering, never on overlapping timeouts of a healthy one Closes #78 --- .../EcencyApi.Tests/HiveRpcFailoverTests.cs | 51 +++++++++++++++++++ .../EcencyApi/Infrastructure/HiveRpcClient.cs | 1 + .../Infrastructure/NodeHealthTracker.cs | 19 +++++-- 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs b/dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs index 6a381e9d..a7b7a848 100644 --- a/dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs +++ b/dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs @@ -345,6 +345,57 @@ public async Task ANodeOutOfItsPark_AdmitsOneProbeAtATime() Assert.Equal(4, client.HealthSnapshot()[0]!["calls"]!.GetValue()); } + [Fact] + public async Task AHealthyNodeWithOverlappingTimeouts_IsNotParked() + { + // Production shape: the node that answers ~98% of calls has three + // heavy queries time out at once. Three "consecutive" failures, yet it + // is answering everything else; parking it would push the load onto + // weaker nodes. Only a node that is not answering gets parked. + var hang = false; + await using var busy = new StubNode(() => hang ? -1 : 200); + await using var spare = new StubNode(() => 200); + long now = 0; + var client = new HiveRpcClient(new[] { busy.Url, spare.Url }, timeoutMs: 200, failoverThreshold: 1, clock: () => now); + + for (var i = 0; i < 40; i++) + { + await client.Call("condenser_api", "get_accounts", new JsonArray()); + } + // Three overlapping timeouts (sequential here is the strictest form of + // "consecutive"; the stub's single-threaded loop makes them serial). + hang = true; + for (var i = 0; i < 3; i++) + { + now += 31_000; // past the recent-failure demotion, so it is retried + await client.Call("condenser_api", "get_accounts", new JsonArray()); + } + hang = false; + var view = client.HealthSnapshot()[0]!; + Assert.Equal(3, view["consecutive_failures"]!.GetValue()); + Assert.Equal(0, view["parked_for_ms"]!.GetValue()); + Assert.True(view["failure_rate"]!.GetValue() < 0.5); + + // With an alternative available the ranking stops trying it, so its + // fraction can only climb where it keeps being tried: a node that is the + // only option and keeps failing crosses the fraction and is parked. + var dying = false; + await using var only = new StubNode(() => dying ? -1 : 200); + var lone = new HiveRpcClient(new[] { only.Url }, timeoutMs: 200, failoverThreshold: 1, clock: () => now); + for (var i = 0; i < 10; i++) + { + await lone.Call("condenser_api", "get_accounts", new JsonArray()); + } + dying = true; + for (var i = 0; i < 8; i++) + { + await Assert.ThrowsAnyAsync(() => lone.Call("condenser_api", "get_accounts", new JsonArray())); + } + view = lone.HealthSnapshot()[0]!; + Assert.True(view["failure_rate"]!.GetValue() >= 0.5); + Assert.True(view["parked_for_ms"]!.GetValue() > 0); + } + [Fact] public async Task RateLimits_DoNotCountTowardFailureParking() { diff --git a/dotnet/EcencyApi/Infrastructure/HiveRpcClient.cs b/dotnet/EcencyApi/Infrastructure/HiveRpcClient.cs index 8678bb70..55ab4ba3 100644 --- a/dotnet/EcencyApi/Infrastructure/HiveRpcClient.cs +++ b/dotnet/EcencyApi/Infrastructure/HiveRpcClient.cs @@ -67,6 +67,7 @@ public JsonArray HealthSnapshot() ["recent_failure"] = v.RecentFailure, ["rate_limited_for_ms"] = v.RateLimitedForMs, ["parked_for_ms"] = v.FailureParkedForMs, + ["failure_rate"] = Math.Round(v.FailureRate, 3), }); } return arr; diff --git a/dotnet/EcencyApi/Infrastructure/NodeHealthTracker.cs b/dotnet/EcencyApi/Infrastructure/NodeHealthTracker.cs index c4395ad0..b984b998 100644 --- a/dotnet/EcencyApi/Infrastructure/NodeHealthTracker.cs +++ b/dotnet/EcencyApi/Infrastructure/NodeHealthTracker.cs @@ -40,6 +40,13 @@ public sealed class NodeHealthTracker private const int FailureParkThreshold = 3; private const int FailureParkBaseMs = 30_000; private const int FailureParkMaxMs = 120_000; + // Under concurrency "consecutive" is not "sequential": with hundreds of + // calls in flight, three overlapping timeouts of a heavy query satisfy the + // count while the node is answering everything else. Parking therefore + // also needs the node to have never answered, or to be failing most of its + // recent calls (an EWMA of the failure fraction, alpha 0.1). + private const double FailureRateAlpha = 0.1; + private const double FailureRateParkFloor = 0.5; private sealed class NodeHealth { @@ -50,6 +57,8 @@ private sealed class NodeHealth public long LastRateLimitAtMs; public long FailureParkedUntilMs; public int FailureParkStreak; + // Recent failure fraction (1 = every recent call failed). + public double FailureRate; // Hard failures only (timeouts, refusals, bad answers), never 429s: a // throttled node is responsive and has its own parking. public int ConsecutiveHardFailures; @@ -67,7 +76,7 @@ private sealed class NodeHealth public sealed record NodeView( int Index, long Calls, long Successes, long Failures, long Timeouts, long RateLimits, double? EwmaLatencyMs, int LatencySamples, int ConsecutiveFailures, - bool RecentFailure, long RateLimitedForMs, long FailureParkedForMs); + bool RecentFailure, long RateLimitedForMs, long FailureParkedForMs, double FailureRate); private readonly NodeHealth[] _health; private readonly object _lock = new(); @@ -94,6 +103,7 @@ public void RecordSuccess(int nodeIndex, double elapsedMs) var h = _health[nodeIndex]; h.Calls++; h.Successes++; + h.FailureRate *= 1 - FailureRateAlpha; h.ConsecutiveFailures = 0; h.ConsecutiveHardFailures = 0; h.RateLimitStreak = 0; @@ -122,6 +132,7 @@ public bool RecordFailure(int nodeIndex, double elapsedMs, bool timedOut = false h.Calls++; h.Failures++; if (timedOut) h.Timeouts++; + h.FailureRate = h.FailureRate * (1 - FailureRateAlpha) + FailureRateAlpha; h.ConsecutiveFailures++; h.ConsecutiveHardFailures++; h.LastFailureAtMs = now; @@ -136,7 +147,8 @@ public bool RecordFailure(int nodeIndex, double elapsedMs, bool timedOut = false { RecordLatency(h, elapsedMs); } - if (h.ConsecutiveHardFailures >= FailureParkThreshold) + var notAnswering = h.Successes == 0 || h.FailureRate >= FailureRateParkFloor; + if (h.ConsecutiveHardFailures >= FailureParkThreshold && notAnswering) { var parkMs = Math.Min(FailureParkBaseMs << Math.Min(h.FailureParkStreak, 2), FailureParkMaxMs); h.FailureParkedUntilMs = now + parkMs; @@ -284,7 +296,8 @@ public List Snapshot() return new NodeView(i, h.Calls, h.Successes, h.Failures, h.Timeouts, h.RateLimits, h.EwmaLatencyMs, h.LatencySampleCount, h.ConsecutiveFailures, h.ConsecutiveFailures > 0 && now - h.LastFailureAtMs < RecentFailureWindowMs, - Math.Max(0, h.RateLimitedUntilMs - now), Math.Max(0, h.FailureParkedUntilMs - now)); + Math.Max(0, h.RateLimitedUntilMs - now), Math.Max(0, h.FailureParkedUntilMs - now), + h.FailureRate); }).ToList(); } } From 0ed5f6366f86de3f841ee5997c12194460182a44 Mon Sep 17 00:00:00 2001 From: feruzm Date: Fri, 21 Aug 2026 17:01:51 +0000 Subject: [PATCH 2/2] review: the three timeouts in the test overlap --- dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs b/dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs index a7b7a848..4b0d38e3 100644 --- a/dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs +++ b/dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs @@ -362,14 +362,12 @@ public async Task AHealthyNodeWithOverlappingTimeouts_IsNotParked() { await client.Call("condenser_api", "get_accounts", new JsonArray()); } - // Three overlapping timeouts (sequential here is the strictest form of - // "consecutive"; the stub's single-threaded loop makes them serial). + // Three overlapping timeouts: the calls start together, each holding an + // ordering with the busy node first, and all three fail on it at the + // same time before failing over to the spare. hang = true; - for (var i = 0; i < 3; i++) - { - now += 31_000; // past the recent-failure demotion, so it is retried - await client.Call("condenser_api", "get_accounts", new JsonArray()); - } + await Task.WhenAll(Enumerable.Range(0, 3) + .Select(_ => client.Call("condenser_api", "get_accounts", new JsonArray()))); hang = false; var view = client.HealthSnapshot()[0]!; Assert.Equal(3, view["consecutive_failures"]!.GetValue());