Skip to content

AgentClasses hot-reload ignores member Pool: quota-pool changes are a silent no-op #481

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AdamFrisby merged 1 commit into from
Sep 14, 2026
+310 −1
Merged

AgentClasses hot-reload ignores member Pool: quota-pool changes are a silent no-op #481

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion src/CodeyBox.Api/AgentConfigHotReload.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1097,7 +1097,19 @@ private static string SerializePricing(AgentPricingOptions opts) =>
},
JsonOpts);

private static string SerializeRouterInputs(
/// <summary>
/// Hot-reload fingerprint for the <c>AgentClasses</c> / <c>AgentInstances</c> /
/// <c>AgentScoreModifiers</c> block. Must observe every field
/// <see cref="AgentClassesConfigBuilder.Build"/> consumes — a configured field
/// missing here silently behaves as restart-required (the <c>Pool</c> failure
/// mode: the edit is accepted, no reload fires, the router keeps the old value).
/// The only intentional exclusion is
/// <see cref="TimeOfDayModifierOptions.Comment"/>, a readability annotation the
/// builder never reads. Internal for the fingerprint-coverage test, which proves
/// every settable field on the member/instance/class config POCOs is observed
/// here and fails when a new field is added without being covered.
/// </summary>
internal static string SerializeRouterInputs(
List<AgentClassOptions> classes,
List<AgentInstanceOptions> instances,
AgentScoreModifiersOptions modifiers) =>
Expand All @@ -1115,6 +1127,7 @@ private static string SerializeRouterInputs(
i.SettingsFilePath,
i.DestinationPath,
i.SandboxEnvironmentVariable,
i.Provider,
})
.OrderBy(i => i.Id, StringComparer.OrdinalIgnoreCase)
.ThenBy(i => i.Agent, StringComparer.OrdinalIgnoreCase)
Expand All @@ -1124,11 +1137,13 @@ private static string SerializeRouterInputs(
{
c.Id,
c.DisplayName,
ClaudeSession = c.ClaudeSession?.Enabled,
Members = c.Members
.Select(m => new
{
m.Agent,
m.InstanceId,
m.Pool,
m.Billing,
m.ModelId,
m.CredentialFilePath,
Expand All @@ -1137,11 +1152,13 @@ private static string SerializeRouterInputs(
m.SettingsFilePath,
m.DestinationPath,
m.SandboxEnvironmentVariable,
m.Provider,
m.QualityScore,
m.ReasoningMode,
Capabilities = m.Capabilities
.OrderBy(c => c, StringComparer.OrdinalIgnoreCase)
.ToArray(),
ClaudeSession = m.ClaudeSession?.Enabled,
})
.ToArray(),
})
Expand Down
96 changes: 96 additions & 0 deletions tests/CodeyBox.Tests/AgentConfigHotReloadTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -920,6 +920,102 @@ [new ClaudeInVmSmokeProbe()],
await coordinator.StopAsync(CancellationToken.None);
}

[Fact]
public async Task Coordinator_OnChange_MemberPoolOnlyEdit_AppliesReload()
{
// Regression: SerializeRouterInputs omitted AgentMembership.Pool, so a
// Pool-only edit produced an identical fingerprint, the reload was
// skipped, and the router kept Pool=null with no error or log line.
// Adding, changing, or removing a member's Pool must each reload.
var initial = new CodeyBoxOptions
{
AgentClasses =
[
new AgentClassOptions
{
Id = "frontier",
Members =
[
new AgentMembershipOptions
{
Agent = "claude",
Billing = "Subscription",
QualityScore = 100,
},
],
},
],
};
var monitor = new ManualOptionsMonitor<CodeyBoxOptions>(initial);
var router = new AgentClassRouter(
AgentClassesConfigBuilder.Build(initial.AgentClasses, NullLogger<AgentClassRouter>.Instance),
Array.Empty<IAgentQuotaProbe>(),
new QuotaRouterOptions { MinQuotaPct = 5.0 },
NullLogger<AgentClassRouter>.Instance);
using var orchFixture = OrchestratorFixture.Build(new AgentConcurrencyOptions());
var burnEstimator = new AgentBurnEstimator(
new InertCostStore(), new AgentBurnEstimatorOptions(),
NullLogger<AgentBurnEstimator>.Instance);

var coordinator = new AgentConfigHotReload(
monitor, orchFixture.Orchestrator, router, burnEstimator,
NullLogger<AgentConfigHotReload>.Instance);
await coordinator.StartAsync(CancellationToken.None);

Assert.Null(router.GetClassMembers("frontier")[0].Pool);

monitor.Fire(new CodeyBoxOptions
{
AgentClasses =
[
new AgentClassOptions
{
Id = "frontier",
Members =
[
new AgentMembershipOptions
{
Agent = "claude",
Billing = "Subscription",
QualityScore = 100,
Pool = "opencode-go",
},
],
},
],
});

var added = router.GetClassMembers("frontier");
Assert.Single(added);
Assert.Equal("opencode-go", added[0].Pool);

monitor.Fire(new CodeyBoxOptions
{
AgentClasses =
[
new AgentClassOptions
{
Id = "frontier",
Members =
[
new AgentMembershipOptions
{
Agent = "claude",
Billing = "Subscription",
QualityScore = 100,
},
],
},
],
});

var removed = router.GetClassMembers("frontier");
Assert.Single(removed);
Assert.Null(removed[0].Pool);

await coordinator.StopAsync(CancellationToken.None);
}

private static AgentClassOptions HotReloadClass(string id, params string[] agents)
{
var cls = new AgentClassOptions { Id = id, DisplayName = id };
Expand Down
196 changes: 196 additions & 0 deletions tests/CodeyBox.Tests/ConfigReloadClassificationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,42 @@ public void WorkerPoolFingerprint_IgnoresRestartRequiredFields(string field)
AgentConfigHotReload.SerializeWorkerPool(mutated, legacyConcurrency: null));
}

public static TheoryData<string> RouterFingerprintFields
{
get
{
// Driven by reflection over the config POCOs so a newly added
// settable property becomes a case automatically — and
// MutateRouterField throws for names it does not know, failing the
// new case until the fingerprint covers the field.
var data = new TheoryData<string>();
foreach (var p in typeof(AgentMembershipOptions).GetProperties())
data.Add("Member." + p.Name);
foreach (var p in typeof(AgentInstanceOptions).GetProperties())
data.Add("Instance." + p.Name);
foreach (var p in typeof(AgentClassOptions).GetProperties())
data.Add("Class." + p.Name);
return data;
}
}

[Theory]
[MemberData(nameof(RouterFingerprintFields))]
public void RouterFingerprint_ObservesEveryConfigurableField(string field)
{
// A configured router field missing from SerializeRouterInputs silently
// behaves as restart-required: the edit is accepted, no reload fires,
// and the router keeps the old value (the Pool failure mode). Mutating
// the field solo must move the fingerprint.
var baseline = BaselineRouterInputs();
var mutated = BaselineRouterInputs();
MutateRouterField(mutated, field);

Assert.NotEqual(
AgentConfigHotReload.SerializeRouterInputs(baseline.Classes, baseline.Instances, baseline.Modifiers),
AgentConfigHotReload.SerializeRouterInputs(mutated.Classes, mutated.Instances, mutated.Modifiers));
}

public static TheoryData<string> GuardedKeyPaths
{
get
Expand Down Expand Up @@ -558,6 +594,166 @@ private static void MutateWorkerPoolField(WorkerPoolOptions opts, string field)
}
}

private static (List<AgentClassOptions> Classes, List<AgentInstanceOptions> Instances, AgentScoreModifiersOptions Modifiers)
BaselineRouterInputs() =>
(
[
new AgentClassOptions
{
Id = "frontier",
DisplayName = "Frontier",
ClaudeSession = new AgentClassClaudeSessionOptions { Enabled = true },
Members = [BaselineRouterMember()],
},
],
[
new AgentInstanceOptions
{
Id = "acct-a",
Agent = "claude",
CredentialFilePath = "/cred-a",
TokenEnvironmentVariable = "TOKEN_A",
AuthJsonEnvironmentVariable = "AUTH_A",
SettingsFilePath = "/settings-a",
DestinationPath = "/dest-a",
SandboxEnvironmentVariable = "SANDBOX_A",
Provider = "provider-a",
},
],
new AgentScoreModifiersOptions()
);

private static AgentMembershipOptions BaselineRouterMember() => new()
{
Agent = "claude",
InstanceId = "acct-a",
Pool = "pool-a",
Billing = "Subscription",
ModelId = "model-a",
CredentialFilePath = "/cred-a",
TokenEnvironmentVariable = "TOKEN_A",
AuthJsonEnvironmentVariable = "AUTH_A",
SettingsFilePath = "/settings-a",
DestinationPath = "/dest-a",
SandboxEnvironmentVariable = "SANDBOX_A",
Provider = "provider-a",
QualityScore = 100,
ReasoningMode = "high",
Capabilities = ["tag-a"],
ClaudeSession = new AgentClassClaudeSessionOptions { Enabled = true },
};

private static void MutateRouterField(
(List<AgentClassOptions> Classes, List<AgentInstanceOptions> Instances, AgentScoreModifiersOptions Modifiers) inputs,
string field)
{
// Every case flips exactly one configured value. A property added to
// any of these POCOs without a case here (and without fingerprint
// coverage) fails loudly via the default arm — that is the point.
var member = inputs.Classes[0].Members[0];
var instance = inputs.Instances[0];
var cls = inputs.Classes[0];
switch (field)
{
case "Member.Agent":
member.Agent = "codex";
break;
case "Member.InstanceId":
member.InstanceId = "acct-b";
break;
case "Member.Pool":
member.Pool = "pool-b";
break;
case "Member.Billing":
member.Billing = "PayPerApi";
break;
case "Member.ModelId":
member.ModelId = "model-b";
break;
case "Member.CredentialFilePath":
member.CredentialFilePath = "/cred-b";
break;
case "Member.TokenEnvironmentVariable":
member.TokenEnvironmentVariable = "TOKEN_B";
break;
case "Member.AuthJsonEnvironmentVariable":
member.AuthJsonEnvironmentVariable = "AUTH_B";
break;
case "Member.SettingsFilePath":
member.SettingsFilePath = "/settings-b";
break;
case "Member.DestinationPath":
member.DestinationPath = "/dest-b";
break;
case "Member.SandboxEnvironmentVariable":
member.SandboxEnvironmentVariable = "SANDBOX_B";
break;
case "Member.Provider":
member.Provider = "provider-b";
break;
case "Member.QualityScore":
member.QualityScore = 99;
break;
case "Member.ReasoningMode":
member.ReasoningMode = "low";
break;
case "Member.Capabilities":
member.Capabilities = ["tag-b"];
break;
case "Member.ClaudeSession":
member.ClaudeSession = new AgentClassClaudeSessionOptions { Enabled = false };
break;
case "Instance.Id":
instance.Id = "acct-b";
break;
case "Instance.Agent":
instance.Agent = "codex";
break;
case "Instance.CredentialFilePath":
instance.CredentialFilePath = "/cred-b";
break;
case "Instance.TokenEnvironmentVariable":
instance.TokenEnvironmentVariable = "TOKEN_B";
break;
case "Instance.AuthJsonEnvironmentVariable":
instance.AuthJsonEnvironmentVariable = "AUTH_B";
break;
case "Instance.SettingsFilePath":
instance.SettingsFilePath = "/settings-b";
break;
case "Instance.DestinationPath":
instance.DestinationPath = "/dest-b";
break;
case "Instance.SandboxEnvironmentVariable":
instance.SandboxEnvironmentVariable = "SANDBOX_B";
break;
case "Instance.Provider":
instance.Provider = "provider-b";
break;
case "Class.Id":
cls.Id = "other";
break;
case "Class.DisplayName":
cls.DisplayName = "Other";
break;
case "Class.ClaudeSession":
cls.ClaudeSession = new AgentClassClaudeSessionOptions { Enabled = false };
break;
case "Class.Members":
cls.Members.Add(new AgentMembershipOptions
{
Agent = "codex",
Billing = "Subscription",
QualityScore = 50,
});
break;
default:
throw new InvalidOperationException(
$"MutateRouterField has no mutation for '{field}'. " +
"Cover the new config property in SerializeRouterInputs and add its mutation here.");
}
}

private static (CodeyBoxOptions Startup, CodeyBoxOptions Candidate) BuildGuardedKeyPair(string keyPath)
{
var startup = new CodeyBoxOptions();
Expand Down
Loading