diff --git a/examples/agent-eval-method.mts b/examples/agent-eval-method.mts new file mode 100644 index 0000000..41e6c30 --- /dev/null +++ b/examples/agent-eval-method.mts @@ -0,0 +1,108 @@ +/** + * A complete optimization method for a game, in the shape the comparator takes. + * + * `compareOptimizationMethods` runs several of these on disjoint train, + * selection and final-test scenarios and reports each one's lift over the + * baseline with a simultaneous, Bonferroni-adjusted bootstrap interval. The + * point of writing a method rather than a loop is that the comparator can then + * answer a question a single run cannot: whether one way of searching beats + * another by more than the game's own noise. + * + * This is the simplest honest method. It authors N candidate programs, scores + * each on the SELECTION scenarios, and keeps the best. The final-test partition + * is never touched, which is the one rule a method may not break. + */ +import type { + ComparisonCost, + OptimizationMethod, + OptimizationMethodResult, +} from '@tangle-network/agent-eval/campaign' +import type { CellResult } from '../matrix-run' +import type { GameScenario } from './agent-eval-arena.mts' + +/** + * Author one candidate and return where its program landed. + * + * Supplied by the caller because building a player is the expensive, credential + * -carrying half, and a method should not decide how an agent is run. + */ +export type AuthorCandidate = (attempt: number) => Promise<{ + /** Path of the program the agent left behind. */ + policy: string + usd: number | null + /** `oauth` bills a plan and reports no per-request figure. */ + authMode: 'api-key' | 'oauth' | null +}> + +/** + * State the spend honestly, including when it cannot be known. + * + * A plan-billed agent reports no per-request cost, so its absence is recorded + * as INCOMPLETE ACCOUNTING rather than folded into a total as zero. A + * comparison that sums an unbilled arm to zero reports the arm nobody could + * meter as the cheapest one. + */ +function costOf(spends: readonly { usd: number | null; authMode: string | null }[]): ComparisonCost { + const known = spends.filter((s) => s.usd !== null) + const unbilled = spends.filter((s) => s.usd === null) + const total = known.reduce((sum, s) => sum + (s.usd ?? 0), 0) + return { + totalCostUsd: total, + // The union refuses to call a partial sum observed. An arm nobody could + // meter makes the whole total `uncaptured`, which is the correct claim: a + // known subtotal is not a total, and reporting it as one would make the + // unmetered arm the cheapest. + costProvenance: unbilled.length === 0 ? { kind: 'observed', usd: total } : { kind: 'uncaptured', usd: null }, + accountingComplete: unbilled.length === 0, + incompleteReasons: unbilled.length === 0 + ? [] + : [`${unbilled.length} of ${spends.length} candidates billed a plan` + + ` (${[...new Set(unbilled.map((s) => s.authMode ?? 'unknown'))].join(', ')})` + + ' and reported no per-request cost'], + } +} + +/** + * Author `attempts` players, keep the one that scores best on selection. + * + * Candidates are scored ONLY on the selection partition. Reading the final test + * to choose a winner is how a comparison reports a lift it did not earn, and it + * is the reason the comparator hands a method two partitions instead of one. + */ +export function bestOfN( + attempts: number, + author: AuthorCandidate, +): OptimizationMethod { + return { + name: `best-of-${attempts}`, + optimize: async (input): Promise => { + const spends: { usd: number | null; authMode: string | null }[] = [] + let best: { policy: string; mean: number } | null = null + + for (let attempt = 1; attempt <= attempts; attempt += 1) { + input.runOptions.signal?.throwIfAborted?.() + const built = await author(attempt) + spends.push({ usd: built.usd, authMode: built.authMode }) + + const scored: number[] = [] + for (const scenario of input.selectionScenarios) { + const row = await input.dispatchWithSurface(built.policy, scenario, { + cellId: `${scenario.id}#${attempt}`, + } as Parameters[2]) + // A cell that could not be built did not play badly. Skipping it + // keeps an unbuildable clock from reading as a weak candidate. + if (row.status === 'played' && row.score !== null) scored.push(row.score) + } + if (scored.length === 0) continue + + const mean = scored.reduce((a, b) => a + b, 0) / scored.length + if (best === null || mean > best.mean) best = { policy: built.policy, mean } + } + + if (best === null) { + throw new Error(`${attempts} candidates were authored and none produced a scoreable run`) + } + return { winnerSurface: best.policy, cost: costOf(spends) } + }, + } +}