From 7c38895652dfb75336bbbc26db5f09455752dc67 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 5 Sep 2026 12:08:11 -0700 Subject: [PATCH 1/6] fix(learning): preserve candidate identity and research feedback --- CHANGELOG.md | 26 + bench/package.json | 2 +- docs/api/index.md | 63 ++- docs/api/primitive-catalog.md | 5 +- docs/api/runtime.md | 17 +- docs/architecture-interpretations.md | 9 +- docs/architecture.md | 43 +- docs/canonical-api.md | 4 +- docs/learning-flywheel.md | 30 +- docs/research/README.md | 1 + .../learning-system-audit-2026-09-05.md | 527 ++++++++++++++++++ package.json | 4 +- pnpm-workspace.yaml | 4 +- src/improvement/agentic-generator.ts | 25 +- src/improvement/code-execution.ts | 4 +- src/improvement/improve-types.ts | 6 +- src/improvement/reflective-generator.ts | 98 +++- src/knowledge/supervised-update.ts | 13 +- src/runtime/observe.ts | 52 +- src/runtime/personify/corpus.ts | 84 +-- src/runtime/strategy-author.ts | 23 +- src/runtime/strategy-evolution.ts | 165 +++++- .../fixtures/agent-improvement-proposal.json | 10 +- .../agent-profile-improvement-proposal.json | 6 +- tests/agentic-generator.test.ts | 51 ++ tests/improvement-driver.test.ts | 177 +++++- tests/kernel/corpus-integrity.test.ts | 190 +++++++ tests/kernel/rsi-wave.test.ts | 15 + tests/kernel/strategy-evolution.test.ts | 126 ++++- tests/kernel/strategy-suite.test.ts | 52 +- tests/knowledge-supervised-update.test.ts | 26 +- tests/runtime-observe.test.ts | 65 ++- tests/version-bump-check.test.ts | 2 +- 33 files changed, 1741 insertions(+), 184 deletions(-) create mode 100644 docs/research/learning-system-audit-2026-09-05.md create mode 100644 tests/kernel/corpus-integrity.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b90f8e179..d804cb88b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## 0.195.0 + +Learning results now use Eval 0.174 and Knowledge 14. +Code improvement preserves Eval's native proposer result type. +The [learning audit](docs/research/learning-system-audit-2026-09-05.md) records the findings and the continuing-learning design. + +### Exact strategy and code candidates + +Strategy checkpoints require `checkpoint.executionRef`, which identifies callbacks, transports, baseline implementations, and external state. +Runtime also hashes profiles, settings, canonical JSON task payloads, and authored module bytes. +Changed inputs reject resume before their saved results can be reused. +Create a new checkpoint when these dependencies change. + +`reflectiveGenerator` now requires `createImprovementProposalSource(context)` instead of a pre-bound proposal source. +Construct the proposer with `repoRoot: context.worktreePath` and use its signal and cost account for drafting. +Draft failures, stale file contents, and failed patch batches now reject the candidate. +The complete patch batch applies atomically. +Tracked diagnosis-only edits no longer become code candidates. + +### Persistent lessons and research + +Observer responses must match the declared schema. +Malformed responses and failed lesson writes remain errors in observation and harvest results. +Corpus records are detached immutable values, and file appends serialize conflicting IDs across processes. +Supervised knowledge updates execute the supplied profile without appending fixed research instructions. + ## 0.193.1 ### Bridge roots refuse unavailable work before allocation diff --git a/bench/package.json b/bench/package.json index db950b720..8f70ae2be 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.8.31", + "version": "0.8.32", "type": "module", "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.", "repository": { diff --git a/docs/api/index.md b/docs/api/index.md index fcbaf26e3..4849dc93a 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -6234,7 +6234,7 @@ Number of generations explored by Runtime's code path. ##### raw -> **raw**: `SelfImproveResult`\<`TScenario`, `TArtifact`\> +> **raw**: `SelfImproveProposerResult`\<`TScenario`, `TArtifact`\> #### Methods @@ -6626,11 +6626,66 @@ Findings to fall back to when the generation had NO failing cells, so a ### ReflectiveGeneratorOptions -#### Properties +#### Methods + +##### createImprovementProposalSource() + +> **createImprovementProposalSource**(`context`): [`ImprovementProposalSource`](analyst-loop.md#improvementproposalsource)\<[`SurfaceImprovementEdit`](agent.md#surfaceimprovementedit)\> + +Bind proposal reads and paid calls to this candidate's worktree and account. + +###### Parameters + +###### context + +###### worktreePath + +`string` + +The candidate worktree — a clean checkout of the current incumbent. + +###### findings + +readonly `ProposalFinding`[] + +Search or production findings explicitly admitted for proposal use. + +###### maxShots + +`number` + +DEPTH: max iterations the generator may take (agentic uses this; the + reflective generator ignores it). -##### improvementProposalSource +###### signal + +`AbortSignal` + +###### generation? + +`number` + +Generation coordinates supplied by Runtime's internal code candidate driver. + +###### candidateIndex? + +`number` + +###### costLedger? + +`CostLedgerHandle` + +Shared run-wide paid-call account supplied by agent-eval 0.117+. + +###### costPhase? + +`string` + +Receipt attribution phase supplied alongside `costLedger`. + +###### Returns -> **improvementProposalSource**: [`ImprovementProposalSource`](analyst-loop.md#improvementproposalsource)\<[`SurfaceImprovementEdit`](agent.md#surfaceimprovementedit)\> +[`ImprovementProposalSource`](analyst-loop.md#improvementproposalsource)\<[`SurfaceImprovementEdit`](agent.md#surfaceimprovementedit)\> *** diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 5d4198018..593643010 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.193.1` and `@tangle-network/agent-eval@0.173.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.195.0` and `@tangle-network/agent-eval@0.174.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -1950,7 +1950,7 @@ Import from `@tangle-network/agent-eval` — 58 exports. ### CAMPAIGN — profile matrix, gates, improvement loop -Import from `@tangle-network/agent-eval/campaign` — 408 exports. +Import from `@tangle-network/agent-eval/campaign` — 409 exports. | Symbol | Kind | Summary | |---|---|---| @@ -2051,6 +2051,7 @@ Import from `@tangle-network/agent-eval/campaign` — 408 exports. | `sequentialPairedGate` | function | Anytime-valid sequential paired gate. Conforms to the existing `Gate` | | `skillOptOptimizationMethod` | function | Run Microsoft's SkillOpt trainer as a complete optimization method. | | `surfaceContentHash` | function | Full SHA-256 content identity for a prompt or finalized code surface. | +| `surfaceDispatchRef` | function | Bind a campaign cache entry to the exact surface and caller-owned execution revision. | | `surfaceHash` | function | Short loop key derived from the same content identity as provenance. | | `tangleTracesRoot` | function | The shared, out-of-repo root for campaign/benchmark run bundles. Keeping run | | `traceAnalystQualityJudge` | function | _(no summary — add a TSDoc line at the declaration)_ | diff --git a/docs/api/runtime.md b/docs/api/runtime.md index b0052ca18..de2bfb70b 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -9468,6 +9468,15 @@ Endurance: write the run state after every completed phase; with `resume`, a > `optional` **resume?**: `boolean` +###### executionRef + +> **executionRef**: `` `sha256:${string}` `` + +Digest of execution dependencies: environment, baseline code, transports, callbacks, +and external state such as a corpus. Update it when any dependency changes. +Runtime hashes profiles, settings, JSON task payloads, and authored bytes separately; +it cannot infer callback behavior or external state. + ##### onPhase? > `optional` **onPhase?**: (`phase`) => `Promise`\<`void`\> @@ -9548,6 +9557,12 @@ Called before each benchmark phase (gen0, gen1…, band-screen, holdout, reprodu > `optional` **file?**: `string` +##### sourceSha256? + +> `optional` **sourceSha256?**: `` `sha256:${string}` `` + +Digest of the exact authored module evaluated in this generation. + ##### gzipBits? > `optional` **gzipBits?**: `number` @@ -26254,7 +26269,7 @@ stay visibly distinct. ### strategyAuthorContract -> `const` **strategyAuthorContract**: "\nYou author an OPTIMIZATION STRATEGY for an agentic loop system. A strategy decides how to\nspend a compute budget to beat a task's deployable check. You compose exactly two steps:\n\n shot(spec?: \{ handle?, messages?, steer?, persona?, tools? \}): Promise\\n Runs ONE worker attempt (a bounded tool loop) over an artifact.\n - omit handle =\> the shot opens its OWN fresh artifact and closes it after (a sample).\n - pass handle =\> the shot CONTINUES that artifact (state accumulates across shots).\n - messages =\> the carried conversation (pass the previous ShotResult.messages to continue).\n - steer =\> a corrective instruction injected before the shot.\n - persona =\> \{ systemPrompt?, model? \} — give THIS shot its own role and/or model\n (multi-agent strategies: a researcher shot then an engineer shot, a panel of k\n personas over one budget). On a fresh shot the systemPrompt replaces the task's; on\n a carried conversation it arrives as a hand-off message. Same conserved budget.\n - tools =\> string\[\] — restrict THIS shot to a subset of the task's tools by\n name (focus an explore shot on read-only tools, an execute shot on write tools).\n Restriction-only; unknown names make the shot fail. ALWAYS select from\n await listTools(handle) — never hardcode. Omitted =\> the shot sees every tool.\n ShotResult = \{ messages, score (0..1 on the task's check), passes, total, completions, toolErrors \}\n Returns null if the attempt failed infra-wise.\n\n critique(messages): Promise\\n A firewalled trace-analyst reads the attempt's trajectory and returns ONE corrective\n instruction (or null when it judges the work complete). Costs ~1 completion.\n\n consult(messages, instruction): Promise\\n The RAW analyst channel: the same firewalled critic answers YOUR instruction over the\n trajectory verbatim (no reformatting) — use it when you need a specific reply format\n (a decision, a prediction). Costs ~1 completion.\n\n surface.open(task) / surface.close(handle)\n Open a persistent artifact you manage yourself (remember to close in a finally).\n close is idempotent — closing an already-closed handle is a safe no-op.\n\n listTools(handle): Promise\\>\n The tools THIS task actually offers. TOOL SETS VARY PER TASK — if you restrict a\n shot with \`tools\`, you MUST pick names from await listTools(handle); hardcoding\n names from an example kills your shots on every task whose tools differ.\n\nRules:\n- ALWAYS await every shot/critique/surface call — a floating promise that rejects\n crashes the whole benchmark run.\n- Stay within ~budget total shots; every shot/critique spends from a conserved pool.\n- For a FRESH attempt OMIT \`messages\` entirely (never pass \`\[\]\` — an empty array is a\n fresh conversation too, but be explicit). To CONTINUE, pass the previous\n ShotResult.messages unchanged.\n- Return \{ score, resolved, completions, progression, shots \} — score = the BEST checkpoint\n you reached (keep-best, never final-state), progression = score after each shot.\n- The module must be EXACTLY this shape (no other imports, no commentary outside code):\n\nimport \{ defineStrategy \} from '@tangle-network/agent-runtime/kernel'\nexport default defineStrategy('your-strategy-name', async (\{ surface, task, budget, shot, critique, listTools \}) =\> \{\n // your composition (listTools comes from the destructured context — it is NOT a global)\n\})\n" +> `const` **strategyAuthorContract**: "\nYou author an OPTIMIZATION STRATEGY for an agentic loop system. A strategy decides how to\nspend a compute budget to beat a task's deployable check. You compose exactly two steps:\n\n shot(spec?: \{ handle?, messages?, steer?, profile?, tools? \}): Promise\\n Runs ONE worker attempt (a bounded tool loop) over an artifact.\n - omit handle =\> the shot opens its OWN fresh artifact and closes it after (a sample).\n - pass handle =\> the shot CONTINUES that artifact (state accumulates across shots).\n - messages =\> the carried conversation (pass the previous ShotResult.messages to continue).\n - steer =\> a corrective instruction injected before the shot.\n - profile =\> a complete AgentProfile — give THIS shot its own instructions,\n model, skills, tools, hooks, and subagents. Include name, harness, and\n model: \{ provider, default \}; put standing instructions in prompt.systemPrompt.\n For example: \{ ...opts.workerProfile, name: 'researcher',\n prompt: \{ ...opts.workerProfile.prompt,\n systemPrompt: 'Inspect the evidence before proposing a change.' \} \}.\n Choose an available model from the current execution setup. Omit profile to use\n the worker's exact profile. Every shot spends from the same conserved budget.\n - tools =\> string\[\] — restrict THIS shot to a subset of the task's tools by\n name (focus an explore shot on read-only tools, an execute shot on write tools).\n Restriction-only; unknown names make the shot fail. ALWAYS select from\n await listTools(handle) — never hardcode. Omitted =\> the shot sees every tool.\n ShotResult = \{ messages, score (0..1 on the task's check), passes, total, completions, toolErrors \}\n Returns null if the attempt failed infra-wise.\n\n critique(messages): Promise\\n A firewalled trace-analyst reads the attempt's trajectory and returns ONE corrective\n instruction (or null when it judges the work complete). Costs ~1 completion.\n\n consult(messages, instruction): Promise\\n The RAW analyst channel: the same firewalled critic answers YOUR instruction over the\n trajectory verbatim (no reformatting) — use it when you need a specific reply format\n (a decision, a prediction). Costs ~1 completion.\n\n surface.open(task) / surface.close(handle)\n Open a persistent artifact you manage yourself (remember to close in a finally).\n close is idempotent — closing an already-closed handle is a safe no-op.\n\n listTools(handle): Promise\\>\n The tools THIS task actually offers. TOOL SETS VARY PER TASK — if you restrict a\n shot with \`tools\`, you MUST pick names from await listTools(handle); hardcoding\n names from an example kills your shots on every task whose tools differ.\n\nRules:\n- ALWAYS await every shot/critique/surface call — a floating promise that rejects\n crashes the whole benchmark run.\n- Stay within ~budget total shots; every shot/critique spends from a conserved pool.\n- For a FRESH attempt OMIT \`messages\` entirely (never pass \`\[\]\` — an empty array is a\n fresh conversation too, but be explicit). To CONTINUE, pass the previous\n ShotResult.messages unchanged.\n- Return \{ score, resolved, completions, progression, shots \} — score = the BEST checkpoint\n you reached (keep-best, never final-state), progression = score after each shot.\n- The module must be EXACTLY this shape (no other imports, no commentary outside code):\n\nimport \{ defineStrategy \} from '@tangle-network/agent-runtime/kernel'\nexport default defineStrategy('your-strategy-name', async (\{ surface, task, opts, budget, shot, critique, listTools \}) =\> \{\n // your composition (listTools comes from the destructured context — it is NOT a global)\n\})\n" The compressed consumable a skill carries: everything an author needs to emit a loop. diff --git a/docs/architecture-interpretations.md b/docs/architecture-interpretations.md index 175888a44..f1aae1c3f 100644 --- a/docs/architecture-interpretations.md +++ b/docs/architecture-interpretations.md @@ -78,7 +78,7 @@ The discipline that the architecture leans on — *selector ≠ judge*, judge wr | **Test-time-compute / search** | Driver = search controller, selector = ranking, judge = oracle reward | Only if a *learned* controller beats fixed best-of-N | Controller is open-loop; refine loses to flat sampling at matched budget | | **Active learning / experimental design** | Driver = acquisition function picking the next most-informative source | **Yes — it makes the goal measurable**; the best frame for the research use case | Needs a *calibrated* gap signal; today "gap" is an LLM vibe | | **Program synthesis** | Driver = JIT emitting a topology program; runAgentRounds = interpreter | Only if the ISA grows `seq`/nesting and the emitter reads an IR | It's a **3-opcode flat enum**, not a DSL; GEPA tunes a prompt comment, not the emitter | -| **Two-timescale / RSI** | Inner answers; outer rewrites the answerer from traces + judge | Only with the missing wire **and** a cross-benchmark transfer test | RSI is the **shape, not the system**; no transfer test exists | +| **Domain learning / meta-learning** | Improve specialists, working evaluations, and the process that trains them | Repeatable gains within the intended domain; process transfer is a separate claim | Existing components need a joined learning process and evidence appropriate to each level | | **Skeptic / Occam** | self-refine (loses) steering best-of-N (wins) | No — vocabulary, not capability | Overclaims past "untested ≠ disproven" for a trace-fed driver | ### 3.1 Test-time-compute / search @@ -143,7 +143,12 @@ Breaks: the load-bearing assumption — a **calibrated** gap signal — is absen ### 3.4 Two-timescale / recursive self-improvement -Inner fast loop drives an answer now; outer slow loop (`improve()` with an official GEPA or SkillOpt method) rewrites policy from accumulated traces + judge scores, measures the exact candidate on final-test tasks hidden from the method, and requires an explicit activation. The recursion is real *in shape* — the optimiser is an atom editing an atom's policy — but cross-benchmark transfer remains unproven. The frame's value is its sharp corpus-vs-policy split: **wiki growth is an input to inference; only prompt/tool/policy rewrites are RSI.** The research-acquisition loop is RSI only if findings about *which acquisition move paid off* rewrite the driver's acquisition policy and the resulting profile wins on fresh tasks. +The fast loop improves an answer; the domain learning process improves specialists, working evaluations, and future experimental decisions. +`improve()` supplies bounded search and final comparisons, while activation remains explicit. +Stored knowledge can improve domain work, and changes to acquisition policy can improve the process that produces that knowledge. +Each claim needs its own task outcome evidence. +A meta-agent can learn how to construct these domain learning processes without requiring the resulting specialists to generalize. +Cross-domain reuse of the learning process is a further experiment, not a prerequisite for useful domain learning. ### 3.5 Skeptic / Occam (adversarial) diff --git a/docs/architecture.md b/docs/architecture.md index a3d45ef7f..c0d65e132 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -28,6 +28,10 @@ if a section drifts from one of these, the claim wins and the section is wrong. The target is a persistent research and engineering system that completes complex software and produces independently checked research. It can revise its methods, tools, and organization, then retain changes that improve later work. Recursive self-improvement includes improving how those changes are discovered and tested. +Domain specialists can succeed without generalizing outside their intended tasks. +A domain learning process can repeatedly improve specialists and its working evaluations across many rollouts and objectives. +A meta-agent can improve that process and learn to construct effective learning processes in other domains. +The reusable knowledge can be how to learn, even when the resulting specialists remain domain-specific. 1. **The atom is a decision, not a spawn.** At every level an agent faces the same question: given the solution so far, the feedback so far, and the budget left, what @@ -44,15 +48,13 @@ Recursive self-improvement includes improving how those changes are discovered a Research can require proof checking, independent replication, or new experiments when no answer key exists. Agents can author working checks, while independent assessment tests the final claims and records unverified assumptions. The quality and cost of that evidence are part of the research problem. -4. **The improvement that counts is the policy getting better across runs.** Two things - improve on two clocks (§2). *Within* a run the **solution** climbs (the artifact gets - better round over round). *Across* runs the **decision policy** climbs — it remembers - which decisions, on which kinds of problems, produced good multi-objective outcomes, - and chooses better next time. **That across-run curve is RSI, and it is THE success - criterion** (Gate B — defined in [learning-flywheel.md](./learning-flywheel.md), §2 - here). A single within-run result beating a blind baseline at equal compute (Gate A) - is **one narrow diagnostic**, not the goal — do not read it as the verdict on the - product. +4. **Measure improvement at the level being changed.** + A specialist improves on fresh tasks within its domain. + A domain learner improves when it reliably produces better specialists, working evaluations, or research outcomes. + A meta-agent improves when its learning decisions produce better domain learning processes. + Transfer of a learning process to new domains is a further claim; specialist transfer is optional. + Each claim needs independent evidence and complete resource accounting over its declared horizon. + The within-run comparison (Gate A) and across-run comparison (Gate B) test specific claims, not the entire product. --- @@ -189,7 +191,7 @@ the enforcing code, in §13.3 (`assertTraceDerivedFindings`). --- -## 5. GEPA at every level +## 5. Optimization at each learning level The optimizer `O` improves any `Agent`'s `context`+prompt and the program shape, from the shared corpus, **held-out gated** (train ∩ selection ∩ final test = ∅, enforced by @@ -201,10 +203,10 @@ by its own deployable checker (tests · clock · scanner · cost meter), with th write-only judge as the fixed anchor on the *correctness* axis so the recursion can't Goodhart. **Status:** the loop today carries a single `score` per attempt (§6's `adapter.judge`) — collapsing the vector at the boundary is the open gap to close before -the optimizer can trade objectives honestly. The analyst-prompt coordinate measured -flat; the live outer-loop lever is **program/strategy space** (`defineStrategy` + -`authorStrategy`) — see -[docs/research/optimization-space.md](./research/optimization-space.md) and the ledger. +the optimizer can trade objectives honestly. +Candidate surfaces include complete profiles, executable strategies, working evaluations, curricula, and the learning method itself. +Use the current evidence ledger to choose a method and experiment for the intended domain and objective. +A result for one coordinate or task family does not decide the value of every other learning process. --- @@ -266,11 +268,10 @@ being squatted on. ## 8. The moat (honest) -The inference-program scaffold (compound AI systems / DSPy-style) is becoming -**table stakes** — others will have it. The defensible bet is the **cross- -benchmark learning flywheel + recursive self-improvement**, anchored by the -external write-only judge, where a controller **learns the program and transfers -across benchmarks**. Infra is the cost of entry; transfer is the company. +The technical bet includes specialist capability, repeatable domain learning, and agents that improve how learning processes are constructed. +Accumulated experiment records and executable learning procedures can improve future domain work. +Those procedures can also transfer to other domains without requiring the specialists they produce to generalize. +The value and transfer of each learned procedure require their own evidence. --- @@ -290,13 +291,15 @@ The mechanisms below are experiment options; an earlier option winning is not a 2. **Give the driver execution access through `sandbox-agent` (auto-research).** 3. **GEPA** the driver/analyst `context`+prompts, held-out gated. 4. **Composition lift** — `fork`/coordinator/nested (driver-of-drivers). -5. **Cross-benchmark transfer** — one learned controller, many benchmarks. The moat. +5. **Domain learning and meta-learning** — repeatedly improve specialists and working evaluations; separately test learning-process reuse in new domains. Test combinations when the claimed benefit depends on interactions between delegation, retained knowledge, continued work, or learned decisions. Remove components in controlled comparisons to identify their contribution. An isolated component losing does not establish that the combination cannot help. Measure the hardest independently checked work completed, results across resource budgets, and improvement across successive projects. +Evaluate working-evaluation changes against independent outcomes and their effect on subsequent learning. +Use fresh tasks within a domain for specialist and domain-learning claims; use fresh domains for learning-process transfer claims. Use equal actual resources within each comparison and keep model versions and task difficulty controlled. Report learning costs separately and include them in total costs over the declared project horizon. diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 775c64b43..342bfb8d4 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,9 +4,9 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.193.1.** +> **Version 0.195.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. -> `agent-eval` must satisfy `>=0.173.0 <0.174.0`. +> `agent-eval` must satisfy `>=0.174.0 <0.175.0`. > `sandbox` must satisfy `>=0.36.4 <0.38.0`. > Portable profile and tool-part types come from `@tangle-network/agent-interface` `^2.3.0`. > diff --git a/docs/learning-flywheel.md b/docs/learning-flywheel.md index 909b58fbf..fe9522909 100644 --- a/docs/learning-flywheel.md +++ b/docs/learning-flywheel.md @@ -1,4 +1,4 @@ -# The Continual Cross-Benchmark Learning Flywheel +# Continual Domain Learning and Meta-Learning > **In plain terms:** This is a design-rationale doc — it explains *why* this project is built > to get better the more it runs, not how to use the package day to day. It's for a developer @@ -19,17 +19,16 @@ > > - **Inner loop (within-run):** a controller steers a worker over k attempts on a single > task — refine/fanout/stop. Useful, but NOT the product, and not where the moonshot lives. -> - **Outer loop (the FLYWHEEL — the product):** every eval run, across every benchmark, -> generates `(state, trace, steer, outcome, cost)` data that accumulates into a durable -> corpus; the **controller** learns from *all of it*; that improves future runs across -> *all* benchmarks; which generates more data. +> - **Outer loop:** domain work generates `(state, trace, steer, outcome, cost)` records. +> A domain learner uses those records to improve specialists, working evaluations, and its own experimental decisions. +> A meta-agent can learn how to construct and improve those domain learning processes. > -> It is **NOT only within-run self-improvement.** Self-improvement is **cross-run and -> cross-benchmark**, compounding over time. A run that shows zero within-run effect still -> feeds the corpus; the learnable structure emerges in the aggregate. The asset is the -> corpus and the controller it trains — never any single result. +> Sustained improvement within one domain is valuable in its own right. +> A specialist need not transfer to another domain. +> The transferable knowledge can be the procedure that trains specialists and improves their evaluations. +> Failed experiments can inform that procedure when their evidence survives and affects later decisions. -> **Success — the one definition (Gate B).** The flywheel works iff, across repeated runs on a +> **Across-run policy improvement (Gate B).** One test of the domain learner asks whether, across repeated runs on a > persistent, checkable, long-horizon task family, the deployed controller's verifier-graded > **multi-objective** score improves **run-over-run** (run N+1 starts above run N at **matched > per-run compute**), the only changed variable is that the controller learned from the accumulated @@ -38,11 +37,12 @@ > **deployable checker** — never the answer oracle or the write-only judge. *Multi-objective* is > load-bearing: success is a vector (correct · fast · secure · cheap), with evidence scoped to each objective. > Tests, clocks, scanners, and cost meters provide partial measurements; record each check's coverage and unverified assumptions. -> This OUTER-loop slope is THE success criterion. The +> This tests one learning claim; evaluation quality, learning-process quality, and process transfer require separate comparisons. +> The > within-run "trace+findings-fed controller beats the blind same-compute baseline under a non-oracle > selector at **equal compute**" question is a separate, narrower diagnostic — **Gate A**, the > comparison for within-run steering, scoped by [architecture.md §9](./architecture.md#9-build-order-and-experiment-scope). -> Compare actual resource use in both tests, including learning costs over the declared project horizon. +> Compare actual resource use in both tests, including learning and evaluation-development costs over the declared horizon. > The budget may fund one deep trajectory, several shallow attempts, or a mixture. ## The flywheel @@ -79,6 +79,12 @@ that passed a checker, not facts — ## The lifting generalization: recursive self-improvement +The object being improved can be a complete domain learning process. +It can produce specialized AgentProfiles, improved working evaluations, and the next experimental policy. +The process can learn within one domain before, or without, being reused elsewhere. +When it is reused, measure whether it constructs a useful learner in the new domain rather than expecting the old specialist to generalize. +Working evaluations may evolve, while independent assessment tests whether those changes better detect meaningful success and failure. + The flywheel is one instance of a more general object. Name the loop: ``` diff --git a/docs/research/README.md b/docs/research/README.md index e1065666b..44b4a2754 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -18,6 +18,7 @@ The research files below are source history and focused design inputs, not compe | Doc | What it holds | |-----|---------------| +| [learning-system-audit-2026-09-05.md](./learning-system-audit-2026-09-05.md) | Current-source audit of learning across Runtime, Eval, and Knowledge, with reproduced failures and a unification design. | | [rsi-atom-masterplan.md](./rsi-atom-masterplan.md) | Historical self-designing-atom plan. Distributed execution work is superseded by `agent-managed-compute/`. | | [optimization-space.md](./optimization-space.md) | The 6-axis optimization taxonomy + canon-compatibility audit (the portfolio map the canonical spine references). Per-layer evidence now lives in `.evolve/current.json`. | | [leapfrog-program.md](./leapfrog-program.md) | The research program's honest formal core (v2 — breakthrough framing retracted; what survived). | diff --git a/docs/research/learning-system-audit-2026-09-05.md b/docs/research/learning-system-audit-2026-09-05.md new file mode 100644 index 000000000..d52d5e86a --- /dev/null +++ b/docs/research/learning-system-audit-2026-09-05.md @@ -0,0 +1,527 @@ +# Learning system audit: Runtime, Eval, and Knowledge + +**Verdict: make the learning process itself a first-class object, with separate evidence for specialist quality, domain learning, evaluation quality, and meta-learning.** +The audit reproduced wrong candidate decisions and disconnected research feedback on the current main branches. +These failures justify repairs to the existing paths. +They do not justify reducing the ambition to prompt tuning or rejecting learning across projects. + +The target includes specialized agents, repeatable learning within a domain, and agents that improve how those specialists are discovered and trained. +Its changing state includes instructions, tools, code, knowledge, coordination, resource decisions, and the procedure that changes those parts. +A specialist can succeed within its intended domain without generalizing beyond it. +The transferable result can instead be the procedure that constructs a successful learning process in a new domain. +Cross-domain transfer is a further objective, not the sole definition of success. + +## Method and evidence boundary + +The audit fetched all three main branches before creating isolated checkouts. +Source references below describe these exact starting revisions unless a repair is explicitly named. + +| Repository | Audited main | Package version | +| --- | --- | --- | +| [agent-runtime](https://github.com/tangle-network/agent-runtime/tree/a16d8a3b91481b140cb552e373d5bde98b34af05) | `a16d8a3b91481b140cb552e373d5bde98b34af05` | `0.193.1` | +| [agent-eval](https://github.com/tangle-network/agent-eval/tree/f8e3da285b6286386699a196733e9c0c27c20cfd) | `f8e3da285b6286386699a196733e9c0c27c20cfd` | `0.173.3` | +| [agent-knowledge](https://github.com/tangle-network/agent-knowledge/tree/390f2da9883e55cc86a8985167324d8b1f10894a) | `390f2da9883e55cc86a8985167324d8b1f10894a` | `13.0.1` | + +`R`, `E`, and `K` in source references mean these Runtime, Eval, and Knowledge revisions. +Runtime coverage includes all 24 production files under `src/improvement`, plus execution, adoption, strategy evolution, observation, and memory serving. +Eval coverage includes native search, complete methods, Python adapters, final comparisons, provenance, costs, and related learning utilities. +Knowledge coverage includes KB candidates, retrieval, memory experiments, research loops, source evidence, and Runtime composition. +Public callsites and nearby applications were searched to distinguish implemented APIs from demonstrated adoption. + +Defect probes used real public functions with deterministic callbacks, real Git worktrees, and real filesystem storage. +KB snapshot probes ran in a local Linux container because the exact snapshot implementation intentionally refuses macOS. +These tests measure implementation behavior, not model intelligence or defect prevalence. +No paid model experiment, production deployment, or current same-task comparison against another learning system ran during this audit. + +Primary research was checked through paper abstracts, official repository documentation, package metadata, and selected implementation contracts. +This is a mechanism comparison, not a systematic literature review or a benchmark ranking. +Upstream performance claims were not independently reproduced and are not assigned to this system. + +## What the system must learn + +Different outcomes currently share the word “improvement.” +They need distinct evidence even when they use the same execution and storage components. + +| Outcome | Example | Evidence that would establish it | +| --- | --- | --- | +| Better work within one task | Diagnose a failed test, change code, and finish correctly | A checked final result and the actual resources consumed | +| Better domain specialist | Improve an `AgentProfile` for a defined task family | Improvement on fresh tasks within that domain; cross-domain performance is optional | +| Better domain learning process | Repeatedly discover better specialists across rollouts and objectives | Continued useful gains and retained ability under the declared domain objectives and resources | +| Useful accumulated knowledge | Reuse a discovery when solving a different project | Retrieval/use records joined to later task outcomes, with retention and transfer checks | +| Better evaluation engineering | Generate cases or checks that expose failures the prior evaluation missed | Better agreement with independently established outcomes, coverage, discrimination, and resistance to easy exploits | +| Better learning itself | Change how experiments are selected, interpreted, or consolidated | A changed learner produces better specialists or domain learning processes across repeated trials | +| Transfer of learning procedure | Construct a working learning process in a new domain | The learned procedure produces better domain specialists than the declared alternative there | + +An optimizer selecting a new prompt establishes none of these outcomes by itself. +A record in a lesson store establishes that text was retained. +A knowledge page with a citation establishes a provenance relationship. +Each can be useful without constituting a demonstrated capability gain. + +The ambition should include searching over the whole executable agent and its learning procedure. +The candidate may add a tool, reorganize work, change memory access, retain a useful result, or replace the search method. +Different interventions can require each other to produce value. +For example, retaining a research result helps only if later planning retrieves and uses it. +Testing each component alone cannot reject the combined mechanism. + +Search should retain informative failures and promising alternatives even when they are unsuitable for immediate adoption. +Requiring every exploratory step to improve a short-run scalar would prevent some useful discoveries. +Adopting an active version remains a separate decision with an explicit outcome and resource policy. +That distinction permits broad exploration without turning an unsupported claim into a deployed improvement. + +### The domain learning process is a reusable artifact + +A domain learner takes a domain objective, available execution tools, current specialists, prior experience, and resources. +It produces evaluated specialist candidates, improved working evaluations, experiment records, and an updated learning state. +Its behavior includes which problems to generate, what to change, how to measure it, and when to change the learning strategy. +This is a richer object than one call that improves one profile on fixed cases. + +The meta-agent can learn to construct and run that process repeatedly. +What transfers might be an evaluation-design tactic, a curriculum, a diagnosis method, an experimental policy, or a complete executable learning procedure. +The resulting specialists may remain entirely domain-specific. +Transfer of specialist behavior and transfer of the procedure that trains specialists require different experiments. + +There are therefore at least three changeable artifacts: the specialist, the working evaluation, and the learning policy. +They can improve together, but they should not collapse into one ambiguous score. +An evaluation candidate is valuable when it better detects meaningful success and failure, even if it lowers the current specialist's measured score. +A learning-policy candidate is valuable when it produces better subsequent learning, even if its own first specialist is weaker. +Within-domain improvement is also valuable when neither artifact transfers elsewhere. + +Automated evaluation hill climbing should target evaluation quality and downstream learning quality. +Optimizing the current specialist's score by weakening its tests would measure a different, undesirable objective. +Independent outcomes, fresh failure cases, controlled defects, and external assessment can test evaluation changes. +The working evaluation can evolve; its authority to declare its own success must remain bounded by evidence outside that optimization. + +## What `improve(...)` actually does + +The Runtime name covers two different execution paths. +Knowledge improvement has another public entry point. +The profile surfaces already share a method implementation; they are not nine independently written optimizers. + +```mermaid +flowchart TD + P[Exact AgentProfile] --> I[Runtime improve: profile] + I --> M[Eval complete OptimizationMethod] + M --> D[Practice and selection executions] + D --> M + M --> F[Selected profile and fresh final comparison] + C[Repository checkout] --> IC[Runtime improve: code] + IC --> N[Eval native candidate search] + N --> CF[Selected code and fresh final comparison] + K[Knowledge snapshot] --> KI[Knowledge improvement lifecycle] + KI --> KF[Detached knowledge candidate] + S[Strategy program] --> SE[Runtime strategy evolution] + SE --> SF[Separate archive and final decision] + F --> A[Runtime exact experiment and adoption] + CF --> A + KF --> A + H[Authored or imported candidate] --> A +``` + +The arrows into exact experiments require caller composition. +They do not imply that every API automatically shares execution records or deployment state. + +### Profile improvement + +`R:src/improvement/method-execution.ts:291` parses and freezes the full baseline profile, then extracts the requested change surface. +The selected complete `OptimizationMethod` owns search and candidate selection. +Its inputs include disjoint practice and selection cases; final cases remain outside its input. +Each proposed value is materialized back into a full immutable profile before the caller executes it. +The result contains the selected candidate, final uncertainty, cost, task identities, and the candidate population when the method supplies it. + +This is a useful bounded experiment. +The call does not choose the next research question, query accumulated experience, schedule the next experiment, or deploy its result. +It does not establish that the learning method improved. +The profile path requires complete reported cost and a positive lower confidence bound beyond the configured minimum lift for `ship`. +That policy answers a particular scalar comparison; it is not a universal definition of progress toward the broader product objective. + +### Code improvement + +`R:src/improvement/code-execution.ts:167` creates an isolated baseline checkout and supplies a code generator to Eval's native search. +Later generations inherit the accepted incumbent code, and returned code includes its full difference from the original baseline. +The default generator executes a supplied coding-agent profile. +It can use failure summaries or raw trace paths and retain its best checked attempt. +Rejected worktrees are removed; the caller disposes the returned worktree when finished. + +Runtime ownership of worktrees is appropriate. +Separate representations of final evidence, cost, resume identity, and adoption are not inherent requirements of code search. +The code result also has narrower lineage fields than the profile result. +The exact-code bundle builder already verifies and embeds the finalized patch bytes. + +### Surfaces and their actual reach + +| Surface | Value searched | What changes | Boundary or limitation | +| --- | --- | --- | --- | +| `prompt` | String | `prompt.systemPrompt` | Does not include every instruction field | +| `skills` | One inline document | One named `resources.skills` entry | The named coordinate cannot create or delete a skill | +| `tools` | JSON | Allowed tool configuration | Tool implementations remain code | +| `mcp` | JSON | Tool-server configuration | Server implementations remain code | +| `hooks` | JSON | Hook definitions | The execution backend must support the changed behavior | +| `subagents` | JSON | Subagent definitions | Backend support remains necessary | +| `agent-profile` | Full JSON or named components | The complete profile | Full JSON permits structural edits; component maps preserve their declared keys | +| `memory` | Inline text | `resources.instructions` | This is curated text, not memory retrieval, consolidation, or forgetting | +| `rollout-policy` | JSON | The `structural-rollout` extension | Four fixed settings; an executor must actually consume the extension | +| `code` | Git code candidate | Repository implementations | The caller must execute the candidate checkout | +| Knowledge | Separate Knowledge API | KB content or knowledge policy | Not a Runtime `ImproveSurface` coordinate | + +Sources: `R:src/improvement/profile-surface.ts:32`, `:185`; `improve-types.ts:21`; `rollout-policy.ts`. +The full-profile materializer checks baseline and component round trips and freezes exact values. +That work should remain. +It already permits joint profile changes and should be extended in place when a concrete candidate cannot be represented. + +Changing serialized configuration is not sufficient to show that execution changed. +Runtime already has backend capability descriptions in `src/agent/profile-materialization.ts` that can reject unsupported profile fields. +An arbitrary callback passed to `improve` does not automatically use those checks. +The audited source has no non-test execution consumer for the structural rollout-policy read helper. +That finding bounds demonstrated in-tree use; it does not rule out external consumers. + +## Inventory beyond `improve` + +The fragmentation includes search, measurement, stored experience, and adoption. +Counting exported function names alone would confuse useful adapters with duplicated learning systems. + +| Existing mechanism | Actual responsibility | What it does not establish | +| --- | --- | --- | +| Eval `runOptimization` | Native population search, incumbent updates, per-case candidate frontier | Continuing learning across invocations | +| Eval `runImprovementLoop` | Native search plus final comparison and release support | A distinct search algorithm | +| Eval `selfImprove({proposer})` | Native search convenience API and reporting | A complete external method's semantics | +| Eval `selfImprove({method})` | Complete-method entry point; repaired by this audit | Automatic production adoption | +| Eval `compareOptimizationMethods` | Complete methods followed by final comparisons | Equal actual resources merely because there is one shared cap | +| GEPA adapter | Official reflective search and six recipe compositions | That the caller supplied useful traces and artifact descriptions | +| SkillOpt adapter | Official bounded text/skill search | SkillOpt Sleep deployment-time learning or CodeSurface search | +| Generic text method | Caller-controlled search through evaluated text candidates | A new built-in learning algorithm | +| DSPy adapter | Judge feedback for external DSPy compilation | Automatic integration with the TypeScript method lifecycle | +| `PairwiseSteeringOptimizer` | Aggregate ranking of supplied scored variants | Paired statistics or candidate generation | +| `Researcher` / `CallbackResearcher` | Caller-provided diagnosis, proposal, plan, and evaluation callbacks | An autonomous researcher implementation | +| `runRLCampaign` | Reward/preference extraction and training-data export | Model weight training | +| Curriculum utilities | Allocate cases using supplied observations | Persistent curriculum execution or validated task generation | +| `runAdaptationCurve` | Compare caller scores at different demonstration counts | Adaptation implementation or complete task execution evidence | +| Off-policy estimators | Estimate policy outcomes from logged probabilities | Policy training or credible results without adequate logged support | +| Runtime `runStrategyEvolution` | Authored strategy programs, tournaments, archive, checkpoint, promotion | Shared complete-method accounting or cross-project transfer | +| Runtime `observe` and `Corpus` | Store behavior-derived recommendations and inject selected records | Independent proof that a recommendation works | +| `runAnalystLoop` and file proposer | Turn typed findings into detached proposed patches | Applying, measuring, or adopting the resulting candidate | +| Knowledge retrieval/RAG/policy search | Use the existing Eval complete-method interface | Four unrelated optimizer implementations | +| Knowledge memory experiments | Evaluate retained and retrieved facts over memory configurations | Downstream task-solving improvement | +| Knowledge research driver | Maintain claims, support, contradictions, and questions for one goal | A learned research policy or cross-goal transfer | +| Runtime authored-candidate proposal | Measure exact supplied profiles without requiring an optimizer | Automatic synthesis of those candidates | +| Runtime candidate experiments/activation | Compare and adopt exact executable versions | Search quality or a continuing learning schedule | +| `withIntelligence` | Delivery and observation integration | A closed continuing learner by itself | + +## Reproduced failures and their consequences + +Each measured row describes a controlled reproduction, not a frequency estimate. +The costs of these failures in production are unmeasured because production incidence was not queried. +The repair and verification record is maintained with the changes and regression tests. + +| ID | Priority | Failure on audited main | Evidence boundary and source | Required repair | +| --- | --- | --- | --- | --- | +| E1 | P1 | A complete method selects WIN, but `selfImprove` reranks it against pooled development cases and returns BASE; WIN receives zero final executions | Measured: selection WIN=1, BASE=.6; pooled WIN=.25, BASE=.6. `E:src/contract/self-improve.ts:671`; `run-optimization.ts:534` | Execute the complete method directly and compare its selected result | +| E2 | P2 | A complete method returns its valid unchanged baseline and gets a duplicate-candidate error | Measured through `selfImprove`; `E:run-optimization.ts:418` | Accept no improvement as a valid method outcome | +| E3 | P1 | Final comparison drops 2 failed candidate executions from a designed 4-cell set and reports score 1, lift +.5 | Measured through `compareOptimizationMethods`; `E:compare-optimization-methods.ts:332` | Require exact case, repetition, and judge coverage | +| E4 | P1 | Reusing one run directory with GOOD then BAD returns BAD with GOOD's score 1 and `ship`, with zero new executions | Measured through native `selfImprove`; `E:run-optimization.ts:239`, `:454`; `run-improvement-loop.ts:171` | Include the measured surface in every cache identity | +| E5 | P1 | A baseline measured by an old judge at 0 is reused against a new candidate at .5 although the new judge would give the baseline 1 | Measured through premeasured baseline import; `E:run-optimization.ts:688` | Require the complete evaluator and execution revision | +| E6 | P1 | An optimizer reports $17 estimated/incomplete cost; the run reports $0 and complete accounting | Measured through `selfImprove`; `E:self-improve.ts:676`, `:828` | Reconcile reported search cost with recorded calls and retain uncertainty | +| E7 | P2 | Native history labels the point estimate .25 as the exact interval [.25, .25] for case scores 1, 0, 0, 0 | Measured through native `selfImprove`; `E:src/campaign/presets/run-optimization.ts:553` | Return null for unestimated uncertainty; retain actual final-test statistics | +| R1 | P2 | A modified tracked diagnosis file is classified as substantive code because Git's leading status space is trimmed | Measured with real Git; `R:src/improvement/agentic-generator.ts:1019` | Parse Git status without deleting path characters | +| R2 | P1 | Strategy resume returns an old promoted result after task payloads, objective, model, environment, and final offset change | Measured: zero author calls and benchmark phases; `R:src/runtime/strategy-evolution.ts:397` | Bind resume to exact serializable inputs and explicit callback revision | +| R3 | P2 | Strategy author instructions teach `shot({persona})`, but the executable API accepts `shot({profile})` | Source-verified mismatch: `R:src/runtime/strategy-author.ts:29`; `strategy.ts:847` | Teach the actual complete-profile API and execute its example | +| R4 | P1 | Malformed observation JSON becomes an empty findings array and “clean run” | Measured through `observe`; `R:src/runtime/observe.ts:235` | Validate the complete response and preserve parse failures | +| R5 | P1 | Lesson storage fails, but harvest reports one observed run, one finding, zero learned records, and no failures | Measured through harvest/observe; `R:src/runtime/observe.ts:203` | Propagate acknowledged storage errors to existing failure reporting | +| R6 | P1 | Conflicting same-ID file appends both succeed, then subsequent reads reject the corrupted log | Measured with concurrent file stores; `R:src/runtime/personify/corpus.ts` | Lock the full read/check/append transaction across processes | +| R7 | P2 | Mutating the caller's tags after append changes the stored lesson | Measured through `InMemoryCorpus`; same source | Retain detached immutable records | +| R8 | P1 | Reflective generation ignores an aborted signal, hides draft/apply failures, and reports an incomplete patch batch as applied | Measured with real patches; `R:src/improvement/reflective-generator.ts:28` | Use candidate-bound drafting, cancellation, base checks, and atomic patch application | +| K1 | P1 | KB acquisition and update receive no findings because diagnosis runs afterward | Measured Linux lifecycle: acquire → update → diagnose; `K:src/kb-improvement/evaluation.ts` | Carry one lifecycle state from diagnosis through construction and final evaluation | +| K2 | P2 | An unchanged empty KB with no outcome tests gets five dimensions equal to 1 and stages as candidate-ready | Measured Linux KB probe; `K:src/kb-improvement/evaluation.ts:510` | Omit unmeasured dimensions and identify the scope of configured checks | +| K3 | P1 | Research stops after 1 of 4 allowed rounds while the actual driver reports incomplete, producing no further steering | Measured real research driver plus verified loop; `K:src/verified-research-loop.ts:327` | Respect driver completion and fold remaining research work | +| K4 | P2 | Runtime appends a worker policy to the exact supplied supervisor; that policy forbids writes while the task requires writes | Source-verified and exercised by adapter tests; `R:src/knowledge/supervised-update.ts:132`; `profiles/researcher.ts:349` | Execute the caller profile unchanged | + +K2 is not a live promotion bypass. +`candidate-ready` stages a detached candidate that passed the configured checks. +The actual defect is assigning successful numerical measurements to outcomes that were never measured. +Structural validity can remain a legitimate staging requirement. + +The E1 repair removes an unnecessary search layer. +An external complete method is not a one-candidate generator for another optimizer. +The result must expose the method's real outcome and final campaigns, without inventing native generations to satisfy an old result type. +Native proposer history remains valid for the native path. + +## Fragmentation that matters + +| Boundary | Evidence of fragmentation | Consequence | Direction | +| --- | --- | --- | --- | +| Search versus selection | Complete methods were reranked by native search | The chosen method's decision was discarded | Preserve method ownership of search and selection | +| Measurement identity | Native search, external search, and final comparisons construct identity differently | One candidate or judge can inherit another's result | Share exact candidate and evaluator identity | +| Evidence completeness | Native final comparison rejects missing cells; method comparison averaged survivors | Identical failures produce different promotion evidence | Share complete measurement validation | +| Cost | Method reports and call receipts were not consistently reconciled | Expensive or incomplete search appears free | One run account with explicit imported/reported costs | +| Research lifecycle | KB mutation and final evaluation restarted separate RAG state | Diagnosis missed construction and final callbacks lost prior context | Continue one state through the existing phases | +| Memory | Inline profile instructions, Corpus lessons, flat MCP memory, provider memory, and KB pages differ | “Memory improved” can refer to unrelated operations | Share identities and task outcome evidence while retaining storage adapters | +| Adoption | Profiles, code, KB snapshots, and memory configurations have different convenience paths | Applications coordinate a measured combination themselves | Use the existing complete candidate version as the adoption unit | +| Continuing learning | Stored runs and proposals do not automatically inform the next experiment | Accumulation can stop at logging | Make the domain learning process consume evidence and update specialists, working evaluations, and its own decisions | + +The three-package boundary is sound. +Runtime owns execution and effectful adoption, Eval owns measurements and decisions, and Knowledge owns source-backed state and retrieval. +Merging the packages would not fix any reproduced defect. +The useful unification lies in what crosses those boundaries. + +### Memory is the least coherent part + +`improve({surface:'memory'})` changes inline instructions. +Runtime's `Corpus` retains analyst recommendations and ranks them using confidence and recency. +The public memory MCP server defines another flat item format and lexical search. +Its retrieval log contains query, time, k, returned IDs, and scores. +It cannot supply the session assignments and withholding probabilities required by Knowledge's existing causal-memory utilities. + +Knowledge has richer provider adapters, source snapshots, use receipts, and session-randomized withholding. +The audited Runtime source has no non-test call to `applySessionStickyRetrievalHoldout`, `runAgentMemoryLearningExperiment`, `runAgentMemoryImprovement`, or `createKnowledgeTools`. +This is an in-tree adoption finding, not proof that no external application uses them. + +The memory experiment evaluates `memory.getContext(...).text` and retrieved hits. +Its `executeStep` callback returns no scored task outcome (`K:src/memory/experiment/cell.ts:441`). +Those tests can measure retention, retrieval, and forgetting. +They cannot establish that memory helps an agent solve a task. +Preserve them as diagnostic tests and join retrieval evidence to an actual task result on the serving path. + +### Research has durable state but a fixed method + +The claim ledger binds exact sources and a specific goal, survives resumes, and merges concurrent updates. +These are useful state guarantees. +Its default reasoning method contains narrower heuristics: distinct URL hosts represent independent support, normalized text represents claim identity, and word overlap can close a question. +Extraction truncates source context and prior claims, while deeper questions use fixed templates. +Router failure can change the extraction method without reporting that change in the accepted-source verdict. + +Those mechanisms can be cheap filters or observable features. +They should not be mistaken for general scientific judgment, novelty assessment, or a learned experimental policy. +The research procedure should be authored and changeable, with source identity and storage integrity enforced independently. +The corrected completion hook keeps the stopping decision caller-owned instead of hard-coding another research policy into the loop. + +### The second final test has a reason today + +`proposeAgentProfileImprovement` searches through `executor.optimize`, then measures release tasks through `executor.measure`. +The code explicitly rejects reuse of practice, selection, or the first final-test cases for that release comparison. +The generic bundle path has equivalent freshness checks (`R:src/intelligence/improvement-cycle.ts:599`). + +Deleting that comparison today would weaken the evidence. +The first callback does not necessarily produce the exact execution records required for adoption. +Also, once its final result influences further work, that result becomes selection information. + +The permanent simplification is to use the same exact measurement contract from the start. +One sealed final comparison could then serve adoption when the execution conditions and evidence requirements match. +Different deployment conditions or additional selection still require fresh evidence. +Reuse an actual compatible measurement; do not reuse a score merely because both APIs call it final. + +## Comparison with current research + +| Primary source | Mechanism relevant to this system | Current local implementation | Consequence for the design | +| --- | --- | --- | --- | +| [GEPA paper](https://arxiv.org/abs/2507.19457), [official code](https://github.com/gepa-ai/gepa) | Trace-guided reflective edits, candidate diversity, complementary combinations, complete search | Real `optimize_anything` integration; engine, sequential, adaptive-sequential, best-of, vote, and omni recipes | Keep the complete-method contract and expose useful execution evidence to it | +| [SkillOpt paper](https://arxiv.org/abs/2605.23904), [official code](https://github.com/microsoft/SkillOpt) | Bounded edits, rejected-attempt context, slower method updates, explicit acceptance rules | Real `ReflACTTrainer` integration over strings | The imported optimizer is substantive; it is not the entire continuing learner | +| [SkillOpt Sleep at audited head](https://github.com/microsoft/SkillOpt/blob/79124b37e9a6371e13b753f8bcd7adb1e493ade1/docs/sleep/README.md) | Harvest experience → mine tasks → replay → consolidate → stage → adopt; optional archive replay | No corresponding joined Runtime/Knowledge path established | Study the experience-to-next-update connection, not just the text optimizer | +| [ACE paper](https://arxiv.org/abs/2510.04618), [official code](https://github.com/ace-agent/ace) | Incremental context updates and detailed retained lessons during online/offline adaptation | Corpus and KB storage exist; usefulness is not automatically validated | Retain precise experience and measure its effect rather than repeatedly replacing it with a short summary | +| [ADAS paper](https://arxiv.org/abs/2408.08435), [official code](https://github.com/ShengranHu/ADAS) | Search over executable agent designs using earlier discoveries | Full profiles, code candidates, and authored strategies are expressible | Preserve architecture search and connect its outcomes to the common experiment record | +| [Darwin Gödel Machine paper](https://arxiv.org/abs/2505.22954), [official code](https://github.com/jennyzzt/dgm) | Iterative self-modification with an archive of agent programs and empirical tests | Code search and a separate strategy archive exist | Retain alternatives and evaluate modified learning procedures on subsequent work | +| [AlphaEvolve primary report](https://deepmind.google/discover/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/) | Program generation, automated domain evaluation, and a population that informs later proposals | The code/experiment components can express this pattern | Search sophistication only matters when evaluation measures the intended domain outcome | + +These sources operate in different domains with different models, budgets, and evaluation designs. +Their published gains are not comparable cells in one experiment. +The audit therefore makes no numerical ranking of their capabilities against this system. + +### Release freshness is not implementation parity + +Live PyPI metadata returned GEPA `0.1.4` and SkillOpt `0.2.0`, matching the declared integration versions. +The source pins are older than the inspected development heads. + +| Package | Local tested source | Inspected upstream head | Commits after local pin | +| --- | --- | --- | ---: | +| GEPA | `f919db0a622e2e9f9204779b81fe00cc1b2d808f` | `0632cdb5dcc052e690eab439e1b4a7e3e9cfe407` | 48 | +| SkillOpt | `61735e3922efc2b90c6d6cab561e62e98452ca90` | `79124b37e9a6371e13b753f8bcd7adb1e493ade1` | 234 | + +The counts come from GitHub's compare API; both comparisons report zero commits behind the local pin. +They measure revision distance, not missing capability or performance loss. +GEPA changes include trace-aware evaluation and separating validation-cache entries from training rollouts. +SkillOpt Sleep describes itself as a preview and distinguishes development-branch features from the published package. +The local SkillOpt compatibility code also documents missing prompt files in the wheel and requires the tested source installation. + +Upstream update work should compare exact behavior and replay compatibility tests before changing pins. +Blindly following main would discard the value of current source-bound compatibility checks. +DSPy `3.2.1` uses a separate GEPA `0.0.27` environment; it should not be silently merged with the `0.1.4` bridge. + +Other integration limits matter more than package names. +External evaluation describes only scenario IDs unless the caller provides case and artifact descriptions. +Canonical search-history receipts are optional, and SkillOpt does not provide the same receipt coverage as native search or GEPA. +Composed GEPA recipes restart; direct GEPA and SkillOpt have explicit version-specific resume support. +A callback evaluation can execute multiple repetitions, so an evaluation count is not a model-call count or an equal-resource comparison. + +**SOTA verdict:** the system uses current research methods and has useful exact-execution machinery. +Its complete domain-learning or meta-learning behavior is not established as state of the art. +The largest demonstrated gaps concern composition and evidence. +Specialist improvement, repeated within-domain learning, evaluator improvement, and learning-process transfer each need an appropriate capability comparison. + +## A more powerful unification + +Use the existing exact candidate and experiment path as the common unit. +Do not introduce another generic learning manager above the current APIs. + +```mermaid +flowchart LR + X[Execute domain tasks and projects] --> E[Store outcomes, costs, traces, and use evidence] + E --> L[Domain learner chooses experiments and improves working evaluations] + L --> C[Candidate: specialist, working evaluation, or learning process] + C --> D[Shared development execution and measurement] + D --> L + D --> A[Retain candidates, parents, failures, and findings] + A --> L + D --> F[Freeze selected version] + F --> T[Independent comparison for the declared learning objective] + T --> V[Adopt exact complete version] + V --> X + A --> ML[Meta-agent learns how to construct and improve domain learners] + ML --> L +``` + +This diagram is the proposed joined behavior, not a claim that all arrows are implemented today. +The following existing components provide its starting points. + +| Concern | Existing owner | Required extension or join | +| --- | --- | --- | +| Exact agent state | `buildAgentCandidateBundle`, exact profile materialization, Knowledge candidate references | Make the complete measured combination the search and adoption unit where needed | +| Search policy | `OptimizationMethod`, native search, profiled authors | Adapt alternative searches to common measurement and records without forcing their internals into one algorithm | +| Development execution | Runtime candidate execution and caller adapters | Use one exact executable candidate path for search and final measurements when supported | +| Evidence | Eval campaigns, search receipts, cost account, Knowledge use receipts | Bind candidate, objective, evaluator, tasks, outcomes, spend, and evidence completeness consistently | +| Persistent learning | Existing candidate history, Corpus, source ledger, Knowledge stores | Make prior failures and useful discoveries explicit inputs to the next learning decision | +| Adoption | Existing proposal, experiment, activation, and restoration contracts | Switch one exact runnable version; retain storage-specific application adapters | +| Learning the method | Profiles, strategy-author contract, code candidates | Evaluate a candidate learner by the future work and agent versions it produces | +| Learning evaluation engineering | Eval judges, calibration data, curricula, and case-generation utilities | Evaluate changes to working evaluations against independent outcomes and their effect on domain learning | + +### Candidate identity must include execution meaning + +A candidate is the exact runnable combination, not just a prompt string or a KB path. +Runtime's bundle builder already covers profile, code, execution policy, knowledge, and memory. +It verifies finalized code bytes rather than trusting a path label. +Reuse that representation and the backend support checks wherever possible. + +Opaque callbacks still require an explicit revision covering code and captured configuration. +JavaScript function names and closure inspection cannot establish that identity. +The current `executionRef` contract correctly assigns this responsibility to callers. +The audit found first-party recipes that leave mutable task lookup data outside that identity, especially ID-only SWE scenarios. +Source-bound execution adapters reduce this burden; they do not make arbitrary mutable external state automatically reproducible. + +### Preserve algorithm freedom while sharing factual records + +GEPA, SkillOpt, native population search, code search, and future training methods may need different internal state and selection rules. +They should agree on the meaning of candidate identity, objective, development observations, selected outcome, resources, and final evidence. +A selected baseline, failed search, interrupted search, and unproven exploratory candidate are valid outcomes. +None requires fabricated generations, an empty success response, or a deployment exception masquerading as a research result. + +The native code path can eventually be exposed as a complete method while retaining Runtime's worktree ownership. +Strategy evolution should retain its executable programs and archive while using the common comparison and resource record. +Replacing those implementation-specific mechanics with another general-purpose orchestration layer would increase fragmentation. + +### Make memory causally relevant to work + +Memory records need to distinguish observations, proposed explanations, checked results, and demonstrated useful transfers. +These are evidence states, not confidence adjectives. +Preserve source/version identity, contradiction handling, scope, and the exact candidate that produced or used a record. +Share the retrieval/use evidence format across serving paths before adding another memory backend. + +Knowledge's session withholding can help estimate lesson usefulness when joined to actual task rewards and valid assignment probabilities. +It cannot establish causality from a flat retrieval log after the fact. +A useful lesson may also require a changed planning or retrieval policy; compare necessary combinations before removing components. +Persistent experiment failures should inform later proposals as well as successful artifacts. + +### Let the learner change its own procedure + +The domain learning method should be an executable policy that can select problems, design interventions, improve working evaluations, and consolidate useful results. +Its profile and implementation are candidate surfaces too. +The same experiment machinery can compare two learning methods by the subsequent specialists, learning processes, or discoveries they produce. +Repeated fresh tasks within one domain can test domain-learning quality. +Fresh domains can separately test whether the method learns to construct useful domain learners. +Both comparisons need the full cost of learning, inference, evaluation engineering, checking, and retained-state maintenance. + +An improved evaluator can also be a research candidate, but it cannot certify itself by redefining success. +Changes to result checking require independent calibration and a fixed external acceptance criterion for the experiment. +Checks for research claims can combine executable tests, source inspection, replication, expert assessment, and explicit uncertainty. +Requiring every valuable question to have a simple deterministic score would unnecessarily restrict the pursuit. + +## What to delete, simplify, and retain + +| Decision | Target | Reason | +| --- | --- | --- | +| Delete | Complete method wrapped as one native proposal | It adds a second selection policy and caused incorrect outcomes | +| Delete | Independent RAG states merged only after callbacks finish | The merge cannot restore information callbacks never received | +| Delete | Perfect values for absent measurements | Missing evidence must remain missing | +| Delete | Hidden worker instructions appended during supervisor execution | They override the actual change surface and can contradict the task | +| Delete | Silent partial-patch and lesson-storage success | They erase the failure signal needed for learning | +| Simplify | Final comparison coverage and candidate identity | These are shared measurement rules, not algorithm choices | +| Simplify after equivalent execution is joined | Duplicate release measurement | Reuse exact compatible evidence; preserve fresh tests when conditions differ | +| Simplify through common records | Strategy evolution and code search lifecycle | Preserve search mechanisms while removing duplicated decision/accounting conventions | +| Consolidate after supported callers migrate | Flat MCP memory and richer Knowledge serving | Different stores can share one retrieval/use contract | +| Retain | Exact immutable profiles, code trees, KB snapshots, and source checks | They make results refer to something reproducible | +| Retain | Separate development and final evidence | Search must not quietly learn the final answers | +| Retain | Candidate diversity, archives, joint changes, and recursive work | They express the ambition; the audit did not disprove their value | +| Retain | Package layering and storage adapters | They isolate meaningful responsibilities | +| Retain | Authored-candidate measurement and activation | These already accept changes without mandating an optimizer | + +No new learning scheduler, memory service, optimizer facade, or universal scalar is justified by the reproduced bugs. +The useful work is joining existing paths and removing conflicting meanings. + +## Experiments for the different learning claims + +This is an experiment specification, not a result or an executed paid run. +For continued domain learning, use repeated task or project sequences within that domain. +Earlier discoveries and procedures should be useful without requiring literal answer reuse. +Include fresh domain tasks, multiple objectives when relevant, and retained-ability checks. +For meta-learning, compare the procedures that construct and improve domain learners. +Only the claim of transfer requires unfamiliar domains; specialist success does not. +For the generic experiment API, one independent measurement unit can contain a whole sequence, with explicit initial state and accumulated intermediate state. +Do not treat isolated per-task memory resets as a test of accumulation across projects. + +First execute the complete learning mechanism and verify that evidence changes subsequent decisions. +Then compare frozen learning with active learning and remove components to identify their contributions. + +| Comparison | Question answered | Important control | +| --- | --- | --- | +| Same initial system, frozen versus active learning | Does the complete process improve later outcomes? | Matched sequences and all actual learning costs | +| Repeated domain learning trials | Does the process reliably discover better specialists in its intended domain? | Fresh domain tasks, explicit objectives, and repeated initial conditions | +| Fixed versus improved working evaluation | Does evaluation engineering improve detection and subsequent learning? | Independent case labels/checks, new failure cases, and full evaluation-development cost | +| Retained knowledge with fixed method versus full learner | Do changed decisions add value beyond storing information? | Same access to accumulated evidence | +| Changed method with reset knowledge versus full learner | Does accumulated knowledge contribute in combination? | Declare the reset and account for reconstruction cost | +| Two learner versions on fresh sequences | Did the improvement procedure itself improve? | Independent outcome checks outside both methods' development work | +| Learned process versus reference process in new domains | Can the meta-agent reconstruct effective learning elsewhere? | Same domain information and resources; do not require the old specialist to transfer | + +Record every measured dimension for every project, including missing values and failed executions. + +| Dimension | Required observation | +| --- | --- | +| Delivered outcome | Task result, independent check, failures, and uncertainty | +| Capability expansion | Previously unsolved problem classes or difficulty levels reached | +| Resources | Learning, execution, retrieval, checking, storage, total spend, and elapsed time | +| Transfer | Performance on unfamiliar projects and domains | +| Retention | Performance on earlier abilities after updates | +| Memory contribution | Exact retrieved items, use records, and withholding assignments when applicable | +| Decision changes | Which prior evidence changed the next experiment or working method | +| Evaluation changes | Which cases, checks, objectives, or calibration decisions changed, and their independent quality | +| Search behavior | Proposed/selected/rejected candidates, ancestry, and unresolved hypotheses | +| Research quality | Claim support, contradictions, unanswered questions, and stopping behavior | +| Adoption | Exact version measured and exact version used by subsequent work | + +Report per-project values and min/median/p90/max with the sample count for each available numeric dimension. +Pair observations at the independent project-sequence level rather than pretending repeated steps are independent projects. +Separate actual resource asymmetries before interpreting any capability difference. +Choose the sample size and stopping rule from a calibrated measure and the smallest effect relevant to the declared objective. +The audit supplies no estimate of that effect and therefore does not invent a required sample count. + +A null result can reject the tested mechanism only after confirming it actually ran and that the measure could detect useful progress. +It does not reject all combinations, longer horizons, harder project regimes, or alternative implementations. +Conversely, a positive result on one sequence does not establish general recursive improvement. + +## Limits and decisions + +The audit found no current artifact establishing the complete domain-learning and meta-learning behavior of this exact assembled system. +That is a bounded finding about inspected evidence, not a claim that no private experiment exists. +Historical reports in these repositories describe different versions, task sets, resource conditions, or incomplete mechanisms. +They must retain those boundaries when used to guide current work. + +The implementation can already represent more ambitious candidates than the high-level API names suggest. +The highest-priority repair is trustworthy composition of those candidates with execution, learning evidence, and adoption. +The capability questions concern better specialists, repeatable domain learning, better evaluation engineering, and improvements to the learning process itself. +Transfer of that process is an additional ambitious claim with its own comparison. +Both are necessary: dependable measurements make ambitious exploration interpretable, while ambitious experiments prevent correctness work from becoming the entire pursuit. diff --git a/package.json b/package.json index 5c5c0ac29..5ccf2e391 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.193.1", + "version": "0.195.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -180,7 +180,7 @@ "license": "MIT", "packageManager": "pnpm@11.17.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.173.0 <0.174.0", + "@tangle-network/agent-eval": ">=0.174.0 <0.175.0", "@tangle-network/agent-interface": "^2.3.0", "@tangle-network/sandbox": ">=0.36.4 <0.38.0" }, diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b7db6fb0f..eb8ad4e1a 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -21,9 +21,9 @@ catalog: '@modelcontextprotocol/sdk': 1.30.0 '@tangle-network/agent-core': '>=0.9.6 <0.10.0' '@types/node': 26.4.0 - '@tangle-network/agent-eval': '>=0.173.0 <0.174.0' + '@tangle-network/agent-eval': '>=0.174.0 <0.175.0' '@tangle-network/agent-interface': ^2.3.0 - '@tangle-network/agent-knowledge': ^13.0.1 + '@tangle-network/agent-knowledge': ^14.0.0 '@tangle-network/agent-profile-materialize': '>=0.19.0 <0.20.0' '@tangle-network/agent-trace-contract': ^1.0.2 '@tangle-network/sandbox': '>=0.36.4 <0.38.0' diff --git a/src/improvement/agentic-generator.ts b/src/improvement/agentic-generator.ts index 3951c49f1..199d2dc95 100644 --- a/src/improvement/agentic-generator.ts +++ b/src/improvement/agentic-generator.ts @@ -1001,7 +1001,7 @@ function worktreeDirty(worktreePath: string): boolean { } function worktreeChangedPaths(worktreePath: string): string[] { - const result = spawnSync('git', ['status', '--porcelain', '--untracked-files=all'], { + const result = spawnSync('git', ['status', '--porcelain=v1', '-z', '--untracked-files=all'], { cwd: worktreePath, encoding: 'utf-8', }) @@ -1015,9 +1015,22 @@ function worktreeChangedPaths(worktreePath: string): string[] { `agenticGenerator: git status exited ${result.status} in ${worktreePath}: ${result.stderr.trim()}`, ) } - return result.stdout - .split('\n') - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => line.slice(3).trim()) + const records = result.stdout.split('\0') + const paths: string[] = [] + for (let i = 0; i < records.length; i += 1) { + const record = records[i] + if (!record) continue + paths.push(record.slice(3)) + // Rename and copy entries have a second NUL-delimited path without a status prefix. + if (record[0] === 'R' || record[1] === 'R' || record[0] === 'C' || record[1] === 'C') { + const source = records[++i] + if (!source) { + throw new Error( + `agenticGenerator: git status omitted a rename/copy path in ${worktreePath}`, + ) + } + paths.push(source) + } + } + return paths } diff --git a/src/improvement/code-execution.ts b/src/improvement/code-execution.ts index 4ba60d05b..10a7cbccf 100644 --- a/src/improvement/code-execution.ts +++ b/src/improvement/code-execution.ts @@ -13,7 +13,7 @@ import { type Scenario, type SelfImproveBudget, type SelfImproveOptions, - type SelfImproveResult, + type SelfImproveProposerResult, type SurfaceProposer, selfImprove, } from '@tangle-network/agent-eval/contract' @@ -260,7 +260,7 @@ export async function runCodeImprovement( const budget: SelfImproveBudget = gate === 'none' ? { ...sharedOptions.budget, generations: 0 } : { ...sharedOptions.budget } - let raw: SelfImproveResult + let raw: SelfImproveProposerResult try { raw = await selfImprove({ ...sharedOptions, diff --git a/src/improvement/improve-types.ts b/src/improvement/improve-types.ts index cc1d5ec6d..e3cd715e2 100644 --- a/src/improvement/improve-types.ts +++ b/src/improvement/improve-types.ts @@ -11,7 +11,7 @@ import type { Scenario, SelfImproveBudget, SelfImproveOptions, - SelfImproveResult, + SelfImproveProposerResult, } from '@tangle-network/agent-eval/contract' import type { AgentImprovementSurface, @@ -389,7 +389,7 @@ interface ImproveResultBase { /** Frozen candidate only. Live state is changed through an approved activation. */ candidate: TCandidate /** Final-test decision for this search result. */ - decision: SelfImproveResult['gateDecision'] + decision: SelfImproveProposerResult['gateDecision'] /** Final-test lift when one was measured. */ lift?: number /** Paired final-test confidence interval for method-based profile runs. */ @@ -432,7 +432,7 @@ export interface ImproveMethodResult extends ImproveResultBase extends ImproveResultBase { mode: 'code' - raw: SelfImproveResult + raw: SelfImproveProposerResult } export type ImproveResult = diff --git a/src/improvement/reflective-generator.ts b/src/improvement/reflective-generator.ts index f42c9fd51..6aad8c0ff 100644 --- a/src/improvement/reflective-generator.ts +++ b/src/improvement/reflective-generator.ts @@ -13,44 +13,116 @@ */ import { spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { readFileSync, realpathSync } from 'node:fs' +import { isAbsolute, relative, resolve, sep } from 'node:path' import type { SurfaceImprovementEdit } from '../agent/improvement-adapter' import type { ImprovementProposalSource } from '../analyst-loop/types' import type { CandidateGenerator } from './improvement-driver' export interface ReflectiveGeneratorOptions { - improvementProposalSource: ImprovementProposalSource + /** Bind proposal reads and paid calls to this candidate's worktree and account. */ + createImprovementProposalSource( + context: Parameters[0], + ): ImprovementProposalSource } /** Cheap no-sandbox `CandidateGenerator` (the `shots=1` setting): draft surface edits via the improvement adapter and apply them as one coherent candidate. */ export function reflectiveGenerator(opts: ReflectiveGeneratorOptions): CandidateGenerator { return { kind: 'reflective', - async generate({ worktreePath, findings }) { - const batch = await opts.improvementProposalSource.proposeFromFindings(findings) + async generate(context) { + const { worktreePath, findings, signal } = context + signal.throwIfAborted() + const source = opts.createImprovementProposalSource(context) + const batch = await source.proposeFromFindings(findings) + signal.throwIfAborted() + if (batch.errors.length > 0) { + throw new AggregateError( + batch.errors.map((error) => new Error(`${error.findingId}: ${error.message}`)), + `reflectiveGenerator: proposal failed: ${batch.errors.map((error) => error.message).join('; ')}`, + ) + } if (batch.edits.length === 0) return { applied: false, summary: '' } - let applied = 0 for (const edit of batch.edits) { - if (applyPatch(edit.patch, worktreePath)) applied++ + assertPatchTarget(edit, worktreePath) + assertCurrentBase(edit, worktreePath) } - if (applied === 0) return { applied: false, summary: '' } + applyPatches(batch.edits, worktreePath) + signal.throwIfAborted() const summary = batch.edits.length === 1 ? batch.edits[0]!.summary - : `analyst: ${applied} surface edit${applied === 1 ? '' : 's'}` + : `analyst: ${batch.edits.length} surface edits` return { applied: true, summary } }, } } -/** Apply a proposed patch inside the isolated candidate worktree. - * candidate worktree (a fresh checkout of baseRef, so `-p0` paths match). */ -function applyPatch(patch: string, cwd: string): boolean { - const result = spawnSync('git', ['apply', '--whitespace=fix', '-p0', '-'], { +function assertPatchTarget(edit: SurfaceImprovementEdit, cwd: string): void { + // Reverse statistics include the source path of a rename or copy. + for (const reverse of [false, true]) { + const result = spawnSync( + 'git', + ['apply', '--numstat', '-z', '-p0', ...(reverse ? ['--reverse'] : []), '-'], + { cwd, input: edit.patch, encoding: 'utf8' }, + ) + if (result.error) throw result.error + if (result.status !== 0) { + throw new Error(`reflectiveGenerator: invalid patch: ${result.stderr.trim()}`) + } + const paths = result.stdout + .split('\0') + .filter(Boolean) + .map((entry) => entry.slice(entry.indexOf('\t', entry.indexOf('\t') + 1) + 1)) + const target = resolve(cwd, edit.target.repoRelativePath) + if (paths.length === 0 || paths.some((path) => resolve(cwd, path) !== target)) { + throw new Error('reflectiveGenerator: patch paths do not match the declared target') + } + } +} + +function assertCurrentBase(edit: SurfaceImprovementEdit, cwd: string): void { + const root = realpathSync(cwd) + const path = resolve(root, edit.target.repoRelativePath) + assertWithinWorktree(root, path) + let content: string + try { + assertWithinWorktree(root, realpathSync(path)) + content = readFileSync(path, 'utf8') + } catch (error) { + if ( + edit.target.intent !== 'create-new' || + !(error instanceof Error && 'code' in error && error.code === 'ENOENT') + ) { + throw error + } + content = '' + } + const actual = createHash('sha256').update(content, 'utf8').digest('hex') + if (actual !== edit.baseSha256) { + throw new Error(`reflectiveGenerator: stale proposal base for ${edit.target.repoRelativePath}`) + } +} + +function assertWithinWorktree(root: string, path: string): void { + const local = relative(root, path) + if (local === '..' || local.startsWith(`..${sep}`) || isAbsolute(local)) { + throw new Error('reflectiveGenerator: proposal target is outside the candidate worktree') + } +} + +function applyPatches(edits: SurfaceImprovementEdit[], cwd: string): void { + // Git applies the complete batch atomically unless --reject is requested. + const result = spawnSync('git', ['apply', '-p0', '-'], { cwd, - input: patch, + input: edits.map((edit) => `${edit.patch.trimEnd()}\n`).join(''), encoding: 'utf-8', }) - return result.status === 0 + if (result.error) throw result.error + if (result.status !== 0) { + throw new Error(`reflectiveGenerator: patch batch failed: ${result.stderr.trim()}`) + } } diff --git a/src/knowledge/supervised-update.ts b/src/knowledge/supervised-update.ts index 7eeab25d8..81ae96335 100644 --- a/src/knowledge/supervised-update.ts +++ b/src/knowledge/supervised-update.ts @@ -1,6 +1,5 @@ import { agentProfileSchema } from '@tangle-network/agent-interface' import type { RagKnowledgeUpdateResult } from '@tangle-network/agent-knowledge' -import { RESEARCHER_SYSTEM_PROMPT } from '../profiles/researcher' import type { DeliverableSpec } from '../runtime/supervise/completion-gate' import { assertExecutableAgentProfile } from '../runtime/supervise/model-policy' import type { ExecutorConfig } from '../runtime/supervise/runtime' @@ -131,19 +130,9 @@ export async function runSupervisedKnowledgeUpdate( ): Promise { const exactSupervisor = agentProfileSchema.parse(options.supervisorProfile) as SupervisorProfile assertExecutableAgentProfile(exactSupervisor, 'runSupervisedKnowledgeUpdate') - const baseInstructions = exactSupervisor.prompt?.systemPrompt ?? RESEARCH_SUPERVISOR_SYSTEM_PROMPT - const workerContract = RESEARCHER_SYSTEM_PROMPT - const systemPrompt = workerContract - ? `${baseInstructions}\n\nEach researcher worker you spawn follows this contract:\n${workerContract}` - : baseInstructions - - const profile: SupervisorProfile = { - ...exactSupervisor, - prompt: { ...exactSupervisor.prompt, systemPrompt }, - } const run = options.runSupervised ?? supervise const task = formatSupervisedKnowledgeTask(options) - const supervised = await run(profile, task, { + const supervised = await run(exactSupervisor, task, { ...options.superviseOptions, budget: options.budget, backend: options.backend, diff --git a/src/runtime/observe.ts b/src/runtime/observe.ts index f81d06a49..42acdc4c8 100644 --- a/src/runtime/observe.ts +++ b/src/runtime/observe.ts @@ -169,11 +169,11 @@ export async function observe(input: ObserveInput, opts: ObserveOptions): Promis parsed.map((f) => makeProposalFinding({ analyst_id: observerId, - area: `${f.area}`, + area: f.area, severity: f.severity, claim: f.claim, recommended_action: f.recommended_action, - confidence: typeof f.confidence === 'number' ? f.confidence : 0.5, + confidence: f.confidence, evidence_refs: [], // The observer reads behavior, never a final evaluation result. derived_from_judge: false, @@ -201,7 +201,12 @@ export async function observe(input: ObserveInput, opts: ObserveOptions): Promis evidence: [{ kind: 'finding', uri: f.finding_id }], } const r = await opts.corpus.append(record) - if (r.succeeded) learned.push(record) + if (!r.succeeded) { + throw new Error( + `observe corpus append failed for '${record.id}' after storing ${learned.length}/${findings.length} findings: ${r.error}`, + ) + } + learned.push(record) } } @@ -236,12 +241,43 @@ function parseFindings(content: string): RawFinding[] { let obj: unknown try { obj = JSON.parse(content) - } catch { - const m = content.match(/\{[\s\S]*\}/) - obj = m ? JSON.parse(m[0]) : { findings: [] } + } catch (error) { + throw new Error('observe response: expected a JSON object with findings', { cause: error }) + } + if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) { + throw new Error('observe response: expected a JSON object with findings') + } + const response = obj as Record + if ( + !Array.isArray(response.findings) || + Object.keys(response).some((key) => key !== 'findings') + ) { + throw new Error('observe response: expected only a findings array') + } + const schema = findingsSchema.schema.properties.findings.items + for (const [index, value] of response.findings.entries()) { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`observe response: findings[${index}] must be an object`) + } + const finding = value as Record + const invalidField = + Object.keys(finding).some((key) => !schema.required.some((field) => field === key)) || + ['area', 'claim', 'recommended_action'].some( + (key) => typeof finding[key] !== 'string' || finding[key].trim().length === 0, + ) || + !schema.properties.severity.enum.some((severity) => severity === finding.severity) || + !schema.properties.audience.enum.some((audience) => audience === finding.audience) || + typeof finding.confidence !== 'number' || + !Number.isFinite(finding.confidence) || + finding.confidence < 0 || + finding.confidence > 1 + if (invalidField) { + throw new Error( + `observe response: findings[${index}] does not match the required finding schema`, + ) + } } - const arr = (obj as { findings?: unknown }).findings - return Array.isArray(arr) ? (arr as RawFinding[]) : [] + return response.findings as RawFinding[] } /** Operator-facing report, split by who should act. The agent block is the diff --git a/src/runtime/personify/corpus.ts b/src/runtime/personify/corpus.ts index bfe2cbe5c..daacdea96 100644 --- a/src/runtime/personify/corpus.ts +++ b/src/runtime/personify/corpus.ts @@ -24,6 +24,7 @@ import { prepareJsonlAppend, writeAllBytes, } from '../../durable/jsonl-file' +import { detachedFrozen } from '../supervise/snapshot' import type { Corpus, CorpusFilter, @@ -171,14 +172,16 @@ export class InMemoryCorpus implements Corpus { async append( record: CorpusRecord, ): Promise<{ succeeded: true } | { succeeded: false; error: string }> { + let snapshot: CorpusRecord try { assertCorpusRecord(record, 'append: record') + snapshot = detachedFrozen(record) } catch (err) { return { succeeded: false, error: err instanceof Error ? err.message : String(err) } } - const existing = this.byId.get(record.id) + const existing = this.byId.get(snapshot.id) if (existing) { - if (recordsEqual(existing, record)) return { succeeded: true } + if (recordsEqual(existing, snapshot)) return { succeeded: true } return { succeeded: false, error: @@ -186,7 +189,7 @@ export class InMemoryCorpus implements Corpus { 'a learned fact is append-only — re-mint the id or reconcile before re-appending', } } - this.byId.set(record.id, freeze(record)) + this.byId.set(snapshot.id, snapshot) return { succeeded: true } } @@ -215,27 +218,46 @@ export class FileCorpus implements Corpus { ): Promise<{ succeeded: true } | { succeeded: false; error: string }> { try { assertCorpusRecord(record, 'append: record') + const snapshot = detachedFrozen(record) + const fs = await import('node:fs/promises') + const { dirname } = await import('node:path') + const { tryAcquireAtomicFileLock } = await import('@tangle-network/agent-eval/ledger-core') + await fs.mkdir(dirname(this.path), { recursive: true }) + // Create the file before resolving it so aliases share a lock on the first append too. + const file = await fs.open(this.path, 'a') + await file.close() + const canonicalPath = await fs.realpath(this.path) + const deadline = Date.now() + 5000 + for (;;) { + const acquisition = tryAcquireAtomicFileLock({ lockPath: `${canonicalPath}.lock` }) + if (!acquisition.acquired) { + if (Date.now() >= deadline) { + throw new Error(`corpus append lock unavailable after 5000ms: ${canonicalPath}`) + } + await new Promise((resolve) => setTimeout(resolve, 25)) + continue + } + try { + const stored = await this.load(canonicalPath) + const existing = stored.get(snapshot.id) + if (existing) { + if (recordsEqual(existing, snapshot)) return { succeeded: true } + return { + succeeded: false, + error: + `corpus conflict: id '${snapshot.id}' is already stored in ${canonicalPath} with a different ` + + 'record; a learned fact is append-only — re-mint the id or reconcile before re-appending', + } + } + await this.appendLine(canonicalPath, snapshot) + return { succeeded: true } + } finally { + acquisition.lock.release() + } + } } catch (err) { return { succeeded: false, error: err instanceof Error ? err.message : String(err) } } - let stored: Map - try { - stored = await this.load() - } catch (err) { - return { succeeded: false, error: err instanceof Error ? err.message : String(err) } - } - const existing = stored.get(record.id) - if (existing) { - if (recordsEqual(existing, record)) return { succeeded: true } - return { - succeeded: false, - error: - `corpus conflict: id '${record.id}' is already stored in ${this.path} with a different ` + - 'record; a learned fact is append-only — re-mint the id or reconcile before re-appending', - } - } - await this.appendLine(record) - return { succeeded: true } } async query(filter: CorpusFilter): Promise> { @@ -243,11 +265,11 @@ export class FileCorpus implements Corpus { return applyFilter([...stored.values()], filter) } - private async load(): Promise> { + private async load(path = this.path): Promise> { const fs = await import('node:fs/promises') let text: string try { - text = await fs.readFile(this.path, 'utf8') + text = await fs.readFile(path, 'utf8') } catch (err) { if (isNoEntError(err)) return new Map() throw err @@ -264,17 +286,15 @@ export class FileCorpus implements Corpus { 'an append-only corpus must never hold a conflicting re-append under one id', ) } - byId.set(parsed.id, freeze(parsed)) + byId.set(parsed.id, detachedFrozen(parsed)) } return byId } - private async appendLine(record: CorpusRecord): Promise { + private async appendLine(path: string, record: CorpusRecord): Promise { const fs = await import('node:fs/promises') - const path = await import('node:path') - await fs.mkdir(path.dirname(this.path), { recursive: true }) - const needsSeparator = await prepareJsonlAppend(this.path) - const fh = await fs.open(this.path, 'a') + const needsSeparator = await prepareJsonlAppend(path) + const fh = await fs.open(path, 'a') try { await writeAllBytes(fh, `${needsSeparator ? '\n' : ''}${JSON.stringify(record)}\n`) await fh.sync() @@ -340,9 +360,3 @@ export async function renderCorpusToInstructions( function renderLine(record: CorpusRecord): string { return record.rationale ? `${record.claim} (${record.rationale})` : record.claim } - -// ── Internal helpers ─────────────────────────────────────────────────────────── - -function freeze(record: CorpusRecord): CorpusRecord { - return Object.freeze({ ...record }) -} diff --git a/src/runtime/strategy-author.ts b/src/runtime/strategy-author.ts index 69dc68588..300bc69aa 100644 --- a/src/runtime/strategy-author.ts +++ b/src/runtime/strategy-author.ts @@ -4,9 +4,10 @@ * optimization strategy as code; the caller gates it like any human-built candidate * (runBenchmark + a frozen holdout). * - * Structurally safe by construction: the authored body composes shot()/critique() and - * spends through the Supervisor's conserved pool — it can be wrong, but it cannot - * Goodhart the check (it never sees the verifiers) and it cannot win by overspending. + * Brokered shot()/critique() calls spend through the Supervisor's conserved pool. + * Source validation checks obvious forbidden operations; it is not an execution sandbox. + * Code that uses other execution paths can access state or spend outside that pool. + * Callers must provide isolation when authored code requires an enforced boundary. * * The authored module is written to `outDir` and dynamically imported — run under a * TS-capable loader (tsx) since models often emit type annotations. @@ -26,16 +27,20 @@ export const strategyAuthorContract = ` You author an OPTIMIZATION STRATEGY for an agentic loop system. A strategy decides how to spend a compute budget to beat a task's deployable check. You compose exactly two steps: - shot(spec?: { handle?, messages?, steer?, persona?, tools? }): Promise + shot(spec?: { handle?, messages?, steer?, profile?, tools? }): Promise Runs ONE worker attempt (a bounded tool loop) over an artifact. - omit handle => the shot opens its OWN fresh artifact and closes it after (a sample). - pass handle => the shot CONTINUES that artifact (state accumulates across shots). - messages => the carried conversation (pass the previous ShotResult.messages to continue). - steer => a corrective instruction injected before the shot. - - persona => { systemPrompt?, model? } — give THIS shot its own role and/or model - (multi-agent strategies: a researcher shot then an engineer shot, a panel of k - personas over one budget). On a fresh shot the systemPrompt replaces the task's; on - a carried conversation it arrives as a hand-off message. Same conserved budget. + - profile => a complete AgentProfile — give THIS shot its own instructions, + model, skills, tools, hooks, and subagents. Include name, harness, and + model: { provider, default }; put standing instructions in prompt.systemPrompt. + For example: { ...opts.workerProfile, name: 'researcher', + prompt: { ...opts.workerProfile.prompt, + systemPrompt: 'Inspect the evidence before proposing a change.' } }. + Choose an available model from the current execution setup. Omit profile to use + the worker's exact profile. Every shot spends from the same conserved budget. - tools => string[] — restrict THIS shot to a subset of the task's tools by name (focus an explore shot on read-only tools, an execute shot on write tools). Restriction-only; unknown names make the shot fail. ALWAYS select from @@ -73,7 +78,7 @@ Rules: - The module must be EXACTLY this shape (no other imports, no commentary outside code): import { defineStrategy } from '@tangle-network/agent-runtime/kernel' -export default defineStrategy('your-strategy-name', async ({ surface, task, budget, shot, critique, listTools }) => { +export default defineStrategy('your-strategy-name', async ({ surface, task, opts, budget, shot, critique, listTools }) => { // your composition (listTools comes from the destructured context — it is NOT a global) }) ` diff --git a/src/runtime/strategy-evolution.ts b/src/runtime/strategy-evolution.ts index 34c38be28..16e14542c 100644 --- a/src/runtime/strategy-evolution.ts +++ b/src/runtime/strategy-evolution.ts @@ -25,8 +25,14 @@ */ import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { pathToFileURL } from 'node:url' import { gzipSync } from 'node:zlib' -import type { AgentProfile } from '@tangle-network/agent-interface' +import type { AgentProfile, Sha256Digest } from '@tangle-network/agent-interface' +import { + canonicalCandidateDigest, + immutableCandidateValue, + sha256Bytes, +} from '../candidate-execution/digest' import type { RuntimeHooks } from '../runtime-hooks' import { profileChatClient } from './profile-chat-client' import { type PromotionVerdict, promotionGate } from './promotion-gate' @@ -142,6 +148,11 @@ export interface StrategyEvolutionConfig { checkpoint?: { path: string resume?: boolean + /** Digest of execution dependencies: environment, baseline code, transports, callbacks, + * and external state such as a corpus. Update it when any dependency changes. + * Runtime hashes profiles, settings, JSON task payloads, and authored bytes separately; + * it cannot infer callback behavior or external state. */ + executionRef: Sha256Digest } /** Called before each benchmark phase (gen0, gen1…, band-screen, holdout, reproduce). * The seam for environment recycling — no artifacts span phases, so a runner may @@ -153,6 +164,9 @@ export interface StrategyEvolutionConfig { /** The on-disk phase ledger — everything needed to skip completed phases on resume. */ interface EvolutionCheckpoint { + fingerprint: Sha256Digest + trainDigest: Sha256Digest + holdoutPoolDigest?: Sha256Digest gen0?: BenchmarkReport gen0Champion?: ChampionPick generations: EvolutionGeneration[] @@ -172,6 +186,8 @@ export interface ChampionPick { export interface EvolutionCandidate { name: string file?: string + /** Digest of the exact authored module evaluated in this generation. */ + sourceSha256?: Sha256Digest gzipBits?: number codeChars?: number /** Present when this author attempt failed (recorded, never silent). */ @@ -392,30 +408,89 @@ export async function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promis const byName = new Map(baselines.map((s) => [s.name, s])) const codeByName = new Map() - // Endurance: the phase ledger. Resume refuses a checkpoint from a different design - // (silently mixing configs would corrupt every downstream comparison). - const fingerprint = { - trainN: cfg.trainN, - holdoutN: cfg.holdoutN, - budget, - generations, - populationSize, + for (const [label, count] of Object.entries({ trainN: cfg.trainN, holdoutN: cfg.holdoutN })) { + if (!Number.isSafeInteger(count) || count < 1) { + throw new Error(`evolution: ${label} must be a positive integer`) + } + } + const holdoutOffset = cfg.holdoutOffset ?? 0 + if (!Number.isSafeInteger(holdoutOffset) || holdoutOffset < 0) { + throw new Error('evolution: holdoutOffset must be a non-negative integer') + } + if (baselines.length === 0 || byName.size !== baselines.length) { + throw new Error('evolution: baselines must have non-empty, unique names') } + if (cfg.checkpoint && !/^sha256:[0-9a-f]{64}$/.test(cfg.checkpoint.executionRef)) { + throw new Error('evolution checkpoint: executionRef must be a lowercase sha256:<64 hex> digest') + } + const fingerprint = cfg.checkpoint + ? canonicalCandidateDigest({ + schemaVersion: 2, + executionRef: cfg.checkpoint?.executionRef ?? null, + environment: cfg.environment.name, + trainN: cfg.trainN, + holdoutN: cfg.holdoutN, + holdoutOffset, + budget, + concurrency, + generations, + populationSize, + baselines: baselines.map((baseline) => baseline.name), + objective: cfg.objective ?? 'score', + scoreTolerance: cfg.scoreTolerance ?? 0.05, + champion: policy, + championEpsilon: epsilon, + minPairedTasks: cfg.minPairedTasks ?? null, + band: cfg.band ?? null, + lossesDetail: cfg.lossesDetail ?? 'exact', + reproducerCheck: cfg.reproducerCheck ?? null, + modelPreflight: cfg.modelPreflight !== false, + modelPreflightTimeoutMs: cfg.modelPreflightTimeoutMs ?? null, + worker: { + routerBaseUrl: cfg.worker.routerBaseUrl, + profile: cfg.worker.workerProfile, + analystProfile: cfg.worker.analystProfile ?? cfg.worker.workerProfile, + corpusTags: cfg.worker.corpusTags ?? [], + corpusReadback: cfg.worker.corpusReadback ?? null, + }, + author: { + profile: cfg.author.profile, + fallbackProfile: cfg.author.fallbackProfile ?? null, + executorBackend: cfg.author.executor.backend, + }, + authorContract: strategyAuthorContract, + }) + : undefined let ckpt: EvolutionCheckpoint | undefined if (cfg.checkpoint?.resume && existsSync(cfg.checkpoint.path)) { - const raw = JSON.parse(readFileSync(cfg.checkpoint.path, 'utf8')) as EvolutionCheckpoint & { - fingerprint?: typeof fingerprint - } - if (JSON.stringify(raw.fingerprint) !== JSON.stringify(fingerprint)) { + const raw = JSON.parse(readFileSync(cfg.checkpoint.path, 'utf8')) as EvolutionCheckpoint + if (raw.fingerprint !== fingerprint) { throw new Error( - `evolution resume: checkpoint design mismatch — checkpoint ${JSON.stringify(raw.fingerprint)} vs config ${JSON.stringify(fingerprint)}; delete ${cfg.checkpoint.path} or match the config`, + `evolution resume: checkpoint design mismatch at ${cfg.checkpoint.path}; use a new checkpoint or restore the original execution dependencies`, ) } ckpt = raw } - const save = (state: EvolutionCheckpoint): void => { + + const train = checkedTaskSlice( + await cfg.tasks(0, cfg.trainN), + cfg.trainN, + 'train', + !!cfg.checkpoint, + ) + const trainDigest = cfg.checkpoint ? canonicalCandidateDigest(train) : undefined + if (ckpt && ckpt.trainDigest !== trainDigest) { + throw new Error('evolution resume: train task payloads changed') + } + let holdoutPoolDigest: Sha256Digest | undefined + const save = ( + state: Omit, + ): void => { if (cfg.checkpoint) - writeFileSync(cfg.checkpoint.path, JSON.stringify({ ...state, fingerprint }, null, 1)) + writeFileSync( + cfg.checkpoint.path, + JSON.stringify({ ...state, fingerprint, trainDigest, holdoutPoolDigest }, null, 1), + ) } let modelsPreflighted = false @@ -439,7 +514,6 @@ export async function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promis return report } - const train = await cfg.tasks(0, cfg.trainN) // One probe round-trip lists the domain's tools so the author can write tool-focused // shots (shot({tools})) — names + descriptions, never the implementations. const probeTask = train[0] @@ -491,15 +565,24 @@ export async function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promis // so report keys stay stable across the restart). for (const row of generationRows) { for (const c of row.candidates) { - if (!c.file || c.error) continue - const mod = (await import(`file://${c.file}`)) as { default?: Strategy } + if (c.error) continue + if (!c.file || !c.sourceSha256) { + throw new Error(`evolution resume: missing source identity for '${c.name}'`) + } + const bytes = readFileSync(c.file) + if (sha256Bytes(bytes) !== c.sourceSha256) { + throw new Error(`evolution resume: authored source changed for '${c.name}' (${c.file})`) + } + const sourceUrl = pathToFileURL(c.file) + sourceUrl.searchParams.set('sha256', c.sourceSha256) + const mod = (await import(sourceUrl.href)) as { default?: Strategy } if (!mod.default || typeof mod.default.driver !== 'function') { throw new Error( `evolution resume: ${c.file} no longer exports a Strategy — cannot restore "${c.name}"`, ) } byName.set(c.name, renameStrategy(mod.default, c.name)) - codeByName.set(c.name, readFileSync(c.file, 'utf8')) + codeByName.set(c.name, bytes.toString('utf8')) } } let authoredOk = generationRows.reduce( @@ -557,6 +640,7 @@ export async function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promis candidates.push({ name: unique, file: authored.file, + sourceSha256: sha256Bytes(Buffer.from(authored.code)), gzipBits: gzipSync(Buffer.from(authored.code)).length * 8, codeChars: authored.code.length, }) @@ -609,7 +693,21 @@ export async function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promis // The promotion decision: ONE fresh slice the search never touched, drawn after all // authoring is done. The gate, not the search policy, owns this verdict. - const holdoutOffset = cfg.trainN + (cfg.holdoutOffset ?? 0) + const poolN = cfg.band?.holdoutPoolN ?? cfg.holdoutN + const pool = checkedTaskSlice( + await cfg.tasks(cfg.trainN + holdoutOffset, poolN), + poolN, + 'holdout', + !!cfg.checkpoint, + ) + const trainIds = new Set(train.map((task) => task.id)) + if (pool.some((task) => trainIds.has(task.id))) { + throw new Error('evolution: train and holdout task IDs must be disjoint') + } + holdoutPoolDigest = cfg.checkpoint ? canonicalCandidateDigest(pool) : undefined + if (ckpt?.holdout && ckpt.holdoutPoolDigest !== holdoutPoolDigest) { + throw new Error('evolution resume: holdout task payloads changed') + } let holdoutTasks: AgenticTask[] = [] let bandInfo: EvolutionBandInfo | undefined if (ckpt?.holdout && ckpt.verdict) { @@ -617,7 +715,6 @@ export async function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promis // the reproducer still needs to bench on them. bandInfo = ckpt.band if (cfg.reproducerCheck && codeByName.has(incumbent.name)) { - const pool = await cfg.tasks(holdoutOffset, cfg.band?.holdoutPoolN ?? cfg.holdoutN) const gateIds = new Set(ckpt.holdout.perTask.map((r) => r.taskId)) holdoutTasks = pool.filter((t) => gateIds.has(t.id)) } @@ -630,7 +727,6 @@ export async function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promis const reference = baselines[0] if (!reference) throw new Error('evolution band: baselines[0] required as the screening reference') - const pool = await cfg.tasks(holdoutOffset, cfg.band.holdoutPoolN) const screen = await bench('band-screen', pool, [reference]) const refScores = screen.perTask .filter((r) => r.cells?.[reference.name]) @@ -645,7 +741,7 @@ export async function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promis holdoutTasks = kept.slice(0, cfg.holdoutN) bandInfo = { screened: refScores.length, inBand: kept.length, refScores } } else { - holdoutTasks = await cfg.tasks(holdoutOffset, cfg.holdoutN) + holdoutTasks = pool } let holdout: BenchmarkReport let verdict: PromotionVerdict @@ -757,3 +853,24 @@ export async function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promis trajectory, } } + +function checkedTaskSlice( + tasks: AgenticTask[], + count: number, + label: string, + snapshot: boolean, +): AgenticTask[] { + if (!Number.isSafeInteger(count) || count < 1 || tasks.length !== count) { + throw new Error( + `evolution: ${label} must supply exactly ${count} tasks; received ${tasks.length}`, + ) + } + const ids = new Set() + for (const task of tasks) { + if (typeof task.id !== 'string' || !task.id.trim() || ids.has(task.id)) { + throw new Error(`evolution: ${label} task IDs must be non-empty and unique`) + } + ids.add(task.id) + } + return snapshot ? immutableCandidateValue(tasks) : tasks +} diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 2de6d2aad..2ab604398 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:6d74288ce43d6b1e2a73377160d62b7ba4e6b294c8bfc5fdc2eac8ce8aba175e", + "digest": "sha256:a413e580934a5dee10daf7d986cb1e848270df8e5b393b939fa59858897ec850", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.193.1" + "runtimeVersion": "0.195.0" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:565f134e0589f18ce455cfd7fef2d76cbf5af335bc8e38befc4b587e640768bf", - "runId": "agent-runtime-0.193.1-proposal-fixture", + "recordDigest": "sha256:21d950ab2d78ded6fd73f68425e809d5993568fe2b2a6116f5eca8e002b17b17", + "runId": "agent-runtime-0.195.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.193.1-proposal-fixture" + "runId": "agent-runtime-0.195.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 49ede6339..9c8118a04 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:4f1f9175b8ae98f66fa860a54f215006e0a8c011db73d26623b2c7d2902ba9fb", + "digest": "sha256:35a184a48c067af92c7344d4eb702b37abd2054da368e04ed2f92a774904b600", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.193.1" + "runtimeVersion": "0.195.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:d897dd5e49008c6352decb6befa70952e578dff0069351194185b4b202411085", + "recordDigest": "sha256:9bcedcf24cc758e65c6dc3ae8c94a3598565fec439e24eff8e8d93fa395d4b58", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } diff --git a/tests/agentic-generator.test.ts b/tests/agentic-generator.test.ts index ebcad0c43..00a3cc35f 100644 --- a/tests/agentic-generator.test.ts +++ b/tests/agentic-generator.test.ts @@ -488,6 +488,57 @@ describe('agenticGenerator exact Runtime execution', () => { ) }) + it.each(['unstaged', 'staged'])('rejects %s edits to only a tracked diagnosis', async (state) => { + const diagnosisPath = '.improve/raw-trace-diagnosis.md' + mkdirSync(join(repoRoot, '.improve')) + writeFileSync(join(repoRoot, diagnosisPath), 'Previous diagnosis.\n') + git(['add', diagnosisPath], repoRoot) + git(['commit', '-q', '-m', 'test: track diagnosis'], repoRoot) + const dispositions: AgenticGeneratorShotDisposition[] = [] + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: routedExecutor(({ worktreePath }) => { + writeFileSync(join(worktreePath, diagnosisPath), `${TRACE_PATH}\nA revised diagnosis.\n`) + if (state === 'staged') git(['add', diagnosisPath], worktreePath) + }), + buildPrompt, + onShotDisposition: (_receipt, disposition) => dispositions.push(disposition), + }) + const worktreePath = await candidateWorktree(`diagnosis-only-${state}`) + + const result = await generator.generate(generateArgs(worktreePath, RAW_TRACE_FINDINGS)) + + expect(result.applied).toBe(false) + expect(dispositions.map((disposition) => disposition.kind)).toEqual(['rejected']) + expect(readFileSync(join(worktreePath, 'app.ts'), 'utf8')).toBe('export const x = 1\n') + }) + + it('accepts a renamed source file alongside a tracked diagnosis edit', async () => { + const diagnosisPath = '.improve/raw-trace-diagnosis.md' + mkdirSync(join(repoRoot, '.improve')) + writeFileSync(join(repoRoot, diagnosisPath), 'Previous diagnosis.\n') + git(['add', diagnosisPath], repoRoot) + git(['commit', '-q', '-m', 'test: track diagnosis'], repoRoot) + const renamedPath = ' renamed\napp.ts ' + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: routedExecutor(({ worktreePath }) => { + git(['mv', 'app.ts', renamedPath], worktreePath) + writeFileSync( + join(worktreePath, diagnosisPath), + `${TRACE_PATH}\nRenamed the source file.\n`, + ) + }), + buildPrompt, + }) + const worktreePath = await candidateWorktree('renamed-source') + + const result = await generator.generate(generateArgs(worktreePath, RAW_TRACE_FINDINGS)) + + expect(result.applied).toBe(true) + expect(readFileSync(join(worktreePath, renamedPath), 'utf8')).toBe('export const x = 1\n') + }) + it('requires a substantive edit and exact trace citation in raw-trace mode', async () => { const prompts: string[] = [] const generator = agenticGenerator({ diff --git a/tests/improvement-driver.test.ts b/tests/improvement-driver.test.ts index 1926b48aa..47ebecec8 100644 --- a/tests/improvement-driver.test.ts +++ b/tests/improvement-driver.test.ts @@ -1,4 +1,5 @@ import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { join } from 'node:path' import { CostLedger, makeProposalFinding, type ProposalFinding } from '@tangle-network/agent-eval' @@ -10,7 +11,10 @@ import { type WorktreeAdapter, } from '@tangle-network/agent-eval/campaign' import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import type { SurfaceImprovementEdit } from '../src/agent/improvement-adapter' +import { + createSurfaceImprovementProposer, + type SurfaceImprovementEdit, +} from '../src/agent/improvement-adapter' import type { ImprovementEditBatch, ImprovementProposalSource } from '../src/analyst-loop/types' import type { CandidateGenerator } from '../src/improvement/improvement-driver' import { improvementDriver } from '../src/improvement/improvement-driver' @@ -61,15 +65,24 @@ function ctxWith(findings: ReadonlyArray): ProposeContext) { return improvementDriver({ - generator: reflectiveGenerator({ improvementProposalSource: adapter }), + generator: reflectiveGenerator({ createImprovementProposalSource: () => adapter }), worktree: gitWorktreeAdapter({ repoRoot }), baseRef: 'main', }) @@ -149,16 +162,166 @@ describe('improvementDriver — reflective generator', () => { expect(await reflectiveDriver(adapter).propose(ctxWith(FINDINGS))).toEqual([]) }) - it('discards the worktree and proposes nothing when no patch applies', async () => { + it('reports patch failure and discards the candidate worktree', async () => { // A patch against content that does not match → git apply fails. const badPatch = '--- prompt.md\n+++ prompt.md\n@@ -1 +1 @@\n-NONEXISTENT line\n+whatever\n' const adapter = stubAdapter({ edits: [editFixture(badPatch)], skipped: 0, errors: [] }) - expect(await reflectiveDriver(adapter).propose(ctxWith(FINDINGS))).toEqual([]) + await expect(reflectiveDriver(adapter).propose(ctxWith(FINDINGS))).rejects.toThrow( + /patch batch failed/, + ) // No orphaned worktree left behind. expect(git(['worktree', 'list'], repoRoot).split('\n').length).toBe(1) }) + it('rejects a mixed patch batch without applying its valid edit', async () => { + const badPatch = '--- prompt.md\n+++ prompt.md\n@@ -1 +1 @@\n-NONEXISTENT line\n+other\n' + const generator = reflectiveGenerator({ + createImprovementProposalSource: () => + stubAdapter({ + edits: [editFixture(GOOD_PATCH), editFixture(badPatch)], + skipped: 0, + errors: [], + }), + }) + await expect( + generator.generate({ + worktreePath: repoRoot, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }), + ).rejects.toThrow(/patch batch failed/) + expect(readFileSync(join(repoRoot, 'prompt.md'), 'utf8')).toBe('lax rubric\n') + expect(git(['status', '--porcelain'], repoRoot)).toBe('') + }) + + it('rejects draft errors and stale base hashes before applying patches', async () => { + for (const batch of [ + { + edits: [editFixture(GOOD_PATCH)], + skipped: 0, + errors: [{ findingId: 'f2', subject: 'rubric', message: 'draft failed' }], + }, + { edits: [editFixture(GOOD_PATCH, 'old content')], skipped: 0, errors: [] }, + ]) { + const generator = reflectiveGenerator({ + createImprovementProposalSource: () => stubAdapter(batch), + }) + await expect( + generator.generate({ + worktreePath: repoRoot, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }), + ).rejects.toThrow(/draft failed|stale proposal base/) + expect(readFileSync(join(repoRoot, 'prompt.md'), 'utf8')).toBe('lax rubric\n') + } + }) + + it('rejects undeclared patch paths, including a different rename source', async () => { + const patches = [ + '--- other.md\n+++ other.md\n@@ -1 +1 @@\n-before\n+after\n', + 'diff --git other.md prompt.md\nsimilarity index 100%\nrename from other.md\nrename to prompt.md\n', + ] + for (const patch of patches) { + const generator = reflectiveGenerator({ + createImprovementProposalSource: () => + stubAdapter({ edits: [editFixture(patch)], skipped: 0, errors: [] }), + }) + await expect( + generator.generate({ + worktreePath: repoRoot, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }), + ).rejects.toThrow(/patch paths do not match the declared target/) + expect(readFileSync(join(repoRoot, 'prompt.md'), 'utf8')).toBe('lax rubric\n') + } + }) + + it('honors cancellation before proposal creation and after drafting', async () => { + for (const preAborted of [true, false]) { + const controller = new AbortController() + let created = 0 + if (preAborted) controller.abort(new Error('cancelled')) + const generator = reflectiveGenerator({ + createImprovementProposalSource() { + created++ + return { + async proposeFromFindings() { + controller.abort(new Error('cancelled')) + return { edits: [editFixture(GOOD_PATCH)], skipped: 0, errors: [] } + }, + } + }, + }) + await expect( + generator.generate({ + worktreePath: repoRoot, + findings: FINDINGS, + maxShots: 1, + signal: controller.signal, + }), + ).rejects.toThrow('cancelled') + expect(created).toBe(preAborted ? 0 : 1) + expect(readFileSync(join(repoRoot, 'prompt.md'), 'utf8')).toBe('lax rubric\n') + } + }) + + it('drafts from each incumbent through the real surface proposer and shares its paid-call context', async () => { + const costLedger = new CostLedger() + const contents: string[] = [] + const driver = improvementDriver({ + generator: reflectiveGenerator({ + createImprovementProposalSource({ worktreePath, signal, costLedger: account, costPhase }) { + expect(worktreePath).not.toBe(repoRoot) + expect(signal.aborted).toBe(false) + expect(account).toBe(costLedger) + expect(costPhase).toBe('search.proposal') + return createSurfaceImprovementProposer({ + repoRoot: worktreePath, + surfaces: { + systemPrompt: '.', + rubric: 'prompt.md', + tools: 'tools', + personas: 'personas', + knowledge: '.knowledge', + }, + async draftPatch({ currentContent }) { + contents.push(currentContent) + return { + patch: `--- prompt.md\n+++ prompt.md\n@@ -1 +1 @@\n-${currentContent.trim()}\n+${currentContent.trim()}!\n`, + summary: 'tighten rubric', + rationale: 'test candidate-bound reads', + } + }, + }) + }, + }), + worktree: gitWorktreeAdapter({ repoRoot }), + baseRef: 'main', + }) + const context = { + ...ctxWith(FINDINGS.map((finding) => ({ ...finding, subject: 'system-prompt:prompt' }))), + costLedger, + costPhase: 'search.proposal', + } + try { + const [first] = await driver.propose(context) + if (!first || typeof first === 'string') throw new Error('expected first code candidate') + const [second] = await driver.propose({ ...context, generation: 1, currentSurface: first }) + if (!second || typeof second === 'string') throw new Error('expected second code candidate') + expect(contents).toEqual(['lax rubric\n', 'lax rubric!\n']) + expect(readFileSync(join(second.worktreeRef, 'prompt.md'), 'utf8')).toBe('lax rubric!!\n') + expect(readFileSync(join(repoRoot, 'prompt.md'), 'utf8')).toBe('lax rubric\n') + } finally { + await driver.cleanup() + } + }, 120_000) + it('still proposes populationSize candidates on EMPTY findings when the generator opts in (proposesWithoutFindings)', async () => { // The meta-harness contract: an agentic coder draws its signal from the repo // + raw traces on disk, so it must run even when the distiller yielded no diff --git a/tests/kernel/corpus-integrity.test.ts b/tests/kernel/corpus-integrity.test.ts new file mode 100644 index 000000000..4df7eeaf5 --- /dev/null +++ b/tests/kernel/corpus-integrity.test.ts @@ -0,0 +1,190 @@ +import { spawn } from 'node:child_process' +import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { tryAcquireAtomicFileLock } from '@tangle-network/agent-eval/ledger-core' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { FileCorpus } from '../../src/runtime/personify/corpus' +import type { CorpusRecord } from '../../src/runtime/personify/wave-types' + +const record: CorpusRecord = { + schemaVersion: '1.0.0', + id: 'finding', + runId: 'run', + producedAt: '2026-09-05T00:00:00Z', + area: 'verification', + claim: 'Run the requested check.', + tags: ['original'], + confidence: 0.9, + evidence: [{ kind: 'trace', uri: 'trace:original' }], +} +const corpusModule = new URL('../../src/runtime/personify/corpus.ts', import.meta.url).href + +function writer(path: string, claim: string, leaveLock = false) { + const source = ` + import { FileCorpus } from ${JSON.stringify(corpusModule)} + import { tryAcquireAtomicFileLock } from '@tangle-network/agent-eval/ledger-core' + const [path, recordJson, leaveLock] = process.argv.slice(1) + process.once('message', async () => { + try { + const result = leaveLock === 'true' + ? tryAcquireAtomicFileLock({ lockPath: path + '.lock' }) + : await new FileCorpus(path).append(JSON.parse(recordJson)) + process.send({ result: leaveLock === 'true' ? { acquired: result.acquired } : result }, + () => process.exit(0)) + } catch (error) { + console.error(error) + process.exit(1) + } + }) + process.send('ready') + ` + const child = spawn( + process.execPath, + [ + '--import', + import.meta.resolve('tsx'), + '--input-type=module', + '-e', + source, + path, + JSON.stringify({ ...record, claim }), + String(leaveLock), + ], + { cwd: process.cwd(), stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }, + ) + let errors = '' + let result: unknown + child.stderr?.on('data', (data) => { + errors += String(data) + }) + const ready = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('message', (message) => { + if (message === 'ready') resolve() + else reject(new Error(`unexpected child readiness: ${JSON.stringify(message)}`)) + }) + child.once('exit', () => reject(new Error(`corpus child exited before readiness: ${errors}`))) + }) + child.on('message', (message) => { + if (message && typeof message === 'object' && 'result' in message) result = message.result + }) + const completed = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('exit', (code) => { + if (code === 0 && result !== undefined) resolve(result) + else reject(new Error(`corpus child exited ${code}: ${errors}`)) + }) + }) + return { child, ready, completed } +} + +describe('FileCorpus transaction integrity', () => { + let dir: string + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'corpus-integrity-')) + }) + afterEach(async () => { + await rm(dir, { recursive: true, force: true }) + }) + + it('serializes conflicting appends across instances and symlink aliases', async () => { + const realDir = join(dir, 'real') + const aliasDir = join(dir, 'alias') + await mkdir(realDir) + await symlink(realDir, aliasDir, 'dir') + const path = join(realDir, 'corpus.jsonl') + const outcomes = await Promise.all([ + new FileCorpus(path).append(record), + new FileCorpus(join(aliasDir, 'corpus.jsonl')).append({ + ...record, + claim: 'Conflicting advice.', + }), + ]) + expect(outcomes.filter((result) => result.succeeded)).toHaveLength(1) + expect(outcomes.find((result) => !result.succeeded)).toMatchObject({ + error: expect.stringContaining('corpus conflict'), + }) + expect(await new FileCorpus(path).query({})).toHaveLength(1) + expect((await readFile(path, 'utf8')).trim().split('\n')).toHaveLength(1) + }) + + it('deduplicates concurrent identical appends and snapshots values before awaiting storage', async () => { + const path = join(dir, 'corpus.jsonl') + const mutable = structuredClone(record) + const pending = new FileCorpus(path).append(mutable) + Reflect.set(mutable.tags, 0, 'changed') + const outcomes = await Promise.all([ + pending, + new FileCorpus(path).append(record), + new FileCorpus(path).append(record), + ]) + expect(outcomes).toEqual([{ succeeded: true }, { succeeded: true }, { succeeded: true }]) + const saved = await new FileCorpus(path).query({}) + expect(saved).toEqual([record]) + expect(Reflect.set(saved[0]!.tags, 0, 'changed')).toBe(false) + expect((await readFile(path, 'utf8')).trim().split('\n')).toHaveLength(1) + }) + + it.each([false, true])( + 'coordinates separate writer processes (identical=%s)', + async (identical) => { + const path = join(dir, 'corpus.jsonl') + const children = [ + writer(path, record.claim), + writer(path, identical ? record.claim : 'Conflicting advice.'), + ] + try { + await Promise.all(children.map((child) => child.ready)) + for (const child of children) child.child.send('append') + const outcomes = await Promise.all(children.map((child) => child.completed)) + expect( + outcomes.filter((result) => (result as { succeeded: boolean }).succeeded), + ).toHaveLength(identical ? 2 : 1) + expect(await new FileCorpus(path).query({})).toHaveLength(1) + expect((await readFile(path, 'utf8')).trim().split('\n')).toHaveLength(1) + } finally { + for (const child of children) child.child.kill() + } + }, + 15000, + ) + + it('recovers an exited process lock before appending', async () => { + const path = join(dir, 'corpus.jsonl') + const child = writer(path, record.claim, true) + try { + await child.ready + child.child.send('lock') + expect(await child.completed).toEqual({ acquired: true }) + expect(await new FileCorpus(path).append(record)).toEqual({ succeeded: true }) + } finally { + child.child.kill() + } + }, 15000) + + it('returns a typed failure when a live owner exceeds the bounded lock wait', async () => { + const path = join(dir, 'corpus.jsonl') + const acquired = tryAcquireAtomicFileLock({ lockPath: `${path}.lock` }) + if (!acquired.acquired) throw new Error('test lock was unavailable') + try { + expect(await new FileCorpus(path).append(record)).toEqual({ + succeeded: false, + error: expect.stringContaining('lock unavailable after 5000ms'), + }) + } finally { + acquired.lock.release() + } + expect(await new FileCorpus(path).append(record)).toEqual({ succeeded: true }) + }, 10000) + + it('returns storage errors through the append outcome', async () => { + await writeFile(join(dir, 'blocked'), 'file blocks directory creation') + expect(await new FileCorpus(join(dir, 'blocked', 'corpus.jsonl')).append(record)).toMatchObject( + { + succeeded: false, + error: expect.any(String), + }, + ) + }) +}) diff --git a/tests/kernel/rsi-wave.test.ts b/tests/kernel/rsi-wave.test.ts index aaa44b97e..fd03c78c7 100644 --- a/tests/kernel/rsi-wave.test.ts +++ b/tests/kernel/rsi-wave.test.ts @@ -162,6 +162,21 @@ describe('cross-run corpus (G2)', () => { if (!conflict.succeeded) expect(conflict.error).toMatch(/conflict/) }) + it('detaches and freezes nested corpus fields', async () => { + const corpus = new InMemoryCorpus() + const tags = ['original'] + const evidence = [{ kind: 'trace', uri: 'trace:original' }] + expect(await corpus.append(record({ tags, evidence }))).toEqual({ succeeded: true }) + + tags.push('injected') + evidence[0]!.uri = 'trace:mutated' + const [saved] = await corpus.query({}) + expect(saved?.tags).toEqual(['original']) + expect(saved?.evidence).toEqual([{ kind: 'trace', uri: 'trace:original' }]) + expect(Reflect.set(saved!.tags, 0, 'changed')).toBe(false) + expect(Reflect.set(saved!.evidence![0]!, 'uri', 'trace:changed')).toBe(false) + }) + it('renderCorpusToInstructions projects facts into a FRESH profile (input unchanged)', async () => { const corpus = new InMemoryCorpus() await corpus.append(record({ id: 'a', confidence: 0.95, claim: 'fact A', rationale: 'why A' })) diff --git a/tests/kernel/strategy-evolution.test.ts b/tests/kernel/strategy-evolution.test.ts index f1a4cbcf0..c5c335629 100644 --- a/tests/kernel/strategy-evolution.test.ts +++ b/tests/kernel/strategy-evolution.test.ts @@ -13,6 +13,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { minimumPairsForPairedDeltaTest } from '@tangle-network/agent-eval' import { afterEach, describe, expect, it, vi } from 'vitest' +import { canonicalCandidateDigest } from '../../src/candidate-execution/digest' import type { BenchmarkReport } from '../../src/runtime/run-benchmark' import type { AgenticSurface, AgenticTask } from '../../src/runtime/strategy' import { sample } from '../../src/runtime/strategy' @@ -606,7 +607,11 @@ describe('checkpoint and resume', () => { populationSize: 1, baselines: [sample], minPairedTasks: 6, - checkpoint: { path: ckptPath, resume: true }, + checkpoint: { + path: ckptPath, + resume: true, + executionRef: canonicalCandidateDigest({ fixture: 'shot-counting-evolution-v1' }), + }, outDir: mkdtempSync(join(tmpdir(), 'evolution-test-')), ...extra, }) @@ -657,6 +662,125 @@ describe('checkpoint and resume', () => { ).rejects.toThrow(/design mismatch/) }) + it.each([ + { objective: 'cost' }, + { holdoutOffset: 100 }, + { worker: { ...worker, workerProfile: testAgentProfile('different-worker') } }, + { environment: { ...shotCountingSurface(), name: 'different-environment' } }, + { band: { holdoutPoolN: 12 } }, + { minPairedTasks: 20 }, + ])('rejects changed behavior before resumed evaluation: %j', async (change) => { + stubWorkerRouter() + const path = join(mkdtempSync(join(tmpdir(), 'evolution-ckpt-')), 'ckpt.json') + await runStrategyEvolution(baseCfg(scriptedChat([fenced(twoShotDepthModule)]).chat, path)) + const phases: string[] = [] + const author = scriptedChat(['SHOULD NEVER BE CALLED']) + await expect( + runStrategyEvolution( + baseCfg(author.chat, path, { + ...change, + onPhase: async (phase: string) => { + phases.push(phase) + }, + }), + ), + ).rejects.toThrow(/design mismatch/) + expect(phases).toEqual([]) + expect(author.seen).toEqual([]) + }) + + it('requires execution identity and rejects a changed callback dependency reference', async () => { + const path = join(mkdtempSync(join(tmpdir(), 'evolution-ckpt-')), 'ckpt.json') + const author = scriptedChat([fenced(twoShotDepthModule)]) + await expect( + runStrategyEvolution( + baseCfg(author.chat, path, { + checkpoint: { path, resume: true }, + }), + ), + ).rejects.toThrow(/executionRef must be/) + expect(author.seen).toEqual([]) + stubWorkerRouter() + await runStrategyEvolution(baseCfg(author.chat, path)) + await expect( + runStrategyEvolution( + baseCfg(author.chat, path, { + checkpoint: { + path, + resume: true, + executionRef: canonicalCandidateDigest({ fixture: 'changed-transport' }), + }, + }), + ), + ).rejects.toThrow(/design mismatch/) + }) + + it.each(['train', 'holdout'])( + 'rejects changed %s payloads with the same task IDs', + async (slice) => { + stubWorkerRouter() + const path = join(mkdtempSync(join(tmpdir(), 'evolution-ckpt-')), 'ckpt.json') + await runStrategyEvolution(baseCfg(scriptedChat([fenced(twoShotDepthModule)]).chat, path)) + const author = scriptedChat(['SHOULD NEVER BE CALLED']) + const phases: string[] = [] + const changedTasks = async (offset: number, n: number) => + (await sliceTasks([])(offset, n)).map((task) => + (slice === 'train' ? offset === 0 : offset > 0) + ? { ...task, userPrompt: 'A different problem under the same task ID.' } + : task, + ) + await expect( + runStrategyEvolution( + baseCfg(author.chat, path, { + tasks: changedTasks, + onPhase: async (phase: string) => { + phases.push(phase) + }, + }), + ), + ).rejects.toThrow(new RegExp(`${slice} task payloads changed`)) + expect(phases).toEqual([]) + expect(author.seen).toEqual([]) + }, + ) + + it('rejects changed authored module bytes before importing the resumed candidate', async () => { + stubWorkerRouter() + const path = join(mkdtempSync(join(tmpdir(), 'evolution-ckpt-')), 'ckpt.json') + const report = await runStrategyEvolution( + baseCfg(scriptedChat([fenced(twoShotDepthModule)]).chat, path), + ) + const file = report.generations[0]!.candidates[0]!.file! + writeFileSync(file, 'throw new Error("changed candidate was imported")\n') + await expect( + runStrategyEvolution(baseCfg(scriptedChat(['UNUSED']).chat, path)), + ).rejects.toThrow(/authored source changed/) + }) + + it.each(['overlap', 'duplicate', 'short'])( + 'rejects invalid task partitions: %s', + async (kind) => { + stubWorkerRouter() + const author = scriptedChat([fenced(twoShotDepthModule)]) + const phases: string[] = [] + const cfg = baseCfg(author.chat, join(tmpdir(), 'unused-checkpoint'), { + checkpoint: undefined, + tasks: async (offset: number, n: number) => { + const tasks = await sliceTasks([])(kind === 'overlap' ? 0 : offset, n) + if (kind === 'duplicate') return tasks.map((task) => ({ ...task, id: 'duplicate' })) + return kind === 'short' ? tasks.slice(1) : tasks + }, + onPhase: async (phase: string) => { + phases.push(phase) + }, + }) + await expect(runStrategyEvolution(cfg)).rejects.toThrow( + /must be disjoint|must be non-empty and unique|must supply exactly/, + ) + expect(phases).not.toContain('holdout') + }, + ) + it('onPhase fires before every benchmark phase in order', async () => { stubWorkerRouter() const phases: string[] = [] diff --git a/tests/kernel/strategy-suite.test.ts b/tests/kernel/strategy-suite.test.ts index 2c4034677..f174e5b72 100644 --- a/tests/kernel/strategy-suite.test.ts +++ b/tests/kernel/strategy-suite.test.ts @@ -435,9 +435,55 @@ describe('promotionGate', () => { // ── The author/optimizer addressability surface ─────────────────────────────────── describe('addressable optimization coordinates', () => { - it('the author contract exposes persona (multi-agent strategies are authorable)', () => { - expect(strategyAuthorContract).toContain('persona') - expect(strategyAuthorContract).toContain('systemPrompt') + it('the author contract teaches a shot profile that changes the actual worker', async () => { + const captured = stubRouter() + const profile = testAgentProfile('strategy-author', { + harness: 'cli-base', + model: { provider: 'offline', default: 'author-model' }, + }) + const code = `import { defineStrategy } from '@tangle-network/agent-runtime/kernel' +export default defineStrategy('specialist', async ({ shot, opts }) => { + await shot({ profile: { ...opts.workerProfile, name: 'researcher', + model: { ...opts.workerProfile.model, default: 'specialist-model' }, + prompt: { ...opts.workerProfile.prompt, systemPrompt: 'SPECIALIST_INSTRUCTION' } } }) + return { score: 0, resolved: false, completions: 1, progression: [], shots: 1 } +})` + const { strategy } = await authorStrategy({ + profile, + executor: { + backend: 'router', + routerBaseUrl: 'http://router.test/v1', + routerKey: 'test-key', + complete: async (request) => { + const prompt = JSON.stringify(request.messages) + expect(prompt).toContain('steer?, profile?, tools?') + expect(prompt).toContain('opts.workerProfile') + expect(prompt).not.toContain('persona?') + return { + choices: [{ message: { content: `\`\`\`ts\n${code}\n\`\`\`` } }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + model: profile.model?.default, + } + }, + }, + environmentName: 'fixture', + lossesJson: '[]', + budget: 1, + outDir: mkdtempSync(join(tmpdir(), 'authored-profile-test-')), + }) + await runAgentic({ + surface: fixtureSurface(() => ({ passes: 1, total: 1 })), + task, + ...worker, + strategy, + budget: 1, + }) + expect(captured).toHaveLength(1) + expect(captured[0]).toMatchObject({ model: 'specialist-model' }) + expect(captured[0]!.messages).toContainEqual({ + role: 'system', + content: 'SPECIALIST_INSTRUCTION', + }) }) it('analystProfile routes the critique call to the critic model, not the worker', async () => { diff --git a/tests/knowledge-supervised-update.test.ts b/tests/knowledge-supervised-update.test.ts index f8222f60e..cf64dba2c 100644 --- a/tests/knowledge-supervised-update.test.ts +++ b/tests/knowledge-supervised-update.test.ts @@ -81,12 +81,32 @@ describe('knowledge supervisor integration', () => { expect(result.metadata.root).toBe('/kb/candidate') expect(captured?.task).toContain('Goal: candidate goal') expect(captured?.task).toContain('Knowledge base root: /kb/candidate') - expect(captured?.profile.name).toBe('knowledge-research-supervisor') - expect(captured?.profile.prompt?.systemPrompt).toContain( - 'Each researcher worker you spawn follows this contract', + expect(captured?.profile).toEqual(supervisorProfile) + expect(captured?.task).toContain( + 'Update files under the knowledge base root only. Stop when the readiness check passes.', ) }) + it('executes the supplied supervisor prompt unchanged', async () => { + const profile: SupervisorProfile = { + ...supervisorProfile, + prompt: { systemPrompt: 'Apply source-backed updates with the configured knowledge tools.' }, + } + await runSupervisedKnowledgeUpdate({ + root: '/kb/candidate', + goal: 'fill support gaps', + readiness: () => true, + budget: { maxIterations: 2, maxTokens: 1000 }, + supervisorProfile: profile, + runSupervised: async (executed, task) => { + expect(executed).toEqual(profile) + expect(task).toContain('Goal: fill support gaps') + expect(task).toContain('Knowledge base root: /kb/candidate') + return winner() + }, + }) + }) + it('formats supervisor tasks with the KB root, findings, and metadata', () => { const task = formatSupervisedKnowledgeTask({ root: '/kb/candidate', diff --git a/tests/runtime-observe.test.ts b/tests/runtime-observe.test.ts index 270a38839..dca2c7c96 100644 --- a/tests/runtime-observe.test.ts +++ b/tests/runtime-observe.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' +import { harvestCorpus } from '../src/runtime/harvest-corpus' import { observe } from '../src/runtime/observe' const observerProfile = { @@ -78,6 +79,68 @@ describe('runtime observe', () => { }, { profile: observerProfile, executor: observerExecutor(content) }, ), - ).rejects.toThrow(/observe findings: every finding must match AnalystFinding/) + ).rejects.toThrow(/observe response: findings\[0\] does not match/) + }) + + it.each([ + 'provider refused the request', + 'null', + '[]', + '{}', + '{"findings":null}', + '{"findings":[],"error":"provider unavailable"}', + '{"findings":[null]}', + ])('rejects an invalid observer response: %s', async (content) => { + await expect( + observe( + { task: 'Inspect the run.', output: 'done', trace: [] }, + { profile: observerProfile, executor: observerExecutor(content) }, + ), + ).rejects.toThrow(/observe response:/) + }) + + it('accepts an explicit empty findings array', async () => { + const result = await observe( + { task: 'Inspect the run.', output: 'done', trace: [] }, + { profile: observerProfile, executor: observerExecutor('{"findings":[]}') }, + ) + expect(result.findings).toEqual([]) + expect(result.report).toContain('clean run') + }) + + it('reports failed corpus persistence through the harvest per-run failure channel', async () => { + const content = JSON.stringify({ + findings: [ + { + area: 'verification', + severity: 'high', + claim: 'A check was skipped.', + recommended_action: 'Run the check.', + audience: 'agent', + confidence: 0.9, + }, + ], + }) + const report = await harvestCorpus({ + runs: [ + { task: 'Inspect the run.', output: 'done', trace: [], runId: 'failed-save' }, + { task: 'Inspect the run.', output: 'done', trace: [], runId: 'saved' }, + ], + profile: observerProfile, + executor: observerExecutor(content), + corpus: { + append: async (record) => + record.runId === 'failed-save' + ? { succeeded: false, error: 'storage unavailable' } + : { succeeded: true }, + query: async () => [], + }, + }) + expect(report).toMatchObject({ + runsObserved: 1, + findings: 1, + learned: 1, + failures: [{ runId: 'failed-save', error: expect.stringContaining('storage unavailable') }], + }) }) }) diff --git a/tests/version-bump-check.test.ts b/tests/version-bump-check.test.ts index ccb3439e9..db9149e50 100644 --- a/tests/version-bump-check.test.ts +++ b/tests/version-bump-check.test.ts @@ -601,7 +601,7 @@ describe('a change to the exported symbols requires a version bump', () => { await expect(check(root, base)).rejects.toMatchObject({ stderr: expect.stringContaining('breaking change needing a minor bump'), }) - }) + }, 120_000) it('does not fire when the exported symbols do not move', async () => { const root = await createRepo() From f84f52b29108eadd216bccac8fe7d202c29fe367 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 5 Sep 2026 12:30:28 -0700 Subject: [PATCH 2/6] docs(learning): remove obsolete architecture verdicts --- docs/architecture-interpretations.md | 299 +++++++-------------------- docs/architecture.md | 26 +-- docs/learning-flywheel.md | 24 +-- 3 files changed, 100 insertions(+), 249 deletions(-) diff --git a/docs/architecture-interpretations.md b/docs/architecture-interpretations.md index f1aae1c3f..7e396d0d4 100644 --- a/docs/architecture-interpretations.md +++ b/docs/architecture-interpretations.md @@ -1,242 +1,103 @@ # Architecture — Five Interpretations and the Coherence Verdict -Companion to [architecture.md](./architecture.md) (the spine) and [learning-flywheel.md](./learning-flywheel.md) (the moat thesis). Where `architecture.md` states *what the system is meant to be*, this doc stress-tests *whether it coheres* — by reading the same atom through five independent lenses, including an adversarial one, and recording where each framing holds and where it breaks. The five lenses converge on one diagnosis and one decision gate; that convergence is the point. - -`Status`: both of this doc's load-bearing gaps have since been resolved — the analyst→driver edge is live on the **agent-driver** (a parent `AgentProfile` reads `observe()` findings and steers its child via `createCoordinationTools` over the `Scope`/`Supervisor`), and **Gate A (§5) has been run**: cleared at small n, then retracted to a tie at power (numbers: `.evolve/current.json` + the memory ledger). The lens analysis below is kept as the stress-test it was; the per-claim corrections are inline. See the evidence anchors (§7) for file:line. - ---- - -## 1. The honest one-liner - -Strip the vocabulary and the built system is **best-of-N sampling + a selector + offline prompt-tuning (GEPA)**, with an *intrinsic self-refine* toggle bolted on — and **the refine toggle is the half that loses**. The "recursive adaptive driver" that would make it more than that is real in shape but not wired. The entire gap is one missing edge: - -> The driver never reads the analyst's findings. It decides from an exit code, not a diagnosis. - -Everything below is an elaboration of that sentence from a different angle. - -*(Status: the diagnosis→steer edge lives on the agent-driver — a parent `AgentProfile` reads -`observe()` findings and steers its child via `createCoordinationTools` over the -`Scope`/`Supervisor`. The within-run question the gate poses has been answered there, -positively at small n then retracted to a TIE at power — §5.)* - ---- - -## 2. Master diagram — the atom, the two timescales, the missing wire - -``` - OUTER LOOP (slow, cross-task) = OPTIMISATION - ┌────────────────────────────────────────────────────────────┐ - │ traces + corpus ─▶ runAnalystLoop / GEPA │ - │ │ │ - │ proposeFromFindings │ - │ ╱ ╲ │ - │ knowledge proposals surface proposals │ - │ (wiki pages) (prompt / tool / rubric) │ - │ = CORPUS = POLICY ◀─ the ONLY RSI path │ - │ │ │ │ - │ ┌────────┴─┐ held-out ┌─────▼──────┐ │ - │ │ JUDGE │════delta═══▶│ gate: ship? │ │ - │ │write-only│ (never read └─────┬──────┘ │ - │ └──────────┘ by inner) │ promote (OFFLINE only) │ - └──────────────────────────────────┼──────────────────────────┘ - │ new policy - INNER LOOP (fast, within-task) = INFERENCE - ┌───────────────────────────────────▼──────────────────────────┐ - │ plan() ─▶ {refine | fanout | stop} ─▶ workers ─▶ selector │ - │ ▲ reads history verdict.score ✓ │ │ - │ │ └▶ TODAY = JUDGE │ - │ ╳ analyses[] → plan(): the kernel-side wire was DELETED; │ - │ the edge now lives on the agent-driver (observe()→steer) │ - └───────────────────────────────────────────────────────────────┘ - The ╳ was the gap when the lenses ran: the driver decided from a - return code. The string-prompt planner that carried it is gone; the - diagnosis→steer edge now lives on the Scope/Supervisor agent-driver. -``` - -Two structural facts as of the original audit, with their current status: - -1. The diagnosis→decision edge lives on the **agent-driver**: - a parent `AgentProfile` consumes `observe()` findings (`AnalystFinding`, the substrate - type) and steers its child via `createCoordinationTools` (`src/mcp/tools/coordination.ts`) - over the `Scope`/`Supervisor` — so an agent decides from the diagnosis, not the verdict - score alone. Honest status: the steer path is live on the Supervisor substrate (§5). -2. The selector ranked with the **judge's score** — an oracle. The deployable, no-oracle - selector has since been **built and measured**: a **verifier-grounded** selector is - positive on a deployable-checker domain (HumanEval: verifier-pick captures the full - oracle ceiling and beats self-consistency, BH-significant), while answer-agreement - selectors are negative (finsearch, aec). The selector needs a runnable checker, not - answer-vote. Numbers: `.evolve/current.json` + the memory ledger. - -The discipline that the architecture leans on — *selector ≠ judge*, judge write-only — is exactly what keeps the outer loop from optimising toward its own grader. The temptation to wire the judge into ranking (it is the cheapest, strongest selector) is the thing the design must resist; the moat depends on resisting it. - ---- +This document tests the design in [architecture.md](./architecture.md) through five lenses. +The [learning audit](./research/learning-system-audit-2026-09-05.md) records the inspected revisions, reproduced defects, repairs, and capability limits. -## 3. Five interpretations +## 1. The learning objective -| Lens | One-liner | Does it add anything over the boring baseline? | Where it breaks | -|---|---|---|---| -| **Test-time-compute / search** | Driver = search controller, selector = ranking, judge = oracle reward | Only if a *learned* controller beats fixed best-of-N | Controller is open-loop; refine loses to flat sampling at matched budget | -| **Active learning / experimental design** | Driver = acquisition function picking the next most-informative source | **Yes — it makes the goal measurable**; the best frame for the research use case | Needs a *calibrated* gap signal; today "gap" is an LLM vibe | -| **Program synthesis** | Driver = JIT emitting a topology program; runAgentRounds = interpreter | Only if the ISA grows `seq`/nesting and the emitter reads an IR | It's a **3-opcode flat enum**, not a DSL; GEPA tunes a prompt comment, not the emitter | -| **Domain learning / meta-learning** | Improve specialists, working evaluations, and the process that trains them | Repeatable gains within the intended domain; process transfer is a separate claim | Existing components need a joined learning process and evidence appropriate to each level | -| **Skeptic / Occam** | self-refine (loses) steering best-of-N (wins) | No — vocabulary, not capability | Overclaims past "untested ≠ disproven" for a trace-fed driver | - -### 3.1 Test-time-compute / search - -A search over candidates: the driver chooses width/depth/stop, the selector ranks, the judge is the held-out oracle. This frame *predicts the empirics exactly* — parallel sampling with a sound selector wins (Brown 2024, Wang 2022, Lightman 2023); intrinsic sequential self-refine degrades on hard tasks (Huang 2023, Kamoi 2024, Stechly 2024). - -``` - root task (search node) - │ driver = controller picks a move - sample/fork (best-of-N, width N) ── the WIN today - ╱ │ ╲ - c1 c2 c3 ← workers (rollouts) - v=.4 v=.8 v=.6 ← selector value - ╲ │ ╱ - argmax v ─▶ pick c2 - │ steer/seq (deepen one path) ── LOSES today - c2' v=.65 (refine can BREAK a good leaf) - │ stop - ════════ search boundary ════════ - JUDGE = held-out oracle (write-only; must NOT be the v above) -``` - -Breaks: the load-bearing asset is a **sound value function**, and there is no evidence `verdict.score` is calibrated to true reward. Until the no-oracle selector is measured, "best-of-N wins" is unverified *for this system*. The controller is open-loop (reads a thin history summary, no analyst signal), so "adaptive topology" is dominated by fixed best-of-N at matched budget. - -### 3.2 Active learning / experimental design (the most useful frame for the research surface) - -The knowledge-acquisition loop is an agent reducing coverage-uncertainty by choosing the next most-informative source. "Driver decides topology" becomes the precise, measurable "an acquisition function picks the next experiment." - -``` - ┌──────────────────────────────────────────┐ - │ BELIEF STATE = corpus / LLM wiki │ - └──────────────────────────────────────────┘ - │ ▲ ingest+merge - ▼ │ (run experiment) - ┌────────────────────┐ ┌───────────┐ - │ GAP ESTIMATOR │ │ WORKER │ - │ wiki critic: │ │ read / │ - │ orphan = low cov │ │ clip / │ - │ contra = high var │ │ sweep / │ - │ stale = decay │ │ scrape │ - └────────────────────┘ └───────────┘ - │ gaps / uncertainty ▲ chosen source - ▼ │ - ┌────────────────────────────┐ │ - │ ACQUISITION FN (= driver) │───────┘ - │ argmax expected info gain │ - │ uses TRACE signal only — │ - │ NOT the held-out judge │ - └────────────────────────────┘ - │ stop when coverage ≥ target or ΔEIG < ε - ▼ - ····· WRITE-ONLY JUDGE = held-out task (never steers) ····· -``` - -Holds: it converts the unfalsifiable "good topology" into a textbook objective (expected information gain) with a literature, a calibration test, and a principled stop rule. It retro-explains rung-0: a miscalibrated acquisition function underperforming random sampling is the canonical active-learning failure. It even maps onto shipped infra — `proposeSynthesisTargets` (variance / coverage / failure-cluster / difficulty-gap → priority) is a real, statistically-grounded acquisition function, pointed at the eval dataset; the corpus-axis version is a port, not a greenfield method. - -Breaks: the load-bearing assumption — a **calibrated** gap signal — is absent. The wiki critic emits LLM-judged contradictions/staleness/orphans and a single self-reported `confidence` scalar. That is a vibe, not a posterior variance, and honouring *selector ≠ judge* gets *harder*: if "gap" is an LLM judgment, it can implicitly encode "what the judge will reward." The frame demands the gap signal be **structural** (graph topology, citation/embedding density, redundancy-discounted coverage), not opinion. - -### 3.3 Program synthesis / interpreter - -`runAgentRounds` is a fetch-execute-halt trampoline; the planner is a JIT that emits one instruction per round. The vocabulary describes the real control flow — but as a *language* it is barely one: the implemented ISA is a 3-value flat union `{refine, fanout, stop}`, emitted one-at-a-time, with no `seq`, no nesting, no emittable `select`. The two ops that would make it non-vacuous (`select`, `seq`) are interpreter builtins the agent cannot author; GEPA rewrites a static directive string (a `#define`), not the emit function; and the emitter compiles from a return-code-plus-truncated-stdout summary, not an IR. Today: a JIT in shape, a switch statement in substance. *(Status: the richer program space this lens asks for is the canonical path: `defineStrategy` (`src/runtime/strategy.ts`), where a strategy is ordinary code composing `shot()`/`critique()` with arbitrary sequencing and branching, authored by `authorStrategy` (`src/runtime/strategy-author.ts`).)* - -### 3.4 Two-timescale / recursive self-improvement - -The fast loop improves an answer; the domain learning process improves specialists, working evaluations, and future experimental decisions. -`improve()` supplies bounded search and final comparisons, while activation remains explicit. -Stored knowledge can improve domain work, and changes to acquisition policy can improve the process that produces that knowledge. -Each claim needs its own task outcome evidence. -A meta-agent can learn how to construct these domain learning processes without requiring the resulting specialists to generalize. -Cross-domain reuse of the learning process is a further experiment, not a prerequisite for useful domain learning. - -### 3.5 Skeptic / Occam (adversarial) - -``` -GRAND (as pitched): ACTUALLY WIRED (Occam): - task ─▶ recursive atom task ─▶ planner LLM - ▼ reads traces+findings, ▼ sees only prior outputs - decides topology ▲ + JUDGE score - ▼ │(NOT ▼ - workers ─▶ analyst ─────┘ WIRED) refine/fanout/stop ← self-refine - ▼ ▼ + a toggle - selector (trace-only) ─▶ winner workers ─▶ outputs - ▲ ▼ - JUDGE write-only pick best ◀ uses JUDGE (oracle) ✗ - -COLLAPSES TO TWO BORING THINGS THAT ALREADY WORK: - (a) best-of-N ─▶ sound verifier ─▶ pick [WINS] - (b) RAG / corpus-build ─▶ retrieve+dedup+cite [WINS] -``` - -The strongest good-faith case: what's wired is the losing half (self-refine) steering the winning half (best-of-N), with the winning half's critical component (a sound selector) faked by the judge. Renaming the stack a "recursive atom that decides topology" adds vocabulary, not capability. The one honest concession the skeptic owes: **"unbuilt and untested" ≠ "disproven."** A planner that reads real traces + analyst findings is a genuinely different object from intrinsic self-refine; rung-0 only falsified the static planner. - ---- +The system should improve specialists, domain learning processes, working evaluations, and the methods that construct those processes. +A specialist can succeed within its intended domain without generalizing elsewhere. +The transferable knowledge can be the process that constructs and trains a successful specialist. -## 4. Does it cohere? +These claims require different comparisons. +Better task execution, repeated domain learning, better evaluation, and transfer of a learning process are separate outcomes. +One result cannot establish or reject all four. -**As built: no.** All five lenses — including the adversary — land here independently. The system is intrinsic self-refine (the half the literature and rung-0 say loses) steering best-of-N (the half that wins), with the winning half's load-bearing component — a sound, non-oracle selector — substituted by the judge in every measurement so far. That is not a new method class; it is a `while` loop with one tunable branch in compiler vocabulary. +## 2. Common execution and evidence -**As designed: conditionally yes — gated on exactly one measurement.** Five independent framings wrote the *same* gate (§5), which is why it is trusted rather than asserted. The strongest case *for* the grand version is the research-acquisition surface specifically, because acquisition is **non-myopic and stateful** — every ingest permanently changes the knowledge base for all future queries. Best-of-N has no concept of "this expansion improves the substrate." That is precisely the regime where a driver conditioning on coverage-gaps could beat blind sampling, and where the cross-query flywheel is real rather than aspirational. +Runtime executes exact profiles, code candidates, and authored strategies. +Eval runs searches and measurements. +Knowledge preserves source-backed state and supplies retrieval and research operations. +The package boundaries express useful responsibilities. ---- +The common unit should identify the candidate, its retained state, the objective, execution conditions, observations, and resources. +Search methods can share those facts while retaining different selection rules, archives, and exploration policies. +A complete method should not be reranked by a second optimizer that changes its decision. -## 5. Gate A — the decision gate for the recursive-driver layer +An existing execution callback can run a domain learning episode and return the specialists it produced. +That callback still must execute those specialists, retain their exact state, and account for all inner work. +The ability to represent the callback does not establish that a complete learning process has been demonstrated. + +## 3. Five interpretations -Test within-run steering with this diagnostic: +| Lens | Object being improved | Comparison that tests it | Weak assumption to challenge | +| --- | --- | --- | --- | +| Search during task execution | How an agent explores, continues, and selects task solutions | Compare execution policies on the same tasks and actual resources | More attempts, better checking, or extra information can explain an apparent policy gain | +| Experimental design | Which problems, sources, or experiments the learner selects | Compare subsequent domain outcomes under alternative acquisition policies | Score variance or an expressed knowledge gap need not identify useful practice | +| Program synthesis | The executable agent or learning algorithm | Execute exact candidate programs and compare their checked outputs | A syntactically valid program need not activate the mechanism it claims | +| Domain learning and meta-learning | The procedure that produces specialists and improves evaluations | Compare repeated learning episodes; test process transfer separately when claimed | The resulting specialist need not generalize outside its intended domain | +| Evaluation engineering | Tests, task generators, judge instructions, and outcome collection | Compare detection of independently established success and failure, then downstream learning | Agreement with a judge or easier tests can improve a score without improving domain outcomes | + +Runtime's strategy programs already permit arbitrary sequencing and branching through ordinary code. +Complete profile and code candidates permit changes beyond prompt wording. +The analysis must inspect how those candidates execute before treating the search space as a fixed menu of actions. + +## 4. Does it cohere? -> On a held-out benchmark, at **equal worker-compute budget** (`k` counts worker ROLLOUTS — each may be a full multi-turn/stateful trajectory, not a single shot), does a **trace + analyst-findings-fed** driver, scored by a **sound non-oracle selector**, beat **blind random@k** selected by that *same* selector — by a statistically significant margin (n large enough for p < 0.05) that **survives test-retest of the selector**? +The division of responsibilities is coherent. +The reproduced defects concern inconsistent candidate identity, measurement completeness, cost accounting, and disconnected feedback. +Those defects justify changes to the existing paths. +They do not justify merging the packages or imposing one search algorithm. -Use [architecture.md §9](./architecture.md#9-build-order-and-experiment-scope) for resource accounting, mechanism activation, and the conditions for rejecting the tested design. +The broader learning claim remains empirical. +Stored recommendations, generated cases, calibration statistics, and optimization methods are useful ingredients. +A complete domain learning episode must show how those ingredients change later decisions and produce better domain outcomes. +A candidate learning method must be judged by what it produces, including unsuccessful episodes and their costs. -**Measured: cleared at small n, then RETRACTED to a TIE at power (POWER-16).** On -EnterpriseOps-Gym itsm, depth-steered continuation (analyst-fed, `observe()`) beat blind -breadth at equal compute under keep-best checkpoint scoring — but the effect collapsed -to a tie when powered, and the program pivoted off this anchor (numbers: -`.evolve/current.json` + the memory ledger). The gate ran on the `Scope`/`Supervisor` + -`defineStrategy` substrate (`src/runtime/strategy.ts`). The domain-boundary law held: -**negative on stateless retrieval** (FinSearchComp), **null-to-negative on stateless -codegen** (HumanEval), **positive on stateful agentic domains** with a correctable -middle band scored keep-best (EOPS). +Preserve informative failures, candidate diversity, and joint interventions. +An exploratory step can be useful before it produces a deployable improvement. +A null component result cannot reject a mechanism that requires multiple components to interact. -**Gate A tests one mechanism under specified conditions.** -Product success is **Gate B**: improvement across runs against an unchanged controller ([learning-flywheel.md](./learning-flywheel.md)). -The historical results above do not establish whether that improvement occurs. +## 5. Gate A — a diagnostic for within-run steering ---- +Compare a driver that consumes traces and findings with a declared alternative on the same tasks and actual resources. +Use the same deployable method to select each result. +Record the information, intermediate checks, termination conditions, and costs available to each policy. +Establish that feedback actually changes a decision before interpreting the comparison. -## 6. What this means for the research-acquisition surface +This tests steering within a task under the specified conditions. +It does not decide whether a domain learner improves specialists, evaluations, or future experimental decisions across runs. +Use [architecture.md §9](./architecture.md#9-build-order-and-experiment-scope) for the comparison and rejection conditions. -The **minimal honest version** survives every critique and yields the proven more-compute win immediately: +## 6. Domain learning and evaluation engineering -1. **Fan-out retriever** — N parallel collection branches over `{deep-read paper, transcribe clip, web-sweep, targeted image/data scrape}`. Plain best-of-N over actions. -2. **A deployable, non-oracle selector** scoring each ingest on *trace-observable structural* signal — citation coverage, contradiction-lint pass, staleness, novelty-vs-existing-wiki. This is the missing piece that makes best-of-N actually pay, and it is the same build as landing the *selector ≠ judge* firewall. -3. **The `llm-wiki` maintainer+critic** as the dedup / cite / lint sink (already exists as a skill). +A domain learner can choose sources, generate practice, change specialists, improve working evaluations, and retain useful results. +Structural source checks can inform that process, but citation counts and lexical overlap do not establish scientific truth or useful learning. +Judge calibration statistics analyze supplied observations; callers must execute the judge and apply any proposed change. -Run the §5 diagnostic to assess within-run steering under those conditions. -Apply [architecture.md §9](./architecture.md#9-build-order-and-experiment-scope) before retaining, rejecting, or combining the tested mechanisms. -Test learning across projects separately before claiming that capability compounds. +Working evaluations can guide learning and can themselves change. +Independent final assessment must remain outside the adaptive decisions whose improvement it tests. +A changed evaluation can be better when it exposes failures and lowers the current specialist's score. +Its value depends on detection quality and subsequent domain outcomes. ---- +For the across-run comparison described in [learning-flywheel.md](./learning-flywheel.md), use repeated domain episodes with retained state and explicit objectives. +Compare actual learning and execution resources over the declared horizon. +Check retained abilities as well as newly solved tasks. +Only a claim about process transfer requires a corresponding comparison in new domains. ## 7. Evidence anchors -- `src/mcp/tools/coordination.ts` — `createCoordinationTools`: the agent-driver's MCP - (spawn · observe · steer · stop). The diagnosis→decision edge runs over the - `Scope`/`Supervisor` (`src/runtime/supervise/`). -- `src/runtime/run-loop.ts` — the surviving leaf kernel; `defaultSelectWinner` (`:983`) / - `branchPoint` (`:797`); `RunAgentRoundsOptions.selectWinner` (`:104`) is the selector-injection seam. -- `src/runtime/strategy.ts` / `src/runtime/strategy-author.ts` — `defineStrategy` / - `authorStrategy`: the program space where the Gate-A strategies run. -- `src/analyst-loop/` — `runAnalystLoop`; the trace observer feeding the canonical loop - is `observe()` (`src/runtime/observe.ts`), consumed by the agent-driver. -- Prompt-space optimization lives in an agent-eval `OptimizationMethod`, invoked through Runtime's `improve()`; the analyst-prompt - coordinate has shown no significant lift on held-back problems in controlled runs to date — see `.evolve/current.json` and the memory ledger for the current evidence state. -- `bench/src/selector.ts` + `bench/src/corpus-replay.mts --selector` — the deployable - selector and its offline replay harness. -- `bench/src/refine-loop.ts` — shared k-shot loop. -- random@k / pass@k computation (the original headline `random@3` was judge-selected, an - oracle upper bound): the measurement path is `bench/src/corpus-replay.mts` + - `corpus-report.mts` over the corpus. - -**Literature.** Parallel sampling + sound selector wins: Brown 2024 (repeated sampling), Wang 2022 (self-consistency), Lightman 2023 (process reward). Intrinsic self-refine degrades on hard tasks: Huang 2023, Kamoi 2024, Stechly 2024. The loop is not a new method class — it is a known combination whose winning half is not yet honestly built. +- `src/improvement/improve.ts`: profile and code improvement entry points. +- `src/improvement/method-execution.ts`: exact profiles executed through complete optimization methods. +- `src/runtime/strategy.ts` and `src/runtime/strategy-author.ts`: executable strategies and their authoring contract. +- `src/runtime/strategy-evolution.ts`: strategy search, retained candidates, and checkpoint identity. +- `src/runtime/observe.ts` and `src/runtime/personify/corpus.ts`: trace-derived recommendations and persistent records. +- `src/mcp/tools/coordination.ts`: agent-driven observation, delegation, and steering. +- `src/candidate-execution/`: execution of exact candidate combinations. +- `src/intelligence/improvement-cycle.ts`: measured proposals and adoption preparation. +- [Learning audit](./research/learning-system-audit-2026-09-05.md): cross-package source references, regression evidence, and research comparison. + +The [earlier interpretation](https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/docs/architecture-interpretations.md) records the original critique and its subsequent corrections. +Its experiment conclusions apply to their recorded tasks, versions, information, and resource conditions. +The current audit did not rerun those experiments. +Consult their original records in `.evolve/current.json` and `memory/` before using them to reject or repeat a mechanism. diff --git a/docs/architecture.md b/docs/architecture.md index 8a28231f3..5c666d638 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -335,23 +335,15 @@ sequential steer used sparingly. ## 11. Empirical status — lives in the ledger -Every measured number — the FinSearchComp rung-0 arms, the Gate-A -clear-then-retraction under the POWER-16 rule, the GEPA-over-analyst-prompt null, -the selector results, and the SOTA comparison tables — lives in -`.evolve/current.json` (the live science state) and the memory ledger. This doc -keeps only the two distilled findings that are mechanism, not state: - -**The domain-boundary law:** within-run steering is **negative on stateless -retrieval** (FinSearchComp rung-0), **null-to-negative on stateless codegen** -(HumanEval steer gate null at equal k; exec-grounded self-repair −17.1pp, -CI [−26.8, −7.3]), and **positive on stateful agentic domains** with a correctable -middle band, scored keep-best (EOPS). The boundary variable is state + the -inability to cheaply resample. - -**Honesty law:** our loop is **not a new method class** — sequential-refine = -Reflexion / CRITIC / FLARE; fanout-vote = self-consistency / -best-of-N-with-verifier. We benchmark *against* those and claim no novelty for -the scaffold; the moat is transfer (§8). +Keep measured results in the dated evidence records and the active state in `.evolve/current.json`. +Interpret each result against its exact tasks, implementation, information, resources, and stopping rule. +A result for stateless retrieval, code generation, or one stateful benchmark does not establish a universal domain boundary. +A within-run comparison does not decide the value of repeated domain learning or evaluation engineering. + +Known inference methods remain useful comparison baselines. +Combining those methods does not establish algorithmic novelty or a capability gain. +The product claims in §8 concern specialist outcomes, repeatable domain learning, better evaluation, and improvement of the learning process. +Each requires evidence at the claimed level. --- diff --git a/docs/learning-flywheel.md b/docs/learning-flywheel.md index fe9522909..8885f6f9a 100644 --- a/docs/learning-flywheel.md +++ b/docs/learning-flywheel.md @@ -70,12 +70,10 @@ The asset is the **corpus**, not any single result. A run that shows no within-run effect still contributes data; the learnable structure emerges in the aggregate. -The read side is not free: **naively priming** the worker context with prior-run prose -records measures **negative** (−11.6pp with a worsening slope; the context-pollution and -instance-transfer falsifiers both fired). The surviving read-side design is -**verifier-gated, relevance-weighted accretion of certified programs** — store strategies -that passed a checker, not facts — -[docs/research/leapfrog-program.md §S3](./research/leapfrog-program.md). +Retained information has value only when its use improves later work. +The [recorded accretion experiment](./research/leapfrog-program.md) compared specific forms of prior-run context and checked program reuse. +Its negative context result does not reject all facts, retrieval policies, or combinations of memory and planning changes. +Measure the exact information retrieved, its use, and subsequent task outcomes before selecting a retention policy. ## The lifting generalization: recursive self-improvement @@ -110,13 +108,13 @@ Ln : improve "how to improve" at level n−1 (same tuple, lifted) `(π, τ, J, D, O)` structure; only the object-of-optimization changes. It is real (not vapor) only under three constraints: -1. **External anchor.** A fixed `J` at the base. Without it the recursion Goodharts — each - level games the metric. The **write-only judge is the keystone of the entire stack**; that - is *why* the integrity rule (judge never feeds steering/selection) is non-negotiable. -2. **Shared corpus `D`.** Improvements persist and are evidenced *across* levels — a level-1 - gain shows up in the corpus the level-0 runs produced. -3. **Evidence per learning level.** Level *n* contributes only when it **measurably lifts level n−1 on `J`**. - This requirement concerns evidence; construction and comparison follow [architecture.md §9](./architecture.md#9-build-order-and-experiment-scope). +1. **Independent assessment.** Each improvement claim needs assessment outside the adaptive decisions being tested. + Working evaluations can guide learning and can change; they cannot establish their own improvement merely by making success easier. +2. **Retained evidence and lineage.** Record which exact prior results and candidate states informed later work. + Storage can remain domain-specific while experiments share evidence contracts. +3. **Evidence per learning level.** Judge a candidate learner by the subsequent specialists, evaluations, or learning processes it produces. + Use repeated outcomes over the declared horizon; an exploratory step need not improve immediately. + Construction and comparison follow [architecture.md §9](./architecture.md#9-build-order-and-experiment-scope). **This subsumes everything in this repo and this design:** the worker, the `f(trace)` steer, the controller-as-signatures, GEPA, `meta-harness`, AND the **skill-governor** (which skill to From e18af4cfd2cb7f7f05e4a8b29cecc8f6e15d28ed Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 5 Sep 2026 12:36:39 -0700 Subject: [PATCH 3/6] docs(learning): separate learning claims and evidence --- docs/architecture.md | 10 +- docs/learning-flywheel.md | 434 +++++++++++--------------------------- docs/roadmap-rsi.md | 5 +- 3 files changed, 133 insertions(+), 316 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 5c666d638..5cadf1631 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -200,8 +200,9 @@ recipe and evaluation ID. This is the **outer flywheel**: the controller is lear not hand-written. Optimize against the **multi-objective vector** (§0.5.2) — *correct, fast, secure, cheap* — Pareto, **not** a pre-collapsed scalar; each component is graded by its own deployable checker (tests · clock · scanner · cost meter), with the external -write-only judge as the fixed anchor on the *correctness* axis so the recursion can't -Goodhart. **Status:** the loop today carries a single `score` per attempt (§6's +write-only judge as an independent check on the *correctness* axis. +That check still requires validation against the domain objective; a fixed score can remain an inadequate proxy. +**Status:** the loop today carries a single `score` per attempt (§6's `adapter.judge`) — collapsing the vector at the boundary is the open gap to close before the optimizer can trade objectives honestly. Candidate surfaces include complete profiles, executable strategies, working evaluations, curricula, and the learning method itself. @@ -261,8 +262,9 @@ being squatted on. never feeds a steer or a selection. - **Selector (distinct):** the deployable, learnable component that picks among candidates at inference (vote / verifier-rerank). A verifier-grounded selector (`verifierGroundedSelect` in - `bench/src/selector.ts`) is built and measured — see `docs/architecture-interpretations.md` §2 - for its current evidence status. The law stands regardless: the selector is never the judge. + `bench/src/selector.ts`) is implemented. + Consult its dated experiment records and subsequent corrections before making a capability claim. + Final assessment remains separate from the policy that chooses an answer. --- diff --git a/docs/learning-flywheel.md b/docs/learning-flywheel.md index 8885f6f9a..772225d15 100644 --- a/docs/learning-flywheel.md +++ b/docs/learning-flywheel.md @@ -1,170 +1,112 @@ # Continual Domain Learning and Meta-Learning -> **In plain terms:** This is a design-rationale doc — it explains *why* this project is built -> to get better the more it runs, not how to use the package day to day. It's for a developer -> who wants the big-picture research bet before reading the code. The one idea to take home: -> every test run saves a full record of what the agent did and how well it scored, and a -> learning component studies *all* of those saved records to steer future runs better — so the -> real asset is the growing library of run records plus the component trained on it, never any -> single test result. - -> **Start with [`architecture.md`](./architecture.md)** — it's the main map of how the system -> fits together: one recursive `Agent` building block, two speeds of improvement (fast within a -> single run, slow across many runs), evals plugged in as adapters, and the rule that the -> component choosing the best answer is never the one scoring it. This doc is the deeper dive -> into the theory and the long-term competitive edge — the `(π,τ,J,D,O)` recursion and the -> hard-won discipline behind it. Where the two disagree, `architecture.md` wins. - -> The core thesis of this project. There are **two loops, and the product is the outer one.** -> -> - **Inner loop (within-run):** a controller steers a worker over k attempts on a single -> task — refine/fanout/stop. Useful, but NOT the product, and not where the moonshot lives. -> - **Outer loop:** domain work generates `(state, trace, steer, outcome, cost)` records. -> A domain learner uses those records to improve specialists, working evaluations, and its own experimental decisions. -> A meta-agent can learn how to construct and improve those domain learning processes. -> -> Sustained improvement within one domain is valuable in its own right. -> A specialist need not transfer to another domain. -> The transferable knowledge can be the procedure that trains specialists and improves their evaluations. -> Failed experiments can inform that procedure when their evidence survives and affects later decisions. - -> **Across-run policy improvement (Gate B).** One test of the domain learner asks whether, across repeated runs on a -> persistent, checkable, long-horizon task family, the deployed controller's verifier-graded -> **multi-objective** score improves **run-over-run** (run N+1 starts above run N at **matched -> per-run compute**), the only changed variable is that the controller learned from the accumulated -> corpus, the gain survives a **frozen-controller control** (re-running an earlier controller shows -> no slope), it is significant at adequate n (paired-bootstrap + BH), and it is graded by a -> **deployable checker** — never the answer oracle or the write-only judge. *Multi-objective* is -> load-bearing: success is a vector (correct · fast · secure · cheap), with evidence scoped to each objective. -> Tests, clocks, scanners, and cost meters provide partial measurements; record each check's coverage and unverified assumptions. -> This tests one learning claim; evaluation quality, learning-process quality, and process transfer require separate comparisons. -> The -> within-run "trace+findings-fed controller beats the blind same-compute baseline under a non-oracle -> selector at **equal compute**" question is a separate, narrower diagnostic — **Gate A**, the -> comparison for within-run steering, scoped by [architecture.md §9](./architecture.md#9-build-order-and-experiment-scope). -> Compare actual resource use in both tests, including learning and evaluation-development costs over the declared horizon. -> The budget may fund one deep trajectory, several shallow attempts, or a mixture. +This document describes the learning objective, not a claim that the complete process has been demonstrated. +[Architecture.md](./architecture.md) defines execution responsibilities and experiment scope. +The [learning audit](./research/learning-system-audit-2026-09-05.md) distinguishes implemented behavior, reproduced defects, and unmeasured capability. + +A specialist, its domain learning process, and the method that constructs that process are all useful objects of improvement. +A specialist can remain specific to its domain. +The reusable knowledge can be how to build and run a successful learning process elsewhere. + +| Object | What changes | Evidence of improvement | +| --- | --- | --- | +| Specialist | Its profile, code, tools, knowledge, memory use, or execution policy | Better results on fresh work within its intended domain | +| Domain learner | Problem selection, diagnosis, candidate construction, working evaluations, and retention decisions | Better specialists or domain outcomes across repeated learning episodes | +| Meta-learner | How domain learning processes are constructed and improved | Better learning processes on the domains and objectives covered by the claim | + +Transfer of the specialist and transfer of its learning process are different claims. +Useful repeated learning within one domain requires neither. +Working evaluations can improve at each level, subject to independent assessment of their quality. ## The flywheel -``` - ┌──────────────────────────────────────────────────────────────────────┐ - │ │ - ▼ │ - RUN evals across MANY benchmarks (coding, research, terminal, browser, …) │ - │ each run = a driver/controller steering a worker over k attempts │ - │ │ - ▼ │ - RECORD the full tuple per attempt → a durable, queryable CORPUS │ - (state · prompt/steer · TRACE · output · judge verdict · cost/turns) │ - │ │ - ▼ │ - LEARN the controller from the WHOLE corpus (offline, cross-benchmark) │ - trace-aware, multi-objective GEPA/optimizer over the steer/topology │ - signatures — optimize for SUCCESS and CLEAN/FAST trace │ - │ │ - ▼ │ - BETTER controller → ships into the next runs ───────────────────────────► ┘ -``` - -The asset is the **corpus**, not any single result. A run that shows no within-run effect -still contributes data; the learnable structure emerges in the aggregate. - -Retained information has value only when its use improves later work. -The [recorded accretion experiment](./research/leapfrog-program.md) compared specific forms of prior-run context and checked program reuse. -Its negative context result does not reject all facts, retrieval policies, or combinations of memory and planning changes. -Measure the exact information retrieved, its use, and subsequent task outcomes before selecting a retention policy. +A domain learner consumes an objective, execution tools, current specialists, prior experience, and resources. +It produces evaluated specialist candidates, working evaluations, experiment records, and updated learning state. +Its decisions include what to investigate, what to change, how to measure it, and what to retain. + +The process becomes continuous when retained evidence changes a later decision or candidate. +Writing a record alone does not establish learning. +An outer evaluation must execute the produced specialist and its exact retained state. +Scoring the learner's explanation of that specialist tests a different outcome. + +Retained information is useful when its use improves later work. +Record what was retrieved, how it was used, and the subsequent result. +A useful change can require memory and planning changes together. +Preserve informative failures and alternative candidates even when they do not qualify for immediate adoption. ## The lifting generalization: recursive self-improvement -The object being improved can be a complete domain learning process. -It can produce specialized AgentProfiles, improved working evaluations, and the next experimental policy. -The process can learn within one domain before, or without, being reused elsewhere. -When it is reused, measure whether it constructs a useful learner in the new domain rather than expecting the old specialist to generalize. -Working evaluations may evolve, while independent assessment tests whether those changes better detect meaningful success and failure. - -The flywheel is one instance of a more general object. Name the loop: - -``` -L = (π, τ, J, D, O) - π policy — produces behavior - τ trace — the behavior + its full execution record - J judge — EXTERNAL, write-only score (the anchor) - D corpus — accumulated (τ, score), shared memory - O optimizer — D → π′ (a better policy) -``` - -**The lift:** `O` is itself a policy → `L` can take `L` as its `π`. The loop is -**self-similar across levels**, where level *n*'s policy is *"how to optimize level n−1"*: - -``` -L0 : improve the WORKER's behavior on a task (π = worker) -L1 : improve the CONTROLLER / steer-function f (π = L0's optimizer) ← the flywheel -L2 : improve the OPTIMIZER that learns f (π = L1's optimizer) ← meta-harness/meta-GEPA -Ln : improve "how to improve" at level n−1 (same tuple, lifted) -``` - -**Recursive self-improvement = this loop closed on itself.** Every level is the *identical* -`(π, τ, J, D, O)` structure; only the object-of-optimization changes. - -It is real (not vapor) only under three constraints: -1. **Independent assessment.** Each improvement claim needs assessment outside the adaptive decisions being tested. - Working evaluations can guide learning and can change; they cannot establish their own improvement merely by making success easier. -2. **Retained evidence and lineage.** Record which exact prior results and candidate states informed later work. - Storage can remain domain-specific while experiments share evidence contracts. -3. **Evidence per learning level.** Judge a candidate learner by the subsequent specialists, evaluations, or learning processes it produces. - Use repeated outcomes over the declared horizon; an exploratory step need not improve immediately. - Construction and comparison follow [architecture.md §9](./architecture.md#9-build-order-and-experiment-scope). - -**This subsumes everything in this repo and this design:** the worker, the `f(trace)` steer, -the controller-as-signatures, GEPA, `meta-harness`, AND the **skill-governor** (which skill to -run next = an L1 policy; learning the governor from skill-run outcomes = L2) are all slices of -one structure — *a uniform recursive optimization stack over policies-with-traces, anchored by -external judges, backed by a shared corpus.* That is what "imagining bigger" resolves to. - -**Benchmark BOTH — in fact, ALL levels.** Every level is an independent toggle, so you -*ablate* to measure each level's marginal lift on `J`: - -``` - within-run refine {on,off} × cross-run learned controller {on,off} × meta {on,off} -``` - -The corpus + external judge make every level measurable in isolation and in combination — -which is how you *prove* a recursive system is real instead of asserting it. - -## Vocabulary (one node type, recursive) - -- **Worker** — does the task (opencode / a browser agent / a coding agent). A black-box - multi-turn agent: one "attempt" is a full agentic rollout, not one LLM turn. -- **Controller (driver)** — shapes *how* the task gets done across attempts. Expressed as a - program of **signatures** (DSPy/ax sense): - - `steerPolicy : (trace, history) → steer` ← the optimizable core (the "f") - - `topologyPolicy : history → refine | fanout | stop` - - `stopPolicy : history → continue | done` - The worker is an **opaque tool** the controller calls. Driver and worker are the same node - type in two modes (execute vs. author-sub-topology); the recursion bottoms out at execution. -- **Judge** — the benchmark's terminal scorer. **Write-only**: it scores the controller's - final chosen output and is NEVER an input to steering/selection (else it's an oracle = - cheating). Deterministic (SWE/terminal) or verified-stable LLM (research). - -## The steer is `f(trace)` — a searchable space of signatures - -`steer` is not a fixed string. It is `f(prior trace, prior answer, history) → context`, and -`f` is a **pluggable, benchmarkable knob** — the "variety of signatures": - -| `f` | what the next attempt is told | carries failure info? | -|---|---|---| -| `∅` (random@k) | the bare task again (k independent tries) | no — compute control | -| fixed directive (hand / GEPA-learned) | a static instruction | no | -| `LLM(trace)` (analyst) | a targeted steer from the actual failure | **yes** ← where signal likely lives | -| compressed trace report | key metrics/errors, denoised | yes | -| **agentic driver** | a full agent investigates (subagents, code audit) → steer | yes (max power, max cost) | - -The same `f(trace)` plugs into **two places**: (1) runtime — what the worker sees next; and -(2) **GEPA reflection input** — what the optimizer sees to rewrite the steer (canonical, -trace-aware GEPA). Benchmarking `f`s = finding the best trace representation. - -**Candidate input must name its source.** +One description of a learning loop is `L = (π, τ, J, D, O)`: + +| Term | Meaning | +| --- | --- | +| `π` | The executable object being improved | +| `τ` | Its behavior and execution record | +| `J` | Independent assessment for the current improvement claim | +| `D` | Retained experience and candidate lineage | +| `O` | The procedure that uses experience to propose and select changed objects | + +The procedure `O` can itself become the object `π` of another experiment. +Its result is then judged through the specialists or learning processes it produces. +Working evaluations can be part of that changeable procedure. +An independent assessment remains outside the adaptive decisions it tests. + +This recursion describes a possible composition, not evidence of recursive improvement. +Three obligations remain at every level: + +1. Execute the exact candidate and retain its state and outcome records. +2. Include all inner work, incomplete episodes, and unmeasured costs in the outer account. +3. Assess the claimed outcome independently of the search decisions that selected the candidate. + +A fixed final score can still be an inadequate proxy for the domain objective. +Validate its coverage and failure detection; independence alone does not establish validity. +Storage can remain domain-specific while experiments share identity and evidence contracts. +A single physical store or one search algorithm is not required. + +## Evaluation engineering is part of learning + +A learner can generate cases, construct judging instructions, create executable checks, and change its practice distribution. +Those working evaluations can guide its next intervention. +An improved evaluation can expose more failures and lower the current specialist's score. +Its value depends on detection quality and subsequent domain learning, not an easier score for the current specialist. + +Use independent outcomes, controlled defects, new failure cases, and external assessment to test evaluation changes. +Calibration statistics analyze observations supplied by a caller. +They do not execute judges, collect domain outcomes, or train a specialist by themselves. +Disagreement and score variance can identify questions to investigate without proving which practice will improve the agent. + +Cases used to construct evaluations or choose learning policies become development evidence for those decisions. +Keep final assessment separate at the level whose improvement is claimed. +Fresh tasks within a domain can assess a specialist or domain learner. +A claim about process transfer requires an appropriate comparison in new domains. + +## Across-run evidence + +The across-run comparison, also called Gate B, tests learning over a declared sequence or horizon. +Compare active learning with an appropriate frozen or reference process under the same objectives and recorded resource conditions. +Evaluate repeated episodes from explicit initial states. +Useful exploration need not improve every intermediate version. + +Gate A tests the narrower question of steering within one task. +Passing that test is not a prerequisite for every form of domain learning. +Use [architecture.md §9](./architecture.md#9-build-order-and-experiment-scope) to choose the comparison and conditions for rejecting the tested mechanism. + +| Dimension | Observation | +| --- | --- | +| Domain outcome | Checked task results, failures, and uncertainty | +| Resources | Learning, execution, evaluation development, retrieval, checking, and retained-state costs | +| Retention | Earlier abilities after updates | +| Learning decisions | Prior evidence that changed an experiment, candidate, or procedure | +| Evaluation quality | Independently checked detection, ranking, coverage, and downstream learning | +| Transfer, when claimed | Reconstruction of useful learning in unfamiliar domains | +| Adoption | The exact version measured and the exact version used later | + +Compare combinations when the claimed benefit depends on interacting components. +Remove components to identify their contributions after proving the complete mechanism executes. +A null result needs adequate measurement sensitivity and observed mechanism activation before it can reject the tested explanation. + +## Candidate input must name its source + Every finding used to generate a candidate is a `ProposalFinding`. `proposal_origin: 'production'` means the finding came from observed production behavior. `proposal_origin: 'search'` means it came from development work during candidate search. @@ -174,158 +116,30 @@ Final evaluation results have no allowed proposal origin and never feed candidat `derived_from_judge` remains descriptive metadata. Search-time judge feedback is valid when it is explicitly marked `proposal_origin: 'search'`. -The final judge result is still isolated from search. +The final judge result remains isolated from search. A separate final-test partition is required because source labels alone cannot prevent overfitting. -## Architecture layers (ranked by leverage) - -1. **Eval + corpus substrate (the GATE).** Cheap, reliable, **trace-rich** evaluation; the - `RunRecord` corpus written by *every* run; deterministic judges where possible; an - **offline replay + reward-model layer** so the controller space can be searched WITHOUT a - live rollout per candidate (agent-eval `./rl`: `buildRlDataset`, off-policy estimation, - reward modeling). *This is the bottleneck. Without it, nothing above is reachable — - GEPA can search any space only if you can afford the metric evals.* -2. **Controller-as-signature-program.** steer/topology/stop as jointly-optimizable - signatures; worker as opaque tool. The compiled-program controller lives - as a `defineStrategy`/`authorStrategy` program (`src/runtime/strategy.ts`) driven over - the `Scope`/`Supervisor`. -3. **Trace-aware, multi-objective optimizer.** GEPA/MIPRO reflecting on **traces** (not - pass/fail), optimizing for **correctness AND clean/fast trace** (Pareto). `meta-harness` - is the code-level search engine that sits HERE — it evolves controller *code* on a Pareto - frontier, and it only works once layer 1 makes the metric cheap + discriminating. - **Measured (2026-06-09): the analyst-prompt coordinate is flat** — a 3-generation GEPA - run over the `observe()` analyst prompt tied the default exactly on a frozen holdout. - The searchable space that remains live at this layer is the **strategy program itself** - (`defineStrategy` + `authorStrategy`), not the analyst prompt. -4. **Cross-domain.** Optimize ONE controller across coding/research/terminal/browser. If the - learned steering **transfers**, that's the moonshot. If not, you get N per-domain - flywheels — still useful, but the "one controller, many benchmarks" claim *requires* - transfer, and that is the open empirical risk. - -## Discipline (hard-won; violate these and the flywheel learns noise) - -- **The flywheel amplifies whatever you feed it.** Clean `(trace, reward)` tuples → real - structure. Noise (unverified judge, infra-corrupted traces, confounded outcomes) → a bigger - pile of noise with false confidence. **Clean data > more data.** Rigor is what makes the - corpus *learnable*, not bureaucracy. -- **Confounds before causal claims.** A delta where treatment gets more compute than control - is not a causal result. Steering must always be measured against its **`random@k` compute - control** as a sibling benchmark arm, so the isolated effect is `refine@k − random@k` at equal - k. The steer itself is concrete: an analyst-derived per-shot string carried shot-to-shot - (`buildSteerContext` builds it; the strategy loop threads it as `pendingSteer`), never a - free-floating prompt edit. Verify the judge is deterministic (re-judge test). Exclude - infra-errored cells; retry transient drops. (See the false "+20pp = steering proven" — it was - compute + infra + an untested judge.) -- **Pre-register the primary metric; correct the family; spend the holdout once.** The ablation - grid (steering arms × directives × benchmarks, plus compute controls) tests *many* contrasts — - each independent "CI excludes 0" inflates the family-wise false-positive rate (garden of forking - paths). The PRIMARY hypothesis (`steering = refineX − random > 0`) is pre-registered; every - reported contrast is **Benjamini-Hochberg corrected within its family** (`corpus-report.mts`), - and a result counts only if it clears the family FDR — never on its own CI. Separate a reusable - **exploration** set (rank candidates freely, BH-corrected) from a **frozen confirmation holdout** - spent once per *locked* candidate; this is what `compareOptimizationMethods` enforces by keeping - the final-test partition out of the optimization method (memorization read as generalization is the default failure otherwise). -- **"Validates the concept" ≠ "validates the product."** A hand-rolled refine loop proves - refinement helps, NOT that `runAgentRounds`/the controller does. Route through the real kernel. -- **Eval economics is the moonshot bottleneck, not controller cleverness.** Build the offline - corpus/replay so search is affordable. Don't build the optimizer cathedral over a metric - you can only sample a few hundred times with overlapping CIs. -- **Choose a decisive test before escalating cost.** - Follow [architecture.md §9](./architecture.md#9-build-order-and-experiment-scope) for complete mechanisms, combinations, resource accounting, and rejection conditions. - -## Honest status (updated 2026-06-10) - -- **Stateful agentic (EnterpriseOps-Gym itsm, 2026-06-09): Gate A POSITIVE.** On the - canonical loop — `Scope`/`Supervisor` + the `observe()` analyst + `defineStrategy` - (`src/runtime/strategy.ts`), not the `runAgentRounds` path — depth-steered continuation beats - breadth (blind best-of-K) at equal compute under keep-best checkpoint scoring: - **+16.4pp CI [+5.3, +29.8]**, 6 wins / 0 losses, n=16, deepseek-v4-pro; replicated - **+8.3pp** on a disjoint task slice. -- **Stateless codegen (HumanEval, 2026-06-08): null-to-negative.** observe→steer does not - beat blind resampling at equal k (n=82, paired bootstrap; compute alone +12.2pp - significant); exec-grounded self-repair is significantly **negative** (−17.1pp, - CI [−26.8, −7.3]). -- **The domain-boundary law (supersedes any "steering loses everywhere" reading of the - rung-0 entry below):** within-run steering is negative on stateless retrieval - (FinSearchComp), null-to-negative on stateless codegen (HumanEval), **positive on - stateful agentic domains** with a correctable middle band, scored keep-best (EOPS). - The boundary variable is state + the inability to cheaply resample. -- **Analyst-prompt GEPA (2026-06-09): NULL.** A 3-generation prompt search + frozen - holdout tied the default `observe()` analyst exactly (the search winner's +12.6pp was - holdout-overfit). The analyst-prompt coordinate is flat; the live outer-loop lever is - program/strategy space (`defineStrategy` + `authorStrategy`). -- **Corpus read-side priming (naive): NEGATIVE** (−11.6pp, worsening slope) — see the - read-side note under "The flywheel" and - [leapfrog-program.md §S3](./research/leapfrog-program.md). -- Evidence map + ranked portfolio: - [docs/research/optimization-space.md](./research/optimization-space.md). - -### Earlier entries (2026-06-03) - -- **Coding (SWE-bench):** refine ≈ blind (net 1 rescue / 1 break, n=23). Directional, NOT - proven — high blind baseline (~74%, likely *contamination* on popular repos) leaves ~no - correctable middle band, and there was no `random@k` control. SWE-bench is a weak instrument - here. -- **Research (FinSearchComp): rung-0 settled, and the answer is NO.** The first - adequately-powered, confound-controlled, judge-verified 3-way through the real `runAgentRounds` - (n=40, 20 T2 + 20 T3, gpt-5 worker + verified-deterministic judge, 0 infra-excluded): - - blind 37.5% → random@3 **60.0%** → refineHand@3 50.0% → refineGepa@3 45.0%. - - **more-compute** (random − blind) = **+22.5pp**, 95% CI [+7.5, +40.0], p=0.008 (13/40 - discordant) — trying again robustly helps. - - **steering** (refineX − random) is **negative on every slice, both directives**: - refineHand −10.0pp (CI [−25, +5], p=0.25), refineGepa −15.0pp (CI [−27.5, −2.5], p=0.032). - The GEPA harm is nominally significant but does **not** survive BH across the 2 steering - arms (q≈0.064) — so the disciplined claim is *no benefit + a consistent negative trend*, - NOT "significantly harms". Mechanism: the inner opencode agent already self-corrects in its - own rollout; an external refine directive adds a chance to BREAK a correct answer, while - `random@k` (independent retries, any-pass) captures the more-attempts benefit without that - downside. The earlier "+7.1pp held-out" was n=8 noise; this supersedes it. - - > `random@k` / `refineHand@k` / `refineGepa@k` are **condition labels for strategy runs** - > recorded in the corpus (the controller column), not importable symbols — `refineGepa@k` - > names "the refine strategy steered by a GEPA-authored prompt, k attempts." - - Subtype splits (n=20 each) are underpowered — even more-compute is not significant on T3 - alone (CI [−5, +35]). T2 mirrors the aggregate (more-compute +30pp sig; steering ≤0). -- **Terminal-Bench:** adapter+judge + blind-vs-refine wired (reuses tb's open-source opencode - agent + verifier). Bench-orchestrated (tb owns containers) — the exception that does NOT - route through `runAgentRounds`. -- **Net:** the first clean rung-0 measurement **contradicts** the flywheel's core premise on - this domain — a within-run steer does NOT beat compute-matched random; compute does. This is - one benchmark, one worker, two directives (incl. a GEPA-learned one that also fails), so it - bounds the *within-run inner loop*, not the cross-run outer flywheel. But it is a real, - controlled NO where there was only confounded YES before — the instrument now works, and it - says: do not escalate to costlier steers on this benchmark to re-derive that more-compute wins. - -## Build sequence - -1. **Corpus capture** (this is the foundation): every bench run persists full `RunRecord`s - (prompt/steer · trace · output · verdict · cost) into one durable store — *stop the - boolean-only scorecards that delete the fuel.* -2. **Rung-0/1 signal:** does `random@k` get beaten by *any* `f` (fixed, then `LLM(trace)`), - confound-controlled, judge-verified, infra-reliable? -3. **Offline replay + reward model** over the corpus → controller search becomes affordable. -4. **Controller-as-signatures + trace-aware multi-objective GEPA / meta-harness** searches the - `f`/topology space over the corpus; validate winners live. -5. **Cross-domain transfer** — one controller, many benchmarks. The moonshot. - ## Where the pieces live -- Kernel + controller seam: `src/runtime/` — the `runAgentRounds` kernel (`run-loop.ts`, one - leaf execution backend) and the canonical agent-driver: - `createCoordinationTools` (`src/mcp/tools/coordination.ts`) over the `Scope`/`Supervisor` - substrate (`src/runtime/supervise/`), with `runAgentic`/`defineStrategy`/`runPersonified`. -- **The published optimization suite**: `@tangle-network/agent-runtime/kernel` (source: - `src/runtime/`): - `Environment`/`Strategy`/`defineStrategy`/`ShotSpec.profile` (`strategy.ts`), `runBenchmark` - (`run-benchmark.ts`), `createVerifierEnvironment`/`createMcpEnvironment`, - `harvestCorpus`, `authorStrategy` (`strategy-author.ts`), `auditIntent`, and - `promotionGate` (`promotion-gate.ts` — the seeded paired-bootstrap holdout gate over - agent-eval's `heldoutSignificance`: evidence floor 6 paired tasks, the CI lower bound - must clear the threshold). -- Benchmarks + workers + experiments: `bench/` (`benchmarks/*`, `worker-*`, - `terminal-compare.ts`, `corpus-report.mts`). The gen0 → `authorStrategy` → gen1 → - rotating-disjoint-holdout runner (the minimal single-objective Gate-B form) over - `authorStrategy` (`src/runtime/strategy-author.ts`) + the seeded `promotionGate` is open work. -- Substrate optimizer/corpus primitives: `@tangle-network/agent-eval` (`OptimizationMethod`, - `compareOptimizationMethods`, `gepaOptimizationMethod`, `skillOptOptimizationMethod`, - `heldoutSignificance`, `RunRecord`/trace-store, `./rl`). +| Concern | Existing implementation | Composition boundary | +| --- | --- | --- | +| Agent-driven work | `Scope`, `Supervisor`, and `createCoordinationTools` | A supplied profile owns working decisions and recursive authority | +| Profile and code improvement | Runtime `improve()` and Eval complete methods or native proposer search | The caller supplies domain execution and objectives | +| Executable strategy search | `defineStrategy`, `authorStrategy`, and `runStrategyEvolution` | Programs and their archive remain explicit; this is not a complete domain learner by itself | +| Observation and retention | `observe`, `Corpus`, and Knowledge state and retrieval | The caller must connect retained evidence to later decisions and measured outcomes | +| Evaluation engineering | Eval judges, scenario search, calibration, and known-failure checks | Executing a utility does not establish the quality of the resulting evaluation or learning process | +| Exact measurement and adoption | Runtime candidate experiments, proposals, and activation | Search output must remain bound to the exact version measured and adopted | + +The [learning audit](./research/learning-system-audit-2026-09-05.md) gives precise source references and the missing connections. +Use those existing components before adding an orchestration facade. + +## Historical evidence + +The [earlier version](https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/docs/learning-flywheel.md) records earlier steering, prompt-search, and context-reuse results. +The [optimization portfolio](./research/optimization-space.md) and [accretion experiment](./research/leapfrog-program.md) provide additional experiment context. +Read dated records and subsequent corrections before repeating or rejecting a mechanism. + +Those comparisons apply to their recorded tasks, versions, information, and resources. +A null prompt search does not eliminate all prompt changes. +A negative context treatment does not eliminate all memory policies. +A within-task result does not decide whether specialists, evaluations, or learning procedures improve across repeated domain work. diff --git a/docs/roadmap-rsi.md b/docs/roadmap-rsi.md index 7c3f723a2..8f43889a8 100644 --- a/docs/roadmap-rsi.md +++ b/docs/roadmap-rsi.md @@ -4,7 +4,7 @@ Companion to [architecture.md](./architecture.md) (the spine) and [architecture- ## The principle: make it measurable before you build it -**Gate A** tests within-run steering ([architecture-interpretations.md §5](./architecture-interpretations.md#5-gate-a--the-decision-gate-for-the-recursive-driver-layer)): +**Gate A** tests within-run steering ([architecture-interpretations.md §5](./architecture-interpretations.md#5-gate-a--a-diagnostic-for-within-run-steering)): > Does a trace-informed driver beat random attempts under the same answer selection method, at equal actual resources, with enough evidence to distinguish useful improvement? @@ -96,7 +96,8 @@ Runs in **parallel** to Phases 1–2 (bench-only, no kernel code). This is the k - **Build a `RefineLoopSpec`** over `runRefineLoop` (`refine-loop.ts:44-63`): `setup` = open/create the vault; `prompt(round, history)` = the maintainer directive folding prior pages + open contradictions; `runShot` variants = (i) an `llm-wiki` maintainer+critic (ingest → propose page edits → lint contradictions/staleness/orphans) and (ii) a `bad`-CLI browser source-fetcher reusing `bench/src/browser/adapters/bad.ts` for web data/video/images; `judge` = the critic's lint verdict; `teardown` = flush the vault. - **Propose through `runAnalystLoop`, measure through `runKnowledgeImprovementJob`, and write only through `createKnowledgeImprovementActivationExecutor`.** Analysis never mutates the live knowledge tree. -- **The gap signal must be STRUCTURAL** — graph topology, citation/embedding density, redundancy-discounted coverage — **not an LLM vibe.** A miscalibrated acquisition function underperforms random sampling ([interpretations §3.2](./architecture-interpretations.md#32-active-learning--experimental-design)); the structural signal is what makes this active learning rather than coverage-greedy ingestion. +- **Validate gap signals against outcomes.** Structural features and model judgments are possible inputs to problem selection. + Neither establishes useful practice without a downstream comparison ([interpretations §3](./architecture-interpretations.md#3-five-interpretations)). - **No mocks** — real vault, real `bad` runs (repo doctrine). - Source-selection is authored as a `defineStrategy` program (`src/runtime/strategy.ts`) driven over the `Scope`/`Supervisor`. From 5c39baf9640b5b3ad7e2521305a5d5af69056177 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 5 Sep 2026 21:50:05 -0700 Subject: [PATCH 4/6] docs(learning): refine audit source anchors --- docs/research/learning-system-audit-2026-09-05.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/research/learning-system-audit-2026-09-05.md b/docs/research/learning-system-audit-2026-09-05.md index 0f4280a08..dea9c7307 100644 --- a/docs/research/learning-system-audit-2026-09-05.md +++ b/docs/research/learning-system-audit-2026-09-05.md @@ -258,7 +258,7 @@ The repair and verification record is maintained with the changes and regression | E6 | P1 | An optimizer reports $17 estimated/incomplete cost; the run reports $0 and complete accounting. A repair review also found $3 recorded with a $0 complete result | Measured through `selfImprove`; `E:src/contract/self-improve.ts:676`, `:828`; review probes compare method reports with actual recorded calls | Reconcile each method's reported cost with its attributed calls, including parallel methods; retain incomplete accounting | | E7 | P2 | Native history labels the point estimate .25 as the exact interval [.25, .25] for case scores 1, 0, 0, 0 | Measured through native `selfImprove`; `E:src/campaign/presets/run-optimization.ts:553` | Return null for unestimated uncertainty; retain actual final-test statistics | | E8 | P2 | An unchanged result includes the same final campaign twice: 2 executions costing $2 become 4 executions costing $4 in its insight | Measured through `selfImprove`; `E:src/contract/self-improve.ts:835`; independent probe checks dispatches, recorded calls, costs, and tokens | Include an identical final campaign once in both method and proposer results | -| E9 | P2 | Scenario-search history uses real IDs while allocation asks for `*`; the second round repeats 10/10 evaluations despite different observed variance | Measured across 40 evaluations; `E:src/fuzz/explorer.ts:146`; `src/rl/active-curriculum.ts:90`; pooled history gives 8/12 | Pool observations by search cell when allocating; preserve exact IDs in stored records | +| E9 | P2 | Scenario-search history uses real IDs while allocation asks for `*`; the second round repeats 10/10 evaluations despite different observed variance | Measured across 40 evaluations; `E:src/fuzz/explorer.ts:144`; `src/rl/active-curriculum.ts:90`; pooled history gives 8/12 | Pool observations by search cell when allocating; preserve exact IDs in stored records | | R1 | P2 | A modified tracked diagnosis file is classified as substantive code because Git's leading status space is trimmed | Measured with real Git; `R:src/improvement/agentic-generator.ts:1019` | Parse Git status without deleting path characters | | R2 | P1 | Strategy resume returns an old promoted result after task payloads, objective, model, environment, and final offset change | Measured: zero author calls and benchmark phases; `R:src/runtime/strategy-evolution.ts:397` | Bind resume to exact serializable inputs and explicit callback revision | | R3 | P2 | Strategy author instructions teach `shot({persona})`, but the executable API accepts `shot({profile})` | Source-verified mismatch: `R:src/runtime/strategy-author.ts:29`; `strategy.ts:847` | Teach the actual complete-profile API and execute its example | @@ -267,8 +267,8 @@ The repair and verification record is maintained with the changes and regression | R6 | P1 | Conflicting same-ID file appends both succeed, then subsequent reads reject the corrupted log | Measured with concurrent file stores; `R:src/runtime/personify/corpus.ts:213` | Lock the full read/check/append transaction across processes | | R7 | P2 | Mutating the caller's tags after append changes the stored lesson | Measured through `InMemoryCorpus`; `R:src/runtime/personify/corpus.ts:171` | Retain detached immutable records | | R8 | P1 | Reflective generation ignores an aborted signal, hides draft/apply failures, and reports an incomplete patch batch as applied | Measured with real patches; `R:src/improvement/reflective-generator.ts:28` | Use candidate-bound drafting, cancellation, base checks, and atomic patch application | -| K1 | P1 | KB acquisition and update receive no findings because diagnosis runs afterward | Measured Linux lifecycle: acquire → update → diagnose; `K:src/kb-improvement/evaluation.ts` | Carry one lifecycle state from diagnosis through construction and final evaluation | -| K2 | P2 | An unchanged empty KB with no outcome tests gets five dimensions equal to 1 and stages as candidate-ready | Measured Linux KB probe; `K:src/kb-improvement/evaluation.ts:510` | Omit unmeasured dimensions and identify the scope of configured checks | +| K1 | P1 | KB acquisition and update receive no findings because diagnosis runs afterward | Measured Linux lifecycle: acquire → update → diagnose; `K:src/kb-improvement/evaluation.ts:123` | Carry one lifecycle state from diagnosis through construction and final evaluation | +| K2 | P2 | An unchanged empty KB with no outcome tests gets five dimensions equal to 1 and stages as candidate-ready | Measured Linux KB probe; `K:src/kb-improvement/evaluation.ts:498` | Omit unmeasured dimensions and identify the scope of configured checks | | K3 | P1 | Research stops after 1 of 4 allowed rounds while the actual driver reports incomplete, producing no further steering | Measured real research driver plus verified loop; `K:src/verified-research-loop.ts:327` | Respect driver completion and fold remaining research work | | K4 | P2 | Runtime appends a worker policy to the exact supplied supervisor; that policy forbids writes while the task requires writes | Source-verified and exercised by adapter tests; `R:src/knowledge/supervised-update.ts:132`; `profiles/researcher.ts:349` | Execute the caller profile unchanged | | K5 | P2 | A caller requires diagnosis but explicitly enables no phases; candidate work succeeds without executing the required phase | Measured through Linux lifecycle tests; `K:src/kb-improvement/evaluation.ts:51`; `tests/kb-improvement/lifecycle.test.ts:18` on repaired source | Reject required-but-disabled phases before candidate work, including final evaluation phases | @@ -504,6 +504,8 @@ No new learning scheduler, memory service, optimizer facade, or universal scalar The useful work is joining existing paths and removing conflicting meanings. The RAG phase implementation moved into an internal module while its public API stayed in place. That file move is not a deletion; the deleted behavior is reconstructing and merging disconnected lifecycle states. +Canonical design notes no longer repeat obsolete architecture verdicts, universal domain boundaries, or claims that one unsuccessful search eliminates a change surface. +Their earlier versions remain linked with exact Git revisions so the historical evidence and its conditions remain available. ## Experiments for the different learning claims From 7fe89641322dc1d0e60dc8c4d66472d786320558 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 5 Sep 2026 22:16:18 -0700 Subject: [PATCH 5/6] chore(release): align the repaired learning packages --- bench/CHANGELOG.md | 4 +++ .../learning-system-audit-2026-09-05.md | 34 ++++++++++++++++++- pnpm-lock.yaml | 32 ++++++++--------- release/cohort.json | 8 ++--- 4 files changed, 57 insertions(+), 21 deletions(-) diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index c150a4a52..bb468800e 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.8.32 + +The dependency ranges now require Runtime 0.195.0, Eval 0.174, and Knowledge 14. + ## 0.8.31 Manifest-only: the Sandbox dependency range now admits the published 0.37 cohort. diff --git a/docs/research/learning-system-audit-2026-09-05.md b/docs/research/learning-system-audit-2026-09-05.md index dea9c7307..7222581c1 100644 --- a/docs/research/learning-system-audit-2026-09-05.md +++ b/docs/research/learning-system-audit-2026-09-05.md @@ -26,6 +26,9 @@ Runtime main advanced to `bd7a2f3f15a93ee286684c5bc6b88c00df9b72de` during the a The repair branch includes that change, including profile-owned recursive authority and durable acceptance of direct submissions. The source findings below retain their original revision boundary. The final integration checks use the combined implementation. +The repair branch also includes the subsequent skills-only main commit `11b6dea0`. +Its maintained skill checks and the regenerated Runtime build passed after integration. +The later instruction-only main commit `2707e232` also merges cleanly and is included. `R`, `E`, and `K` in source references mean these Runtime, Eval, and Knowledge revisions. Runtime coverage includes all 24 production files under `src/improvement`, plus execution, adoption, strategy evolution, observation, and memory serving. @@ -36,12 +39,41 @@ Public callsites and nearby applications were searched to distinguish implemente Defect probes used real public functions with deterministic callbacks, real Git worktrees, and real filesystem storage. KB snapshot probes ran in a local Linux container because the exact snapshot implementation intentionally refuses macOS. These tests measure implementation behavior, not model intelligence or defect prevalence. -No paid model experiment, production deployment, or current same-task comparison against another learning system ran during this audit. +No paid model experiment, application deployment, or current same-task comparison against another learning system ran during this audit. Primary research was checked through paper abstracts, official repository documentation, package metadata, and selected implementation contracts. This is a mechanism comparison, not a systematic literature review or a benchmark ranking. Upstream performance claims were not independently reproduced and are not assigned to this system. +## Repair verification and compatibility + +The audit records 22 repaired defects: 12 high severity and 10 medium severity. +These are distinct findings, not a production failure rate. +The [audit manifest](../../.agent/critical-audit/2026-09-05-learning-system/manifest.json) retains counts, source anchors, failed attempts, exclusions, and evidence limits. +Focused tests overlap full suites and must not be added to their counts. + +| Package | Implementation check | Release evidence | +| --- | --- | --- | +| Eval 0.174.0 | 5,763 JavaScript tests passed, three skipped; separate official integrations and Python checks passed | [PR 738](https://github.com/tangle-network/agent-eval/pull/738), [CI](https://github.com/tangle-network/agent-eval/actions/runs/33987395502), [npm and PyPI publication](https://github.com/tangle-network/agent-eval/actions/runs/34012602635) | +| Knowledge 14.0.0 | 862 tests passed, seven conditional skips; two separate official optimizer checks passed | [PR 190](https://github.com/tangle-network/agent-knowledge/pull/190), [CI](https://github.com/tangle-network/agent-knowledge/actions/runs/34013062769), [npm publication](https://github.com/tangle-network/agent-knowledge/actions/runs/34013280992) | +| Runtime 0.195.0 | Prior local integration at `5d617002`: 3,561 tests passed, nine skipped, with four workers | Final checks against the published dependencies and package publication remain in progress at this report revision | + +Runtime's prior full run used Node 24.11.1 and the locally built repaired dependencies. +Later main integrations changed only instructions and skills. +The environment refresh removed the original local logs; the manifest explicitly distinguishes prior terminal observations from retained CI evidence. +Knowledge's Ubuntu CI exercises snapshot operations that intentionally refuse macOS. +Paid-network and other conditional skips remain excluded from the corresponding implementation claims. + +Eval now distinguishes complete-method results from native-proposer results through `result.mode`. +Consumers that use native generations must narrow that type before reading them. +Runtime strategy checkpoints require an explicit `checkpoint.executionRef` for dependencies that cannot be serialized. +Reflective code generators require `createImprovementProposalSource(context)` so drafts use the current checkout, cancellation signal, and cost account. +Knowledge 14 preserves lifecycle feedback and omits dimensions that were not measured. +The exact dependency commits and compatible ranges are pinned in `release/cohort.json`, the workspace catalog, and the lockfile. + +Installed-package probes use deterministic callbacks and fixture scores. +They test candidate selection, cache identity, accounting, and state propagation; they do not demonstrate model learning quality. + ## What the system must learn Different outcomes currently share the word “improvement.” diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 56d99eef3..1ce1496ce 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,14 +16,14 @@ catalogs: specifier: '>=0.9.6 <0.10.0' version: 0.9.6 '@tangle-network/agent-eval': - specifier: '>=0.173.0 <0.174.0' - version: 0.173.0 + specifier: '>=0.174.0 <0.175.0' + version: 0.174.0 '@tangle-network/agent-interface': specifier: ^2.3.0 version: 2.3.0 '@tangle-network/agent-knowledge': - specifier: ^13.0.1 - version: 13.0.1 + specifier: ^14.0.0 + version: 14.0.0 '@tangle-network/agent-profile-materialize': specifier: '>=0.19.0 <0.20.0' version: 0.19.0 @@ -58,7 +58,7 @@ importers: version: 0.9.6(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 13.0.1(@tangle-network/agent-eval@0.173.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.3.0) + version: 14.0.0(@tangle-network/agent-eval@0.174.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.3.0) '@tangle-network/agent-profile-materialize': specifier: 'catalog:' version: 0.19.0(@tangle-network/agent-interface@2.3.0) @@ -80,7 +80,7 @@ importers: version: 1.30.0(supports-color@10.2.2)(zod@4.5.4) '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.173.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) + version: 0.174.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) '@tangle-network/agent-interface': specifier: 'catalog:' version: 2.3.0 @@ -134,13 +134,13 @@ importers: dependencies: '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.173.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) + version: 0.174.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) '@tangle-network/agent-interface': specifier: 'catalog:' version: 2.3.0 '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 13.0.1(@tangle-network/agent-eval@0.173.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.3.0) + version: 14.0.0(@tangle-network/agent-eval@0.174.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.3.0) '@tangle-network/agent-runtime': specifier: workspace:^ version: link:.. @@ -1297,20 +1297,20 @@ packages: '@modelcontextprotocol/sdk': optional: true - '@tangle-network/agent-eval@0.173.0': - resolution: {integrity: sha512-rLz26KS66ZhwCrPpsMUdvCiwcbyUCGGyS+SCwDc+6YJNSlCm+66ivry/yTO/WBS7j8yuI6hk2cpDBKEeHU6+uA==} + '@tangle-network/agent-eval@0.174.0': + resolution: {integrity: sha512-JcGQzN4fHll5ItkhEcFs7z8HF7ikwMLnlQhnSZekwJqGy6JbsmjGK87adxOV5SO12x1YRiALNzrQm1Hq6u1YUA==} engines: {node: '>=20.19.0'} hasBin: true '@tangle-network/agent-interface@2.3.0': resolution: {integrity: sha512-4BLA6WIgbK9ZtqWD8+sQvMjAMLhT2yDvRQwN/aH3KaWLe2SQ8aXT6B1+VmunM8rvWd6Q9iFvTGOx/l8TXjKtLA==} - '@tangle-network/agent-knowledge@13.0.1': - resolution: {integrity: sha512-G0y4M83ov1S5N0xWdVOWPhydzm8BEZ8lfuanwe7vGbtSKlzKMZKP6tYIUiff8neWbBNubx8fRS82vlE72Ev5bQ==} + '@tangle-network/agent-knowledge@14.0.0': + resolution: {integrity: sha512-PXcW+uEpXs+quy9r8YKrdVSo1lajz9dmlmEA52b2iG1dvOGkGot3Yct9J5ndEiD6/YePm6lI9yQWofLmX2ryxw==} engines: {node: '>=20.19.0'} hasBin: true peerDependencies: - '@tangle-network/agent-eval': '>=0.173.0 <0.174.0' + '@tangle-network/agent-eval': '>=0.174.0 <0.175.0' '@tangle-network/agent-interface': ^2.0.0 '@tangle-network/agent-profile-materialize@0.19.0': @@ -3559,7 +3559,7 @@ snapshots: optionalDependencies: '@modelcontextprotocol/sdk': 1.30.0(supports-color@10.2.2)(zod@4.5.4) - '@tangle-network/agent-eval@0.173.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4))': + '@tangle-network/agent-eval@0.174.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4))': dependencies: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.5.4) '@hono/node-server': 2.1.1(hono@4.13.5) @@ -3579,9 +3579,9 @@ snapshots: spdx-expression-parse: 5.0.0 zod: 4.5.4 - '@tangle-network/agent-knowledge@13.0.1(@tangle-network/agent-eval@0.173.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.3.0)': + '@tangle-network/agent-knowledge@14.0.0(@tangle-network/agent-eval@0.174.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.3.0)': dependencies: - '@tangle-network/agent-eval': 0.173.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) + '@tangle-network/agent-eval': 0.174.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) '@tangle-network/agent-interface': 2.3.0 '@types/proper-lockfile': 4.1.4 proper-lockfile: 4.1.2 diff --git a/release/cohort.json b/release/cohort.json index 12a78a785..e541c4085 100644 --- a/release/cohort.json +++ b/release/cohort.json @@ -10,14 +10,14 @@ "agentEval": { "name": "@tangle-network/agent-eval", "repository": "tangle-network/agent-eval", - "version": "0.173.0", - "ref": "d998e3e02bb570deffeaa380eadd0f115a2e676e" + "version": "0.174.0", + "ref": "0692922ea739a352f44374f4d5f7def2991aacdc" }, "agentKnowledge": { "name": "@tangle-network/agent-knowledge", "repository": "tangle-network/agent-knowledge", - "version": "13.0.1", - "ref": "35ec563869f08c56c2711a6c8ad9671cdefcc128" + "version": "14.0.0", + "ref": "30878397a947c37fd743929b30aa03728e785c76" } } } From 4a9cb5384e9cbd9aaeb33396d7bcec851e523cb3 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 5 Sep 2026 22:29:29 -0700 Subject: [PATCH 6/6] docs(learning): retain audit findings and release evidence --- .../2026-09-05-learning-system/findings.jsonl | 22 + .../2026-09-05-learning-system/manifest.json | 1070 +++++++++++++++++ .../runtime-inventory.json | 53 + .../2026-09-05-learning-system/summary.md | 93 ++ .agent/skill-runs.jsonl | 2 + 5 files changed, 1240 insertions(+) create mode 100644 .agent/critical-audit/2026-09-05-learning-system/findings.jsonl create mode 100644 .agent/critical-audit/2026-09-05-learning-system/manifest.json create mode 100644 .agent/critical-audit/2026-09-05-learning-system/runtime-inventory.json create mode 100644 .agent/critical-audit/2026-09-05-learning-system/summary.md create mode 100644 .agent/skill-runs.jsonl diff --git a/.agent/critical-audit/2026-09-05-learning-system/findings.jsonl b/.agent/critical-audit/2026-09-05-learning-system/findings.jsonl new file mode 100644 index 000000000..e97f17454 --- /dev/null +++ b/.agent/critical-audit/2026-09-05-learning-system/findings.jsonl @@ -0,0 +1,22 @@ +{"rank":1,"id":"E1","severity":"HIGH","priority":"P1","title":"Complete-method winner is selected again by another optimizer","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-eval","commit":"f8e3da285b6286386699a196733e9c0c27c20cfd","path":"src/contract/self-improve.ts","line":671,"url":"https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/contract/self-improve.ts#L671","lineText":" const proposer: SurfaceProposer = opts.method","fileSha256":"12db3bfaf22817fa2fe507ad4b14a33e071a33b760161aa9d3dcece1110c8c0b"},"additionalSources":[],"triggeringScenario":"A complete method selects WIN on its own selection cases. selfImprove wraps it as a native proposal and reranks it against pooled development cases.","evidence":{"observation":{"selectionScores":{"WIN":1,"BASE":0.6},"pooledScores":{"WIN":0.25,"BASE":0.6},"returnedCandidate":"BASE","selectedWinnerFinalExecutions":0},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["E-full","E-ci"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"A method can find an improvement that the wrapper silently discards before the independent final comparison.","fix":{"summary":"Execute the complete method directly and compare its selected candidate without a second search or fabricated native history.","repository":"agent-eval","paths":["src/contract/self-improve.ts","src/contract/self-improve-method.ts","src/campaign/presets/run-final-comparison.ts"],"repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-eval","path":"tests/contract-self-improve-method-integrity.test.ts","line":112,"purpose":"tests the method-selected winner directly without re-ranking on training cases","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/tests/contract-self-improve-method-integrity.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L112","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/contract-self-improve-method-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('tests the method-selected winner directly without re-ranking on training cases', async () => {","fileSha256":"0ef37f1b7a68a3b80d1011b057ec6818f380aa815e3f351abe3bf702801a6559"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":2,"id":"E3","severity":"HIGH","priority":"P1","title":"Final comparison silently loses failed measurement cells","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-eval","commit":"f8e3da285b6286386699a196733e9c0c27c20cfd","path":"src/campaign/presets/compare-optimization-methods.ts","line":350,"url":"https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/campaign/presets/compare-optimization-methods.ts#L350","lineText":" // Every surface must have a score for every designed test scenario. Filling a","fileSha256":"3578a0376619640637bee77a4f091315fa3b10eb6a1eb7143e58db4e2b7d6a4e"},"additionalSources":[],"triggeringScenario":"One candidate repetition for each of two cases fails in a designed four-cell comparison.","evidence":{"observation":{"designedCandidateCells":4,"omittedFailedCandidateCells":2,"reportedCandidateScore":1,"reportedLift":0.5},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["E-full","E-ci"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"A comparison can recommend a candidate based only on surviving successful executions.","fix":{"summary":"Validate exact case, repetition, and judge coverage through the common final comparison path.","repository":"agent-eval","paths":["src/campaign/coverage.ts","src/campaign/presets/run-final-comparison.ts","src/campaign/presets/compare-optimization-methods.ts"],"repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-eval","path":"tests/contract-self-improve-method-integrity.test.ts","line":328,"purpose":"rejects a final comparison that loses one replica of each candidate case","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/tests/contract-self-improve-method-integrity.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L328","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/contract-self-improve-method-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('rejects a final comparison that loses one replica of each candidate case', async () => {","fileSha256":"0ef37f1b7a68a3b80d1011b057ec6818f380aa815e3f351abe3bf702801a6559"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":3,"id":"E4","severity":"HIGH","priority":"P1","title":"New candidate inherits another candidate's cached result","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-eval","commit":"f8e3da285b6286386699a196733e9c0c27c20cfd","path":"src/campaign/presets/run-optimization.ts","line":239,"url":"https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/campaign/presets/run-optimization.ts#L239","lineText":" : await runCampaign({","fileSha256":"770f3500e1cc71db167d2596dcad16bb0912b40973ea70226af2f9c9d201448d"},"additionalSources":[],"triggeringScenario":"Run GOOD, then run BAD in the same output directory without changing the old cache keys.","evidence":{"observation":{"firstCandidate":"GOOD","secondCandidate":"BAD","secondCandidateReportedScore":1,"secondCandidateVerdict":"ship","newExecutions":0},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["E-full","E-ci"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"A different, unexecuted candidate can be recommended for adoption using another candidate's successful score.","fix":{"summary":"Include the measured surface in campaign and search cache identity.","repository":"agent-eval","paths":["src/campaign/surface-identity.ts","src/campaign/campaign-manifest.ts","src/campaign/presets/run-optimization.ts","src/contract/define-agent-eval.ts"],"repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-eval","path":"tests/contract-self-improve-method-integrity.test.ts","line":356,"purpose":"cannot ship a new native candidate using a previous candidate cache","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/tests/contract-self-improve-method-integrity.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L356","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/contract-self-improve-method-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('cannot ship a new native candidate using a previous candidate cache', async () => {","fileSha256":"0ef37f1b7a68a3b80d1011b057ec6818f380aa815e3f351abe3bf702801a6559"},{"repository":"agent-eval","path":"tests/contract-define-agent-eval.test.ts","line":135,"purpose":"does not reuse baseline scores for another surface in the same run directory","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/tests/contract-define-agent-eval.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-define-agent-eval.test.ts#L135","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/contract-define-agent-eval.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('does not reuse baseline scores for another surface in the same run directory', async () => {","fileSha256":"16c0ede885ce22582a7a9fe6f5d165d198271ff801de0c5d41aa779410a5aacd"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":4,"id":"E5","severity":"HIGH","priority":"P1","title":"Imported baseline ignores changed judge identity","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-eval","commit":"f8e3da285b6286386699a196733e9c0c27c20cfd","path":"src/campaign/presets/run-optimization.ts","line":688,"url":"https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/campaign/presets/run-optimization.ts#L688","lineText":"function validatedPremeasuredBaseline(args: {","fileSha256":"770f3500e1cc71db167d2596dcad16bb0912b40973ea70226af2f9c9d201448d"},"additionalSources":[],"triggeringScenario":"Import a baseline measured by an old judge into a search using a different judge revision.","evidence":{"observation":{"oldJudgeBaselineScore":0,"newJudgeCandidateScore":0.5,"newJudgeBaselineScore":1},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["E-full","E-ci"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"An apparent improvement can come entirely from comparing two different judging procedures.","fix":{"summary":"Require the full evaluator and execution revision for an imported baseline.","repository":"agent-eval","paths":["src/campaign/presets/run-optimization.ts"],"repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-eval","path":"tests/contract-self-improve-method-integrity.test.ts","line":386,"purpose":"rejects a premeasured baseline from a different judge revision","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/tests/contract-self-improve-method-integrity.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L386","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/contract-self-improve-method-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('rejects a premeasured baseline from a different judge revision', async () => {","fileSha256":"0ef37f1b7a68a3b80d1011b057ec6818f380aa815e3f351abe3bf702801a6559"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":5,"id":"E6","severity":"HIGH","priority":"P1","title":"Method spending disappears or is attributed incorrectly","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-eval","commit":"f8e3da285b6286386699a196733e9c0c27c20cfd","path":"src/contract/self-improve.ts","line":676,"url":"https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/contract/self-improve.ts#L676","lineText":" const result = await opts.method!.optimize(","fileSha256":"12db3bfaf22817fa2fe507ad4b14a33e071a33b760161aa9d3dcece1110c8c0b"},"additionalSources":[{"repository":"agent-eval","commit":"f8e3da285b6286386699a196733e9c0c27c20cfd","path":"src/contract/self-improve.ts","line":828,"url":"https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/contract/self-improve.ts#L828","lineText":" const cost = result.cost","fileSha256":"12db3bfaf22817fa2fe507ad4b14a33e071a33b760161aa9d3dcece1110c8c0b"}],"triggeringScenario":"A complete method reports estimated or incomplete external spend, or its reported zero conflicts with attributed call receipts.","evidence":{"observation":{"original":{"methodEstimatedUsd":17,"resultUsd":0,"resultAccountingComplete":true},"repairReview":{"recordedUsd":3,"resultUsd":0,"resultAccountingComplete":true},"repairedIndependentProbe":{"totalUsd":3,"ledgerUsd":3,"accountingComplete":false,"receipts":1},"receiptKind":"Synthetic USD amounts in deterministic callbacks; no paid model run."},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["E-full","E-ci","E-review-probes"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"A costly or incompletely metered method can appear free and fully accounted, invalidating resource comparisons and adoption decisions.","fix":{"summary":"Reconcile method reports with their attributed calls, retain incomplete accounting, and isolate concurrent methods' costs.","repository":"agent-eval","paths":["src/campaign/optimization-cost.ts","src/contract/self-improve-method.ts","src/campaign/presets/compare-optimization-methods.ts"],"repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-eval","path":"tests/contract-self-improve-method-integrity.test.ts","line":138,"purpose":"retains external method spend and incomplete accounting in the total","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/tests/contract-self-improve-method-integrity.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L138","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/contract-self-improve-method-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('retains external method spend and incomplete accounting in the total', async () => {","fileSha256":"0ef37f1b7a68a3b80d1011b057ec6818f380aa815e3f351abe3bf702801a6559"},{"repository":"agent-eval","path":"tests/contract-self-improve-method-integrity.test.ts","line":162,"purpose":"counts metered method and final spending exactly once","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/tests/contract-self-improve-method-integrity.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L162","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/contract-self-improve-method-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('counts metered method and final spending exactly once', async () => {","fileSha256":"0ef37f1b7a68a3b80d1011b057ec6818f380aa815e3f351abe3bf702801a6559"},{"repository":"agent-eval","path":"tests/contract-self-improve-method-integrity.test.ts","line":215,"purpose":"retains newly metered spend when the method reports zero (parameterized)","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/tests/contract-self-improve-method-integrity.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L215","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/contract-self-improve-method-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" 'retains newly metered spend when a method reports zero (unknown usage: %s)',","fileSha256":"0ef37f1b7a68a3b80d1011b057ec6818f380aa815e3f351abe3bf702801a6559"},{"repository":"agent-eval","path":"tests/contract-self-improve-method-integrity.test.ts","line":266,"purpose":"does not mark equivalent floating-point cost sums as underreported","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/tests/contract-self-improve-method-integrity.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L266","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/contract-self-improve-method-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('does not mark equivalent floating-point cost sums as underreported', async () => {","fileSha256":"0ef37f1b7a68a3b80d1011b057ec6818f380aa815e3f351abe3bf702801a6559"},{"repository":"agent-eval","path":"tests/campaign/compare-optimization-methods.test.ts","line":335,"purpose":"reconciles each concurrent method against its attributed call costs","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/tests/campaign/compare-optimization-methods.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/campaign/compare-optimization-methods.test.ts#L335","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/campaign/compare-optimization-methods.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('reconciles concurrent methods independently from prior spending and incomplete reports', async () => {","fileSha256":"8f06831f753c971063153543a435ca7d041abe71556a9d19810132b429dd77f5"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":6,"id":"R2","severity":"HIGH","priority":"P1","title":"Strategy resume reuses obsolete promoted work","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-runtime","commit":"a16d8a3b91481b140cb552e373d5bde98b34af05","path":"src/runtime/strategy-evolution.ts","line":397,"url":"https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/runtime/strategy-evolution.ts#L397","lineText":" const fingerprint = {","fileSha256":"c11db192207bba99a07290ea6b9a8bfa7cc0aaae1d49aa8e64be6ea6a0db8ef0"},"additionalSources":[],"triggeringScenario":"Resume after changing task payloads, objectives, execution configuration, callback dependencies, final offsets, or authored module bytes.","evidence":{"observation":{"returnedResult":"old promoted result","authorCalls":0,"benchmarkPhases":0},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["R-full","R-original-focused"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"The result can refer to a different experiment or program without executing the changed experiment.","fix":{"summary":"Bind checkpoints to exact serializable experiment inputs, explicit executionRef, and authored module bytes.","repository":"agent-runtime","paths":["src/runtime/strategy-evolution.ts"],"repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-runtime","path":"tests/kernel/strategy-evolution.test.ts","line":672,"purpose":"rejects changed behavior-bearing experiment inputs (parameterized)","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/kernel/strategy-evolution.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/strategy-evolution.test.ts#L672","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/kernel/strategy-evolution.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" ])('rejects changed behavior before resumed evaluation: %j', async (change) => {","fileSha256":"061c99c6ba009515caaf58684eaf20c30aee24e9557c9ee0c41d2abd38913648"},{"repository":"agent-runtime","path":"tests/kernel/strategy-evolution.test.ts","line":692,"purpose":"requires execution identity and rejects a changed callback dependency reference","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/kernel/strategy-evolution.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/strategy-evolution.test.ts#L692","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/kernel/strategy-evolution.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('requires execution identity and rejects a changed callback dependency reference', async () => {","fileSha256":"061c99c6ba009515caaf58684eaf20c30aee24e9557c9ee0c41d2abd38913648"},{"repository":"agent-runtime","path":"tests/kernel/strategy-evolution.test.ts","line":719,"purpose":"rejects changed training and final task payloads (parameterized)","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/kernel/strategy-evolution.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/strategy-evolution.test.ts#L719","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/kernel/strategy-evolution.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" 'rejects changed %s payloads with the same task IDs',","fileSha256":"061c99c6ba009515caaf58684eaf20c30aee24e9557c9ee0c41d2abd38913648"},{"repository":"agent-runtime","path":"tests/kernel/strategy-evolution.test.ts","line":747,"purpose":"rejects changed authored module bytes before importing the resumed candidate","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/kernel/strategy-evolution.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/strategy-evolution.test.ts#L747","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/kernel/strategy-evolution.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('rejects changed authored module bytes before importing the resumed candidate', async () => {","fileSha256":"061c99c6ba009515caaf58684eaf20c30aee24e9557c9ee0c41d2abd38913648"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":7,"id":"R4","severity":"HIGH","priority":"P1","title":"Malformed observation output becomes a clean run","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-runtime","commit":"a16d8a3b91481b140cb552e373d5bde98b34af05","path":"src/runtime/observe.ts","line":235,"url":"https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/runtime/observe.ts#L235","lineText":"function parseFindings(content: string): RawFinding[] {","fileSha256":"c01ad11275beff071207098167f10b1bdd42d4e05aff78f3004a812859d8f11a"},"additionalSources":[],"triggeringScenario":"The observer model returns malformed JSON or an invalid findings structure.","evidence":{"observation":{"returnedFindings":[],"reportedInterpretation":"clean run"},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["R-full","R-original-focused"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"The learning process loses the failure signal and falsely treats invalid observation output as success.","fix":{"summary":"Validate the complete response and expose parse or validation failure; accept explicit valid empty findings.","repository":"agent-runtime","paths":["src/runtime/observe.ts"],"repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-runtime","path":"tests/runtime-observe.test.ts","line":59,"purpose":"rejects malformed model findings before returning them to callers","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/runtime-observe.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/runtime-observe.test.ts#L59","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/runtime-observe.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('rejects malformed model findings before returning them to callers', async () => {","fileSha256":"9e4d7241fe26b201d8c59472d8c6e1ab45be9e465d3781749f5165af8af07c4c"},{"repository":"agent-runtime","path":"tests/runtime-observe.test.ts","line":102,"purpose":"accepts an explicit empty findings array","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/runtime-observe.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/runtime-observe.test.ts#L102","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/runtime-observe.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('accepts an explicit empty findings array', async () => {","fileSha256":"9e4d7241fe26b201d8c59472d8c6e1ab45be9e465d3781749f5165af8af07c4c"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":8,"id":"R5","severity":"HIGH","priority":"P1","title":"Failed lesson storage disappears from harvest results","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-runtime","commit":"a16d8a3b91481b140cb552e373d5bde98b34af05","path":"src/runtime/observe.ts","line":203,"url":"https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/runtime/observe.ts#L203","lineText":" const r = await opts.corpus.append(record)","fileSha256":"c01ad11275beff071207098167f10b1bdd42d4e05aff78f3004a812859d8f11a"},"additionalSources":[],"triggeringScenario":"Corpus append returns a typed persistence failure during observation harvesting.","evidence":{"observation":{"observedRuns":1,"findings":1,"learnedRecords":0,"reportedFailures":0},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["R-full","R-original-focused"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"An apparent learning pass can lose every lesson without reporting the storage failure.","fix":{"summary":"Propagate the acknowledged storage error through the existing per-run failure channel.","repository":"agent-runtime","paths":["src/runtime/observe.ts"],"repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-runtime","path":"tests/runtime-observe.test.ts","line":111,"purpose":"reports failed corpus persistence through the harvest per-run failure channel","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/runtime-observe.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/runtime-observe.test.ts#L111","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/runtime-observe.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('reports failed corpus persistence through the harvest per-run failure channel', async () => {","fileSha256":"9e4d7241fe26b201d8c59472d8c6e1ab45be9e465d3781749f5165af8af07c4c"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":9,"id":"R6","severity":"HIGH","priority":"P1","title":"Concurrent lesson writes corrupt the shared record log","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-runtime","commit":"a16d8a3b91481b140cb552e373d5bde98b34af05","path":"src/runtime/personify/corpus.ts","line":213,"url":"https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/runtime/personify/corpus.ts#L213","lineText":" async append(","fileSha256":"675a46a7504e4fde6f1366dedc2acfd989e26ad86fcb18b3db25aedaa67b76c9"},"additionalSources":[],"triggeringScenario":"Two file Corpus instances or processes append different records under the same ID.","evidence":{"observation":{"conflictingIds":1,"successfulAppendAcknowledgements":2,"subsequentRead":"rejected as corrupt"},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["R-full","R-original-focused"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"Both callers receive success even though later learning cannot read the store.","fix":{"summary":"Serialize the read/check/append transaction across processes, canonicalize aliases, and retain typed storage failures.","repository":"agent-runtime","paths":["src/runtime/personify/corpus.ts"],"repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-runtime","path":"tests/kernel/corpus-integrity.test.ts","line":91,"purpose":"serializes conflicting appends across instances and symlink aliases","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/kernel/corpus-integrity.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/corpus-integrity.test.ts#L91","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/kernel/corpus-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('serializes conflicting appends across instances and symlink aliases', async () => {","fileSha256":"3e4038f557a8de7ab565893de117deb834caa916e8e11c17006d3265fd2986fb"},{"repository":"agent-runtime","path":"tests/kernel/corpus-integrity.test.ts","line":130,"purpose":"serializes conflicting appends across separate processes","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/kernel/corpus-integrity.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/corpus-integrity.test.ts#L130","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/kernel/corpus-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" 'coordinates separate writer processes (identical=%s)',","fileSha256":"3e4038f557a8de7ab565893de117deb834caa916e8e11c17006d3265fd2986fb"},{"repository":"agent-runtime","path":"tests/kernel/corpus-integrity.test.ts","line":153,"purpose":"recovers an exited process lock before appending","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/kernel/corpus-integrity.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/corpus-integrity.test.ts#L153","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/kernel/corpus-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('recovers an exited process lock before appending', async () => {","fileSha256":"3e4038f557a8de7ab565893de117deb834caa916e8e11c17006d3265fd2986fb"},{"repository":"agent-runtime","path":"tests/kernel/corpus-integrity.test.ts","line":166,"purpose":"returns a typed failure when a live owner exceeds the bounded lock wait","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/kernel/corpus-integrity.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/corpus-integrity.test.ts#L166","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/kernel/corpus-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('returns a typed failure when a live owner exceeds the bounded lock wait', async () => {","fileSha256":"3e4038f557a8de7ab565893de117deb834caa916e8e11c17006d3265fd2986fb"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":10,"id":"R8","severity":"HIGH","priority":"P1","title":"Reflective edits ignore failure and candidate boundaries","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-runtime","commit":"a16d8a3b91481b140cb552e373d5bde98b34af05","path":"src/improvement/reflective-generator.ts","line":28,"url":"https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/improvement/reflective-generator.ts#L28","lineText":" async generate({ worktreePath, findings }) {","fileSha256":"32b3b92d8ca5405846e6c7b35ac1c86c8255f0cfd0d5f6de715f6c84be77f22d"},"additionalSources":[],"triggeringScenario":"Drafting is aborted, produces errors or stale bases, or returns a mixed/undeclared patch batch.","evidence":{"observation":{"beforeRepair":["aborted signal ignored","draft and application failures hidden","partial batch reported as applied"],"independentRepairedPathProbe":{"rejected":true,"declaredTargetUnchanged":true,"undeclaredTargetUnchanged":true}},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["R-full","R-reflective","R-reflective-independent"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"The system can measure partially applied or unintended code while claiming the requested candidate was constructed.","fix":{"summary":"Draft against each exact incumbent, propagate cancellation and errors, validate bases and declared paths, and apply the complete batch atomically.","repository":"agent-runtime","paths":["src/improvement/reflective-generator.ts","src/improvement/code-execution.ts","src/improvement/improve-types.ts"],"repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-runtime","path":"tests/improvement-driver.test.ts","line":177,"purpose":"rejects a mixed patch batch without applying its valid edit","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/improvement-driver.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/improvement-driver.test.ts#L177","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/improvement-driver.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('rejects a mixed patch batch without applying its valid edit', async () => {","fileSha256":"9b06b4499245a9305e27f02016e2aaba56804d29229a7cfad937d927d7811519"},{"repository":"agent-runtime","path":"tests/improvement-driver.test.ts","line":199,"purpose":"rejects draft errors and stale base hashes before applying patches","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/improvement-driver.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/improvement-driver.test.ts#L199","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/improvement-driver.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('rejects draft errors and stale base hashes before applying patches', async () => {","fileSha256":"9b06b4499245a9305e27f02016e2aaba56804d29229a7cfad937d927d7811519"},{"repository":"agent-runtime","path":"tests/improvement-driver.test.ts","line":223,"purpose":"rejects undeclared patch paths, including a different rename source","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/improvement-driver.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/improvement-driver.test.ts#L223","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/improvement-driver.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('rejects undeclared patch paths, including a different rename source', async () => {","fileSha256":"9b06b4499245a9305e27f02016e2aaba56804d29229a7cfad937d927d7811519"},{"repository":"agent-runtime","path":"tests/improvement-driver.test.ts","line":245,"purpose":"honors cancellation before proposal creation and after drafting","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/improvement-driver.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/improvement-driver.test.ts#L245","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/improvement-driver.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('honors cancellation before proposal creation and after drafting', async () => {","fileSha256":"9b06b4499245a9305e27f02016e2aaba56804d29229a7cfad937d927d7811519"},{"repository":"agent-runtime","path":"tests/improvement-driver.test.ts","line":274,"purpose":"drafts from each incumbent through the real surface proposer and shares its paid-call context","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/improvement-driver.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/improvement-driver.test.ts#L274","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/improvement-driver.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('drafts from each incumbent through the real surface proposer and shares its paid-call context', async () => {","fileSha256":"9b06b4499245a9305e27f02016e2aaba56804d29229a7cfad937d927d7811519"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":11,"id":"K1","severity":"HIGH","priority":"P1","title":"Knowledge update runs before the diagnosis it needs","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-knowledge","commit":"390f2da9883e55cc86a8985167324d8b1f10894a","path":"src/kb-improvement/evaluation.ts","line":123,"url":"https://github.com/tangle-network/agent-knowledge/blob/390f2da9883e55cc86a8985167324d8b1f10894a/src/kb-improvement/evaluation.ts#L123","lineText":" const lifecycles: RunRagKnowledgeImprovementLoopResult[] = []","fileSha256":"844f31ba69feec83ac83f3c84814c2ebb46b5551ad4a6803c8c74d6a7d4d545b"},"additionalSources":[{"repository":"agent-knowledge","commit":"390f2da9883e55cc86a8985167324d8b1f10894a","path":"src/kb-improvement/evaluation.ts","line":125,"url":"https://github.com/tangle-network/agent-knowledge/blob/390f2da9883e55cc86a8985167324d8b1f10894a/src/kb-improvement/evaluation.ts#L125","lineText":" const updateLifecycle = await runCandidateUpdateLifecycle(","fileSha256":"844f31ba69feec83ac83f3c84814c2ebb46b5551ad4a6803c8c74d6a7d4d545b"},{"repository":"agent-knowledge","commit":"390f2da9883e55cc86a8985167324d8b1f10894a","path":"src/kb-improvement/evaluation.ts","line":163,"url":"https://github.com/tangle-network/agent-knowledge/blob/390f2da9883e55cc86a8985167324d8b1f10894a/src/kb-improvement/evaluation.ts#L163","lineText":" const evaluationLifecycle = await runCandidateEvaluationLifecycle(","fileSha256":"844f31ba69feec83ac83f3c84814c2ebb46b5551ad4a6803c8c74d6a7d4d545b"},{"repository":"agent-knowledge","commit":"390f2da9883e55cc86a8985167324d8b1f10894a","path":"src/kb-improvement/evaluation.ts","line":173,"url":"https://github.com/tangle-network/agent-knowledge/blob/390f2da9883e55cc86a8985167324d8b1f10894a/src/kb-improvement/evaluation.ts#L173","lineText":" lifecycle = mergeLifecycleResults(options.goal, lifecycles)","fileSha256":"844f31ba69feec83ac83f3c84814c2ebb46b5551ad4a6803c8c74d6a7d4d545b"},{"repository":"agent-knowledge","commit":"390f2da9883e55cc86a8985167324d8b1f10894a","path":"src/kb-improvement/evaluation.ts","line":201,"url":"https://github.com/tangle-network/agent-knowledge/blob/390f2da9883e55cc86a8985167324d8b1f10894a/src/kb-improvement/evaluation.ts#L201","lineText":" const lifecycle = await runRagKnowledgeImprovementLoop({","fileSha256":"844f31ba69feec83ac83f3c84814c2ebb46b5551ad4a6803c8c74d6a7d4d545b"}],"triggeringScenario":"Knowledge candidate acquisition and update run in a separate lifecycle before diagnosis and before final callbacks reconstruct another state.","evidence":{"observation":{"callbackOrderBefore":["acquire","update","diagnose"],"diagnosisAvailableToUpdate":false},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["K-linux-8","K-ci","K-publish","K-registry"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"The candidate constructor cannot respond to diagnosed gaps, and merging states afterward cannot restore missing callback input.","fix":{"summary":"Carry one lifecycle state from diagnosis through construction and final measurement; extract reusable internal phase execution.","repository":"agent-knowledge","paths":["src/kb-improvement/evaluation.ts","src/kb-improvement/selected-candidate.ts","src/rag-improvement-loop.ts","src/rag-improvement-phases.ts"],"repairCommit":"30878397a947c37fd743929b30aa03728e785c76","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-knowledge","path":"tests/kb-improvement/lifecycle.test.ts","line":40,"purpose":"carries diagnosis and update results through development into final measurement","repairCommit":"30878397a947c37fd743929b30aa03728e785c76","worktreePath":"/Users/drew/webb/_wt/knowledge-learning-audit-20260905/tests/kb-improvement/lifecycle.test.ts","url":"https://github.com/tangle-network/agent-knowledge/blob/30878397a947c37fd743929b30aa03728e785c76/tests/kb-improvement/lifecycle.test.ts#L40","revisionStatus":"merged_source","command":"node node_modules/vitest/vitest.mjs run tests/kb-improvement/lifecycle.test.ts","requiredEnvironment":"Linux snapshot implementation","lineText":" it('carries diagnosis and update results through development into final measurement', async () => {","fileSha256":"c2566de3e9eb4cd52a9b77b95af7bd2fda0522a8fbe088fd7033a3ccaae82dbd"},{"repository":"agent-knowledge","path":"tests/kb-improvement/selected-candidate.test.ts","line":20,"purpose":"remeasures the exact selected subset before ordinary promotion","repairCommit":"30878397a947c37fd743929b30aa03728e785c76","worktreePath":"/Users/drew/webb/_wt/knowledge-learning-audit-20260905/tests/kb-improvement/selected-candidate.test.ts","url":"https://github.com/tangle-network/agent-knowledge/blob/30878397a947c37fd743929b30aa03728e785c76/tests/kb-improvement/selected-candidate.test.ts#L20","revisionStatus":"merged_source","command":"node node_modules/vitest/vitest.mjs run tests/kb-improvement/selected-candidate.test.ts","requiredEnvironment":"Linux snapshot implementation","lineText":" it('remeasures the exact selected subset before ordinary promotion', async () => {","fileSha256":"bac61ae725811f5db40ed558f748d7fce079225b96187406a4ebfcfae55caae7"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":12,"id":"K3","severity":"HIGH","priority":"P1","title":"Storage readiness stops an incomplete research process","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-knowledge","commit":"390f2da9883e55cc86a8985167324d8b1f10894a","path":"src/verified-research-loop.ts","line":327,"url":"https://github.com/tangle-network/agent-knowledge/blob/390f2da9883e55cc86a8985167324d8b1f10894a/src/verified-research-loop.ts#L327","lineText":" ready = isReady(readiness?.report)","fileSha256":"17c84d50e1dc5bd765ea36ea8c3701d19a8e68d3903e53294e6257f834d45212"},"additionalSources":[],"triggeringScenario":"Knowledge storage meets its checks while the research driver still has unresolved work.","evidence":{"observation":{"executedRounds":1,"allowedRounds":4,"driverComplete":false,"furtherSteering":0},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["K-host-74","K-ci","K-publish","K-registry"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"Research can stop before the executing driver resolves its claims and questions.","fix":{"summary":"Respect driver completion and persist pending steering while preserving storage-only driver behavior.","repository":"agent-knowledge","paths":["src/verified-research-loop.ts"],"repairCommit":"30878397a947c37fd743929b30aa03728e785c76","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-knowledge","path":"tests/loops/research-driving-loop.test.ts","line":249,"purpose":"is NOT complete while the key claim has only one independent source, regardless of source count","repairCommit":"30878397a947c37fd743929b30aa03728e785c76","worktreePath":"/Users/drew/webb/_wt/knowledge-learning-audit-20260905/tests/loops/research-driving-loop.test.ts","url":"https://github.com/tangle-network/agent-knowledge/blob/30878397a947c37fd743929b30aa03728e785c76/tests/loops/research-driving-loop.test.ts#L249","revisionStatus":"merged_source","command":"node node_modules/vitest/vitest.mjs run tests/loops/research-driving-loop.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('is NOT complete while the key claim has only one independent source, regardless of source count', async () => {","fileSha256":"8439a2086337abeb1d10fbf82655ddbecf81115e7eed01689bce52c22bbea35b"},{"repository":"agent-knowledge","path":"tests/loops/research-driving-loop.test.ts","line":331,"purpose":"prepares and checkpoints steering with no storage gaps until the driver completes","repairCommit":"30878397a947c37fd743929b30aa03728e785c76","worktreePath":"/Users/drew/webb/_wt/knowledge-learning-audit-20260905/tests/loops/research-driving-loop.test.ts","url":"https://github.com/tangle-network/agent-knowledge/blob/30878397a947c37fd743929b30aa03728e785c76/tests/loops/research-driving-loop.test.ts#L331","revisionStatus":"merged_source","command":"node node_modules/vitest/vitest.mjs run tests/loops/research-driving-loop.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('prepares and checkpoints steering with no storage gaps until the driver completes', async () => {","fileSha256":"8439a2086337abeb1d10fbf82655ddbecf81115e7eed01689bce52c22bbea35b"},{"repository":"agent-knowledge","path":"tests/loops/research-driving-loop.test.ts","line":306,"purpose":"keeps storage-only drivers unchanged and skips work when both checks already pass","repairCommit":"30878397a947c37fd743929b30aa03728e785c76","worktreePath":"/Users/drew/webb/_wt/knowledge-learning-audit-20260905/tests/loops/research-driving-loop.test.ts","url":"https://github.com/tangle-network/agent-knowledge/blob/30878397a947c37fd743929b30aa03728e785c76/tests/loops/research-driving-loop.test.ts#L306","revisionStatus":"merged_source","command":"node node_modules/vitest/vitest.mjs run tests/loops/research-driving-loop.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('keeps storage-only drivers unchanged and skips work when both checks already pass', async () => {","fileSha256":"8439a2086337abeb1d10fbf82655ddbecf81115e7eed01689bce52c22bbea35b"},{"repository":"agent-knowledge","path":"tests/loops/research-driving-loop.test.ts","line":383,"purpose":"never reports ready without readiness specs even when the driver is complete","repairCommit":"30878397a947c37fd743929b30aa03728e785c76","worktreePath":"/Users/drew/webb/_wt/knowledge-learning-audit-20260905/tests/loops/research-driving-loop.test.ts","url":"https://github.com/tangle-network/agent-knowledge/blob/30878397a947c37fd743929b30aa03728e785c76/tests/loops/research-driving-loop.test.ts#L383","revisionStatus":"merged_source","command":"node node_modules/vitest/vitest.mjs run tests/loops/research-driving-loop.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('never reports ready without readiness specs even when the driver is complete', async () => {","fileSha256":"8439a2086337abeb1d10fbf82655ddbecf81115e7eed01689bce52c22bbea35b"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":13,"id":"E2","severity":"MEDIUM","priority":"P2","title":"Unchanged method result is rejected as a duplicate","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-eval","commit":"f8e3da285b6286386699a196733e9c0c27c20cfd","path":"src/campaign/presets/run-optimization.ts","line":418,"url":"https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/campaign/presets/run-optimization.ts#L418","lineText":" if (admittedCandidateHashes.has(hash) || generationHashes.has(hash)) {","fileSha256":"770f3500e1cc71db167d2596dcad16bb0912b40973ea70226af2f9c9d201448d"},"additionalSources":[],"triggeringScenario":"A complete method returns the valid unchanged baseline.","evidence":{"observation":{"result":"duplicate-candidate error"},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["E-full","E-ci"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"A valid no-improvement outcome fails instead of returning an honest unchanged result.","fix":{"summary":"Accept the unchanged selected baseline as a complete-method outcome without inventing candidate history.","repository":"agent-eval","paths":["src/contract/self-improve-method.ts","src/contract/self-improve-reporting.ts"],"repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-eval","path":"tests/contract-self-improve-method-integrity.test.ts","line":46,"purpose":"accepts an unchanged selected baseline without inventing candidate history","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/tests/contract-self-improve-method-integrity.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L46","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/contract-self-improve-method-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('accepts an unchanged selected baseline without inventing candidate history', async () => {","fileSha256":"0ef37f1b7a68a3b80d1011b057ec6818f380aa815e3f351abe3bf702801a6559"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":14,"id":"E7","severity":"MEDIUM","priority":"P2","title":"Point estimate is labeled as an exact confidence interval","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-eval","commit":"f8e3da285b6286386699a196733e9c0c27c20cfd","path":"src/campaign/presets/run-optimization.ts","line":553,"url":"https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/campaign/presets/run-optimization.ts#L553","lineText":" ci95: s.composite === null ? null : [s.composite, s.composite],","fileSha256":"770f3500e1cc71db167d2596dcad16bb0912b40973ea70226af2f9c9d201448d"},"additionalSources":[],"triggeringScenario":"A native candidate receives case scores 1, 0, 0, and 0.","evidence":{"observation":{"caseScores":[1,0,0,0],"mean":0.25,"reportedInterval":[0.25,0.25],"repairedUnestimatedInterval":null},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["E-full","E-ci"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"Consumers see false certainty and can mistake an ordinary sample mean for measured uncertainty.","fix":{"summary":"Represent unestimated uncertainty as null and retain actual final-comparison statistics.","repository":"agent-eval","paths":["src/campaign/presets/run-optimization.ts"],"repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-eval","path":"tests/contract-self-improve-method-integrity.test.ts","line":425,"purpose":"does not label a native candidate mean as an estimated confidence interval","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/tests/contract-self-improve-method-integrity.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L425","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/contract-self-improve-method-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('does not label a native candidate mean as an estimated confidence interval', async () => {","fileSha256":"0ef37f1b7a68a3b80d1011b057ec6818f380aa815e3f351abe3bf702801a6559"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":15,"id":"E8","severity":"MEDIUM","priority":"P2","title":"Unchanged final campaign is reported twice","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-eval","commit":"f8e3da285b6286386699a196733e9c0c27c20cfd","path":"src/contract/self-improve.ts","line":835,"url":"https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/contract/self-improve.ts#L835","lineText":" const insight = await analyzeRuns({","fileSha256":"12db3bfaf22817fa2fe507ad4b14a33e071a33b760161aa9d3dcece1110c8c0b"},"additionalSources":[],"triggeringScenario":"An unchanged candidate shares the same final campaign as the baseline.","evidence":{"observation":{"actual":{"executions":2,"costUsd":2},"reportedBeforeRepair":{"executions":4,"costUsd":4},"repairedIndependentProbe":{"calls":2,"receipts":2,"totalUsd":2,"verdict":"hold","insightRuns":2,"insightCostUsd":2,"inputTokens":20,"outputTokens":10},"receiptKind":"Synthetic costs and token receipts."},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["E-full","E-ci","E-review-probes"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"Reports double the same work, distorting costs and outcome summaries.","fix":{"summary":"Include identical final campaigns once in both complete-method and native-proposer reporting.","repository":"agent-eval","paths":["src/contract/self-improve-reporting.ts","src/contract/self-improve.ts","src/contract/self-improve-method.ts"],"repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-eval","path":"tests/contract-self-improve-method-integrity.test.ts","line":71,"purpose":"counts a shared unchanged campaign once in %s insight (parameterized)","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/tests/contract-self-improve-method-integrity.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L71","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/contract-self-improve-method-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" 'counts a shared unchanged campaign once in %s insight',","fileSha256":"0ef37f1b7a68a3b80d1011b057ec6818f380aa815e3f351abe3bf702801a6559"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":16,"id":"E9","severity":"MEDIUM","priority":"P2","title":"Curriculum allocation cannot see its own observations","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-eval","commit":"f8e3da285b6286386699a196733e9c0c27c20cfd","path":"src/fuzz/explorer.ts","line":144,"url":"https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/fuzz/explorer.ts#L144","lineText":" scenarioId: r.scenarioId,","fileSha256":"ae9a61fe331fd70bb171858a27b9153daaba72440b009479a41e8025d562740b"},"additionalSources":[{"repository":"agent-eval","commit":"f8e3da285b6286386699a196733e9c0c27c20cfd","path":"src/fuzz/explorer.ts","line":148,"url":"https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/fuzz/explorer.ts#L148","lineText":" this.cells.map((c) => ({ variantId: c.id, scenarioId: '*' })),","fileSha256":"ae9a61fe331fd70bb171858a27b9153daaba72440b009479a41e8025d562740b"},{"repository":"agent-eval","commit":"f8e3da285b6286386699a196733e9c0c27c20cfd","path":"src/rl/active-curriculum.ts","line":90,"url":"https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/rl/active-curriculum.ts#L90","lineText":" const k = `${o.variantId}::${o.scenarioId}`","fileSha256":"3121c51bbe2859f84660b61e01b04f5871526735a3d85343c9b1ab36cfd7e070"}],"triggeringScenario":"Search history stores concrete scenario IDs while allocation looks up wildcard search cells.","evidence":{"observation":{"totalEvaluations":40,"secondRoundAllocationBefore":[10,10],"secondRoundAllocationAfter":[8,12]},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["E-full","E-ci"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"Later scenario-search rounds ignore observed variance and repeat their initial allocation.","fix":{"summary":"Pool allocation observations by search cell while retaining exact scenario IDs in stored records.","repository":"agent-eval","paths":["src/fuzz/explorer.ts"],"repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-eval","path":"src/fuzz/fuzz-agent.test.ts","line":198,"purpose":"uses observed cell variance to change allocation in the second round","repairCommit":"6c23e86025eec42b4d5f551f9cb4896f474cb16b","worktreePath":"/Users/drew/webb/_wt/eval-learning-audit-20260905/src/fuzz/fuzz-agent.test.ts","url":"https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/src/fuzz/fuzz-agent.test.ts#L198","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run src/fuzz/fuzz-agent.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('uses observed cell variance to change allocation in the second round', async () => {","fileSha256":"2814fd793a24249f2eb5e489ed45e6ab4c22e02388b5bedf52b97c0524471dca"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":17,"id":"R1","severity":"MEDIUM","priority":"P2","title":"Git status parsing turns diagnosis-only edits into code changes","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-runtime","commit":"a16d8a3b91481b140cb552e373d5bde98b34af05","path":"src/improvement/agentic-generator.ts","line":1020,"url":"https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/improvement/agentic-generator.ts#L1020","lineText":" .map((line) => line.trim())","fileSha256":"88f13783ba9880d5635d8a2491de7625bba989379c803e24f44716243a5ed2c6"},"additionalSources":[],"triggeringScenario":"A tracked diagnosis file is modified with a leading-space Git porcelain status.","evidence":{"observation":{"actualEdit":"diagnosis only","reportedEdit":"substantive code","contexts":["staged edit","unstaged edit","rename alongside diagnosis edit"]},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["R-full","R-original-focused"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"A coding attempt can be accepted even though it changed only its diagnostic note.","fix":{"summary":"Parse Git porcelain records without trimming status columns or path bytes.","repository":"agent-runtime","paths":["src/improvement/agentic-generator.ts"],"repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-runtime","path":"tests/agentic-generator.test.ts","line":491,"purpose":"rejects a tracked diagnosis-only edit in staged and unstaged states (parameterized)","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/agentic-generator.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/agentic-generator.test.ts#L491","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/agentic-generator.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it.each(['unstaged', 'staged'])('rejects %s edits to only a tracked diagnosis', async (state) => {","fileSha256":"e30574ebbb2c080dff37d146ed25f48bf70c01471243f5ffc14fc6b7bb142930"},{"repository":"agent-runtime","path":"tests/agentic-generator.test.ts","line":516,"purpose":"accepts a renamed source file alongside a tracked diagnosis edit","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/agentic-generator.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/agentic-generator.test.ts#L516","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/agentic-generator.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('accepts a renamed source file alongside a tracked diagnosis edit', async () => {","fileSha256":"e30574ebbb2c080dff37d146ed25f48bf70c01471243f5ffc14fc6b7bb142930"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":18,"id":"R3","severity":"MEDIUM","priority":"P2","title":"Strategy author is taught the wrong worker profile argument","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-runtime","commit":"a16d8a3b91481b140cb552e373d5bde98b34af05","path":"src/runtime/strategy-author.ts","line":29,"url":"https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/runtime/strategy-author.ts#L29","lineText":" shot(spec?: { handle?, messages?, steer?, persona?, tools? }): Promise","fileSha256":"9e2beb0955ab77fa7fdcd5251387237bc9e3d1c3e8ecb413caea2f683b7c17d1"},"additionalSources":[{"repository":"agent-runtime","commit":"a16d8a3b91481b140cb552e373d5bde98b34af05","path":"src/runtime/strategy.ts","line":847,"url":"https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/runtime/strategy.ts#L847","lineText":"export interface ShotSpec {","fileSha256":"51571341aca90f04892e1b3505d47c5f67ff5d67d0a2caf430acd9102da5eb2f"}],"triggeringScenario":"An author follows shot({persona}) from the supplied contract, while the executable strategy API consumes shot({profile}).","evidence":{"observation":{"authoredArgument":"persona","consumedArgument":"profile","repairCheck":"The taught profile changes the actual offline worker."},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Source-inspected API mismatch plus an actual offline worker execution regression; no real model author was tested.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["R-full","R-profile-integration"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"The documented change can fail to alter the executing worker.","fix":{"summary":"Teach the complete-profile argument accepted by the actual strategy API and execute the example in a regression.","repository":"agent-runtime","paths":["src/runtime/strategy-author.ts"],"repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-runtime","path":"tests/kernel/strategy-suite.test.ts","line":438,"purpose":"the author contract teaches a shot profile that changes the actual worker","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/kernel/strategy-suite.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/strategy-suite.test.ts#L438","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/kernel/strategy-suite.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('the author contract teaches a shot profile that changes the actual worker', async () => {","fileSha256":"a1c682684ef02143104309346f2b7e265a9d5527ed359b989fa700e8805407ff"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":19,"id":"R7","severity":"MEDIUM","priority":"P2","title":"Caller mutation changes an already stored lesson","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-runtime","commit":"a16d8a3b91481b140cb552e373d5bde98b34af05","path":"src/runtime/personify/corpus.ts","line":171,"url":"https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/runtime/personify/corpus.ts#L171","lineText":" async append(","fileSha256":"675a46a7504e4fde6f1366dedc2acfd989e26ad86fcb18b3db25aedaa67b76c9"},"additionalSources":[],"triggeringScenario":"A caller changes the tags array after appending a record, including while file storage awaits.","evidence":{"observation":{"storedState":"changes with caller-owned tags"},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["R-full","R-original-focused"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"A recorded lesson no longer means what was accepted at append time.","fix":{"summary":"Snapshot and freeze nested record fields before returning or awaiting storage.","repository":"agent-runtime","paths":["src/runtime/personify/corpus.ts"],"repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-runtime","path":"tests/kernel/rsi-wave.test.ts","line":165,"purpose":"detaches and freezes nested corpus fields","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/kernel/rsi-wave.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/rsi-wave.test.ts#L165","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/kernel/rsi-wave.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('detaches and freezes nested corpus fields', async () => {","fileSha256":"01654a4c4dee8cdd68e17c5f736d6feff599cb68737283cde73ad022f28f66c5"},{"repository":"agent-runtime","path":"tests/kernel/corpus-integrity.test.ts","line":112,"purpose":"deduplicates concurrent identical appends and snapshots values before awaiting storage","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/kernel/corpus-integrity.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/corpus-integrity.test.ts#L112","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/kernel/corpus-integrity.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('deduplicates concurrent identical appends and snapshots values before awaiting storage', async () => {","fileSha256":"3e4038f557a8de7ab565893de117deb834caa916e8e11c17006d3265fd2986fb"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":20,"id":"K2","severity":"MEDIUM","priority":"P2","title":"Unmeasured knowledge outcomes are reported as perfect","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-knowledge","commit":"390f2da9883e55cc86a8985167324d8b1f10894a","path":"src/kb-improvement/evaluation.ts","line":498,"url":"https://github.com/tangle-network/agent-knowledge/blob/390f2da9883e55cc86a8985167324d8b1f10894a/src/kb-improvement/evaluation.ts#L498","lineText":" const blockingReadiness =","fileSha256":"844f31ba69feec83ac83f3c84814c2ebb46b5551ad4a6803c8c74d6a7d4d545b"},"additionalSources":[],"triggeringScenario":"An unchanged empty KB has no configured outcome checks and is staged as candidate-ready.","evidence":{"observation":{"unmeasuredDimensionsReportedAsOne":5,"stagedState":"candidate-ready","livePromotionBypass":false},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled Linux snapshot lifecycle. Candidate-ready is detached staging, not live promotion; structural validity remains a legitimate staging check.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["K-linux-8","K-ci","K-publish","K-registry"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"A structurally valid detached candidate can appear to have passed outcome checks that never ran.","fix":{"summary":"Omit unmeasured dimensions and record the actual scope of configured checks.","repository":"agent-knowledge","paths":["src/kb-improvement/evaluation.ts"],"repairCommit":"30878397a947c37fd743929b30aa03728e785c76","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-knowledge","path":"tests/kb-improvement/lifecycle.test.ts","line":155,"purpose":"runs diagnosis without consuming final cases or reporting unmeasured checks","repairCommit":"30878397a947c37fd743929b30aa03728e785c76","worktreePath":"/Users/drew/webb/_wt/knowledge-learning-audit-20260905/tests/kb-improvement/lifecycle.test.ts","url":"https://github.com/tangle-network/agent-knowledge/blob/30878397a947c37fd743929b30aa03728e785c76/tests/kb-improvement/lifecycle.test.ts#L155","revisionStatus":"merged_source","command":"node node_modules/vitest/vitest.mjs run tests/kb-improvement/lifecycle.test.ts","requiredEnvironment":"Linux snapshot implementation","lineText":" it('runs diagnosis without consuming final cases or reporting unmeasured checks', async () => {","fileSha256":"c2566de3e9eb4cd52a9b77b95af7bd2fda0522a8fbe088fd7033a3ccaae82dbd"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":21,"id":"K4","severity":"MEDIUM","priority":"P2","title":"Knowledge adapter rewrites the supplied supervisor policy","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-runtime","commit":"a16d8a3b91481b140cb552e373d5bde98b34af05","path":"src/knowledge/supervised-update.ts","line":134,"url":"https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/knowledge/supervised-update.ts#L134","lineText":" const baseInstructions = exactSupervisor.prompt?.systemPrompt ?? RESEARCH_SUPERVISOR_SYSTEM_PROMPT","fileSha256":"fc47d7e64a0645e87b24c548d3829ff3c8fc8f58a0b270ebab26a86f4023e87d"},"additionalSources":[{"repository":"agent-runtime","commit":"a16d8a3b91481b140cb552e373d5bde98b34af05","path":"src/profiles/researcher.ts","line":349,"url":"https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/profiles/researcher.ts#L349","lineText":"export const RESEARCHER_SYSTEM_PROMPT = [","fileSha256":"40034986865be33c9d4af7ada5f10a829dbb0d89e5ee3a50ebd1c4798afd47c0"}],"triggeringScenario":"The caller supplies a supervisor that must edit a candidate KB, and the adapter appends worker instructions that prohibit writes.","evidence":{"observation":{"requiredAction":"write candidate KB","appendedPolicy":"worker read-only policy","repairCheck":"Exact supplied prompt and promptless profile pass through the adapter unchanged."},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Source-inspected instruction conflict plus actual Runtime adapter invocation with a captured callback; no real model execution.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["R-full","R-profile-integration"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"The adapter contradicts the task and silently changes the supposed optimization surface.","fix":{"summary":"Execute the exact caller-authored supervisor profile without appending a worker policy.","repository":"agent-runtime","paths":["src/knowledge/supervised-update.ts"],"repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-runtime","path":"tests/knowledge-supervised-update.test.ts","line":90,"purpose":"executes the supplied supervisor prompt unchanged","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/knowledge-supervised-update.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/knowledge-supervised-update.test.ts#L90","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/knowledge-supervised-update.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('executes the supplied supervisor prompt unchanged', async () => {","fileSha256":"432aa246cd9bf738d6a23badf959a87004d1936eff8cfbf2eeb299d905f01dad"},{"repository":"agent-runtime","path":"tests/knowledge-supervised-update.test.ts","line":53,"purpose":"runs supervised knowledge updates against the candidate KB root","repairCommit":"0d2c84be6741a7074756b988eed1bc17d5fd3cff","worktreePath":"/Users/drew/webb/_wt/runtime-learning-audit-20260905/tests/knowledge-supervised-update.test.ts","url":"https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/knowledge-supervised-update.test.ts#L53","revisionStatus":"committed_source","command":"node_modules/.bin/vitest run tests/knowledge-supervised-update.test.ts","requiredEnvironment":"Offline repository test environment","lineText":" it('runs supervised knowledge updates against the candidate KB root', async () => {","fileSha256":"432aa246cd9bf738d6a23badf959a87004d1936eff8cfbf2eeb299d905f01dad"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} +{"rank":22,"id":"K5","severity":"MEDIUM","priority":"P2","title":"Explicitly disabled required phases are silently skipped","status":"confirmed","resolution":"resolved","evidenceClassification":"measured","source":{"repository":"agent-knowledge","commit":"390f2da9883e55cc86a8985167324d8b1f10894a","path":"src/kb-improvement/evaluation.ts","line":50,"url":"https://github.com/tangle-network/agent-knowledge/blob/390f2da9883e55cc86a8985167324d8b1f10894a/src/kb-improvement/evaluation.ts#L50","lineText":"export function assertKnowledgeImprovementOptions(options: KnowledgeImprovementOptions): void {","fileSha256":"844f31ba69feec83ac83f3c84814c2ebb46b5551ad4a6803c8c74d6a7d4d545b"},"additionalSources":[],"triggeringScenario":"The caller requires diagnosis or answer-quality evaluation but provides enabledPhases: [].","evidence":{"observation":{"enabledPhases":[],"requiredPhaseExecuted":false,"candidateWork":"succeeds before repair"},"observationUnit":"Controlled source inspection or deterministic execution of the identified failure condition.","boundary":"Controlled reproduction through the real public implementation, using deterministic callbacks or real Git/filesystem operations; no production incidence estimate.","sampleCount":null,"sampleCountStatus":"No production sample; exact probe counts are in observation when retained.","durableReport":{"path":"docs/research/learning-system-audit-2026-09-05.md","section":"Reproduced failures and their consequences","sha256":"a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458","commit":"7fe89641322dc1d0e60dc8c4d66472d786320558","url":"https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md"},"proofIds":["K-linux-8","K-ci","K-publish","K-registry"],"originalRawProbeStatus":"Scratch probes and raw logs were lost after the environment refresh; source anchors, durable regressions, and retained observations remain."},"userImpact":"The candidate can bypass an explicitly required development or final evaluation phase.","fix":{"summary":"Reject required-but-disabled phases before candidate work begins.","repository":"agent-knowledge","paths":["src/kb-improvement/evaluation.ts"],"repairCommit":"30878397a947c37fd743929b30aa03728e785c76","verification":"Regression references below and manifest proof records; no package-release approval implied."},"regressions":[{"repository":"agent-knowledge","path":"tests/kb-improvement/lifecycle.test.ts","line":17,"purpose":"rejects required-but-disabled gap-diagnosis and answer-quality phases (parameterized)","repairCommit":"30878397a947c37fd743929b30aa03728e785c76","worktreePath":"/Users/drew/webb/_wt/knowledge-learning-audit-20260905/tests/kb-improvement/lifecycle.test.ts","url":"https://github.com/tangle-network/agent-knowledge/blob/30878397a947c37fd743929b30aa03728e785c76/tests/kb-improvement/lifecycle.test.ts#L17","revisionStatus":"merged_source","command":"node node_modules/vitest/vitest.mjs run tests/kb-improvement/lifecycle.test.ts","requiredEnvironment":"Linux snapshot implementation","lineText":" it.each(['gap-diagnosis', 'answer-quality'] as const)(","fileSha256":"c2566de3e9eb4cd52a9b77b95af7bd2fda0522a8fbe088fd7033a3ccaae82dbd"}],"cost":{"failure":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Production frequency, affected workload, and monetary loss were not measured.","assumptions":[],"estimate":null},"repair":{"value":null,"unit":"USD/month","status":"unmeasured","reason":"Recurring operation and maintenance cost after this repair were not measured.","assumptions":[],"estimate":null},"comparison":"Both sides use USD/month. Null is unknown, not zero. Synthetic probe receipts are not monthly costs."}} diff --git a/.agent/critical-audit/2026-09-05-learning-system/manifest.json b/.agent/critical-audit/2026-09-05-learning-system/manifest.json new file mode 100644 index 000000000..7e743a7eb --- /dev/null +++ b/.agent/critical-audit/2026-09-05-learning-system/manifest.json @@ -0,0 +1,1070 @@ +{ + "schemaVersion": 1, + "auditId": "2026-09-05-learning-system", + "createdAtUtc": "2026-09-06 05:05:12 UTC", + "skill": "/critical-audit", + "scope": { + "objective": "Audit the learning process, improve(...), improvement surfaces, fragmentation, current research mechanisms, and the case for simplification or stronger unification across Runtime, Eval, and Knowledge.", + "artifactTask": "Persist the completed audits and repair reviews. No new source review, capability experiment, or release action was initiated for these artifacts.", + "repositories": { + "R": { + "name": "agent-runtime", + "remote": "tangle-network/agent-runtime", + "root": "/Users/drew/webb/_wt/runtime-learning-audit-20260905", + "initialCommit": "a16d8a3b91481b140cb552e373d5bde98b34af05", + "initialVersion": "0.193.1", + "repairCommit": "0d2c84be6741a7074756b988eed1bc17d5fd3cff", + "testedCommit": "5d61700236d43c9abcc4adc645130749d8239ca6" + }, + "E": { + "name": "agent-eval", + "remote": "tangle-network/agent-eval", + "root": "/Users/drew/webb/_wt/eval-learning-audit-20260905", + "initialCommit": "f8e3da285b6286386699a196733e9c0c27c20cfd", + "initialVersion": "0.173.3", + "repairCommit": "6c23e86025eec42b4d5f551f9cb4896f474cb16b", + "testedCommit": "6c23e86025eec42b4d5f551f9cb4896f474cb16b" + }, + "K": { + "name": "agent-knowledge", + "remote": "tangle-network/agent-knowledge", + "root": "/Users/drew/webb/_wt/knowledge-learning-audit-20260905", + "initialCommit": "390f2da9883e55cc86a8985167324d8b1f10894a", + "initialVersion": "13.0.1", + "repairCommit": "30878397a947c37fd743929b30aa03728e785c76", + "testedCommit": "30878397a947c37fd743929b30aa03728e785c76", + "repairState": "Merged and published as 14.0.0; terminal owner proof is retained below.", + "reviewedSourceCommit": "29998a07db4fd8c1ca7e89bc3713911ebf128bca" + } + }, + "initialBranches": "The three origin/main revisions were fetched before isolated audit worktrees were created.", + "runtimeIntegration": { + "sourceTestedCommit": "5d61700236d43c9abcc4adc645130749d8239ca6", + "currentRepairCommit": "0d2c84be6741a7074756b988eed1bc17d5fd3cff", + "latestIntegratedMainCommit": "2707e2321e7b26b0e71efacb86ebdf3ddd23adac", + "postTestChanges": "Main integration changes AGENTS.md and CLAUDE.md only. Later report and released dependency pins are at 7fe89641; their local package and cohort checks passed separately.", + "integrationCommit": "7f05654a69441db1bd135e8545787af785cde4a2", + "latestReportAndCohortCommit": "7fe89641322dc1d0e60dc8c4d66472d786320558" + }, + "inventory": { + "runtime": { + "previouslyObservedUniqueFiles": 55, + "production": 38, + "tests": 7, + "callers": 5, + "documentation": 5, + "allImprovementProductionFilesRead": 24, + "inventoryPath": "runtime-inventory.json", + "rawOriginalListAvailable": false + }, + "eval": { + "inspectedFileCount": null, + "status": "unmeasured; no complete inspected-file inventory was retained" + }, + "knowledge": { + "inspectedFileCount": null, + "status": "unmeasured; no complete inspected-file inventory was retained" + }, + "externalCallers": "Nearby application/workspace snapshots, not fetched main revisions." + }, + "coverage": "Full Runtime improvement directory; selected execution, adoption, strategy, observation, memory, Eval method/native/final/cost/identity paths, Knowledge candidates and research loops, and related tests/callers as documented in the main report.", + "uninspected": "No claim of exhaustive whole-repository coverage. No production deployment, paid model learning comparison, or independently reproduced external research benchmark." + }, + "workflow": { + "kind": "Aggregation of completed independent repository audits and cross-reviews", + "artifactVerification": "Retained prior 31-source/45-regression validation; re-resolved eight Knowledge regression anchors at the merged commit, checked the committed report hash and Runtime instruction-only main integration, and read terminal Knowledge CI, publication, and owner registry proof.", + "reviewerRunDuringArtifactCreation": false, + "reasoning": "Rank confirmed defects by the consequence of wrong candidate decisions, invalid evidence, data loss, or broken required learning behavior. Production likelihood remains unmeasured.", + "sourceReviewStatus": "The completed repair reviews found no remaining blocker among the 22 identified source defects.", + "limitations": [ + "Original scratch probes, local terminal logs, and the 55-path inventory were removed by the environment refresh.", + "Prior terminal counts remain observations from the session continuation record. They are not claimed as freshly re-executed checks.", + "Runtime release owner must append current CI, merge, and publication proof independently. Knowledge 14.0.0 terminal release evidence is retained below." + ] + }, + "counts": { + "findings": 22, + "resolved": 22, + "unresolved": 0, + "dropped": 0, + "severity": { + "CRITICAL": 0, + "HIGH": 12, + "MEDIUM": 10, + "LOW": 0 + } + }, + "sourceRepairReview": { + "verdict": "APPROVE", + "scope": "The 22 identified source defects and their retained regression coverage.", + "remainingIdentifiedSourceBlockers": 0, + "doesNotApprove": [ + "State-of-the-art performance", + "Successful continuing domain learning", + "Meta-learning capability", + "Production learning effectiveness", + "Package publication", + "Untested dependency combinations" + ] + }, + "releaseProof": { + "status": "pending", + "owner": "Parent integration/release work", + "separateFromSourceReview": true, + "eval": { + "status": "live", + "ownerProofPath": "/Users/drew/webb/_wt/eval-learning-audit-20260905/.agent/release-0.174.0/release-proof.json", + "evidenceBoundary": "Read the Eval owner's retained release-proof.json. Registry checks were executed by the Eval owner.", + "proof": { + "observedAt": "2026-09-06T05:05:01.345114+00:00", + "status": "live", + "version": "0.174.0", + "tag": "v0.174.0", + "reviewedSource": "6c23e86025eec42b4d5f551f9cb4896f474cb16b", + "mergedCommit": "0692922ea739a352f44374f4d5f7def2991aacdc", + "tree": "5bed29c76642eb6ff2fa03a604a6c4214fd79db3", + "pr": "https://github.com/tangle-network/agent-eval/pull/738", + "mergeAuthority": "Configured owner exception (--admin), authorized by parent after independent review and successful CI; no GitHub review approval.", + "ci": "https://github.com/tangle-network/agent-eval/actions/runs/33987395502", + "publish": "https://github.com/tangle-network/agent-eval/actions/runs/34012602635", + "jobs": [ + { + "name": "verify", + "status": "completed", + "conclusion": "success" + }, + { + "name": "publish-npm", + "status": "completed", + "conclusion": "success" + }, + { + "name": "publish-pypi", + "status": "completed", + "conclusion": "success" + } + ], + "tests": [ + { + "suite": "JavaScript", + "passed": 5763, + "skipped": 3 + }, + { + "suite": "Python official sources", + "passed": 207, + "skipped": 4 + }, + { + "suite": "TypeScript official integrations", + "passed": 2, + "skipped": 0 + }, + { + "suite": "Python published GEPA", + "passed": 45, + "skipped": 1 + }, + { + "suite": "TypeScript published GEPA", + "passed": 1, + "skipped": 0 + }, + { + "suite": "DSPy", + "passed": 18, + "skipped": 0 + } + ], + "npm": { + "version": "0.174.0", + "gitHead": "0692922ea739a352f44374f4d5f7def2991aacdc", + "size": 3818482, + "integrity": "sha512-JcGQzN4fHll5ItkhEcFs7z8HF7ikwMLnlQhnSZekwJqGy6JbsmjGK87adxOV5SO12x1YRiALNzrQm1Hq6u1YUA==", + "url": "https://registry.npmjs.org/@tangle-network/agent-eval/-/agent-eval-0.174.0.tgz" + }, + "npmBehaviorSmoke": { + "observedAt": "2026-09-06T05:02:30.649Z", + "nodeVersion": "24.11.1", + "mode": "method", + "winner": "WIN", + "lift": 0.4, + "selectedFinalExecutions": 4, + "unchangedFinalExecutions": 2, + "unchangedInsightCount": 2, + "cacheIdentityChanged": true, + "repeatCacheHits": 5, + "repeatExecutions": 0 + }, + "pypi": { + "version": "0.174.0", + "files": [ + { + "filename": "agent_eval_rpc-0.174.0-py3-none-any.whl", + "packagetype": "bdist_wheel", + "size": 76183, + "sha256": "b651612ad8dcc392febe73f8bf9440911c29fcdccbb8322ffc0bcfb8654dc04a", + "url": "https://files.pythonhosted.org/packages/30/d4/ea6c5bb6f488e32a546ec4d92e2a205b05b70508419e1b95d1b6bcea68ef/agent_eval_rpc-0.174.0-py3-none-any.whl" + }, + { + "filename": "agent_eval_rpc-0.174.0.tar.gz", + "packagetype": "sdist", + "size": 365357, + "sha256": "8fcf4ff30e69a0371d1b4390fb5c7b16acb554a6e48b2bb0a14a70c5486c6ee0", + "url": "https://files.pythonhosted.org/packages/10/38/df575536b7188509d4b9f73c3cdd3ecde513bf805be6a286d4075b168234/agent_eval_rpc-0.174.0.tar.gz" + } + ] + }, + "pythonImport": "Installed 13 packages in 15ms\n{'python': '3.12.10', 'package': '0.174.0', 'runtime': '0.174.0', 'module': '/Users/drew/.cache/uv/archive-v0/Cqmaqr-WTlhZCM7N4YqyI/lib/python3.12/site-packages/agent_eval_rpc/__init__.py'}", + "limits": [ + "Behavior smoke uses deterministic callbacks and fixture scores; it does not measure model quality.", + "Custom optimization methods remain responsible for external work they do not report.", + "0.174.0 has result-contract changes; consumers must narrow result.mode and migrate native-generation assumptions." + ] + } + }, + "runtime": { + "status": "pending Runtime CI, merge, and publication; final local package and cohort checks passed at 7fe89641", + "proofIds": [ + "R-final-package", + "R-cohort" + ] + }, + "knowledge": { + "status": "live", + "ownerProofPath": "/tmp/learning-system-audit-20260905/knowledge-final-validation.md", + "evidenceBoundary": "Artifact updater read the retained owner proof and queried terminal GitHub jobs/logs. Registry installation and behavior checks were executed by the Knowledge owner.", + "proof": { + "version": "14.0.0", + "tag": "v14.0.0", + "reviewedSource": "29998a07db4fd8c1ca7e89bc3713911ebf128bca", + "mergedCommit": "30878397a947c37fd743929b30aa03728e785c76", + "pr": "https://github.com/tangle-network/agent-knowledge/pull/190", + "ci": "https://github.com/tangle-network/agent-knowledge/actions/runs/34013062769", + "mainCi": "https://github.com/tangle-network/agent-knowledge/actions/runs/34013245408", + "publish": "https://github.com/tangle-network/agent-knowledge/actions/runs/34013280992", + "publishJobs": [ + { + "name": "verify", + "status": "completed", + "conclusion": "success" + }, + { + "name": "publish-npm", + "status": "completed", + "conclusion": "success" + } + ], + "npm": { + "version": "14.0.0", + "dist.integrity": "sha512-PXcW+uEpXs+quy9r8YKrdVSo1lajz9dmlmEA52b2iG1dvOGkGot3Yct9J5ndEiD6/YePm6lI9yQWofLmX2ryxw==", + "dist.tarball": "https://registry.npmjs.org/@tangle-network/agent-knowledge/-/agent-knowledge-14.0.0.tgz", + "dist.shasum": "1c32c40c8600987ebc9d47f77f052eb4529cb025", + "gitHead": "30878397a947c37fd743929b30aa03728e785c76" + }, + "installedIntegrity": { + "integrityMatch": true, + "resolvedTarballMatch": true, + "gitHead": "30878397a947c37fd743929b30aa03728e785c76", + "integrity": "sha512-PXcW+uEpXs+quy9r8YKrdVSo1lajz9dmlmEA52b2iG1dvOGkGot3Yct9J5ndEiD6/YePm6lI9yQWofLmX2ryxw==", + "versions": { + "@tangle-network/agent-knowledge": { + "version": "14.0.0", + "copies": 1 + }, + "@tangle-network/agent-eval": { + "version": "0.174.0", + "copies": 1 + }, + "@tangle-network/agent-core": { + "version": "0.9.6", + "copies": 1 + }, + "@tangle-network/agent-interface": { + "version": "2.0.0", + "copies": 1 + }, + "zod": { + "version": "4.5.4", + "copies": 1 + } + } + }, + "behaviorSmoke": { + "package": "@tangle-network/agent-knowledge", + "version": "14.0.0", + "evalPeer": ">=0.174.0 <0.175.0", + "platform": "darwin", + "node": "v24.11.1", + "researchRounds": 2, + "ready": true, + "callbacks": [ + "worker:1", + "prepare", + "fold", + "checkpoint", + "published:1", + "worker:2", + "checkpoint", + "published:2" + ], + "disabledRequiredPhaseRejected": null, + "unsupportedCandidatePlatformRejected": true, + "providerCalls": 0 + }, + "cliVersion": "14.0.0", + "dependencyTree": "knowledge-release-consumer@ /private/tmp/learning-system-audit-20260905/knowledge-live-consumer\n\u251c\u2500\u252c @tangle-network/agent-eval@0.174.0\n\u2502 \u251c\u2500\u252c @asteasolutions/zod-to-openapi@9.1.0\n\u2502 \u2502 \u2514\u2500\u2500 zod@4.5.4 deduped\n\u2502 \u251c\u2500\u252c @tangle-network/agent-core@0.9.6\n\u2502 \u2502 \u251c\u2500\u2500 @tangle-network/agent-interface@2.0.0 deduped\n\u2502 \u2502 \u2514\u2500\u2500 zod@4.5.4 deduped\n\u2502 \u251c\u2500\u2500 @tangle-network/agent-interface@2.0.0 deduped\n\u2502 \u2514\u2500\u2500 zod@4.5.4\n\u251c\u2500\u252c @tangle-network/agent-interface@2.0.0\n\u2502 \u2514\u2500\u2500 zod@4.5.4 deduped\n\u2514\u2500\u252c @tangle-network/agent-knowledge@14.0.0\n \u251c\u2500\u2500 @tangle-network/agent-eval@0.174.0 deduped\n \u251c\u2500\u2500 @tangle-network/agent-interface@2.0.0 deduped\n \u2514\u2500\u2500 zod@4.5.4 deduped\n\n", + "ownerReport": { + "sha256": "24a534ea3e9914b41b094ed9c8e2f10d3a8687ec11bcda8b6dca944f049891a6", + "text": "# Knowledge 14.0.0 final validation\n\nPR: https://github.com/tangle-network/agent-knowledge/pull/190\nPR commit: `29998a07db4fd8c1ca7e89bc3713911ebf128bca`.\nMerged commit and tag `v14.0.0`: `30878397a947c37fd743929b30aa03728e785c76`.\nThe two commits have identical trees, checked with `git diff --exit-code`.\n\n## Final dependency cohort\n\nKnowledge 14.0.0 requires Eval `>=0.174.0 <0.175.0`, tested at 0.174.0.\nThe clean package install has one copy each of Eval 0.174.0, Core 0.9.6, Interface 2.0.0, and Zod 4.5.4.\nThe lockfile changed five lines and their replacements for Eval identity and integrity.\n\n## Host checks\n\nExecution: macOS, Node 24.11.1, pnpm 10.34.5.\n\n- Source typecheck: `node node_modules/typescript/bin/tsc --noEmit`, passed.\n- Contract typecheck: `node node_modules/typescript/bin/tsc --noEmit -p tsconfig.contracts.json`, passed.\n- Lint: `node node_modules/@biomejs/biome/bin/biome check src tests`, 249 files passed.\n- Build: `./node_modules/.bin/tsdown`, passed in 2210 ms.\n- Tests: `node node_modules/vitest/vitest.mjs run tests/rag-improvement-loop.test.ts tests/loops/research-driving-loop.test.ts tests/loops/research-driving-driver.test.ts tests/claim-persistence.test.ts tests/memory/improvement.test.ts`, 85 of 85 passed in five files, 7.32 seconds.\n- `pnpm verify:package`, passed for the final dependency cohort.\nThis includes API record, path containment, skill validation, publint, declaration resolution, clean install, imports, CLI version, and re-pack.\n- The API record is current: 995 exports across six entrypoints.\nOnly the exported ResearchDriver shape changes.\n- The committed `pnpm check:version-bump` passed and requires the authored major version.\n- `git diff --check`, commit hooks, and push hooks passed.\n\nThe built artifact smoke completed two research rounds.\nThe second worker received steering with zero storage gaps.\nThe callback order was worker1, prepare, fold, checkpoint, publish1, worker2, checkpoint, publish2.\nThe result was ready only after driver completion.\nThe host correctly rejects Linux-only exact candidate execution.\nNo provider inference was called.\n\n## Required Ubuntu CI\n\nRun: https://github.com/tangle-network/agent-knowledge/actions/runs/34013062769\n\n| Job | Tests | Files | Duration | Result |\n| --- | --- | --- | --- | --- |\n| ci | 862 passed, 7 conditional skips, 0 failed | 88 passed, 2 skipped, 90 total | tests 35.49 seconds; job 68 seconds | Success |\n| official-optimizers | 2 passed, 0 failed | 1 passed | tests 6.48 seconds; job 80 seconds | Success |\n\nCI uses Ubuntu 24.04 and Node 22.\nThe ci job runs with `AGENT_KNOWLEDGE_RUN_NETWORK_TESTS=1`.\nThe separate official-optimizers job executes both GEPA retrieval optimization and SkillOpt text optimization through the packed package.\nThe CI job also passed clean-package installation against the final cohort.\nThese runs include the exact Linux candidate lifecycle, disabled required phases, candidate integrity, promotion, persistence, and research regressions.\n\n## Landing and publication\n\nLive PR comments, inline comments, and GitHub reviews were empty at the reviewed head.\nThe repository has one required approval and `enforce_admins=false`.\nThe configured owner exception was used under the user's landing authorization after independent source review and all CI jobs passed.\nNo branch protection was changed.\n\nPublish run: https://github.com/tangle-network/agent-knowledge/actions/runs/34013280992\nPublication completed successfully at the merged commit.\nBoth `verify` and `publish-npm` completed successfully.\nMain CI also completed successfully: https://github.com/tangle-network/agent-knowledge/actions/runs/34013245408\n\n## Published package proof\n\nThe fresh registry check returned Knowledge 14.0.0 at `gitHead` `30878397a947c37fd743929b30aa03728e785c76`.\nThe published tarball is `https://registry.npmjs.org/@tangle-network/agent-knowledge/-/agent-knowledge-14.0.0.tgz`.\nIts SHA-1 is `1c32c40c8600987ebc9d47f77f052eb4529cb025`.\nIts integrity is `sha512-PXcW+uEpXs+quy9r8YKrdVSo1lajz9dmlmEA52b2iG1dvOGkGot3Yct9J5ndEiD6/YePm6lI9yQWofLmX2ryxw==`.\nThe clean consumer's package lock matches both the registry integrity and tarball URL.\n\nThe consumer installation used npm, macOS, and Node 24.11.1.\nIt installed the published packages in `/tmp/learning-system-audit-20260905/knowledge-live-consumer`.\nThe install completed with 33 packages added in eight seconds.\nThe package lock has one installed copy each of Knowledge 14.0.0, Eval 0.174.0, Core 0.9.6, Interface 2.0.0, and Zod 4.5.4.\nThe installed CLI's `version` command returned `14.0.0`.\nAn initial `--version` invocation was invalid; the CLI documents `version` as its command.\n\nThe installed package's functional smoke completed two research rounds using real temporary knowledge storage and scripted sources.\nThe second worker received steering with zero storage gaps.\nResearch returned `ready: true` only after driver completion.\nCallbacks occurred in this order: worker1, prepare, fold, checkpoint, publish1, worker2, checkpoint, publish2.\nThe smoke recorded zero provider calls.\nIt confirmed that this macOS host refuses Linux-only exact candidate execution.\nIts disabled-required-phase result is `null` because that branch requires Linux.\nThe successful Ubuntu CI supplies the Linux-only candidate evidence; this smoke makes no Linux execution claim.\n\nFresh evidence files:\n\n- `/tmp/learning-system-audit-20260905/knowledge-registry.json`: npm metadata.\n- `/tmp/learning-system-audit-20260905/knowledge-live-smoke.json`: functional result from the registry installation.\n- `/tmp/learning-system-audit-20260905/knowledge-live-consumer/smoke.mjs`: executed smoke source.\n- `/tmp/learning-system-audit-20260905/knowledge-live-integrity.json`: integrity, tarball, dependency versions, and installed-copy assertions.\n- `/tmp/learning-system-audit-20260905/knowledge-live-dependencies.log`: complete relevant dependency tree.\n- `/tmp/learning-system-audit-20260905/knowledge-live-cli-version.log`: installed CLI version.\n" + }, + "limits": [ + "Registry smoke ran on macOS/Node 24.11.1 with Interface 2.0.0; Runtime uses Interface 2.3.0.", + "The Linux-only required-phase field is null in the macOS smoke; actual Linux candidate evidence comes from Ubuntu CI.", + "PR CI enables five live-source tests that Publish skips. The two runs retain separate counts.", + "These are implementation checks with scripted sources and zero provider calls, not evidence of learning quality." + ] + } + } + }, + "verification": { + "proofs": { + "E-full": { + "repository": "agent-eval", + "revision": "6c23e86025eec42b4d5f551f9cb4896f474cb16b", + "status": "passed", + "executionContext": "Local full JavaScript suite; four workers.", + "command": "pnpm test --maxWorkers=4", + "tests": { + "passed": 5763, + "failed": 0, + "skipped": 3, + "total": 5766 + }, + "files": { + "passed": 392, + "failed": 0, + "skipped": 2, + "total": 394 + }, + "durationSeconds": 140.77, + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/eval-full-tests-controlled.log", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + }, + "limitations": [ + "This is the final 5763-pass local run, not the earlier 5761-pass run.", + "Implementation verification, not a paid learning comparison." + ] + }, + "E-ci": { + "repository": "agent-eval", + "revision": "6c23e86025eec42b4d5f551f9cb4896f474cb16b", + "status": "passed", + "executionContext": "GitHub Actions CI at the repaired source revision.", + "runUrl": "https://github.com/tangle-network/agent-eval/actions/runs/33987395502", + "rawLog": { + "path": "/Users/drew/webb/_wt/eval-learning-audit-20260905/.agent/release-0.174.0/pr-ci.log", + "status": "present_and_read", + "lines": [ + 6212, + 6213, + 7168, + 7192, + 7193, + 7275, + 478, + 500, + 501 + ] + }, + "tests": { + "passed": 5763, + "failed": 0, + "skipped": 3, + "total": 5766 + }, + "files": { + "passed": 392, + "failed": 0, + "skipped": 2, + "total": 394 + }, + "additionalChecks": [ + { + "name": "Python official source", + "passed": 207, + "skipped": 4, + "failed": 0, + "total": 211, + "line": 7168 + }, + { + "name": "TypeScript official integrations", + "passed": 2, + "skipped": 0, + "failed": 0, + "total": 2, + "line": 7193 + }, + { + "name": "DSPy", + "passed": 18, + "skipped": 0, + "failed": 0, + "total": 18, + "line": 7275 + }, + { + "name": "Released GEPA Python", + "passed": 45, + "skipped": 1, + "failed": 0, + "total": 46, + "line": 478 + }, + { + "name": "Released GEPA TypeScript bridge", + "passed": 1, + "skipped": 0, + "failed": 0, + "total": 1, + "line": 501 + } + ], + "limitations": [ + "These are separate suites and are not added into an inflated full-suite count.", + "CI verification does not establish registry publication or learning effectiveness." + ] + }, + "E-focused": { + "repository": "agent-eval", + "revision": "6c23e86025eec42b4d5f551f9cb4896f474cb16b", + "status": "passed", + "tests": { + "passed": 185, + "failed": 0, + "skipped": 0, + "total": 185 + }, + "files": { + "passed": 5, + "failed": 0, + "skipped": 0, + "total": 5 + }, + "durationSeconds": 11.31, + "command": null, + "commandStatus": "Exact focused command not retained in the continuation summary.", + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/eval-focused-release.log", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + } + }, + "E-review-probes": { + "repository": "agent-eval", + "status": "passed", + "executionContext": "Independent deterministic probes through selfImprove with synthetic paid-call receipts.", + "observations": { + "E6": { + "totalUsd": 3, + "ledgerUsd": 3, + "accountingComplete": false, + "receipts": 1 + }, + "E8": { + "calls": 2, + "receipts": 2, + "totalUsd": 2, + "verdict": "hold", + "insightRuns": 2, + "insightCostUsd": 2, + "inputTokens": 20, + "outputTokens": 10 + } + }, + "rawEvidence": [ + { + "path": "/tmp/learning-system-audit-20260905/eval-cost-review.mts", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + }, + { + "path": "/tmp/learning-system-audit-20260905/eval-noop-review.mts", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + }, + { + "path": "/tmp/learning-system-audit-20260905/knowledge-implementation.md", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + } + ], + "durableRegressions": [ + "tests/contract-self-improve-method-integrity.test.ts" + ], + "limitations": [ + "No paid model call or production cost was measured.", + "Probe text and raw output are no longer present." + ] + }, + "R-full": { + "repository": "agent-runtime", + "revision": "5d61700236d43c9abcc4adc645130749d8239ca6", + "status": "passed", + "command": "node_modules/.bin/vitest run --maxWorkers=4", + "executionContext": "Local full Runtime suite; four workers.", + "tests": { + "passed": 3561, + "failed": 0, + "skipped": 9, + "total": 3570 + }, + "files": { + "passed": 280, + "failed": 0, + "skipped": 3, + "total": 283 + }, + "durationSeconds": 205.06, + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/runtime-final-tests.log", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + }, + "limitations": [ + "The later integrated changes were documentation and skills only.", + "Fresh Runtime CI and release evidence are owned by the parent and remain separate." + ] + }, + "R-earlier-full": { + "repository": "agent-runtime", + "revision": null, + "revisionStatus": "Pre-deadline-fix integrated worktree; exact commit not retained.", + "status": "failed", + "command": "node_modules/.bin/vitest run --maxWorkers=4", + "tests": { + "passed": 3559, + "failed": 2, + "skipped": 9, + "total": 3570 + }, + "durationSeconds": 420.09, + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/runtime-integrated-tests.log", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + }, + "failures": [ + { + "path": "tests/kernel/workspace.test.ts", + "line": 136, + "cause": "The second-worker Python/Git integration case exceeded its 20-second limit.", + "repair": "Set this case's deadline to 60000 milliseconds; assertions unchanged." + }, + { + "path": "tests/improvement-cycle.test.ts", + "line": 1098, + "cause": "This integration case exceeded its 15-second override under the full-suite load.", + "repair": "Remove the 15000-millisecond override and use the existing 30000-millisecond describe deadline; assertions unchanged." + } + ] + }, + "R-deadline-focused": { + "repository": "agent-runtime", + "status": "passed", + "command": "node_modules/.bin/vitest run tests/kernel/workspace.test.ts tests/improvement-cycle.test.ts --reporter=verbose", + "tests": { + "passed": 35, + "failed": 0, + "skipped": 1, + "total": 36 + }, + "durationSeconds": 81.73, + "affectedTestDurationSeconds": { + "workspace": 23.586, + "improvementCycle": 11.475 + }, + "skipReason": "Existing optional jj test.", + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/runtime-deadline-focused.log", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + }, + "diagnosis": { + "path": "/tmp/learning-system-audit-20260905/runtime-deadline-diagnosis.md", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + } + }, + "R-original-focused": { + "repository": "agent-runtime", + "status": "passed", + "tests": { + "uniquePassed": 146 + }, + "command": null, + "commandStatus": "The original command list was in runtime-fixes.md and is no longer retained.", + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/runtime-fixes.md", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + }, + "limitations": [ + "Do not add this focused count to the later full-suite count." + ] + }, + "R-profile-integration": { + "repository": "agent-runtime", + "status": "passed", + "command": "node_modules/.bin/vitest run tests/kernel/strategy-suite.test.ts tests/knowledge-supervised-update.test.ts", + "tests": { + "passed": 43, + "failed": 0, + "skipped": 0, + "total": 43 + }, + "files": { + "passed": 2, + "failed": 0, + "skipped": 0, + "total": 2 + }, + "durationSeconds": 7.98, + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/runtime-current-main-review.md", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + }, + "executionContext": "Independent offline integration of the supplied profile contract after current-main merge." + }, + "R-reflective": { + "repository": "agent-runtime", + "status": "passed", + "command": null, + "tests": { + "passed": 16, + "failed": 0, + "skipped": 0, + "total": 16 + }, + "durationSeconds": 29.24, + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/reflective-tests-final.log", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + }, + "reproduceCommand": "node_modules/.bin/vitest run tests/improvement-driver.test.ts", + "commandStatus": "The exact historical invocation flags are not retained; reproduceCommand identifies the durable suite." + }, + "R-reflective-independent": { + "repository": "agent-runtime", + "status": "passed", + "exitCode": 0, + "observations": { + "rejected": true, + "declaredTargetUnchanged": true, + "undeclaredTargetUnchanged": true + }, + "executionContext": "Independent scratch probe through the real reflective patch application path.", + "rawEvidence": [ + { + "path": "/tmp/learning-system-audit-20260905/reflective-review-probe.mts", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + }, + { + "path": "/tmp/learning-system-audit-20260905/eval-engineering-addendum.md", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + } + ], + "durableRegression": "tests/improvement-driver.test.ts:223" + }, + "K-host-74": { + "repository": "agent-knowledge", + "revision": null, + "baseRevision": "390f2da9883e55cc86a8985167324d8b1f10894a", + "status": "passed", + "executionContext": "Host focused checks before the new Eval dependency cohort.", + "tests": { + "passed": 74, + "failed": 0, + "total": 74 + }, + "files": { + "passed": 4, + "total": 4 + }, + "command": null, + "commandStatus": "Exact focused command was in knowledge-implementation.md and is no longer retained.", + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/knowledge-implementation.md", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + }, + "limitations": [ + "This is not a full Knowledge suite or the new-dependency package proof." + ] + }, + "K-other-consumers": { + "repository": "agent-knowledge", + "status": "passed_with_skips", + "executionContext": "Host consumer checks before the new Eval dependency cohort.", + "tests": { + "passed": 7, + "failed": 0, + "skipped": 5, + "total": 12 + }, + "files": { + "passed": 3, + "skipped": 1, + "total": 4 + }, + "skipReason": "Five paid-network cases in one file.", + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/knowledge-implementation.md", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + } + }, + "K-linux-initial": { + "repository": "agent-knowledge", + "status": "failed", + "executionContext": "Local Colima Linux container, Node 24.20.0, Linux aarch64 6.8.0-64-generic; cached node:24-bookworm image be23f54a88d3.", + "tests": { + "passed": 132, + "failed": 1, + "total": 133 + }, + "failure": "A new fixture omitted the required answerQualityCostCeiling.", + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/knowledge-implementation.md", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + } + }, + "K-linux-fixture-corrected": { + "repository": "agent-knowledge", + "status": "passed", + "executionContext": "The same local Linux setup after the fixture correction.", + "tests": { + "passed": 2, + "failed": 0, + "total": 2 + }, + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/knowledge-implementation.md", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + } + }, + "K-linux-8": { + "repository": "agent-knowledge", + "revision": null, + "baseRevision": "390f2da9883e55cc86a8985167324d8b1f10894a", + "status": "passed", + "command": "docker exec -w /work knowledge-learning-tests-20260905 node node_modules/vitest/vitest.mjs run tests/kb-improvement/lifecycle.test.ts tests/kb-improvement/selected-candidate.test.ts", + "executionContext": "Local Colima; cached node:24-bookworm image be23f54a88d3; Node 24.20.0; Linux aarch64 6.8.0-64-generic. Source mounted read-only at /input and copied to /work; no platform override.", + "tests": { + "passed": 8, + "failed": 0, + "total": 8 + }, + "durationSeconds": 11.27, + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/knowledge-implementation.md", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + }, + "limitations": [ + "Before the new Eval 0.174.0 dependency cohort.", + "This focused result cannot be combined with the earlier failed run into a full-suite pass." + ] + }, + "K-previous-package": { + "repository": "agent-knowledge", + "status": "passed", + "checks": [ + "Host source typecheck", + "Contract typecheck", + "Lint over 249 files", + "Build", + "Earlier packed package verification" + ], + "dependencyCohort": { + "agent-eval": "0.173.0", + "agent-interface": "2.0.0" + }, + "rawLog": { + "path": "/tmp/learning-system-audit-20260905/knowledge-implementation.md", + "status": "unavailable_after_environment_refresh", + "resultSource": "Previously observed terminal output retained in the session continuation summary." + }, + "limitations": [ + "This earlier proof covers Eval 0.173.0 only; K-ci and K-publish record completed checks for Eval 0.174.0." + ] + }, + "K-ci": { + "repository": "agent-knowledge", + "revision": "29998a07db4fd8c1ca7e89bc3713911ebf128bca", + "status": "passed_with_conditional_skips", + "executionContext": "Ubuntu 24.04, Node 22; AGENT_KNOWLEDGE_RUN_NETWORK_TESTS=1.", + "command": "pnpm test", + "runUrl": "https://github.com/tangle-network/agent-knowledge/actions/runs/34013062769", + "rawLog": { + "status": "read_via_gh_run_view" + }, + "tests": { + "passed": 862, + "failed": 0, + "skipped": 7, + "total": 869 + }, + "files": { + "passed": 88, + "failed": 0, + "skipped": 2, + "total": 90 + }, + "durationSeconds": 35.49, + "additionalChecks": [ + { + "name": "packed GEPA and SkillOpt integration", + "passed": 2, + "failed": 0, + "filesPassed": 1, + "durationSeconds": 6.48 + } + ], + "dependencyCohort": { + "agent-knowledge": "14.0.0", + "agent-eval": "0.174.0" + }, + "limitations": [ + "Seven conditional skips remain in the broad suite; two official optimizer cases run separately.", + "Do not add the separate official cases to the broad suite as distinct observation units." + ] + }, + "K-publish": { + "repository": "agent-knowledge", + "revision": "30878397a947c37fd743929b30aa03728e785c76", + "status": "passed_with_conditional_skips", + "executionContext": "Ubuntu publish workflow, Node 22 and Python 3.12; live-source network flag is absent.", + "command": "pnpm run test", + "runUrl": "https://github.com/tangle-network/agent-knowledge/actions/runs/34013280992", + "rawLog": { + "status": "read_via_gh_run_view" + }, + "tests": { + "passed": 857, + "failed": 0, + "skipped": 12, + "total": 869 + }, + "files": { + "passed": 87, + "failed": 0, + "skipped": 3, + "total": 90 + }, + "additionalChecks": [ + { + "name": "packed GEPA and SkillOpt integration", + "passed": 2, + "failed": 0, + "filesPassed": 1 + } + ], + "limitations": [ + "Five extra skipped cases are tests/sources-live.test.ts because Publish does not set AGENT_KNOWLEDGE_RUN_NETWORK_TESTS=1.", + "The two official optimizer cases pass in a separate step." + ] + }, + "K-main-ci": { + "repository": "agent-knowledge", + "revision": "30878397a947c37fd743929b30aa03728e785c76", + "status": "passed", + "runUrl": "https://github.com/tangle-network/agent-knowledge/actions/runs/34013245408", + "jobs": [ + { + "name": "ci", + "conclusion": "success" + }, + { + "name": "official-optimizers", + "conclusion": "success" + } + ], + "evidenceBoundary": "Terminal jobs queried directly; test counts are retained separately for the inspected PR and Publish logs." + }, + "K-registry": { + "repository": "agent-knowledge", + "revision": "30878397a947c37fd743929b30aa03728e785c76", + "status": "live", + "evidenceBoundary": "Read retained Knowledge owner outputs; artifact updater did not rerun the registry smoke.", + "ownerProofPath": "/tmp/learning-system-audit-20260905/knowledge-final-validation.md", + "releaseProofRef": "releaseProof.knowledge.proof", + "limitations": [ + "Interface 2.0.0 in this consumer does not verify Runtime Interface 2.3.0.", + "macOS smoke does not execute Linux-only candidate work." + ] + }, + "R-final-package": { + "repository": "agent-runtime", + "revision": "7fe89641322dc1d0e60dc8c4d66472d786320558", + "status": "passed", + "evidenceBoundary": "Parent-reported terminal checks; artifact updater did not re-execute them.", + "checks": [ + "verify:package, including clean installed exports and edge Worker execution", + "source and example typechecks", + "lint: 701 files", + "docs:freshness", + "publish and upstream workflow checks", + "version checks", + "verify:primeintellect" + ], + "limitations": [ + "Runtime CI, merge, and publication remain pending.", + "This does not replace the earlier full-suite observation at 5d617002." + ] + }, + "R-cohort": { + "repository": "agent-runtime", + "revision": "7fe89641322dc1d0e60dc8c4d66472d786320558", + "status": "passed", + "processExitCode": 0, + "processExitSource": "Parent observed terminal exit 0.", + "evidenceBoundary": "Read the parent-produced report; artifact updater did not execute the consumer.", + "rawReport": { + "path": "/tmp/learning-system-audit-20260905/runtime-cohort.json", + "sha256": "c0cfd2feb3ac99d022e1f6120e877b3064da9ff08e57a530f09149c83a537bde", + "status": "present_and_read" + }, + "report": { + "packages": [ + { + "name": "@tangle-network/agent-interface", + "version": "2.3.0", + "sourceCommit": "7b092480b661d146cafbcc4fad1cfbdbc09f385f", + "sha256": "5fd2391a3b8b91eecaba2d0fe81e2c286ff3c6c37f4decfbe0f48d15aeb60345" + }, + { + "name": "@tangle-network/agent-eval", + "version": "0.174.0", + "sourceCommit": "0692922ea739a352f44374f4d5f7def2991aacdc", + "sha256": "907be506b71923b1d601b128a18a981c9f04032e05bac02d9aeecef9b74d886e" + }, + { + "name": "@tangle-network/agent-knowledge", + "version": "14.0.0", + "sourceCommit": "30878397a947c37fd743929b30aa03728e785c76", + "sha256": "5ca2d9f8c17d367587c6005336ea11690a2c7c1f3ec28289afb25458f4e1202d" + }, + { + "name": "@tangle-network/agent-runtime", + "version": "0.195.0", + "sourceCommit": "7fe89641322dc1d0e60dc8c4d66472d786320558", + "sha256": "2662ae3c6b523a2e27ccbecdcf9f5ee2876971486940fcdbfb8e93cae400e12e" + } + ], + "consumer": { + "install": "pnpm install --frozen-lockfile", + "packageCount": 4, + "sandboxVersions": [ + "0.36.4", + "0.37.0" + ], + "exactArchiveResolution": true, + "publicImportCount": 128, + "proposals": [ + { + "sandboxVersion": "0.36.4", + "proposal": { + "proposalKind": "agent-profile-improvement-measured-comparison", + "baseline": 0, + "candidate": 1, + "pairs": 6, + "executions": 12, + "changedSurfaces": [ + "prompt" + ], + "evalSignTestPValue": 0.125, + "knowledgeReadinessId": "packed-cohort" + } + }, + { + "sandboxVersion": "0.37.0", + "proposal": { + "proposalKind": "agent-profile-improvement-measured-comparison", + "baseline": 0, + "candidate": 1, + "pairs": 6, + "executions": 12, + "changedSurfaces": [ + "prompt" + ], + "evalSignTestPValue": 0.125, + "knowledgeReadinessId": "packed-cohort" + } + } + ] + } + }, + "limitations": [ + "Package SHA-256 values identify the exact local archives; they are distinct from registry tarball integrity hashes.", + "The Runtime consumer selects Interface 2.3.0, unlike the Knowledge owner consumer at Interface 2.0.0.", + "Deterministic proposal values, six pairs, and p=0.125 check composition; they do not establish model-quality improvement.", + "Runtime CI, merge, and publication remain pending." + ] + } + }, + "aggregationRule": "Do not sum overlapping focused and full suites or combine corrected focused runs into a full-suite pass.", + "excludedRunCounts": [ + { + "description": "Earlier Eval full local run", + "passed": 5761, + "reason": "Superseded by the controlled 5763-pass run and matching recovered CI." + }, + { + "description": "Independent Knowledge 11/11 Linux probe", + "passed": 11, + "reason": "Conversation-only result without retained raw log; not used as a finding proof." + } + ] + }, + "evidence": { + "durableMainReport": { + "path": "docs/research/learning-system-audit-2026-09-05.md", + "sha256": "a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458", + "commit": "7fe89641322dc1d0e60dc8c4d66472d786320558", + "url": "https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md" + }, + "findingFile": "findings.jsonl", + "summaryFile": "summary.md", + "runtimeInventory": "runtime-inventory.json", + "rawLocalEvidenceStatus": "unavailable_after_environment_refresh", + "preservedEvidence": "Initial GitHub source URLs, committed repair/regression paths and content hashes, the pinned report, recovered Eval CI and release proof, Knowledge terminal CI and owner registry proof, Runtime local cohort report, and explicit earlier observations.", + "caveat": "No missing source list, file hash, runtime cost, prevalence, uncertainty estimate, or live capability result has been reconstructed as an observation." + }, + "impact": { + "unit": "USD/month", + "failureCost": null, + "repairCost": null, + "status": "unmeasured on both sides for every finding", + "reason": "No production incidence, affected workload, or ongoing operating-cost study was performed.", + "probeReceipts": "Synthetic USD receipts reproduce accounting behavior only." + }, + "capabilityAssessment": { + "paidLearningExperiments": 0, + "sotaRanking": "Not established. The research comparison inspected mechanisms and supported integrations, not matched same-task performance.", + "architecturalVerdict": "Make the domain learning process a reusable executable artifact. Reuse exact candidates, shared measurement records, retained evidence, and exact adoption. Preserve independent learning methods and package responsibilities.", + "learningClaims": "Specialist quality, repeatable domain learning, evaluation engineering, learning-method improvement, and transfer require separate evidence.", + "proposedUnificationImplemented": false, + "implementedBoundary": "The 22 concrete defects and associated simplifications are repaired; the broader continuing/meta-learning composition remains a proposed direction." + }, + "updatedAtUtc": "2026-09-06T05:25:08.806615+00:00" +} diff --git a/.agent/critical-audit/2026-09-05-learning-system/runtime-inventory.json b/.agent/critical-audit/2026-09-05-learning-system/runtime-inventory.json new file mode 100644 index 000000000..8949f4a8d --- /dev/null +++ b/.agent/critical-audit/2026-09-05-learning-system/runtime-inventory.json @@ -0,0 +1,53 @@ +{ + "schemaVersion": 1, + "auditId": "2026-09-05-learning-system", + "repository": "agent-runtime", + "initialCommit": "a16d8a3b91481b140cb552e373d5bde98b34af05", + "status": "historical_counts_preserved_original_list_unavailable", + "listUnavailable": true, + "paths": null, + "perFileHashes": null, + "coverageByFile": null, + "counts": { + "previouslyObservedUniquePaths": 55, + "production": 38, + "tests": 7, + "callers": 5, + "documentation": 5, + "productionFilesUnderSrcImprovementCompletelyRead": 24 + }, + "countConsistency": { + "sum": 55, + "expected": 55, + "consistent": true + }, + "evidence": { + "kind": "Previously verified inventory summarized in the session continuation record.", + "originalPath": "/tmp/learning-system-audit-20260905/runtime-inventory.json", + "originalAvailable": false, + "loss": "The environment refresh removed the scratch file before artifact persistence.", + "durableScopeDescription": "https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md#L34", + "durableMainReport": { + "path": "docs/research/learning-system-audit-2026-09-05.md", + "sha256": "a093f4b696dc211b485174f0c8f29b9a0c22d255bee968b1219bcd1425494458", + "commit": "7fe89641322dc1d0e60dc8c4d66472d786320558", + "url": "https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md" + } + }, + "limitations": [ + "The original 55-path list and per-file hashes cannot be recovered from the retained evidence.", + "No list of the remaining 31 files has been reconstructed.", + "The 24-file complete-read count is a prior audit observation, not a new read performed while creating this artifact.", + "External caller inspections used workspace snapshots rather than fetched main." + ], + "latestIntegration": { + "mainCommit": "2707e2321e7b26b0e71efacb86ebdf3ddd23adac", + "integrationCommit": "7f05654a69441db1bd135e8545787af785cde4a2", + "changedFiles": [ + "AGENTS.md", + "CLAUDE.md" + ], + "reportAndCohortCommit": "7fe89641322dc1d0e60dc8c4d66472d786320558", + "scope": "Instruction-only main integration; this metadata update does not add files to the original audit inventory." + } +} diff --git a/.agent/critical-audit/2026-09-05-learning-system/summary.md b/.agent/critical-audit/2026-09-05-learning-system/summary.md new file mode 100644 index 000000000..d7adc58e5 --- /dev/null +++ b/.agent/critical-audit/2026-09-05-learning-system/summary.md @@ -0,0 +1,93 @@ +# Learning system audit + +**Approve the repairs for the 22 identified source defects: 12 HIGH and 10 MEDIUM.** +This verdict does not establish continuing learning, meta-learning, or state-of-the-art performance. +Runtime CI, merge, and publication remain pending separately. +Knowledge 14.0.0 is live; its retained owner proof and terminal CI results are embedded in [manifest.json](manifest.json). +Eval 0.174.0 is live; its retained release proof is embedded in [manifest.json](manifest.json). + +The stronger direction is a reusable domain learning process that changes specialists, working evaluations, and its own procedure. +Reuse exact executable candidates, common measurement records, retained experience, and exact adoption. +Preserve alternative search methods and the three packages' distinct responsibilities. +The broader joined process is proposed, not implemented by these defect repairs. +See the [committed full analysis](https://github.com/tangle-network/agent-runtime/blob/7fe89641322dc1d0e60dc8c4d66472d786320558/docs/research/learning-system-audit-2026-09-05.md). + +The initial revisions are Runtime `a16d8a3b91481b140cb552e373d5bde98b34af05`, Eval `f8e3da285b6286386699a196733e9c0c27c20cfd`, and Knowledge `390f2da9883e55cc86a8985167324d8b1f10894a`. +Every source link below pins that original revision. +Regression links identify committed repaired source; Knowledge links pin merged commit `30878397a947c37fd743929b30aa03728e785c76`. + +The original Runtime inventory recorded 55 files: 38 production, seven tests, five callers, and five documents. +All 24 production files under `src/improvement/` were read completely. +The environment refresh removed that inventory and earlier scratch logs. +[runtime-inventory.json](runtime-inventory.json) preserves the prior counts and explicitly marks its path list and hashes unavailable. +Eval and Knowledge complete inspected-file counts are unknown. +External callers were workspace snapshots, not fetched main. + +These are controlled implementation observations using deterministic callbacks, real Git/filesystem operations, and Linux knowledge snapshots where required. +The two instruction mismatches are identified separately below. +No paid learning experiment or production incidence study ran. +Both cost columns use USD/month; null means unmeasured, not zero. +Synthetic accounting receipts do not estimate monthly costs. + +| Rank / ID | Severity | Original source | Trigger and evidence | Repair | Regression / proof IDs | Failure cost USD/month | Repair cost USD/month | +| --- | --- | --- | --- | --- | --- | --- | --- | +| 1 / E1 | HIGH | [E:src/contract/self-improve.ts:671](https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/contract/self-improve.ts#L671) | Measured: WIN selected at 1 versus 0.6; reranking returns BASE; WIN gets zero final executions. | Execute the complete method directly and compare its selected candidate without a second search or fabricated native history. | [contract-self-improve-method-integrity.test.ts:112](https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L112); E-full, E-ci | null | null | +| 2 / E3 | HIGH | [E:src/campaign/presets/compare-optimization-methods.ts:350](https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/campaign/presets/compare-optimization-methods.ts#L350) | Measured: 2 failed cells disappear from 4; score 1 and lift +0.5 remain. | Validate exact case, repetition, and judge coverage through the common final comparison path. | [contract-self-improve-method-integrity.test.ts:328](https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L328); E-full, E-ci | null | null | +| 3 / E4 | HIGH | [E:src/campaign/presets/run-optimization.ts:239](https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/campaign/presets/run-optimization.ts#L239) | Measured: BAD inherits GOOD score 1 and ship with zero new executions. | Include the measured surface in campaign and search cache identity. | [contract-self-improve-method-integrity.test.ts:356](https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L356); E-full, E-ci | null | null | +| 4 / E5 | HIGH | [E:src/campaign/presets/run-optimization.ts:688](https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/campaign/presets/run-optimization.ts#L688) | Measured: old-judge baseline 0 versus candidate 0.5; new-judge baseline would be 1. | Require the full evaluator and execution revision for an imported baseline. | [contract-self-improve-method-integrity.test.ts:386](https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L386); E-full, E-ci | null | null | +| 5 / E6 | HIGH | [E:src/contract/self-improve.ts:676](https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/contract/self-improve.ts#L676) | Measured synthetic receipts: $17 becomes $0 complete; repair review also found $3 reported as $0. | Reconcile method reports with their attributed calls, retain incomplete accounting, and isolate concurrent methods' costs. | [contract-self-improve-method-integrity.test.ts:138](https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L138); E-full, E-ci, E-review-probes | null | null | +| 6 / R2 | HIGH | [R:src/runtime/strategy-evolution.ts:397](https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/runtime/strategy-evolution.ts#L397) | Measured: changed experiment resumes the obsolete promoted result with zero authors or benchmark phases. | Bind checkpoints to exact serializable experiment inputs, explicit executionRef, and authored module bytes. | [strategy-evolution.test.ts:672](https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/strategy-evolution.test.ts#L672); R-full, R-original-focused | null | null | +| 7 / R4 | HIGH | [R:src/runtime/observe.ts:235](https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/runtime/observe.ts#L235) | Measured: malformed observation JSON becomes an empty clean result. | Validate the complete response and expose parse or validation failure; accept explicit valid empty findings. | [runtime-observe.test.ts:59](https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/runtime-observe.test.ts#L59); R-full, R-original-focused | null | null | +| 8 / R5 | HIGH | [R:src/runtime/observe.ts:203](https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/runtime/observe.ts#L203) | Measured: 1 run, 1 finding, 0 learned records, and 0 reported storage failures. | Propagate the acknowledged storage error through the existing per-run failure channel. | [runtime-observe.test.ts:111](https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/runtime-observe.test.ts#L111); R-full, R-original-focused | null | null | +| 9 / R6 | HIGH | [R:src/runtime/personify/corpus.ts:213](https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/runtime/personify/corpus.ts#L213) | Measured: 2 conflicting same-ID writes are acknowledged; the next read rejects the log. | Serialize the read/check/append transaction across processes, canonicalize aliases, and retain typed storage failures. | [corpus-integrity.test.ts:91](https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/corpus-integrity.test.ts#L91); R-full, R-original-focused | null | null | +| 10 / R8 | HIGH | [R:src/improvement/reflective-generator.ts:28](https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/improvement/reflective-generator.ts#L28) | Measured with Git patches: cancellation and failures disappear; partial or undeclared edits cross candidate boundaries. | Draft against each exact incumbent, propagate cancellation and errors, validate bases and declared paths, and apply the complete batch atomically. | [improvement-driver.test.ts:177](https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/improvement-driver.test.ts#L177); R-full, R-reflective, R-reflective-independent | null | null | +| 11 / K1 | HIGH | [K:src/kb-improvement/evaluation.ts:123](https://github.com/tangle-network/agent-knowledge/blob/390f2da9883e55cc86a8985167324d8b1f10894a/src/kb-improvement/evaluation.ts#L123) | Measured Linux lifecycle: acquisition and update precede diagnosis and lack its findings. | Carry one lifecycle state from diagnosis through construction and final measurement; extract reusable internal phase execution. | [lifecycle.test.ts:40](https://github.com/tangle-network/agent-knowledge/blob/30878397a947c37fd743929b30aa03728e785c76/tests/kb-improvement/lifecycle.test.ts#L40); K-linux-8 | null | null | +| 12 / K3 | HIGH | [K:src/verified-research-loop.ts:327](https://github.com/tangle-network/agent-knowledge/blob/390f2da9883e55cc86a8985167324d8b1f10894a/src/verified-research-loop.ts#L327) | Measured real driver: research stops after 1 of 4 allowed rounds while incomplete. | Respect driver completion and persist pending steering while preserving storage-only driver behavior. | [research-driving-loop.test.ts:249](https://github.com/tangle-network/agent-knowledge/blob/30878397a947c37fd743929b30aa03728e785c76/tests/loops/research-driving-loop.test.ts#L249); K-host-74 | null | null | +| 13 / E2 | MEDIUM | [E:src/campaign/presets/run-optimization.ts:418](https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/campaign/presets/run-optimization.ts#L418) | Measured: unchanged baseline throws a duplicate-candidate error. | Accept the unchanged selected baseline as a complete-method outcome without inventing candidate history. | [contract-self-improve-method-integrity.test.ts:46](https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L46); E-full, E-ci | null | null | +| 14 / E7 | MEDIUM | [E:src/campaign/presets/run-optimization.ts:553](https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/campaign/presets/run-optimization.ts#L553) | Measured: [1,0,0,0] is reported as mean 0.25 with interval [0.25,0.25]. | Represent unestimated uncertainty as null and retain actual final-comparison statistics. | [contract-self-improve-method-integrity.test.ts:425](https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L425); E-full, E-ci | null | null | +| 15 / E8 | MEDIUM | [E:src/contract/self-improve.ts:835](https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/contract/self-improve.ts#L835) | Measured synthetic receipts: 2 executions/$2 become 4 executions/$4. | Include identical final campaigns once in both complete-method and native-proposer reporting. | [contract-self-improve-method-integrity.test.ts:71](https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/tests/contract-self-improve-method-integrity.test.ts#L71); E-full, E-ci, E-review-probes | null | null | +| 16 / E9 | MEDIUM | [E:src/fuzz/explorer.ts:144](https://github.com/tangle-network/agent-eval/blob/f8e3da285b6286386699a196733e9c0c27c20cfd/src/fuzz/explorer.ts#L144) | Measured: 40 evaluations; round-two 10/10 allocation changes to 8/12 after repair. | Pool allocation observations by search cell while retaining exact scenario IDs in stored records. | [fuzz-agent.test.ts:198](https://github.com/tangle-network/agent-eval/blob/6c23e86025eec42b4d5f551f9cb4896f474cb16b/src/fuzz/fuzz-agent.test.ts#L198); E-full, E-ci | null | null | +| 17 / R1 | MEDIUM | [R:src/improvement/agentic-generator.ts:1020](https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/improvement/agentic-generator.ts#L1020) | Measured with Git: a tracked diagnosis-only edit is accepted as substantive code. | Parse Git porcelain records without trimming status columns or path bytes. | [agentic-generator.test.ts:491](https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/agentic-generator.test.ts#L491); R-full, R-original-focused | null | null | +| 18 / R3 | MEDIUM | [R:src/runtime/strategy-author.ts:29](https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/runtime/strategy-author.ts#L29) | Source mismatch plus offline execution: the author teaches persona while shot consumes profile. | Teach the complete-profile argument accepted by the actual strategy API and execute the example in a regression. | [strategy-suite.test.ts:438](https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/strategy-suite.test.ts#L438); R-full, R-profile-integration | null | null | +| 19 / R7 | MEDIUM | [R:src/runtime/personify/corpus.ts:171](https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/runtime/personify/corpus.ts#L171) | Measured: caller tag mutation changes an accepted lesson. | Snapshot and freeze nested record fields before returning or awaiting storage. | [rsi-wave.test.ts:165](https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/kernel/rsi-wave.test.ts#L165); R-full, R-original-focused | null | null | +| 20 / K2 | MEDIUM | [K:src/kb-improvement/evaluation.ts:498](https://github.com/tangle-network/agent-knowledge/blob/390f2da9883e55cc86a8985167324d8b1f10894a/src/kb-improvement/evaluation.ts#L498) | Measured Linux lifecycle: 5 absent checks become 1; detached candidate-ready is not live promotion. | Omit unmeasured dimensions and record the actual scope of configured checks. | [lifecycle.test.ts:155](https://github.com/tangle-network/agent-knowledge/blob/30878397a947c37fd743929b30aa03728e785c76/tests/kb-improvement/lifecycle.test.ts#L155); K-linux-8 | null | null | +| 21 / K4 | MEDIUM | [R:src/knowledge/supervised-update.ts:134](https://github.com/tangle-network/agent-runtime/blob/a16d8a3b91481b140cb552e373d5bde98b34af05/src/knowledge/supervised-update.ts#L134) | Source conflict plus adapter invocation: the exact supervisor receives a write-forbidding worker policy. | Execute the exact caller-authored supervisor profile without appending a worker policy. | [knowledge-supervised-update.test.ts:90](https://github.com/tangle-network/agent-runtime/blob/0d2c84be6741a7074756b988eed1bc17d5fd3cff/tests/knowledge-supervised-update.test.ts#L90); R-full, R-profile-integration | null | null | +| 22 / K5 | MEDIUM | [K:src/kb-improvement/evaluation.ts:50](https://github.com/tangle-network/agent-knowledge/blob/390f2da9883e55cc86a8985167324d8b1f10894a/src/kb-improvement/evaluation.ts#L50) | Measured Linux lifecycle: enabledPhases: [] silently defeats required diagnosis or answer-quality work. | Reject required-but-disabled phases before candidate work begins. | [lifecycle.test.ts:17](https://github.com/tangle-network/agent-knowledge/blob/30878397a947c37fd743929b30aa03728e785c76/tests/kb-improvement/lifecycle.test.ts#L17); K-linux-8 | null | null | + +[findings.jsonl](findings.jsonl) retains all 31 original source anchors, 45 regression anchors, exact observations, fixes, content hashes, and uncertainty. +The manifest resolves every proof ID and preserves earlier failed attempts separately. + +| Proof | Observed result | Retention and boundary | +| --- | --- | --- | +| E-ci | 5763 passed, 3 skipped; 392 files passed, 2 skipped | Recovered [CI run](https://github.com/tangle-network/agent-eval/actions/runs/33987395502); raw log read directly | +| E-full | 5763 passed, 3 skipped; 140.77 seconds | Prior local terminal observation; raw log unavailable | +| E-focused | 185 passed in five files; 11.31 seconds | Prior focused observation; overlaps the full suite | +| E-review-probes | Costs reconcile to $3 incomplete; unchanged result retains two calls and $2 | Synthetic receipts; original scratch probes unavailable | +| R-full | 3561 passed, 9 skipped; 280 files passed, 3 skipped; 205.06 seconds | Prior terminal observation at 5d617002; raw log unavailable | +| R-earlier-full | 3559 passed, 2 timeout failures, 9 skipped; 420.09 seconds | Retained failed attempt; test deadlines corrected without changing assertions | +| R-deadline-focused | 35 passed, 1 optional jj skip; 81.73 seconds | Both affected files; separate from final full-suite count | +| R-original-focused | 146 unique passing tests | Prior observation; original command list unavailable | +| R-profile-integration | 43 passed in two files; 7.98 seconds | Exact-profile behavior; offline execution | +| R-reflective | 16 passed; 29.24 seconds | Real candidate worktrees and patch operations | +| R-reflective-independent | Undeclared patch rejected; both targets unchanged | Prior independent probe, exit zero; raw probe unavailable | +| K-host-74 | 74 passed in four files | Prior host checks; before new dependency cohort | +| K-other-consumers | 7 passed, 5 paid-network cases skipped | Three passing files and one skipped file | +| K-linux-initial | 132 passed, 1 failed out of 133 | Fixture lacked required answerQualityCostCeiling | +| K-linux-fixture-corrected | 2 passed | Focused fixture correction only | +| K-linux-8 | 8 passed; 11.27 seconds | Linux lifecycle and selected-candidate checks after required-phase repair | +| K-previous-package | Typechecks, 249-file lint, build, packed package passed | Earlier Eval 0.173.0 and Interface 2.0.0 cohort only; current proof is K-ci and K-publish | +| K-ci | 862 passed, 7 conditional skips; 88 files passed, 2 skipped; separate official optimizer step: 2 passed | [PR CI](https://github.com/tangle-network/agent-knowledge/actions/runs/34013062769); Node 22, Ubuntu, network flag enabled | +| K-publish | 857 passed, 12 conditional skips; 87 files passed, 3 skipped; separate official optimizer step: 2 passed | [Publish](https://github.com/tangle-network/agent-knowledge/actions/runs/34013280992); five live-source cases skipped because the network flag is absent | +| K-main-ci | Both jobs succeeded at merged 30878397 | [Main CI](https://github.com/tangle-network/agent-knowledge/actions/runs/34013245408); no combined count inferred | +| K-registry | Installed 14.0.0; two research rounds; expected steering and callback order; zero provider calls | Owner smoke: macOS, Node 24.11.1, Interface 2.0.0; Linux-only phase result is null | +| R-final-package | Package, source/example typechecks, 701-file lint, docs/workflow/version/Prime checks passed | Parent terminal results at 7fe89641; Runtime CI and publication pending | +| R-cohort | 128 public imports; four exact archives; Sandbox 0.36.4 and 0.37.0; process exit 0 | Parent report embedded in manifest; Interface 2.3.0; archive SHA-256 values are distinct from registry hashes | + +Focused and full results overlap and must not be added. +The earlier corrected Knowledge focused runs do not constitute a full-suite pass; the final CI and Publish runs are recorded separately. +Public CI, package, and publication checks remain distinct from source repair approval and capability evidence. + +Runtime integrated main `2707e2321e7b26b0e71efacb86ebdf3ddd23adac` through `7f05654a69441db1bd135e8545787af785cde4a2`. +That merge changes AGENTS.md and CLAUDE.md only. +The report and released dependency pins are committed at `7fe89641322dc1d0e60dc8c4d66472d786320558`. +Runtime publication remains pending. diff --git a/.agent/skill-runs.jsonl b/.agent/skill-runs.jsonl new file mode 100644 index 000000000..d7bbacd92 --- /dev/null +++ b/.agent/skill-runs.jsonl @@ -0,0 +1,2 @@ +{"timestamp":"2026-09-06 05:05:12 UTC","auditId":"2026-09-05-learning-system","skill":"/critical-audit","target":"agent-runtime, agent-eval, agent-knowledge learning process; 22 source findings; Runtime prior inventory n=55 files","verdict":"APPROVE","verdictScope":"The 22 identified source defects and their repaired regression coverage.","artifactPath":".agent/critical-audit/2026-09-05-learning-system","releaseStatus":{"eval":"0.174.0 live; retained owner proof embedded in manifest","runtime":"pending parent proof","knowledge":"pending owner proof for new dependency cohort"},"evidenceLimits":["Prior local scratch logs and original Runtime file inventory are unavailable after environment refresh.","Source repair approval does not establish continuing learning, meta-learning, or SOTA performance."],"next":"/stop","nextOwner":"Parent integration and release work continues independently."} +{"timestamp":"2026-09-06T05:25:08.806615+00:00","auditId":"2026-09-05-learning-system","skill":"/critical-audit","event":"release-proof-update","target":"Retain terminal Knowledge 14.0.0 proof and current Runtime integration/local package evidence","verdict":"APPROVE","verdictScope":"Preserves the existing 22-source-defect repair review; no new source review executed.","artifactPath":".agent/critical-audit/2026-09-05-learning-system","releaseStatus":{"eval":"0.174.0 live","knowledge":"14.0.0 live; owner registry proof and terminal CI retained","runtime":"local package/cohort checks passed; CI, merge, publication pending"},"durableReportCommit":"7fe89641322dc1d0e60dc8c4d66472d786320558","runtimeMainCommit":"2707e2321e7b26b0e71efacb86ebdf3ddd23adac","runtimeIntegrationCommit":"7f05654a69441db1bd135e8545787af785cde4a2","knowledgeRegressionAnchorsPinned":8,"evidenceLimits":["Original failed runs and unavailable scratch observations remain distinct.","Knowledge consumer Interface 2.0.0 differs from Runtime consumer Interface 2.3.0.","No new capability experiment or production incidence measurement."],"next":"/stop","nextOwner":"Parent Runtime integration and publication."}