From b74d1eaaa23ce9ee6a699f86c09c6a6fbfa75abc Mon Sep 17 00:00:00 2001 From: deepfates Date: Thu, 17 Sep 2026 12:13:48 -0700 Subject: [PATCH 1/2] Public benchmark surface: one doc, one results table, honest claims --- .github/workflows/ci.yml | 11 +- CONTRIBUTING.md | 3 + README.md | 16 +- benchmarks/RESULTS.md | 79 + benchmarks/data/GEPA_SPLITS_ATTRIBUTION.md | 25 + benchmarks/data/IFBENCH_ATTRIBUTION.md | 37 + benchmarks/data/TREC_ATTRIBUTION.md | 27 + decisions.md | 2 +- docs/BENCHMARKS.md | 223 + docs/CASE_STUDY_TREC.md | 95 +- docs/EVIDENCE.md | 254 +- docs/TRAJECTORIES.md | 36 + docs/TUTORIAL_TICKET_ROUTING.md | 40 +- .../ADAPTER_FIDELITY.md | 0 .../AUTO_EVALUATION_DIFFERENTIAL.md | 0 .../AVATAR_FIDELITY.md | 9 +- .../AX_DIFFERENTIAL.md | 0 .../COMBEE_FIDELITY.md | 0 .../CONFIDENCE_CALIBRATION.md | 4 +- .../FAILURE_RECOVERY_EVIDENCE.md | 13 +- .../INSTRUCTION_OPTIMIZER_FIDELITY.md | 6 +- docs/{internal => differentials}/MMGRPO_C1.md | 3 +- .../MULTIMODAL_FIDELITY.md | 14 +- .../PLAYBOOK_OPTIMIZER.md | 0 .../REACT_V2_FIDELITY.md | 0 docs/differentials/README.md | 40 + .../RESEARCH_LANDSCAPE.md | 2 +- .../RLM_FIDELITY.md | 2 +- .../TUTORIAL_EXAMPLE_PARITY.md | 0 .../UPSTREAM_EXAM.md | 0 .../WEIGHT_COMPOSITION_C1.md | 8 +- docs/internal/BENCHMARK_CATALOG.md | 140 - docs/internal/BENCHMARK_TRUTH.md | 1265 --- docs/internal/COVERAGE_MATRIX.md | 145 - docs/internal/IDENTITY_COMPATIBILITY.md | 25 - .../internal/OPTIMIZER_TRAJECTORY_CONTRACT.md | 30 - docs/internal/PARITY_VALIDATION_PROGRAM.md | 524 -- docs/internal/README.md | 26 - docs/internal/TRUST_AUDIT_2026-08-07.md | 128 - docs/internal/UPSTREAM_FIDELITY_AUDIT.md | 155 - examples/deployment/data/README.md | 20 + .../data/aggregate-recomputed.json | 703 ++ .../data/imp-scored-rows.json | 8337 +++++++++++++++++ .../data/upstream-scored-rows.json | 8064 ++++++++++++++++ .../recompute_compact.exs | 97 +- lib/mix/tasks/imp.benchmark.optimizer_lift.ex | 2 +- mix.exs | 18 +- priv/tutorial/SUPPORT_TICKETS_LICENSE.md | 21 + test/case_study_trec_recomputation_test.exs | 111 + test/current_dspy_state_boundary_test.exs | 2 +- test/documentation_contract_test.exs | 126 +- test/package_contract_test.exs | 8 +- test/public_surface_test.exs | 4 +- test/tutorial_parity_test.exs | 6 +- test/upstream_exam/adapters_test.exs | 2 +- test/upstream_exam/evaluate_test.exs | 2 +- test/upstream_exam/predict_test.exs | 2 +- test/upstream_exam/signatures_test.exs | 2 +- test/upstream_exam/streaming_test.exs | 2 +- test/upstream_exam/teleprompt_test.exs | 2 +- 60 files changed, 18154 insertions(+), 2764 deletions(-) create mode 100644 benchmarks/RESULTS.md create mode 100644 benchmarks/data/GEPA_SPLITS_ATTRIBUTION.md create mode 100644 benchmarks/data/IFBENCH_ATTRIBUTION.md create mode 100644 benchmarks/data/TREC_ATTRIBUTION.md create mode 100644 docs/BENCHMARKS.md rename docs/{internal => differentials}/ADAPTER_FIDELITY.md (100%) rename docs/{internal => differentials}/AUTO_EVALUATION_DIFFERENTIAL.md (100%) rename docs/{internal => differentials}/AVATAR_FIDELITY.md (86%) rename docs/{internal => differentials}/AX_DIFFERENTIAL.md (100%) rename docs/{internal => differentials}/COMBEE_FIDELITY.md (100%) rename docs/{internal => differentials}/CONFIDENCE_CALIBRATION.md (98%) rename docs/{internal => differentials}/FAILURE_RECOVERY_EVIDENCE.md (87%) rename docs/{internal => differentials}/INSTRUCTION_OPTIMIZER_FIDELITY.md (98%) rename docs/{internal => differentials}/MMGRPO_C1.md (89%) rename docs/{internal => differentials}/MULTIMODAL_FIDELITY.md (93%) rename docs/{internal => differentials}/PLAYBOOK_OPTIMIZER.md (100%) rename docs/{internal => differentials}/REACT_V2_FIDELITY.md (100%) create mode 100644 docs/differentials/README.md rename docs/{internal => differentials}/RESEARCH_LANDSCAPE.md (99%) rename docs/{internal => differentials}/RLM_FIDELITY.md (99%) rename docs/{internal => differentials}/TUTORIAL_EXAMPLE_PARITY.md (100%) rename docs/{internal => differentials}/UPSTREAM_EXAM.md (100%) rename docs/{internal => differentials}/WEIGHT_COMPOSITION_C1.md (86%) delete mode 100644 docs/internal/BENCHMARK_CATALOG.md delete mode 100644 docs/internal/BENCHMARK_TRUTH.md delete mode 100644 docs/internal/COVERAGE_MATRIX.md delete mode 100644 docs/internal/IDENTITY_COMPATIBILITY.md delete mode 100644 docs/internal/OPTIMIZER_TRAJECTORY_CONTRACT.md delete mode 100644 docs/internal/PARITY_VALIDATION_PROGRAM.md delete mode 100644 docs/internal/README.md delete mode 100644 docs/internal/TRUST_AUDIT_2026-08-07.md delete mode 100644 docs/internal/UPSTREAM_FIDELITY_AUDIT.md create mode 100644 examples/deployment/data/README.md create mode 100644 examples/matched_instruction_optimizers_trec/data/aggregate-recomputed.json create mode 100644 examples/matched_instruction_optimizers_trec/data/imp-scored-rows.json create mode 100644 examples/matched_instruction_optimizers_trec/data/upstream-scored-rows.json create mode 100644 priv/tutorial/SUPPORT_TICKETS_LICENSE.md create mode 100644 test/case_study_trec_recomputation_test.exs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 277212fe..306e9b56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -202,7 +202,7 @@ jobs: # Its self-test imports gepa_artifact from the checkout root; the # maintainer environment had that on the ambient path, runners don't. - run: PYTHONPATH="$PWD/tmp/gepa-artifact" scripts/setup_corrected_gepa_comparator.sh tmp/gepa-artifact - # Source-exact IFBench scorer venv (docs/internal/BENCHMARK_TRUTH.md). + # Source-exact IFBench scorer venv (docs/BENCHMARKS.md). - run: | python3 -m venv tmp/ifbench-parity-venv tmp/ifbench-parity-venv/bin/python -m pip install -q -r benchmarks/requirements-ifbench-parity.txt @@ -213,10 +213,11 @@ jobs: # spend their test timeout compiling. # Children run in both dev (banking77) and test (hotpotqa) envs. - run: cd examples/deployment && IMP_PATH=../.. mix deps.get && IMP_PATH=../.. mix deps.compile && IMP_PATH=../.. mix compile && IMP_PATH=../.. MIX_ENV=test mix deps.compile && IMP_PATH=../.. MIX_ENV=test mix compile - # Known-red residue at introduction (tracked on imp-sqkr): the sealed - # v1 contract detects mix.lock drift after the bandit CVE bump, and the - # MuSiQue receipts read a machine-local /tmp data dir. This job makes - # that visible on every run rather than hiding it in the fast lane. + # Re-measured 2026-09-17 on a clean tracked tree: neither residue this + # comment used to describe is still red. The mix.lock drift is gone and + # the MuSiQue receipts are excluded by their own tag. The only remaining + # failures came from Imp.UpstreamFidelityTest, which is removed with the + # rest of the conformance-ledger machinery. - run: mix differential.check docs: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2b4e9b48..34c67e80 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,6 +62,9 @@ ticket file in this repository. - Preserve upstream algorithmic semantics when using an upstream name. - Give deliberate Elixir-native alternatives a distinct contract and rationale. - Never turn fixtures, smoke tests, or symbol presence into broad parity claims. +- Every published number needs a row in `benchmarks/RESULTS.md` carrying its + dataset, license, model, provider, date, commit, and the command that + produces it; `docs/BENCHMARKS.md` says what running that command needs. - Keep changes focused and update ExDoc or Livebooks with public API changes. ## Pull Requests diff --git a/README.md b/README.md index 0af252f8..56054a58 100644 --- a/README.md +++ b/README.md @@ -189,13 +189,21 @@ protocol endpoint. See [protocol integration and migration](docs/PRODUCTION_OPER - [Production Operations](docs/PRODUCTION_OPERATIONS.md) — credentials, telemetry, concurrency, persistence, failure handling, and the protocol adapters. +- [Ticket Routing Tutorial](docs/TUTORIAL_TICKET_ROUTING.md) — score a router + on held-out data, improve it with an optimizer, and prove the improvement on + tickets it has never seen. About a cent to run yourself. - [Runnable Livebooks](livebooks/01_real_lm_front_door.livemd) — inspect the same progression in IEx-ready notebooks. +- [Benchmarks](https://github.com/deepfates/imp/blob/main/docs/BENCHMARKS.md) — every number this repository publishes, + the exact command that produces it, what that command costs you, and what + cannot be re-measured at all. The numbers themselves are one row each in + [benchmarks/RESULTS.md](https://github.com/deepfates/imp/blob/main/benchmarks/RESULTS.md). +- [Case study: GEPA and MIPROv2 on TREC](https://github.com/deepfates/imp/blob/main/docs/CASE_STUDY_TREC.md) — a matched + optimizer comparison against pinned DSPy, recomputable from committed rows + but not reproducible, and labeled that way. - [Evidence](https://github.com/deepfates/imp/blob/main/docs/EVIDENCE.md) — - the research side: how far the DSPy parity aim has actually been - established, benchmark reproductions, execution traces and portable - trajectories, and what remains a target. It lives in the source repository, - not in the installed package. + what kind of evidence stands behind which kind of claim, and the record of + the runs that did not work. Run `mix docs` for the exhaustive module and function reference. diff --git a/benchmarks/RESULTS.md b/benchmarks/RESULTS.md new file mode 100644 index 00000000..ef3ee7c8 --- /dev/null +++ b/benchmarks/RESULTS.md @@ -0,0 +1,79 @@ +# Results + +One row per published number. Every row carries the dataset it was measured on, +the dataset's license, the model and provider that produced it, the date, the +commit, and the command. Prose elsewhere in this repository cites a row here +rather than restating a number. + +A row is **re-measurable** if a stranger with an API key can run its command and +get a comparable number, and **recomputable** if the command only recomputes +statistics from committed rows. The distinction is not cosmetic: a recomputable +row cannot tell you whether the original measurement was made correctly, only +that the arithmetic on the retained rows is what the doc says it is. + +See [docs/BENCHMARKS.md](../docs/BENCHMARKS.md) for what each command needs and +for the claims that cannot be re-measured at all. + +## Re-measurable + +| # | Number | Dataset (license) | Model | Provider | Date | Commit | Command | +| --- | --- | --- | --- | --- | --- | --- | --- | +| R1 | Zero-shot held-out accuracy `0.30`–`0.50` over 3 repeats (20 held-out of 60 tickets) | `priv/tutorial/support_tickets.json`, 60 rows, sha256 `7ea5ae7a…` (written for this repository; see [SUPPORT_TICKETS_LICENSE.md](../priv/tutorial/SUPPORT_TICKETS_LICENSE.md)) | `gpt-5.4-mini` | OpenRouter route `openai/gpt-5.4-mini` | 2026-08-22 | `88d61a9c` | `OPENAI_API_KEY=… mix run scripts/tutorial_ticket_routing_experiment.exs` | +| R2 | `LabeledFewShot(k: 8)` held-out accuracy `0.95`–`1.00` over the same 3 repeats; per-repeat lift `+0.45`, `+0.65`, `+0.65` | same as R1 | `gpt-5.4-mini` | OpenRouter route `openai/gpt-5.4-mini` | 2026-08-22 | `88d61a9c` | same as R1 | + +R1 and R2 come from one execution of one command; they are two numbers from the +same three repeats, not independent measurements. That run used 120 requests, +44,365 tokens and `$0.038819` in provider-priced usage for all three repeats — +about `$0.013` and 8–9 seconds per repeat. No row errored and the in-BEAM cache +was cleared before each repeat, so all 120 calls were live. + +Three repeats of a twenty-row evaluation is a coarse instrument. The gap between +the two rows (45–65 points) is far larger than the instrument's resolution +(one row is 5 points), which is why the direction is trustworthy while the exact +endpoints are not. + +## Recomputable only + +| # | Number | Dataset (license) | Model | Provider | Date | Commit | Command | +| --- | --- | --- | --- | --- | --- | --- | --- | +| R3 | Imp GEPA minus its own baseline, held-out accuracy `+0.4000`, 95% CI `[0.2958, 0.5042]`, Holm-adjusted `p = 0.00020` | TREC fine-grained, 20 train / 40 selection / 80 held-out drawn from `benchmarks/data/confidence-calibration-trec-fine.jsonl` (see [TREC_ATTRIBUTION.md](data/TREC_ATTRIBUTION.md)) | task `gpt-5.4-mini`, optimizer `claude-sonnet-4.6` | OpenRouter | 2026-07-26 | sealed in `examples/matched_instruction_optimizers_trec/contract.json` | `mix run --no-start examples/matched_instruction_optimizers_trec/recompute_compact.exs -- examples/matched_instruction_optimizers_trec/contract.json examples/matched_instruction_optimizers_trec/data/imp-scored-rows.json examples/matched_instruction_optimizers_trec/data/upstream-scored-rows.json examples/matched_instruction_optimizers_trec/data/aggregate-recomputed.json` | +| R4 | Imp MIPROv2 minus its own baseline, held-out accuracy `+0.1458`, 95% CI `[0.0458, 0.2458]`, Holm-adjusted `p = 0.00270` | same as R3 | same as R3 | OpenRouter | 2026-07-26 | same as R3 | same as R3 | +| R5 | Imp GEPA minus pinned DSPy 3.2.1 GEPA, held-out accuracy `-0.0083`, 95% CI `[-0.0458, 0.0292]`, above the preregistered `-0.05` noninferiority margin | same as R3 | same as R3 | OpenRouter | 2026-07-26 | same as R3 | same as R3 | + +R3, R4 and R5 are three seeds (`2026072602`, `2026072603`, `2026072604`) of one +sealed experiment that used 6,491 model calls and `$3.13862325`. The command +replays gold-label checks, row scoring, source-clustered bootstrapping, Holm +correction and the noninferiority decision from the committed scored rows. It +does not re-contact a provider. The raw request-level traces (181 MB) were not +published, so nobody outside this repository can check that the committed rows +are what the providers actually returned. See +[docs/CASE_STUDY_TREC.md](../docs/CASE_STUDY_TREC.md). + +| # | Number | Dataset (license) | Model | Provider | Date | Commit | Command | +| --- | --- | --- | --- | --- | --- | --- | --- | +| R6 | Optimize Anything on three ReActV2 tool descriptions: held-out mean over 4 untouched requests `0.95` → `1.0` | 4 held-out agent requests written for `examples/deployment` (see [examples/deployment/data/README.md](../examples/deployment/data/README.md)) | task `gpt-5.4-mini`, reflection `claude-sonnet-4.6` | OpenRouter | 2026-08-23 | `8a6ce8fd` | `mix test test/deployment_agent_optimization_example_test.exs` verifies the retained result; reproducing it needs `OPENROUTER_API_KEY` and `examples/deployment/agent_optimization.exs` | + +R6 used 72 task requests for `$0.054251` and 3 reflection requests for +`$0.010494`, each under a separate one-dollar hard cap. It is one stochastic +treatment over four held-out requests: a move from 19/20 to 20/20 scoring +points. It establishes that the component-optimization, action-observation, +Artifact and restart path runs end to end. It does not establish agent +effectiveness. The retained result and Artifact are under +`examples/deployment/evidence/`. + +## Findings that are not results + +These are recorded because deleting them would misrepresent the record. None of +them can be re-measured from this repository: the raw artifacts that diagnosed +them are not all published, and in two cases the machinery that produced them +has since changed. They are dated observations, not standing claims. + +| Finding | Date | What was observed | +| --- | --- | --- | +| HotPotQA JSON-GEPA, mean lift `-0.015` over 3 seeds (task `gpt-5.4-mini`, reflection `claude-sonnet-4.6`) | 2026 | A completed treatment that did not improve held-out performance. Its 32 semantic metric calls were tiny beside the GEPA artifact's 6,871 for HotpotQA; the 24-row test moves in 0.042 steps; six strict-adapter parse failures scored zero on one seed. | +| Banking77 modeled-MIPRO, two conditions at `+0.0417` and `+0.0208` (2 of 3 improving seeds each) | 2026 | Both missed the preregistered `≥0.05` bar. Proposals, attached demos and acquisitions were real. The 48-row test and a high baseline explain the resolution limit. | +| Grue stateful-agent GEPA, 0 of 3 seeds improved (local `llama3.2:3b`) | 2026 | Every candidate scored 0.0 on every selection row, so the optimizer had no ranking signal and retained the baseline. The treatment could not answer the question it was posed. | +| IFBench scorer defect | fixed at `8c798d2e` | The scorer represented nested rule arguments incorrectly and used a non-pinned language fallback. Optimizer results produced with the faulty scorer are invalid and were withdrawn, not rescored. | +| Matched IFBench 16k rehearsal | 2026-08-20 | Both runtimes sealed six optimization-and-selection cells at 16384-token settings for `$16.42`, then the campaign stopped in the held-out phase on a 4096-token input bound of our own. It produced no held-out verdict. An earlier draft compared imp `0.8542` against upstream `0.8698`; those are different quantities (an internal champion score versus an independent re-scoring) and the comparison is withdrawn in full. | +| ChatAdapter parse failures dominate per-cell variation | 2026-08-20 | Across takes of the above, upstream logged 0–5 parse failures per 32-row evaluation (mean 8.3%), correlating with the take's mean at `r = -0.84`. Any future outcome on this design must be reported as two numbers, parse rate and score-given-parse. | +| GEPA Pareto pruning divergence | found by the recorded-tape GEPA differential | Ties were broken by an Elixir term-printing artifact rather than upstream's stable discovery order. A score comparison at the power above could not have detected it. Caught by `mix differential.check`, which is free and has no sampling noise. | diff --git a/benchmarks/data/GEPA_SPLITS_ATTRIBUTION.md b/benchmarks/data/GEPA_SPLITS_ATTRIBUTION.md new file mode 100644 index 00000000..72fbacb7 --- /dev/null +++ b/benchmarks/data/GEPA_SPLITS_ATTRIBUTION.md @@ -0,0 +1,25 @@ +# GEPA six-family splits attribution + +`gepa-campaign-full/families.json` pins the six families of the GEPA paper's +benchmark — AIMEBench, HotpotQABench, hoverBench, IFBench, LiveBenchMathBench +and Papillon — by family name, program shape, signature, upstream metric, split +row counts and a SHA-256 for each of the train, dev and test splits. + +**The split files themselves are not in this repository.** `families.json` is a +manifest of data that is absent: nothing under `benchmarks/data/` matches those +checksums, for any of the six families. Any claim that rests on running these +splits therefore cannot be re-measured here. See the "Cannot be re-measured" +section of [docs/BENCHMARKS.md](../../docs/BENCHMARKS.md). + +Upstream source, as recorded in the manifest: + +- GEPA artifact repository, commit `cbefbc1aa0f43dd39874ec4bf42211365dbda42e`: + https://github.com/gepa-ai/gepa-artifact +- GEPA paper: https://arxiv.org/abs/2507.19457 + +The underlying corpora have their own licenses, which differ per family: +HotpotQA is CC BY-SA 4.0, HoVer derives from HotpotQA and Wikipedia, and +Papillon uses the PUPA corpus. AIMEBench, IFBench and LiveBenchMathBench are +**unknown**: the artifact repository does not declare a license for them and we +have not resolved one. The manifest's `dataset_aliases` name the Hugging Face +repositories where two of the six can be obtained. diff --git a/benchmarks/data/IFBENCH_ATTRIBUTION.md b/benchmarks/data/IFBENCH_ATTRIBUTION.md new file mode 100644 index 00000000..0e2d883f --- /dev/null +++ b/benchmarks/data/IFBENCH_ATTRIBUTION.md @@ -0,0 +1,37 @@ +# IFBench rows: what is and is not IFBench here + +Two different things in this repository carry the name IFBench. They have +different provenance and only one of them is the benchmark. + +## `ifbench_instruction_following` — not IFBench data + +`mix imp.benchmark.fetch --tasks ifbench_instruction_following` writes +`ifbench_instruction_following-test-0-N.jsonl` into the output directory. Those +rows are **written in this repository**, in +`bench/imp/benchmark_truth/fetcher.ex`, under the internal dataset name +`imp/local-ifbench`, config `verifier-smoke`. They are a handful of +hand-authored instruction/constraint pairs that exercise the constraint +verifier. They are MIT, like the rest of this repository. + +They are not drawn from IFBench, they are not a sample of it, and a score on +them is not an IFBench score. Naming the task after the benchmark it was +modeled on was a mistake that this note exists to correct until the name +changes. + +## The matched IFBench experiments — real IFBench, license unknown + +`examples/matched_gepa_mipro_ifbench*/` and +`examples/matched_ifbench_rehearsal16k/` use the real IFBench task graph, +metric and splits (`IFBench_train[0:300]`, `IFBench_test`), taken from the GEPA +artifact repository: + +- https://github.com/gepa-ai/gepa-artifact +- GEPA paper: https://arxiv.org/abs/2507.19457 +- IFBench scorer dependencies are pinned in + `benchmarks/requirements-ifbench-parity.txt` + +**License: unknown.** The artifact repository declares no license for the +IFBench data, and we have not resolved one with its authors. The corpus files +themselves are not committed here; the experiment contracts pin them by digest +and the harness materializes them. The repository's MIT license makes no +statement about that corpus. diff --git a/benchmarks/data/TREC_ATTRIBUTION.md b/benchmarks/data/TREC_ATTRIBUTION.md new file mode 100644 index 00000000..eac72a3f --- /dev/null +++ b/benchmarks/data/TREC_ATTRIBUTION.md @@ -0,0 +1,27 @@ +# TREC question-classification subset attribution + +`confidence-calibration-trec-fine.jsonl` is a 400-row materialization (200 +calibration, 200 held-out) of the TREC question-classification corpus with +fine-grained labels. `benchmarks/data/confidence-calibration-trec-fine.provenance.json` +records the two source files, their SHA-256 digests, the deduplication and +overlap rules, and the selection seed; +`benchmarks/data/build_confidence_calibration_trec.py` rebuilds it. + +The matched GEPA/MIPROv2 experiment in +`examples/matched_instruction_optimizers_trec/` draws its 20 train, 40 +selection and 80 held-out rows from this file. Those splits are listed by +source id in that example's `contract.json`. + +Source: + +- Project: https://cogcomp.seas.upenn.edu/Data/QA/QC/ +- Files: `train_5500.label`, `TREC_10.label` +- Dataset card: https://huggingface.co/datasets/CogComp/trec +- Papers: Li and Roth, COLING 2002; Hovy et al., HLT 2001 + +**License: unknown.** The pinned dataset card states no license, and the +project page distributes the label files without one. The provenance file +records this as `"unknown (as reported by the pinned dataset card)"` rather +than guessing. The repository's MIT license covers our code and our derived +split files; it makes no statement about the underlying corpus. If you intend +to redistribute these rows, resolve the license with the corpus authors first. diff --git a/decisions.md b/decisions.md index 294b426d..936b1ab0 100644 --- a/decisions.md +++ b/decisions.md @@ -29,7 +29,7 @@ not necessarily when it was made. | 2026-07-20 | `Imp.Adapter.Chat.parse/3` fails loudly when the completion's `[[ ## field ## ]]` sections do not cover every output field: no single-output leniency, no in-parse JSON decode. | `lib/imp/adapter/chat.ex` `do_parse/2`, `test/silent_failure_regressions_test.exs`. DSPy `__call__` answers a failed ChatAdapter parse with a second call through the JSON adapter; a lenient parse returns the whole completion as the answer and that fallback never fires. | In force. | Does not retire. | | 2026-07-18 | `mix check` is the merge signal. Provider-backed (`:live`), integration, protocol, package, differential and evidence-infrastructure runs are separate gates because they need credentials, datasets, external services or spend. A green default run is not evidence for a fidelity claim. | `.github/workflows/ci.yml` header (commit `dc9d95f7`), restated in `AGENTS.md` "Checks" and `CONTRIBUTING.md` (2026-08-22). | In force. | Does not retire. | | 2026-08-22 | Research campaigns and evidence receipts are not release gates. | `.github/workflows/ci.yml` header. | In force. | Does not retire. | -| 2026-08-07 | The pinned-DSPy differential lane (`mix differential.check`) runs per PR with known-red residue kept visible: the sealed v1 contract detects `mix.lock` drift after the bandit CVE bump, and the MuSiQue receipts read a machine-local `/tmp` data directory. | `.github/workflows/ci.yml` `differential` job comment. Visible red beats hidden red. | Known red, deliberate. | Both residues are fixed; the comment and this line go with them. | +| 2026-08-07 | The pinned-DSPy differential lane (`mix differential.check`) runs per PR with known-red residue kept visible: the sealed v1 contract detects `mix.lock` drift after the bandit CVE bump, and the MuSiQue receipts read a machine-local `/tmp` data directory. | `.github/workflows/ci.yml` `differential` job comment. Visible red beats hidden red. | Retired 2026-09-17: re-run on a clean tracked tree, neither residue is red any more. | Retired. | | 2026-07-17 | CI dependency cache: exact `mix.lock` key with no prefix fallback; `package.check` is the only writer; every job warms `~/.hex` to the full lock closure. | `.github/actions/elixir-setup/action.yml`. Four cache-poisoning incidents from partial ancestor caches saved under new keys. | In force. | `package.check`'s offline clean-room no longer needs a complete `~/.hex`, or `actions/cache` starts re-saving on exact hits. | | 2026-07-17 | Evidence-infrastructure tests (`:evidence_infrastructure`) are excluded from the default `mix test`. | `CONTRIBUTING.md`. They need full git history, pinned DSPy Python environments and sometimes provider credentials, none of which a fresh clone has. | In force. | Does not retire. | | 2026-07-10 | ReqLLM is the provider transport boundary; upstream algorithm names keep upstream semantics; deliberate Elixir-native alternatives get a distinct contract and rationale; fixtures and symbol presence never become parity claims. | `CONTRIBUTING.md` "Design Standard". | In force. | Does not retire. | diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md new file mode 100644 index 00000000..3875d0fe --- /dev/null +++ b/docs/BENCHMARKS.md @@ -0,0 +1,223 @@ +# Benchmarks + +Every number this repository publishes is in one table, +[benchmarks/RESULTS.md](https://github.com/deepfates/imp/blob/main/benchmarks/RESULTS.md), +with the command that produces it. This page says what each of those commands +needs from you — key, Python environment, time, rough cost — and, at the end, +what we claim that you cannot check. + +Nothing here is a release score. There is no aggregate, no grade, and no +dashboard. A benchmark tells you about the task, the model and the budget it +ran on; carrying it further is your judgment, not ours. + +## The three kinds of check + +**Differentials** compare Imp against a pinned upstream — DSPy 3.2.1 at commit +`29448ae12756abdd14bd8796c819247ebb83673c`, GEPA 0.1.4 — on deterministic +inputs. They need a Python environment, not an API key. They cost nothing, take +minutes, and have no sampling noise. When they disagree, one of the two +implementations is wrong, and the disagreement is exact enough to point at the +line. This is where fidelity evidence comes from. + +**Provider-free runners** execute Imp's own machinery — optimizers, scorers, +retrievers, the failure campaign — against fixtures or a deterministic oracle. +They need neither a key nor Python. They prove that the machinery runs and that +its outputs are stable; they say nothing about whether a model got the answer +right. + +**Live measurements** call a real provider and cost real money. There are two +of them in this repository that a stranger can run: the ticket-routing tutorial +(about a cent) and the agent-optimization deployment example (about seven +cents). Everything else that was measured live was measured once, by us, and is +recorded as a retained result rather than a command you can repeat cheaply. + +## Live: what you can measure yourself + +### Ticket routing (rows R1, R2) + +```sh +OPENAI_API_KEY=… mix run scripts/tutorial_ticket_routing_experiment.exs +``` + +Also accepts `OPENROUTER_API_KEY` with `openai/gpt-5.4-mini`. Set +`TUTORIAL_REPEATS=1` for a single repeat. + +- **Needs:** an OpenAI or OpenRouter key. No Python, no dataset download; the + sixty tickets ship in the package at `priv/tutorial/support_tickets.json`. +- **Time:** 8–9 seconds per repeat, three repeats by default. +- **Cost:** about `$0.013` per repeat at `gpt-5.4-mini` prices; the script + enforces a hard budget of `$1.00` per repeat and refuses to exceed it. +- **What it measures:** held-out accuracy of a four-way enum router before and + after `LabeledFewShot(k: 8)`, on twenty tickets the optimizer never saw. +- **What it does not measure:** anything about your data, and anything about + the search optimizers. Twenty rows move in 5-point steps. + +This is the one end-to-end effectiveness number an outsider can reproduce from +scratch. [The tutorial](TUTORIAL_TICKET_ROUTING.md) walks through the same +experiment as ordinary library code. + +### Agent optimization (row R6) + +```sh +cd examples/deployment && OPENROUTER_API_KEY=… mix run agent_optimization.exs +``` + +- **Needs:** an OpenRouter key with access to `openai/gpt-5.4-mini` and + `anthropic/claude-sonnet-4.6`. +- **Cost:** about `$0.065`, under two separate one-dollar hard caps. +- **What it measures:** whether Optimize Anything can improve three ReActV2 + tool descriptions, scored on the actual ordered tool calls, results, + termination and final answer of four held-out requests — not on model prose. +- **What it does not measure:** agent effectiveness. Four requests is a + 4-point instrument, and the result moved one point. + +`mix test test/deployment_agent_optimization_example_test.exs` checks the +retained result and Artifact without spending anything. + +## Differentials: free, deterministic, need Python + +All of these run under one command: + +```sh +mix differential.check +``` + +which is `mix test --raise --only dspy_parity`. Before it will run you need the +pinned upstream sources and virtual environments: + +```sh +scripts/setup_dspy_parity_env.sh +scripts/setup_dspy_stable_source.sh +git clone --filter=blob:none https://github.com/gepa-ai/gepa.git tmp/gepa-v0.1.4 +git -C tmp/gepa-v0.1.4 checkout --detach v0.1.4 +PYTHONPATH="$PWD/tmp/gepa-artifact" scripts/setup_corrected_gepa_comparator.sh tmp/gepa-artifact +python3 -m venv tmp/ifbench-parity-venv +tmp/ifbench-parity-venv/bin/python -m pip install -r benchmarks/requirements-ifbench-parity.txt +``` + +Python 3.12, a full (non-shallow) clone, and `zsh` on the path. About fifteen +minutes of setup, under a minute to run, no API key, no cost. The exact +sequence CI uses is the `differential` job in `.github/workflows/ci.yml`. + +Individual differentials can be run on their own, and each has a note saying +what it compares and what it deliberately does not: + +| Command | Compares | Note | +| --- | --- | --- | +| `mix parity.check` | Rendered messages and per-call request envelopes, byte for byte, against DSPy 3.2.1 | [Adapter fidelity](https://github.com/deepfates/imp/blob/main/docs/differentials/ADAPTER_FIDELITY.md) | +| `mix benchmark.instruction_optimizer.contract.check` | MIPROv2 and SIMBA control flow against pinned upstream | [Instruction optimizer fidelity](https://github.com/deepfates/imp/blob/main/docs/differentials/INSTRUCTION_OPTIMIZER_FIDELITY.md) | +| `mix benchmark.gepa.contract.check` | GEPA 0.1.4 reflective datasets, reflection prompts, module rotation, stopping decisions, on a recorded tape | [ComBee-style aggregation](https://github.com/deepfates/imp/blob/main/docs/differentials/COMBEE_FIDELITY.md) | +| `mix imp.benchmark.auto_evaluation_contract` | `SemanticF1` and `CompleteAndGrounded` scoring contracts | [Auto-evaluation differential](https://github.com/deepfates/imp/blob/main/docs/differentials/AUTO_EVALUATION_DIFFERENTIAL.md) | +| `mix imp.benchmark.avatar_actor_differential`, `mix imp.benchmark.avatar_optimizer_differential` | Avatar actor and trajectory optimizer | [Avatar fidelity](https://github.com/deepfates/imp/blob/main/docs/differentials/AVATAR_FIDELITY.md) | +| `mix imp.benchmark.mmgrpo_differential` | `Imp.Optimizer.GRPO` against DSPy's `GRPO.compile/3` | [mmGRPO differential](https://github.com/deepfates/imp/blob/main/docs/differentials/MMGRPO_C1.md) | +| `mix imp.benchmark.bootstrap_finetune_differential`, `mix imp.benchmark.better_together_differential` | BootstrapFinetune and BetterTogether | [Weight composition](https://github.com/deepfates/imp/blob/main/docs/differentials/WEIGHT_COMPOSITION_C1.md) | +| `mix benchmark.rlm.contract.check` | Recursive Language Model operational contracts | [RLM fidelity](https://github.com/deepfates/imp/blob/main/docs/differentials/RLM_FIDELITY.md) | +| `mix imp.benchmark.ax_contract` | Imp against Ax, an independent TypeScript implementation | [Ax differential](https://github.com/deepfates/imp/blob/main/docs/differentials/AX_DIFFERENTIAL.md) | +| `mix imp.benchmark.rag_tool_failure_differential` | A queued action schedule through Imp and DSPy ReAct, including a retriever-tool exception | — | +| `mix imp.benchmark.hotpot_retrieval` | Document ids, top-k contexts, supporting-fact recall and extractive EM/F1 over a shared 100-document corpus | — | +| `mix imp.benchmark.bootstrap_few_shot_differential`, `mix imp.benchmark.random_search_differential` | BootstrapFewShot and BootstrapFewShotWithRandomSearch against pinned upstream | — | +| `mix imp.benchmark.ensemble_differential` | Ensemble selection and reduction | — | +| `mix imp.benchmark.optimize_anything_upstream_differential` | Optimize Anything against the upstream reflective loop | — | +| `mix imp.benchmark.rlm_runtime_differential` | RLM runtime control flow | [RLM fidelity](https://github.com/deepfates/imp/blob/main/docs/differentials/RLM_FIDELITY.md) | + +A differential proving that Imp matches DSPy on an input says nothing about +whether either one helps your program. Those are separate questions and this +page keeps them apart. + +## Provider-free runners: free, no Python + +| Command | What it exercises | +| --- | --- | +| `mix benchmark.optimizer_lift.check` | Deterministic optimizer lift on fixture tasks, including a natural classification lane | +| `mix benchmark.truth.check` | Colors, Iris, Iris-Typo, Heart Disease, hard math and local instruction-following rows end to end through fetch, run and integrity | +| `mix benchmark.failure_campaign.check` | Cancellation, bounded admission, terminal partial-stream failures, checkpoint and tamper recovery, flake rates, leak accounting | +| `mix benchmark.overhead.check` | Per-operation timing against per-case absolute and reference-relative budgets. These are measurements, not a speed claim | +| `mix benchmark.trace.check` | Golden prompt traces | +| `mix benchmark.copro_isolation.check` | COPRO candidate isolation | +| `mix benchmark.bfcl_scorer.check` | A BFCL-shaped scorer against an independent implementation. Not official BFCL performance | +| `mix imp.benchmark.confidence_calibration` | Constrained-label token confidence against empirical calibration | +| `mix imp.benchmark.playbook` | Playbook parameters through optimization and persistence | +| `mix benchmark.fast_slow.check` | The fast/slow learning handoff protocol | +| `mix imp.benchmark.multimodal_quality` | Multimodal encoding and decoding. Explicitly not live multimodal reasoning | +| `mix benchmark.operations_stress.check` | Save/load, cache hit/miss telemetry, redaction, malformed JSON/XML/chat, partial streams. A diagnostic, not a claim | +| `mix benchmark.search.check` | Search execution under bounded concurrency | +| `mix benchmark.rlm.check` | The RLM benchmark over a small fixture split | +| `mix benchmark.rag_tool_failure.check` | A retriever-tool exception through both runtimes. Not a wall-clock timeout comparison | +| `mix imp.benchmark.gepa_dataset` | Materializes what GEPA family data is present. See the absence noted below | + +`mix benchmark.parity.check` and `mix benchmark.parity.full` download GSM8K and +HotPotQA and run both runtimes over them; `parity.full` is 7,405 HotPotQA rows +and is not cheap in time. `mix benchmark.live.check` runs two rows of each +against a live provider and needs a key. `mix imp.benchmark.local_mlx` needs a +local MLX model rather than a provider. + +## Statistics you can recompute, but not reproduce + +### Matched GEPA and MIPROv2 on TREC (rows R3, R4, R5) + +```sh +mix run --no-start \ + examples/matched_instruction_optimizers_trec/recompute_compact.exs -- \ + examples/matched_instruction_optimizers_trec/contract.json \ + examples/matched_instruction_optimizers_trec/data/imp-scored-rows.json \ + examples/matched_instruction_optimizers_trec/data/upstream-scored-rows.json \ + examples/matched_instruction_optimizers_trec/data/aggregate-recomputed.json +``` + +Free, offline, about a minute including compilation. It re-derives the gold +label checks, row scoring, source-clustered bootstrap, Holm correction and the +noninferiority decision from committed scored rows, prints what it computed, +and exits non-zero if any of it disagrees with the committed aggregate. + +What it cannot do is tell you that the committed rows are what the providers +returned. The raw request-level traces are 181 MB and were not published, so +the step between "a model answered" and "a row says it answered this" is taken +on trust. That is the whole distinction between recomputable and reproducible, +and [the case study](CASE_STUDY_TREC.md) states it in its first paragraph. + +## Cannot be re-measured + +Stated plainly, because the alternative is implying these are checkable. + +**The GEPA six-family campaign.** `benchmarks/data/gepa-campaign-full/families.json` +pins AIMEBench, HotpotQABench, hoverBench, IFBench, LiveBenchMathBench and +Papillon with per-split SHA-256 digests and row counts. The split files are +absent — not three of six, none of them. Nothing in this repository can run that +campaign, and no result from it is claimed. See +[the attribution note](https://github.com/deepfates/imp/blob/main/benchmarks/data/GEPA_SPLITS_ATTRIBUTION.md). + +**The matched IFBench 16k rehearsal.** It stopped in the held-out phase and +produced no verdict. Its earlier reported comparison was withdrawn in full +because the two figures were different quantities. The stopped run's records +are a dated observation in RESULTS.md, not a result. + +**The historical negatives** — HotPotQA JSON-GEPA, Banking77 modeled-MIPRO, the +Grue stateful-agent run, the IFBench scorer defect. Each was a real run whose +retained artifacts diagnosed it. They are in RESULTS.md as findings with their +dates. Re-running them would cost money, would not reproduce the same numbers, +and in the IFBench case would run against a scorer that has since been fixed. + +**The `ifbench_instruction_following` task is not IFBench.** Its rows are +written in `bench/imp/benchmark_truth/fetcher.ex`. A score on them is a score +on our verifier smoke rows. See +[the note](https://github.com/deepfates/imp/blob/main/benchmarks/data/IFBENCH_ATTRIBUTION.md). + +**Anything about your task.** Imp does not claim an optimizer helps a program +until a held-out result on that program says so. Two tasks in this repository +have such a result; both are small; neither is yours. + +## Dataset licenses + +| Data | License | +| --- | --- | +| `priv/tutorial/support_tickets.json` | MIT — [written here](https://github.com/deepfates/imp/blob/main/priv/tutorial/SUPPORT_TICKETS_LICENSE.md) | +| `benchmarks/data/hotpotqa-validation-0-10.jsonl` | CC BY-SA 4.0 — [attribution](https://github.com/deepfates/imp/blob/main/benchmarks/data/HOTPOTQA_ATTRIBUTION.md) | +| `benchmarks/data/grpo-usefulness-banking77-v1.json`, `examples/deployment/data/banking77-*.json` | CC BY 4.0 | +| `examples/deployment/data/hotpotqa-gepa/` | CC BY-SA 4.0 | +| `benchmarks/data/rlm/` | declared in `provenance.json` | +| `benchmarks/data/confidence-calibration-trec-fine.jsonl` | **unknown** — [attribution](https://github.com/deepfates/imp/blob/main/benchmarks/data/TREC_ATTRIBUTION.md) | +| GEPA six-family splits | **unknown** for three of six families — [attribution](https://github.com/deepfates/imp/blob/main/benchmarks/data/GEPA_SPLITS_ATTRIBUTION.md) | +| IFBench corpus used by the matched examples | **unknown** — [attribution](https://github.com/deepfates/imp/blob/main/benchmarks/data/IFBENCH_ATTRIBUTION.md) | + +Imp's MIT license covers our code and our derived split files. It does not +replace the license of any upstream corpus. diff --git a/docs/CASE_STUDY_TREC.md b/docs/CASE_STUDY_TREC.md index 08da9e33..10f020d4 100644 --- a/docs/CASE_STUDY_TREC.md +++ b/docs/CASE_STUDY_TREC.md @@ -1,22 +1,31 @@ # Case study: GEPA and MIPROv2 on TREC -This case study answers one deliberately narrow question: on a frozen -classification task, did Imp's GEPA or MIPROv2 implementation improve its own -baseline, and did the winning Imp optimizer remain within a declared margin of -the pinned DSPy implementation? - -The answer was yes for this task. It is not evidence that either optimizer will -improve every program, or that Imp is generally better than DSPy. +**This result is recomputable, not reproducible.** You can re-derive every +statistic below from the scored rows committed in this repository, and the +command that does it is in the next section. You cannot re-run the experiment: +the raw provider responses behind those rows are 181 MB of request-level +evidence that was not published, so the step from "a model answered" to "a row +records this answer" is taken on our word. Nobody outside this repository can +check it. We publish the result anyway, labeled this way, because a withheld +result and an unlabeled one are both worse. + +The narrow question it answers: on a frozen classification task, did Imp's GEPA +or MIPROv2 implementation improve its own baseline, and did the winning Imp +optimizer stay within a declared margin of pinned DSPy? Yes, for this task. It +is not evidence that either optimizer will improve every program, or that Imp +is generally better than DSPy. ## What was compared -The experiment used: - -- a balanced TREC classification task with opaque output labels; +- a balanced TREC fine-grained classification task with opaque output labels + (license **unknown**; see + [the attribution note](https://github.com/deepfates/imp/blob/main/benchmarks/data/TREC_ATTRIBUTION.md)); - disjoint sets of 20 training, 40 selection, and 80 held-out examples; - three fixed seeds; -- GPT-5.4 Mini for task calls and Claude Sonnet 4.6 for optimizer calls; -- Imp GEPA and MIPROv2 against pinned DSPy 3.2.1 and GEPA 0.1.4; and +- GPT-5.4 Mini for task calls and Claude Sonnet 4.6 for optimizer calls, both + through OpenRouter; +- Imp GEPA and MIPROv2 against pinned DSPy 3.2.1 + (`29448ae12756abdd14bd8796c819247ebb83673c`) and GEPA 0.1.4; and - a preregistered `-0.05` noninferiority margin for the winning optimizer. Program selection happened before held-out rows were opened. The completed @@ -24,20 +33,15 @@ treatment used 6,491 model calls and `$3.13862325` in provider-reported cost. ## What happened -Across the three seeds: - -- Imp GEPA improved held-out accuracy over its baseline by `+0.4000`. Its - source-clustered 95% interval was `[0.2958, 0.5042]`, with Holm-adjusted - `p = 0.00020`. -- Imp GEPA differed from pinned DSPy GEPA by `-0.0083`. Its 95% interval was - `[-0.0458, 0.0292]`, above the declared `-0.05` margin. -- Imp MIPROv2 improved held-out accuracy over its baseline by `+0.1458`. Its - interval was `[0.0458, 0.2458]`, with Holm-adjusted `p = 0.00270`. +Rows R3, R4 and R5 in +[benchmarks/RESULTS.md](https://github.com/deepfates/imp/blob/main/benchmarks/RESULTS.md) +carry the three numbers, their intervals, their p-values and their provenance. +In summary: GEPA improved over its own baseline and satisfied the frozen +headline; MIPROv2 also improved over its own baseline; and the winning +optimizer stayed above the declared noninferiority margin against DSPy. -GEPA therefore satisfied the frozen headline. MIPROv2 also produced positive -own-baseline evidence, but its Imp-versus-DSPy difference varied substantially -across the three seeds. Three seeds remain limited evidence about model-sampling -uncertainty. +MIPROv2's Imp-versus-DSPy difference varied substantially across the three +seeds. Three seeds remain limited evidence about model-sampling uncertainty. ## Recompute the result @@ -52,38 +56,47 @@ mix deps.get mix run --no-start \ examples/matched_instruction_optimizers_trec/recompute_compact.exs -- \ examples/matched_instruction_optimizers_trec/contract.json \ - benchmarks/evidence/archive/matched_experiments/trec/imp-scored-rows.json \ - benchmarks/evidence/archive/matched_experiments/trec/upstream-scored-rows.json \ - benchmarks/evidence/archive/matched_experiments/trec/aggregate-recomputed.json + examples/matched_instruction_optimizers_trec/data/imp-scored-rows.json \ + examples/matched_instruction_optimizers_trec/data/upstream-scored-rows.json \ + examples/matched_instruction_optimizers_trec/data/aggregate-recomputed.json ``` -The command should end with: +It prints the three headline quantities it computed — not stored strings; the +script holds no numeric literals — and then either agrees with the committed +aggregate or prints both sides and exits non-zero. ```text -matched TREC compact recomputation passed: GEPA +0.4000, MIPROv2 +0.1458, GEPA Imp-minus-DSPy -0.0083 +Recomputed from imp-scored-rows.json and upstream-scored-rows.json: + GEPA over its own baseline: +0.4000 95% CI [0.2958, 0.5042], Holm-adjusted p = 0.00020 + MIPROv2 over its own baseline: +0.1458 95% CI [0.0458, 0.2458], Holm-adjusted p = 0.00270 + GEPA Imp minus DSPy: -0.0083 95% CI [-0.0458, 0.0292] + noninferiority margin -0.0500, winning optimizer gepa, headline passed: true + +Recomputation agrees with aggregate-recomputed.json in full. ``` -The public recomputation inputs are content-bound as follows: +The recomputation inputs are content-bound as follows: | File | SHA-256 | | --- | --- | -| `recompute_compact.exs` | `cea5f2fc6721b7cdeec1bd7edc0a97048cab644f6bcdd4e15154427b1df739f0` | +| `recompute_compact.exs` | `3b9041e97913abbc6d21f951216c0adc2b844bc075b27abe7af7cdf13acc7fde` | | `contract.json` | `0253960b8c570f0e0dd3a2a84450327f3244338fdff002f40ffed44bd9f15e94` | | `imp-scored-rows.json` | `0b6ab45639dea8f2ac6ea1c0405ef62c414a93228a8d25edd2540694f59826bd` | | `upstream-scored-rows.json` | `11706b6e1e9c7e3a09f677e3a354df69676a61fc02da19a80b2cdd7c4ec45b4c` | | `aggregate-recomputed.json` | `295f7f4a2312ccf57f2fae6a4eaf248e51e442e60588a8b769edb679b421dee1` | The command verifies gold-label checks, row scoring, source-clustered -bootstrapping, Holm correction, and the noninferiority decision from the -committed scored rows. It does not independently verify the private raw provider -responses, routing, cost, or the original selection barrier. Those raw traces -contain 181 MB of request-level evidence and are intentionally not published. +bootstrapping, Holm correction and the noninferiority decision. It does not +verify the private raw provider responses, routing, cost, or the original +selection barrier. + +See [Benchmarks](https://github.com/deepfates/imp/blob/main/docs/BENCHMARKS.md) for what every other number here costs to re-measure. ## How to use this evidence This result establishes task-, model-, and budget-specific evidence for Imp -GEPA and MIPROv2. It belongs beside the clean Banking77 and HotPotQA negatives, -not in place of them. The useful product lesson is that Imp can run a real -optimizer comparison, select before testing, preserve the result, and expose a -compact recomputation boundary. Broader optimizer usefulness remains an open -scientific question. +GEPA and MIPROv2. It belongs beside the Banking77 and HotPotQA negatives in +RESULTS.md, not in place of them. The useful product lesson is that Imp can run +a real optimizer comparison, select before testing, preserve the result, and +expose a recomputation boundary narrow enough to state honestly. Broader +optimizer usefulness remains an open scientific question. diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md index bad6e992..7fa17b58 100644 --- a/docs/EVIDENCE.md +++ b/docs/EVIDENCE.md @@ -1,161 +1,95 @@ -# Evidence: What Imp Has Actually Exercised - -This page separates API existence, semantic fidelity, real operation, and -effectiveness. The distinctions are useful; turning them into a single release -score is not. Follow the linked test, protocol, and retained result for the -claim you care about. - -## The ladder - -Each rung answers a different question. **Asserted** means maintainers intend a -statement at its named scope; it is not proof by itself. A claim that is not -asserted remains a **target**, the index's word for work not yet established. - -| Rung | What it proves | -| --- | --- | -| **C0** | The API exists and is callable. | -| **C1** | Behavior conforms to a pinned authority for the declared scope — an executable differential against real DSPy 3.3.1 for the families that have one (the optimizer and adapter families), a behavioral conformance test otherwise. | -| **C2** | The capability executes operationally through its real boundary (real transport, real process tree, real artifact round-trip). | -| **C3** | Held-out evidence supports effectiveness for the declared task portfolio — a number on data nothing selected for. | -| **C4** | An exact paper protocol is reproduced from public authoritative materials. | -| **C5** | Powered, paired evidence supports comparative advantage. | - -The rungs deliberately separate three questions that marketing language -usually blurs: *is it there* (C0), *is it faithful and operational* (C1–C2), -and *does it actually help* (C3–C5). A faithful port of an optimizer is a -different claim from that optimizer improving your program. - -## Read the current state - -Start with the documented user story and run its real path. The maintainer -release procedure requires a clean consumer to install, optimize, inspect, -persist, restart, and serve a program through the public API. For compatibility -or research claims, inspect the named authority and retained artifact directly. -There is deliberately no generated global readiness dashboard. - -Imp does not claim that an optimizer helps a task until a held-out result says -so. Task-scoped positive, neutral, negative, and stopped results keep their -exact limitations in the result artifact and linked example; a higher rung on -one task never becomes general effectiveness. - -## Where the receipts live - -- The benchmark harness lives under `lib/mix/tasks/imp.benchmark.*`, `bench/`, - and `scripts/`; each task writes its own report under `benchmarks/runs/` - when you run it. -- Upstream pins the differential harness compares against are in - `benchmarks/authorities.json`; datasets and run configuration are in - `benchmarks/data/` and `benchmarks/config/`. -- Execution traces, bounded native run observations, cancellation evidence, - and portable ATIF trajectories are described in - [Execution evidence and ATIF](TRAJECTORIES.md). - -The discipline is simple: a public claim states its scope and points to -observable evidence. Users should not need maintainer bookkeeping to decide -whether the documented workflow works for them. - -## A live executed-agent lifecycle now works - -The packaged `examples/deployment/agent_optimization.exs` is the first ordinary -Imp user story that optimizes an agent which actually takes actions. It exposes +# Evidence + +This page says what kind of evidence stands behind which kind of claim. The +numbers themselves are in +[benchmarks/RESULTS.md](https://github.com/deepfates/imp/blob/main/benchmarks/RESULTS.md), +one row each, with dataset, model, provider, date, commit and command. +[Benchmarks](https://github.com/deepfates/imp/blob/main/docs/BENCHMARKS.md) says what running any of those commands costs you +and which claims cannot be re-measured at all. + +There is deliberately no aggregate score, no readiness grade, and no dashboard. + +## Four different questions + +These get blurred together constantly, usually to a vendor's advantage. Imp +keeps them apart, and the labels below are used consistently across this +repository. + +| | The question | How it is answered here | +| --- | --- | --- | +| **C0** | Does the API exist and run? | Ordinary tests | +| **C1** | Does it behave like the pinned upstream? | A deterministic differential against DSPy 3.2.1 (`29448ae12756abdd14bd8796c819247ebb83673c`) and GEPA 0.1.4 for the families that have one; a behavioral conformance test otherwise | +| **C2** | Does it work through its real boundary? | Real transport, real process tree, real artifact round-trip | +| **C3** | Does it actually help? | A held-out number on data nothing selected for | +| **C4–C5** | Does it replicate a paper, or beat an alternative? | Not claimed for anything in this repository today | + +A faithful port of an optimizer is a different claim from that optimizer +improving your program. C1 says the first; only C3 says the second, and only +for the task, model and budget it was measured on. + +## Where fidelity evidence comes from + +The differentials. They compare Imp against pinned upstream source on +deterministic inputs, run in CI at no cost, and have no sampling noise. They +earn their keep: one of them found that GEPA's Pareto pruning broke ties by an +Elixir term-printing artifact rather than upstream's stable discovery order — a +trajectory-level defect that no end-to-end score comparison at any power we can +afford would have detected. + +`mix differential.check` runs them all. Per-family notes, saying what each one +compares and what it deliberately does not, are in +[docs/differentials/](https://github.com/deepfates/imp/tree/main/docs/differentials). + +## Where effectiveness evidence comes from + +Two held-out results, both small, both in RESULTS.md. + +Rows R1 and R2 are the ticket-routing tutorial: a zero-shot enum router scored +0.30–0.50 on twenty held-out tickets and 0.95–1.00 after `LabeledFewShot(k: 8)`, +over three live repeats at about a cent each. A stranger with an API key can run +that command and get their own numbers. It is the only end-to-end effectiveness +claim here that is reproducible from scratch. + +Row R6 is `examples/deployment/agent_optimization.exs`, the first ordinary Imp +user story that optimizes an agent which actually takes actions. It exposes three sandboxed ReActV2 tool descriptions as program components, lets Optimize -Anything propose natural replacements, selects on separate rows, evaluates on -four untouched requests, writes the selected parameter Artifact, reconstructs -the trusted tool functions in a fresh BEAM, and scores the actual ordered tool -calls, results, termination, and final answer rather than trusting model prose. - -The clean rerun at `8a6ce8fd8f5d67da90ab2defeddf7930b1a2a851` improved the -held-out mean from `0.95` to `1.0`. Its selected billing-action description -removed the baseline's unnecessary account lookup on the refund case, and the -selected Artifact scored `1.0` after application to freshly reconstructed -trusted tools in a second BEAM. The run used 72 task requests and three -reflection requests for `$0.054251` and `$0.010494` respectively, under -separate one-dollar hard caps. - -This is one stochastic treatment over four held-out requests. It exercised a -real component-optimization, action-observation, Artifact, and restart path; it -does not establish general agent effectiveness, external-side-effect safety, -multi-seed optimizer effectiveness, or DSPy/Ax parity. The -[result](../examples/deployment/evidence/agent-optimization-result.json) -(`6293a68b…`) binds the disjoint row identities, ordered action observations, -budgets, source commit, and the retained parameter -[Artifact](../examples/deployment/evidence/agent-optimization-artifact.json) -(`302c0bad…`). A normal example test verifies the pair, applies the Artifact to -trusted code, and scans both files for credential-shaped content. These files -belong to the executable deployment story rather than the comparative research -registry; their narrow result must not be promoted into a broad C3 claim. - -## Historical red results and their classifications - -Under the parity frame — pinned DSPy/GEPA/MIPRO are replicated results, so a -faithful port should match them — each historical negative was diagnosed from -its retained artifacts as either a config that could not show lift even -upstream (benign) or an Imp fidelity defect (bug). - -| Negative | Verdict | Why | -|---|---|---| -| HotPotQA JSON-GEPA, mean lift −0.015 (3 seeds, GPT-5.4-mini task / Sonnet 4.6 reflection) | **Scientific negative for this treatment** | The declared treatment completed and did not improve held-out performance. Its 32 semantic metric calls were tiny beside the GEPA artifact's 6,871 for HotpotQA, the 24-row test moves in 0.042 steps, and six strict-adapter parse failures scored zero on one seed. Those facts motivate a different future treatment; they do not turn this completed negative into a positive or a product defect. | -| Banking77 modeled-MIPRO, two conditions missed the ≥0.05 bar | **Scientific negative against the preregistered bar** | Both conditions had small positive means (+0.0417 and +0.0208; 2/3 improving seeds each), but neither met the declared ≥0.05 criterion. Real proposals, attached demos, and legal acquisitions make this a clean result. The 48-row test and high baseline explain its resolution limit; they do not retroactively change the threshold. | -| Grue stateful-agent GEPA, 0/3 seeds lift (local llama3.2:3b) | **Unresolved no-signal treatment** | Every candidate scored 0.0 on every selection row, so the optimizer had no ranking signal and correctly retained baseline. Reflected candidates were real and varied. This establishes that the treatment could not answer the usefulness question, not that GEPA is ineffective or that the product is broken. | -| IFBench optimization interpretation | **Product/integration defect — fixed and disclosed** | The scorer represented nested rule arguments incorrectly and used a non-pinned language fallback; fixed source-exact at `8c798d2e` with the superseded interpretation explicitly withdrawn. | - -None of these red results is erased. Completed valid treatments keep their -negative verdicts; a no-signal treatment remains unresolved; a diagnosed defect -is repaired at the layer that owned it. A successor experiment must be named -and selected for a reason established before its outcomes are read. - -## The matched campaign: state as of 2026-08-20 - -**Nothing in this section is effectiveness or fidelity evidence.** It records -what an *engineering rehearsal* established, and — as importantly — what an -earlier draft of this section wrongly claimed. - -The 16k rehearsal is scoped by its own contract as `one_seed_engineering_rehearsal_source_faithful_config`: -it exercises machinery and cost at the benchmark authors' settings before any -larger spend, and by preregistration claims nothing about optimizer -effectiveness. On its eleventh launch both runtimes completed baseline, GEPA, -and MIPROv2 and sealed all six optimization-and-selection cells at 16384-token -settings for $16.42. The campaign itself did not complete: it stopped in the -held-out phase on an input-token bound of our own (4096, exceeded by a -4243-token prompt), so it produced no held-out optimizer verdict. Ten prior -launches stopped on harness defects, each dated in its preregistration. - -**Withdrawn.** An earlier version of this section reported "GEPA optimization -moves at source-faithful budget, and imp's magnitude matches upstream's," -citing imp 0.8542 vs upstream 0.8698. That comparison was invalid: imp's figure -was its optimizer's *internal* champion score (a maximum over noisy trials, -biased upward by selection), while upstream's was an *independent re-scoring* -of the champion program. They are different quantities. The claim is withdrawn -in full, and no lift or parity conclusion replaces it — the rehearsal is not -powered to support one. A single paired cell in this design carries roughly -±0.09; the effects at issue are 0.05–0.10. - -**A measurement worth keeping.** Across takes, per-cell score variation is -dominated not by task performance but by **ChatAdapter parse failures scored -zero** — upstream logged 0–5 such rows per 32-row evaluation (mean 8.3%), and -the count correlates with the take's mean at r = -0.84. Any future outcome -must be reported as two numbers, parse rate and score-given-parse; a single -mean silently absorbs a format-robustness effect and cannot answer a question -about optimizers. - -**A repaired parity boundary.** This rehearsal used Imp's explicitly -startup-only Optuna mode, so both arms were limited to 9 post-baseline trials; -that historical treatment remains exactly what its artifact records. Imp now -also implements the pinned modeled Optuna 4.9.0 categorical TPE path and -exercises startup, the first Bayesian opportunity, checkpoint resume, public -compile, and minibatch selection against pinned upstream behavior. A documented -floating-point tie boundary prevents a claim of bit-exact NumPy identity, but -modeled TPE is no longer the blocker to normal source-scale MIPRO use. These -differentials establish search mechanics, not broad live effectiveness; the -latter still requires natural retained optimizer lifecycles and representative -held-out evidence. - -**Where fidelity evidence actually comes from.** The C1 rung, not this campaign. -Deterministic differential tests against pinned DSPy 3.2.1 and gepa 0.1.4 — -including a recorded-tape GEPA component comparison of reflective datasets, -reflection prompts, module rotation, and stopping decisions — run in CI at no -cost and with no sampling noise. That instrument is strictly better suited to -the question, and it earns its keep: it exposed a real divergence in GEPA's -Pareto pruning (ties broken by an Elixir term-printing artifact rather than -upstream's stable discovery order), a trajectory-level defect that an -end-to-end score comparison at this power could never have detected. +Anything propose replacements, selects on separate rows, evaluates on four +untouched requests, writes the selected parameter Artifact, reconstructs the +trusted tool functions in a fresh BEAM, and scores the actual ordered tool +calls, results, termination and final answer rather than trusting model prose. +Its held-out mean moved from 0.95 to 1.0 — one point on a four-point +instrument — with task calls on `gpt-5.4-mini` and reflection calls on +`claude-sonnet-4.6`, both through OpenRouter, on 2026-08-23 at `8a6ce8fd`. + +That is one stochastic treatment over four requests. It exercised a real +component-optimization, action-observation, Artifact and restart path. It does +not establish agent effectiveness, external-side-effect safety, multi-seed +optimizer effectiveness, or DSPy parity. A normal example test verifies the +retained result and Artifact, applies the Artifact to trusted code, and scans +both files for credential-shaped content. + +Rows R3–R5, the matched GEPA and MIPROv2 comparison on TREC, are a third kind: +recomputable from committed scored rows, but not reproducible, because the raw +provider traces were not published. [The case study](CASE_STUDY_TREC.md) says so +in its first paragraph. + +## The record of things that did not work + +RESULTS.md ends with a table of findings that are not results: a HotPotQA +GEPA run with negative mean lift, a Banking77 MIPRO run that missed its +preregistered bar, a stateful-agent run where every candidate scored zero so +the optimizer had no signal at all, a scorer defect that invalidated a set of +our own optimizer results, and a matched IFBench rehearsal that stopped before +producing a verdict and whose earlier reported comparison is withdrawn in full. + +None of these can be re-measured from this repository, and the table says so. +They are kept because a record that only contains successes is not a record. +A completed treatment keeps its negative verdict; a no-signal treatment stays +unresolved; a diagnosed defect is repaired at the layer that owned it and its +results are withdrawn rather than rescored. + +## Execution traces + +Execution traces, bounded native run observations, cancellation evidence and +portable ATIF trajectories are described in +[Execution evidence and ATIF](TRAJECTORIES.md). diff --git a/docs/TRAJECTORIES.md b/docs/TRAJECTORIES.md index 904ba323..2187140d 100644 --- a/docs/TRAJECTORIES.md +++ b/docs/TRAJECTORIES.md @@ -77,3 +77,39 @@ semantics. That warning must not be reported as full v1.8 viewer support; v1.8's audio additions are outside this export's current coverage. Validation and rendering demonstrate interchange for the exercised fixture, not completeness of an arbitrary application's evidence. + +## The optimizer trajectory envelope + +`Imp.Run` events above are the execution record of one program run. +`Imp.Optimizer.Trajectory` is a different envelope: the one the optimizers +share. + +`Imp.Optimizer.Trajectory` is the canonical execution envelope shared by +GEPA, MIPROv2, SIMBA, RLM, ReAct, optimize-anything, and evaluation adapters. +It preserves each runtime's native `trace`, feedback, metadata, and named +parameter values while also projecting ordered provider-neutral events. + +The version 1 envelope includes: + +- text or typed multimodal examples and predictions; +- reasoning, module calls, tool calls/results, partial errors, and evaluator feedback; +- token/request/cost usage and microsecond timing; +- cache key/hit identity, program and rollout identity, and named parameters; +- optimizer-specific metadata that is intentionally not flattened. + +Use `Imp.Optimizer.Trajectory.project/3` at runtime boundaries. A trajectory +batch can be checked with `validate_aligned!/1`; events must be contiguous and +tool results must follow a unique matching call. `dump/1` emits the only +supported cross-runtime JSON representation and redacts credentials in +structured fields before they cross that boundary. Opaque image, audio, and +file bytes remain byte-for-byte intact; callers must not place credentials in +attachment payloads. `Imp.dump/1` and `Imp.load/1` use this same codec. +`load/1` accepts only the exact versioned schema, +known typed values, valid accounting, ordered events, and aligned tool calls. +It returns `{:error, %Imp.Optimizer.Trajectory.DecodeError{}}` for malformed or +future-version data rather than partially restoring it. + +The native `trace` remains available because reflection semantics differ by +optimizer. Consumers should use `events` for cross-runtime inspection and the +native fields when implementing optimizer-specific reflection or mutation. + diff --git a/docs/TUTORIAL_TICKET_ROUTING.md b/docs/TUTORIAL_TICKET_ROUTING.md index ae83d2f3..9268c3ab 100644 --- a/docs/TUTORIAL_TICKET_ROUTING.md +++ b/docs/TUTORIAL_TICKET_ROUTING.md @@ -4,18 +4,19 @@ Let's take the support-ticket router from the [README](../README.md) and do what you cannot do with a prompt string: score it on held-out data, improve it with an optimizer, and prove the improvement on tickets it has never seen. -In the current committed benchmark runs the zero-shot router scored **30–50%** -on twenty held-out tickets. The optimized router scored **95–100%** on the -same twenty in all three repeats. Each full experiment — baseline, -optimization, and held-out evaluation — cost about **$0.013** and ran in -**7–9 seconds** with `gpt-5.4-mini`. - -Those numbers come from a committed, content-addressed run artifact in the -source repository. The repository-only -[experiment script](https://github.com/deepfates/imp/blob/main/scripts/tutorial_ticket_routing_experiment.exs) -retains their research provenance; the packaged tutorial below uses the same -public program, evaluation, and optimizer APIs without shipping that evidence -runner. +The zero-shot router scored **30–50%** on twenty held-out tickets; the +optimized router scored **95–100%** on the same twenty in all three repeats. +Each full experiment — baseline, optimization, and held-out evaluation — cost +about **$0.013** and ran in **8–9 seconds** with `gpt-5.4-mini`. + +Those are rows R1 and R2 in +[benchmarks/RESULTS.md](https://github.com/deepfates/imp/blob/main/benchmarks/RESULTS.md), +which carries the dataset, model, provider, date and commit. You can measure +them yourself: with an API key, +`mix run scripts/tutorial_ticket_routing_experiment.exs` in a source checkout +runs exactly this experiment three times, for about four cents in total. The +packaged tutorial below uses the same public program, evaluation and optimizer +APIs without that runner. The gain has a plain-English reason: our routing labels encode conventions the model cannot guess, and the optimizer put examples of those conventions into the program. @@ -138,10 +139,13 @@ optimized = Imp.evaluate(compiled, testset, metric, max_concurrency: 8, timeout: #=> {0.3, 0.95} ``` -The current three-repeat artifact measured 50% → 95%, 35% → 100%, and -30% → 95%: gains of 45–65 points on held-out tickets. Each full run used -about 14,800 tokens, cost about $0.013, and finished in 7–9 seconds. All 120 -evaluation calls completed without a row error. +The three repeats measured 50% → 95%, 35% → 100%, and 30% → 95%: gains of +45–65 points on held-out tickets. Each run used about 14,800 tokens, cost about +$0.013, and finished in 8–9 seconds; all 120 evaluation calls completed without +a row error and the cache was cleared before each repeat, so every call was +live. Twenty rows move in 5-point steps, so trust the direction and the +magnitude, not the endpoints. Rows R1 and R2 in +[benchmarks/RESULTS.md](https://github.com/deepfates/imp/blob/main/benchmarks/RESULTS.md). It is not a magic button. The remaining misses are genuinely marginal tickets ("Scheduled reports did not run last night" — a platform failure that reads @@ -204,7 +208,7 @@ router = Imp.load!("ticket_router.json") Credentials never enter the artifact; bind the live model at runtime with `Imp.with_lm/2` or a scoped `Imp.context/2`. -The committed evidence runner also writes the optimized parameters as an +The experiment script also writes the optimized parameters as an `Imp.Optimizer.Artifact`, starts a fresh OS process, reconstructs the trusted router, applies only those parameters, and serves four concurrent OTP tasks. All four fresh-process probes routed correctly; the artifact never contained @@ -219,3 +223,5 @@ the provider credential or executable application code. same metric, bigger budget. - [Livebook 03](../livebooks/03_evaluate_and_optimize.livemd) runs this workflow interactively. +- [Benchmarks](https://github.com/deepfates/imp/blob/main/docs/BENCHMARKS.md) lists every number this repository publishes, + what each costs to re-measure, and what cannot be re-measured at all. diff --git a/docs/internal/ADAPTER_FIDELITY.md b/docs/differentials/ADAPTER_FIDELITY.md similarity index 100% rename from docs/internal/ADAPTER_FIDELITY.md rename to docs/differentials/ADAPTER_FIDELITY.md diff --git a/docs/internal/AUTO_EVALUATION_DIFFERENTIAL.md b/docs/differentials/AUTO_EVALUATION_DIFFERENTIAL.md similarity index 100% rename from docs/internal/AUTO_EVALUATION_DIFFERENTIAL.md rename to docs/differentials/AUTO_EVALUATION_DIFFERENTIAL.md diff --git a/docs/internal/AVATAR_FIDELITY.md b/docs/differentials/AVATAR_FIDELITY.md similarity index 86% rename from docs/internal/AVATAR_FIDELITY.md rename to docs/differentials/AVATAR_FIDELITY.md index 02d0dbe9..3365f735 100644 --- a/docs/internal/AVATAR_FIDELITY.md +++ b/docs/differentials/AVATAR_FIDELITY.md @@ -43,8 +43,7 @@ pre-rewrite actor score. Imp also returns an executable diagnostic baseline when either trajectory class is absent and takes bounded examples deterministically. A C1 artifact must preserve these deviations explicitly. -Neither protocol can emit admissible evidence from a dirty checkout. The -artifact binds the committed task, Python sidecar, shared authentication -helper, fixture, authority ledger and manifest, and relevant Imp implementation -source. Capture and registry admission therefore occur only after these files -are committed cleanly. +Neither protocol runs from a dirty checkout. The report binds the committed +task, Python sidecar, shared authentication helper, fixture, pinned upstream +source manifest, and the relevant Imp implementation source, so it can only be +produced after those files are committed. diff --git a/docs/internal/AX_DIFFERENTIAL.md b/docs/differentials/AX_DIFFERENTIAL.md similarity index 100% rename from docs/internal/AX_DIFFERENTIAL.md rename to docs/differentials/AX_DIFFERENTIAL.md diff --git a/docs/internal/COMBEE_FIDELITY.md b/docs/differentials/COMBEE_FIDELITY.md similarity index 100% rename from docs/internal/COMBEE_FIDELITY.md rename to docs/differentials/COMBEE_FIDELITY.md diff --git a/docs/internal/CONFIDENCE_CALIBRATION.md b/docs/differentials/CONFIDENCE_CALIBRATION.md similarity index 98% rename from docs/internal/CONFIDENCE_CALIBRATION.md rename to docs/differentials/CONFIDENCE_CALIBRATION.md index 959b34da..da48041d 100644 --- a/docs/internal/CONFIDENCE_CALIBRATION.md +++ b/docs/differentials/CONFIDENCE_CALIBRATION.md @@ -90,8 +90,8 @@ digest. Re-running the builder with the same arguments produces the same JSONL bytes. The task requires returned OpenAI Chat token logprobs and records the effective -model, API, usage, raw metrics, fit diagnostics, complete mapping, authority -gates, and Brier comparison. Authority requires verified source labels, unique +model, API, usage, raw metrics, fit diagnostics, complete mapping, validity +gates, and Brier comparison. A valid run requires verified source labels, unique evaluation and source IDs, source/group split disjointness, at least 100 records per split, mixed outcomes in both splits, at least two supported calibration bins, an authoritative fit, a complete held-out calibrated report, and a proper diff --git a/docs/internal/FAILURE_RECOVERY_EVIDENCE.md b/docs/differentials/FAILURE_RECOVERY_EVIDENCE.md similarity index 87% rename from docs/internal/FAILURE_RECOVERY_EVIDENCE.md rename to docs/differentials/FAILURE_RECOVERY_EVIDENCE.md index 1aaec1d2..09ad6a08 100644 --- a/docs/internal/FAILURE_RECOVERY_EVIDENCE.md +++ b/docs/differentials/FAILURE_RECOVERY_EVIDENCE.md @@ -4,7 +4,7 @@ The canonical failure campaign exercises Imp runtime behavior through public Elixir APIs. It does not substitute benchmark-only recovery implementations for the production paths. -## Deterministic authority +## The deterministic campaign `mix benchmark.failure_campaign.check` runs ten iterations of cancellation, explicit task timeout, bounded concurrency, terminal partial-stream failure, @@ -14,9 +14,9 @@ exact MIPROv2 and SIMBA checkpoint resume/tamper rejection. The artifact records process, port, supervised task, admission queue, and telemetry-handler deltas. It also records balanced telemetry span counts and a credential scan. Any failed iteration, nonzero added resource count, unbalanced -span, retained handler, or credential hit keeps deterministic authority red. +span, retained handler, or credential hit fails the campaign. -## Selected local operational authority +## The local operational campaign The operational campaign is intentionally local, bounded, and provider-free: @@ -58,7 +58,6 @@ mix imp.benchmark.failure_campaign \ --out benchmarks/runs/failure-recovery ``` -Then validate and retain the result. The artifact's payload -hash, exact Git revision, clean workspace state, attempt counts, time bounds, -exact tool history, canary digest, telemetry, and resource deltas are the -authority inputs. +The report records the payload hash, exact Git revision, clean workspace +state, attempt counts, time bounds, exact tool history, canary digest, +telemetry, and resource deltas. Read those, not a summary verdict. diff --git a/docs/internal/INSTRUCTION_OPTIMIZER_FIDELITY.md b/docs/differentials/INSTRUCTION_OPTIMIZER_FIDELITY.md similarity index 98% rename from docs/internal/INSTRUCTION_OPTIMIZER_FIDELITY.md rename to docs/differentials/INSTRUCTION_OPTIMIZER_FIDELITY.md index d42982c7..7d19d1c8 100644 --- a/docs/internal/INSTRUCTION_OPTIMIZER_FIDELITY.md +++ b/docs/differentials/INSTRUCTION_OPTIMIZER_FIDELITY.md @@ -362,7 +362,7 @@ mix imp.benchmark.instruction_optimizer_experiment \ --out benchmarks/runs/instruction-optimizer-experiment ``` -The admitted economical preflight binds Claude Haiku 4.5 to the +The economical preflight binds Claude Haiku 4.5 to the provider-specific ReqLLM and LiteLLM identifiers, all three AIME split hashes, seed, arm order, optimizer options, DSPy `3.3.0b1`, Optuna `4.9.0`, and independent per-arm request, @@ -372,9 +372,7 @@ The orchestrator derives one Imp campaign and one all-arm DSPy campaign and refuses to merge incomplete or identity-mismatched artifacts. -The admitted one-seed artifact is -`benchmarks/evidence/admitted/instruction_live/e2d79f12c6ef7120df8efacd8a43d03027be65963a41aaff5dd1f87a8bcd1c76.json`. -Both runtimes completed every arm without failures. Baseline, MIPROv2, and +One seed was run live. Both runtimes completed every arm without failures. Baseline, MIPROv2, and SIMBA each scored `2/3` on that runtime's frozen test split, so this establishes live sampled execution and accounting but not optimizer lift or T3 parity. diff --git a/docs/internal/MMGRPO_C1.md b/docs/differentials/MMGRPO_C1.md similarity index 89% rename from docs/internal/MMGRPO_C1.md rename to docs/differentials/MMGRPO_C1.md index 12dc2efb..db2e0f5e 100644 --- a/docs/internal/MMGRPO_C1.md +++ b/docs/differentials/MMGRPO_C1.md @@ -10,9 +10,8 @@ trace collection; the Elixir side executes Imp's public compiler with a local The comparison covers balanced dataset cycling, two-rollout group cardinality, two-predictor attribution, successful program-reward propagation, and the configured structured-format failure reward. Its source binding hashes the -canonical `family.optimizer_mmgrpo` authority projection, the pinned authority +canonical `family.optimizer_mmgrpo` projection, the pinned upstream source manifest, both harness implementations, the fixture, and Imp's GRPO source. -Unrelated authority-ledger edits therefore do not invalidate this family. The protocol does **not** claim exact Python RNG or shuffle order, provider or transport behavior, model quality, training effectiveness, variable-invocation diff --git a/docs/internal/MULTIMODAL_FIDELITY.md b/docs/differentials/MULTIMODAL_FIDELITY.md similarity index 93% rename from docs/internal/MULTIMODAL_FIDELITY.md rename to docs/differentials/MULTIMODAL_FIDELITY.md index 8bf44bb1..2345ea3c 100644 --- a/docs/internal/MULTIMODAL_FIDELITY.md +++ b/docs/differentials/MULTIMODAL_FIDELITY.md @@ -5,13 +5,11 @@ quality. Typed value construction and pre-dispatch content shapes do not prove that ReqLLM serialized those values, that a provider received them, or that a model answered correctly. -## Current Evidence +## What has been observed -The claim-authorizing artifact is -`benchmarks/evidence/admitted/multimodal_live/02d3c35797723e6ce4a7c544a3f602579771430276d6838beb14bcfe091e391e.json` (SHA-256 -`02d3c35797723e6ce4a7c544a3f602579771430276d6838beb14bcfe091e391e`). It was -generated on 2026-07-13 from OpenAI manifest payload SHA-256 -`04daa155d3f97edfff62e329dfdca1248855f22b2271f7e954228432f1ae39d8`. +One live run, on 2026-07-13, from OpenAI manifest payload SHA-256 +`04daa155d3f97edfff62e329dfdca1248855f22b2271f7e954228432f1ae39d8`. Its +record is not published; what follows is the pinned configuration it used. The run used the pinned OpenAI Responses endpoint and model: @@ -52,8 +50,8 @@ native file support. ## Serialized Audit The runner installs a Req request step after ReqLLM provider `encode_body` and -before transport. This is the claim-authorizing request boundary. For every -request, the persisted redacted audit contains: +before transport — the last point at which what Imp actually sends can be +observed. For every request, the redacted audit contains: - sanitized endpoint, API, HTTP method, serialized model, and body SHA-256; - ordered serialized part types; diff --git a/docs/internal/PLAYBOOK_OPTIMIZER.md b/docs/differentials/PLAYBOOK_OPTIMIZER.md similarity index 100% rename from docs/internal/PLAYBOOK_OPTIMIZER.md rename to docs/differentials/PLAYBOOK_OPTIMIZER.md diff --git a/docs/internal/REACT_V2_FIDELITY.md b/docs/differentials/REACT_V2_FIDELITY.md similarity index 100% rename from docs/internal/REACT_V2_FIDELITY.md rename to docs/differentials/REACT_V2_FIDELITY.md diff --git a/docs/differentials/README.md b/docs/differentials/README.md new file mode 100644 index 00000000..203d233c --- /dev/null +++ b/docs/differentials/README.md @@ -0,0 +1,40 @@ +# Differential notes + +One note per family, saying what that differential compares Imp against, what +it found, and what it deliberately does not establish. They are written for +someone reading the differential's output or changing the implementation it +guards — not as a second user manual. + +[docs/BENCHMARKS.md](../BENCHMARKS.md) is the index: it lists every command, +what each one needs, and links back here. `mix differential.check` runs them +all against the pinned upstream sources. + +The pinned upstream is DSPy 3.2.1 at commit +`29448ae12756abdd14bd8796c819247ebb83673c` and GEPA 0.1.4, materialized under +`tmp/` by the setup scripts. Two newer probes compare against DSPy 3.3.1 +(`638e155c`) instead and say so where they appear. + +- **Adapters and rendering** — [ADAPTER_FIDELITY.md](ADAPTER_FIDELITY.md) +- **Instruction optimizers (MIPROv2, SIMBA)** — [INSTRUCTION_OPTIMIZER_FIDELITY.md](INSTRUCTION_OPTIMIZER_FIDELITY.md) +- **GEPA reflection aggregation** — [COMBEE_FIDELITY.md](COMBEE_FIDELITY.md) +- **Weight composition (BootstrapFinetune, BetterTogether)** — [WEIGHT_COMPOSITION_C1.md](WEIGHT_COMPOSITION_C1.md) +- **GRPO / mmGRPO** — [MMGRPO_C1.md](MMGRPO_C1.md) +- **Avatar actor and trajectory optimizer** — [AVATAR_FIDELITY.md](AVATAR_FIDELITY.md) +- **Playbook parameters** — [PLAYBOOK_OPTIMIZER.md](PLAYBOOK_OPTIMIZER.md) +- **Recursive Language Models** — [RLM_FIDELITY.md](RLM_FIDELITY.md) +- **ReAct family and code execution** — [REACT_V2_FIDELITY.md](REACT_V2_FIDELITY.md) +- **Auto-evaluation metrics** — [AUTO_EVALUATION_DIFFERENTIAL.md](AUTO_EVALUATION_DIFFERENTIAL.md) +- **Confidence and calibration** — [CONFIDENCE_CALIBRATION.md](CONFIDENCE_CALIBRATION.md) +- **Multimodal primitives** — [MULTIMODAL_FIDELITY.md](MULTIMODAL_FIDELITY.md) +- **Failure and recovery** — [FAILURE_RECOVERY_EVIDENCE.md](FAILURE_RECOVERY_EVIDENCE.md) +- **Ax, an independent TypeScript implementation** — [AX_DIFFERENTIAL.md](AX_DIFFERENTIAL.md) +- **DSPy's own test suite, ported and accounted for** — [UPSTREAM_EXAM.md](UPSTREAM_EXAM.md), the disposition map for `test/upstream_exam/` +- **Which tutorial and example covers which family** — [TUTORIAL_EXAMPLE_PARITY.md](TUTORIAL_EXAMPLE_PARITY.md) +- **Prior art and neighbouring systems** — [RESEARCH_LANDSCAPE.md](RESEARCH_LANDSCAPE.md), a dated outside view + +A differential proving Imp matches upstream on an input says nothing about +whether either helps your program. Held-out effectiveness numbers live in +[benchmarks/RESULTS.md](../../benchmarks/RESULTS.md), and there are only three. + +Dated observations in these notes remain true at the date they record. They do +not silently become current decisions. diff --git a/docs/internal/RESEARCH_LANDSCAPE.md b/docs/differentials/RESEARCH_LANDSCAPE.md similarity index 99% rename from docs/internal/RESEARCH_LANDSCAPE.md rename to docs/differentials/RESEARCH_LANDSCAPE.md index 28283735..33466037 100644 --- a/docs/internal/RESEARCH_LANDSCAPE.md +++ b/docs/differentials/RESEARCH_LANDSCAPE.md @@ -279,7 +279,7 @@ The review proposed this first group of work: It proposed this later group: 1. Maintain the pinned Ax `23.0.0` executable differential and current - `24.0.4` product-semantic audit in `docs/internal/AX_DIFFERENTIAL.md` as + `24.0.4` product-semantic audit in `docs/differentials/AX_DIFFERENTIAL.md` as independent implementation checks without promoting Ax to scientific authority. 2. Add operational failure campaigns for cancellation, timeout, retries, diff --git a/docs/internal/RLM_FIDELITY.md b/docs/differentials/RLM_FIDELITY.md similarity index 99% rename from docs/internal/RLM_FIDELITY.md rename to docs/differentials/RLM_FIDELITY.md index 9e50548c..2552b0ab 100644 --- a/docs/internal/RLM_FIDELITY.md +++ b/docs/differentials/RLM_FIDELITY.md @@ -291,7 +291,7 @@ now accepts only complete canonical `(id_1, id_2)` lines or a standalone empty-set marker; any mixed prose is invalid. Failed rows are excluded from paired bootstrap comparisons instead of being treated as scored zeros. -The admitted replacement was generated from commit `2b9cacd` on frozen query +The replacement run was generated from commit `2b9cacd` on frozen query `1` at context size `1024` with `claude-sonnet-5`. Imp direct and DSPy direct both explained the correct empty-set conclusion in noncanonical prose and scored `0.0`. Imp RLM recorded `run`, `action_error`, and `submit` events, diff --git a/docs/internal/TUTORIAL_EXAMPLE_PARITY.md b/docs/differentials/TUTORIAL_EXAMPLE_PARITY.md similarity index 100% rename from docs/internal/TUTORIAL_EXAMPLE_PARITY.md rename to docs/differentials/TUTORIAL_EXAMPLE_PARITY.md diff --git a/docs/internal/UPSTREAM_EXAM.md b/docs/differentials/UPSTREAM_EXAM.md similarity index 100% rename from docs/internal/UPSTREAM_EXAM.md rename to docs/differentials/UPSTREAM_EXAM.md diff --git a/docs/internal/WEIGHT_COMPOSITION_C1.md b/docs/differentials/WEIGHT_COMPOSITION_C1.md similarity index 86% rename from docs/internal/WEIGHT_COMPOSITION_C1.md rename to docs/differentials/WEIGHT_COMPOSITION_C1.md index 6a17231a..307c042c 100644 --- a/docs/internal/WEIGHT_COMPOSITION_C1.md +++ b/docs/differentials/WEIGHT_COMPOSITION_C1.md @@ -25,8 +25,8 @@ Independent execution of the pinned runtime confirms that its default ratio ther an empty validation set for one through nine rows and falls into latest-prefix selection. Imp treats that truncation as incidental: with two or more rows and a positive ratio it keeps at least one validation row, while retaining a lone row for training when no split is possible. -This BEAM-native correction has direct consumer coverage; it is outside the admitted shared C1 -observations and is not evidence of BetterTogether effectiveness. +This BEAM-native correction has direct consumer coverage. It is a deliberate divergence from +upstream, not a comparison result, and it is not evidence of BetterTogether effectiveness. Imp's aggregate launch/cancellation deadlines for BootstrapFinetune and bounded asynchronous training lifecycle for BetterTogether are recorded as BEAM-native extensions. They are not @@ -45,6 +45,4 @@ mix imp.benchmark.better_together_differential ``` The tasks refuse dirty-source capture and bind the committed Imp optimizer, task, sidecar, fixture, -authority manifest, and authority ledger hashes. Registry admission remains a separate step: this -tranche intentionally does not edit the shared claims, authorities, or reproductions -registries while parallel work is active. +and pinned upstream source manifest hashes. diff --git a/docs/internal/BENCHMARK_CATALOG.md b/docs/internal/BENCHMARK_CATALOG.md deleted file mode 100644 index fdec62e5..00000000 --- a/docs/internal/BENCHMARK_CATALOG.md +++ /dev/null @@ -1,140 +0,0 @@ -# Benchmark Catalog - -This catalog asks whether Imp is sampling the right task families from the DSPy -literature, docs, and examples. It is a research map, not a product release -gate. Broad comparative statements are scoped in `benchmarks/claims.json`. - -The commands in this document are source-checkout evidence commands for Imp -maintainers. They are not part of the Hex package API or a normal application -install path. - -The goal is a small representative portfolio whose evidence matches the public -claim. Cheap checks falsify mechanics, source, metric, and cost assumptions; they -must not become a permanent ladder or screen tasks by whether early outcomes -look favorable. A full study is justified by claim value and a thin ordinary -path, not by accumulating harness tiers. - -## Workflow - -Each benchmark family should move through the same four stages. - -| Stage | Meaning | Expected cost | -| --- | --- | --- | -| Cataloged | Source, dataset, metric, Imp surface, and DSPy surface are named. | Free | -| Sampled | Imp can fetch or materialize a deterministic `jsonl` sample with a manifest. | Free or cheap network | -| Matched smoke | Imp and Python DSPy run the same 2-20 rows with the same provider/model or provider-free oracle. | Cheap | -| Research slice | Imp and Python DSPy run enough rows to catch systematic failures, usually 100-300 examples or a task-specific equivalent. | Moderate | -| Full lane | A full split or paper-scale run is executed only when it supports a release or paper-quality claim. | Expensive | - -The sampling harness must preserve: - -- source URL and dataset/config/split; -- offset, length, and SHA256 digest; -- input fields and label fields; -- metric contract; -- program shape, such as `Predict`, `ChainOfThought`, ReAct, RAG, or optimizer - compile; -- whether the lane is provider-free, live matched Imp-vs-DSPy, Imp-only - production semantics, or intentionally unavailable. - -## Current Coverage - -| Family | Source lineage | Imp status | Runner evidence | Next useful step | -| --- | --- | --- | --- | --- | -| Math word problems | DSPy paper and docs use GSM8K-style chain-of-thought examples. | Implemented | `mix benchmark.truth.check`, `mix benchmark.parity.check`, `mix benchmark.parity.full` | Keep as canonical low-cost/full lane. | -| Multi-hop QA | DSPy/DSP lineage centers retrieval-heavy HotPotQA and Baleen-style QA. | A source-bound provider-free differential matches Imp and DSPy on all ten pinned validation rows over a shared 100-document corpus, with exact document IDs, top-k contexts, supporting-fact recall, and extractive answer-availability EM/F1. It is not generation-quality evidence. | `mix imp.benchmark.hotpot_retrieval`, `mix benchmark.truth.check`, live parity campaigns | Add provider-backed generation and a larger preregistered sample only when research-tier model-quality evidence is required. | -| Color/classification | DSPy public dataset lineage includes simple Colors-style classification. | Provider-free Colors/Iris/Iris-Typo/Heart Disease samplers and runner smoke exist with accuracy and macro/micro/weighted F1 reports; optimizer-lift artifact also includes a passing natural classification lane. | `mix imp.benchmark.fetch --tasks colors,iris,iris_typo,heart_disease --full --out benchmarks/data`, `mix imp.benchmark.run --colors benchmarks/data/colors-test-0-6.jsonl --iris benchmarks/data/iris-test-0-6.jsonl --iris-typo benchmarks/data/iris_typo-test-0-3.jsonl --heart-disease benchmarks/data/heart_disease-test-0-4.jsonl`, `mix benchmark.truth.check`, `mix benchmark.optimizer_lift.check` | Scale to larger pinned classification/factuality datasets when research-tier evidence is required. | -| RAG/retrieval | DSP and DSPy papers emphasize retrieval + generation for knowledge-intensive QA. | Direct RAG/retriever tests plus a pinned ten-row/100-document matched HotPotQA retrieval differential exist; the differential reports 0.35 supporting-title recall and 0.30 extractive answer-availability EM/F1 for both runtimes, not model generation quality. A separate failure row injects a retriever-tool timeout exception; it is not a wall-clock transport timeout comparison. | `mix test test/imp_facade_test.exs test/external_retriever_test.exs`, `mix imp.benchmark.hotpot_retrieval`, `mix imp.benchmark.rag_tool_failure_differential`, `mix protocol.retriever.check` | Keep larger-corpus retrieval and provider-backed answer quality open. | -| Tool and ReAct programs | DSPy docs present tools and agents as first-class programming workflows. | Direct ReAct/tool/MCP tests are separate from a C1/T1 BFCL-shaped scorer fixture. A C2 six-scenario differential executes one queued action schedule through actual Imp and source-authenticated DSPy ReAct, matching retry observations, fixture idempotency, unknown/failing tools, and terminal traces. The schedule does not test model-selected recovery. | `mix benchmark.trace.check`, `mix test test/react_contract_test.exs test/tool_schema_runtime_test.exs test/mcp_import_test.exs`, `mix imp.benchmark.bfcl_adapted`, `mix imp.benchmark.rag_tool_failure_differential`, `mix integration.check` | Add licensed official/public rows and model-produced calls only if the public claim requires research-tier effectiveness. | -| RLM recursive control | DSPy RLM docs and paper position recursive controller loops as long-context inference over variable space, distinct from RAG. | BEAM-native symbolic execution and live sub-LM proof exist. T1 gates twelve operational contracts against DSPy 3.3.0b1 with source hashing; T0 remains fixture replay. Neither is effectiveness evidence. | `mix benchmark.rlm.contract.check`, `mix benchmark.rlm.check`, `mix test test/rlm_test.exs`, `LIVE_PROVIDER=1 mix live.check` | Add live sampled effectiveness and complete the T3 paper-scale RLM protocol. | -| Program composition and orchestration | DSPy modules compose predictors, ensembles, comparison, refinement, parallel fan-out, and retrieval-aware variants. | Provider-free sampled orchestration benchmark exists for BestOfN, Refine, MultiChainComparison, Ensemble, KNN, and Parallel; selected live orchestration coverage also exists. | `mix imp.benchmark.fetch --tasks composition_orchestration --full --out benchmarks/data`, `mix imp.benchmark.run --composition-orchestration benchmarks/data/composition_orchestration-test-0-3.jsonl`, `mix test test/public_surface_test.exs test/refine_feedback_test.exs` | Extend sampled orchestration to matched Imp/DSPy live-provider comparison once cost and model policy are selected. | -| Adapters, streaming, and structured I/O | DSPy adapters and Ax-style signatures make parsing, schema negotiation, retries, and streaming part of the programming contract. | ExUnit and a test-only operations diagnostic cover malformed JSON/XML/chat, partial streams, and native schema option shape; the timestamped diagnostic is not claim evidence. Trace/schema/ReqLLM/live coverage also exists. | `mix test test/operations_stress_test.exs test/stream_listener_incremental_test.exs`, `mix benchmark.trace.check`, `mix test test/schema_constraints_test.exs test/req_llm_client_test.exs`, `LIVE_PROVIDER=1 mix live.check` | Keep the operations diagnostic outside claims; add source-bound matched live-provider drift evidence only when release policy requires it. | -| Persistence, cache, telemetry, and OTP operations | Production DSP-style systems need save/load, cache behavior, redaction, observability, and supervised concurrency outside notebooks. | ExUnit and the test-only operations diagnostic exercise save/load, cache hit/miss telemetry, and redaction but do not establish an operational claim. The source-bound failure campaign adds cancellation, bounded admission, terminal partial-stream failures, checkpoint/tamper recovery, flake rates, and leak accounting. Its optional local rows add a bounded timeout and exact tool-agent recovery history without external calls. | `mix test test/operations_stress_test.exs test/adversarial_security_stress_test.exs`, `mix benchmark.failure_campaign.check`, `mix check`, `mix integration.check`, `mix benchmark.overhead.check` | Admit clean current-source local operational recovery evidence, then extend lifecycle stress to long-running supervised service soak tests. | -| Multimodal primitives | Modern provider surfaces include image, audio, file, document, and code content blocks. | Deterministic encoding/decoding test coverage exists. The test-only operations diagnostic records that live multimodal reasoning is not claimed and is not evidence for such a claim. | `mix test test/multimodal_adapter_test.exs` | Add source-bound provider evidence only if public docs claim live multimodal reasoning. | -| Optimizer lift | DSPy optimizer docs cover few-shot bootstrapping, instruction/demo search, MIPROv2, GEPA, and finetuning workflows. | Provider-free lift and natural-task evidence exist. A pinned DSPy 3.3.0b1 structural differential gate now covers MIPROv2 and SIMBA control flow, but only a fresh passing artifact satisfies T1. Neither lane proves full optimizer parity. | `mix benchmark.optimizer_lift.check`, `mix benchmark.instruction_optimizer.contract.check` | Keep the structural artifact fresh, then run held-out multi-seed effectiveness campaigns under matched provider, metric, and cost budgets. | -| Optimize Anything non-prompt artifacts | Optimize Anything and GEPA generalize reflective optimization from prompts to measurable text artifacts. | Production Imp runner plus executable code, agent-configuration, and scheduling evaluators exist. The live campaign requires three seeds, positive mean lift, a majority of improving runs, provider usage/cost, embedded checksummed final budget evidence, and explicit request/token/dollar ceilings enforced by pre-request reservation. The local checkpoint uses sync-write plus rename and is neither resumable spend state nor a power-loss guarantee. | `mix benchmark.optimize_anything.check`, `mix imp.benchmark.optimize_anything --live ... --max-cost-usd ... --max-requests ... --max-input-tokens ... --max-output-tokens ... --max-output-tokens-per-request ...` | Broaden to matched public upstream artifact tasks only when making a paper-scale or implementation-comparison claim; never treat a configured ceiling as observed cost. | -| GEPA six-task benchmark | GEPA artifact repo covers AIMEBench, HotpotQABench, hoverBench, IFBench, LiveBenchMathBench, and Papillon. | The repository has the six-family source identities, official split contracts, task programs, metrics, source-exact HoVer Python BM25S retrieval, Imp execution, and replication artifact contract. The complete current matched baseline row is admitted across all six families; no current matched GEPA or MIPROv2 cell is complete. The earlier campaign controller is Imp-side orchestration and is not itself a matched result. The current three-seed/two-runtime plan distinguishes a 1,231,212-transport nominal schedule from a 2,280,420-transport legal failure ceiling after correcting PAPILLON's phase-specific judge work. Imp's source-authenticated merge profile matches the current public DSPy treatment shape; no-merge remains an ablation. | `mix benchmark.gepa_replication.check`, `mix imp.benchmark.gepa_dataset`, `mix imp.benchmark.gepa_campaign`, `mix imp.benchmark.gepa_replication --from-gepa-artifact ...` | Run a bounded optimizer-role canary that exercises actual proposals and task evaluation; if treatment identity, telemetry, persistence, and fresh service hold, seek explicit spend authority for the full merge-enabled GEPA/MIPROv2 table. Treat the table as an adapted current-model reference differential, not exact paper replication. Do not extend the private campaign controller or impose a private universal win threshold. | -| Hallucination/factuality classification | DSPy optimizer comparison papers use CovidQA, PubMedQA, DROP, FinanceBench, and similar labeled QA/factuality tasks. | Not implemented as fetchable benchmark lanes. | None | Add a generic classification/QA sampler and metric adapters for exact/F1/macro-F1. | -| MIPRO / LangProBe tabular classification | The MIPRO paper uses a 4-call Heart Disease clinical-opinion ensemble on 120 train / 183 test rows. LangProBe independently adapts the same program and TensorFlow-derived 303-row file into a 15-train / 136-validation / 152-test optimizer benchmark, with retained results across six model families and several optimizers. | A provider-free four-predictor Imp and pinned-DSPy product-fit proof exercises MIPRO selection, state/Artifact persistence, fresh loading, and fresh service; it does not claim full optimizer opportunity or effectiveness. The LangProBe file is not equivalent to official UCI processed Cleveland labels: its matched rows classify severity 2--4 as positive, the prompt says generic disease presence, and six incomplete official rows are replaced. Its test majority baseline is 74.34%, above every retained LangProBe MIPRO result for the four-call program. | `mix test test/langprobe_heart_disease_product_fit_test.exs --include evidence_infrastructure`, `python3 scripts/langprobe_heart_data_audit.py ...` | Preserve as a provider-free compatibility/accounting anchor; do not fund the five-run live comparison or use it as an external quality claim. | -| ScoNe logical classification | MIPRO optimizer benchmarks include ScoNe. | Not implemented as benchmark lane. | None | Pin a public dataset source and add accuracy metric. | -| HoVer claim verification | MIPRO and GEPA benchmark lineage includes HoVer multi-hop verification. | GEPA exporter and metric adapter use the upstream `retrieved_docs` contract; source-exact BM25/wiki corpus and index are reproducible locally, with an upstream-python campaign path to avoid loading the full corpus in the BEAM. Imp HoVer campaign rows now generate LM queries and report positive usage accounting, but full GEPA claims still require uncapped rows. | `mix imp.benchmark.gepa_dataset`, `IMP_HOVER_UPSTREAM_BM25=1 mix imp.benchmark.gepa_campaign` | Run uncapped HoVer GEPA campaign rows with source-exact upstream BM25 retrieval and positive usage accounting. | -| IFBench instruction following | GEPA benchmark lineage includes verifiable instruction following. | The source-exact scorer boundary is repaired and checked across the frozen constraint families. Earlier Imp optimizer results remain invalid/unverified because optimization used the faulty scorer; that invalidates those outcomes, not IFBench or a new correctly scored arm. | `mix imp.benchmark.fetch --tasks ifbench_instruction_following --full --out benchmarks/data`, `mix imp.benchmark.run --ifbench-instruction-following benchmarks/data/ifbench_instruction_following-test-0-3.jsonl`, `mix benchmark.truth.check` | Include a newly frozen correctly scored arm in the matched six-task table; never rescore the old optimizer trajectory into effectiveness evidence. | -| Hard math/competition reasoning | GEPA and modern optimizer work often uses AIME/MATH-style tasks. | Provider-free AIME/MATH-style smoke exists with normalized exact answer scoring. | `mix imp.benchmark.fetch --tasks hard_math --full --out benchmarks/data`, `mix imp.benchmark.run --hard-math benchmarks/data/hard_math-test-0-3.jsonl`, `mix benchmark.truth.check` | Scale to pinned public MATH/AIME snapshots for research-tier evidence. | -| Privacy-conscious delegation | GEPA/PAPILLON/PUPA lineage evaluates useful delegation without leaking private information. | The repository has the Papillon two-stage program, quality/leakage judge path, source identity, official split contract, and provider-free Artifact lifecycle. A prior live calibration measured scoped format/judge variability but was not optimizer effectiveness evidence. | Six-task GEPA source manifest plus `bench/imp/benchmark_truth/papillon.ex` and the matched table paths described above. | Run the official frozen Papillon arm in the complete matched current-model table; retain private inputs and report quality and leakage separately. | -| LiveBench-Math | GEPA benchmark lineage includes date-versioned LiveBench-Math. | The six-family campaign assets pin the dated task identity, split contract, program, and guarded symbolic scoring bridge. No matched current Imp-versus-DSPy optimizer result exists. | Six-task GEPA source manifest and the matched table paths described above. | Verify the frozen dated snapshot and execute it as one preregistered row of the complete table rather than treating a moving leaderboard as evidence. | -| Long-form writing / STORM-style research | DSPy-related paper list includes writing Wikipedia-like articles from scratch. | Out of current product proof. | None | Track as deferred; do not block production unless Imp claims long-form writing optimization. | -| Extreme multi-label classification | DSPy-related paper list includes in-context learning for XML classification. | Out of current product proof. | None | Track as deferred; useful after core classification sampler exists. | -| Finetuning / BetterTogether / GRPO | DSPy paper list and docs include finetuning plus prompt optimization. | BootstrapFinetune has a protocol-compatible lifecycle plus pinned local MLX SFT effectiveness and fused save/load rebinding. BetterTogether can continue from that weight step into prompt optimization. GRPO has a bundled pinned TRL/MPS LoRA engine, durable multi-step artifacts, validation selection, and fresh-process rebind. One source-disjoint 33-step TREC treatment improved validation but regressed slightly on held-out test, so general usefulness remains open. | `mix protocol.training.check`, `mix imp.benchmark.local_mlx`, `mix benchmark.optimizer_lift.check`, `examples/local_grpo_opaque_banking77/README.md` | Retain paid-provider, BetterTogether parity, GRPO/mmGRPO parity, and general held-out effectiveness as distinct gaps; do not describe the negative TREC result as either a win or an engine façade. | - -## Prioritized Additions - -1. **Staged matched optimizer portfolio.** Preserve the existing matched TREC - result. The cheap current LangProBe/MIPRO Heart Disease candidate was - falsified before live spend: the recognized program is four compositional - LM calls over only 303 public benchmark rows, but every retained optimized - score is below its 74.34% constant-majority baseline. The original MIPRO paper - (`120/183`) and LangProBe (`15/136/152`) use different study splits. A provider-free - ordinary product-fit proof now exercises the four-predictor program in Imp - and pinned DSPy through optimization, persistence, and fresh use. The exact - LangProBe/TensorFlow bytes, program, split, current optimizer/runtime - versions, and a hypothetical five-run matched analysis are pinned, but the - live study is not recommended for execution. Preserve the benchmark as - provider-free design evidence. Then run one serious current-source - multi-stage comparison, with MuSiQue as the strongest prepared candidate but - not a mandatory choice. Evaluate a GEPA paper-family HoVer/HotPotQA arm only - after an outcome-blind cost pilot justifies its much larger opportunity. For - every matched study, make the selected Imp-versus-DSPy outcome the direct - comparison, separately require Imp to improve its own baseline, and retain - difference-in-lifts as a diagnostic rather than a flattering substitute for - final quality. Preserve negative outcomes and do not replace them with more - catalog machinery. -2. **Classification/factuality sampler.** Support Colors, Iris/Iris-Typo, - Heart Disease, PubMedQA/CovidQA-like binary or multiclass rows, and macro-F1 - metrics. -3. **Retrieval-indexed QA and verification sampler.** Add a tiny corpus + query - + answer/label format for HotPotQA/Baleen-style QA and HoVer-style - verification - that can run Imp retrieval and DSPy retrieval over the same rows. -4. **Tool/ReAct/RLM task sampler.** Add measurable tool-use and recursive - controller tasks so ReAct and RLM are tested by outcomes, traces, budget - adherence, and policy behavior rather than only fixture replay. -5. **Scaled optimizer lift.** Extend the current natural classification, QA, - retrieval, and instruction-following lift lanes to larger sampled datasets - when Imp needs model-quality claims beyond provider-free product proof. -6. **Research-scale instruction and hard-math data.** Extend the current - provider-free IFBench and hard-math smoke rows to pinned public snapshots - when release policy requires research-tier model-quality evidence. - -## Sources - -Primary sources to keep the catalog grounded: - -- DSPy repository and paper list: - -- DSPy docs: - -- DSPy optimizer docs: - -- DSPy ICLR paper: - -- MIPROv2 docs: - -- MIPRO paper: - -- LangProBe paper and source benchmark: - , - -- UCI Heart Disease dataset authority: - -- GEPA paper: - -- Demonstrate-Search-Predict paper: - - -Secondary sources, useful for candidate benchmark families but not release -authority by themselves: - -- Comparative DSPy optimizer studies that mention CovidQA, PubMedQA, DROP, and - FinanceBench-style labeled QA/factuality tasks. -- GEPA and optimize-anything materials for reflective artifact optimization and - hard reasoning benchmark candidates. diff --git a/docs/internal/BENCHMARK_TRUTH.md b/docs/internal/BENCHMARK_TRUTH.md deleted file mode 100644 index 92372e98..00000000 --- a/docs/internal/BENCHMARK_TRUTH.md +++ /dev/null @@ -1,1265 +0,0 @@ -# Benchmark Truth - -Imp has two benchmark lanes, one outside-view benchmark catalog, and one -release-level validation program. - -Imp keeps benchmark evidence behind Mix tasks instead of treating benchmark -helpers as part of the application API. Deterministic production fixtures prove -that core mechanics keep working: structured parsing, tools, program -optimization, and artifact optimization. - -The benchmark truth tasks are the research-evidence lane. They run Imp -programs over canonical DSPy-style dataset rows, write auditable result JSON, -and separate fixture-mode harness proof from live-provider evidence. - -`BENCHMARK_CATALOG.md` maps the broader DSPy paper/docs/example benchmark -universe to Imp's current samplers and gaps. `PARITY_VALIDATION_PROGRAM.md` -defines the full release evidence standard. A full-row live benchmark is one -important lane, but it is not sufficient by itself. Full parity claims also -require provider-free golden trace parity, optimizer lift parity, -RAG/tool/agent semantics, and provider-free performance benchmarks. - -## Canonical Minimum - -The first benchmark truth suite targets the datasets and task families most -closely tied to DSPy examples and papers: - -- GSM8K: math word problems for chain-of-thought reasoning. -- HotPotQA: multi-hop question answering, with context/retrieval pressure. -- Color-style classification: retained as a simple low-cost smoke task in the - dataset layer, not yet a benchmark truth gate. - -DSPy's public docs list HotPotQA, GSM8K, and Color as built-in datasets. The -DSPy paper lineage evaluates math word problems and multi-hop QA, especially -GSM8K and HotPotQA. Imp should not claim benchmark parity until it has run -real provider/model comparisons over fixed train/dev/test manifests. - -The broader benchmark backlog is intentionally larger than this minimum. See -`docs/internal/BENCHMARK_CATALOG.md` for classification/factuality, retrieval-indexed -QA, hard math, optimizer-lift, tool-use, and deferred long-form writing lanes. -Current source-checkout smoke evidence includes local IFBench-style rows with -executable constraint verifiers and local AIME/MATH-style rows with normalized -exact answer scoring. - -## Fetch Data - -```sh -mix imp.benchmark.fetch --tasks gsm8k,hotpotqa --length 20 --out benchmarks/data -``` - -For a full canonical split fetch: - -```sh -mix imp.benchmark.fetch --tasks gsm8k,hotpotqa --full --out benchmarks/data -``` - -`--full` currently means GSM8K test `1319` rows and HotPotQA distractor -validation `7405` rows. Full fetches use HuggingFace's Parquet exports by -default so they can retrieve the canonical splits without hammering the rows -API. Small `--length` fetches use the rows API and record every source page URL -in the manifest. A single API page is not treated as a full dataset. - -The fetcher uses HuggingFace's datasets-server rows API and writes: - -- `*.jsonl` normalized rows -- `*.manifest.json` source URL, dataset/config/split, offset, row count, - timestamp, input keys, and SHA256 digest - -Generated data lives under `benchmarks/data/` and is ignored by git. Commit -small fixtures only when they are needed for deterministic tests. - -## Check Data Integrity - -```sh -mix imp.benchmark.integrity \ - --gsm8k benchmarks/data/gsm8k-test-0-1319.jsonl \ - --hotpotqa benchmarks/data/hotpotqa-validation-0-7405.jsonl \ - --out benchmarks/runs/integrity \ - --require-clean -``` - -This writes a `benchmark-data-integrity-*.json` artifact. The check fails on -missing required fields and HotPotQA rows whose declared supporting-fact pages -are absent from the flattened context. It also records non-blocking warnings -when an extractive answer string is not present in the context. Those warnings -are useful because they identify rows where an exact-match score may reward -parametric knowledge rather than retrieval-grounded reasoning. - -## Run Fixture Proof - -```sh -mix benchmark.truth.check -``` - -This runs the benchmark harness with checked-in GSM8K/HotPotQA-shaped fixtures -and an oracle LM. It proves: - -- loaders accept benchmark-shaped records -- GSM8K canonical-answer extraction works -- HotPotQA context flattening works -- Imp programs can be evaluated over the real benchmark artifact schema -- result JSON includes dataset digests, per-row scores, git SHA, Elixir, and OTP -- result JSON includes baseline-vs-optimized smoke comparisons for - `LabeledFewShot`, `BootstrapFewShot`, `COPRO`, `MIPROv2`, `SIMBA`, and - `GEPA` over the sampled rows when at least two examples are available - -It does not prove model quality. - -## Run Golden Trace Parity - -```sh -mix benchmark.trace.check -``` - -This is the provider-free Imp-vs-DSPy parity lane. It replays checked-in -fixture responses through Imp and the Python DSPy sidecar, then writes a -`golden-trace-parity-*.json` artifact. The current corpus covers: - -- `Predict` with field-labelled chat output -- `ChainOfThought` -- typed output coercion -- JSON adapter output -- ReAct lookup tool trajectory normalized across DSPy trajectory fields and - Imp provider tool calls -- multi-tool ReAct trajectory normalization -- ReAct tool-argument error status parity -- missing-field error status parity -- normalized prediction parity -- retained Imp and DSPy message histories for prompt-template review -- Imp semantic checks for incremental field streaming, save/load credential - redaction, ReqLLM cache hits, and provider text/tool-call stream chunk replay - -This lane is intentionally stricter and cheaper than live benchmark parity: -prediction and expected-error parity must pass without provider nondeterminism. -It does not claim byte-identical prompt/message-template parity; Imp keeps an -Elixir-native provider-tool prompt shape and records both message histories so -template differences stay reviewable instead of hidden. - -## Run Provider-Free Overhead Parity - -```sh -mix benchmark.overhead.check -``` - -The canonical alias requires a clean checkout. During development, -`mix imp.benchmark.overhead --no-require-clean ...` may produce a diagnostic -artifact, but it remains ineligible for a performance statement until the -integrated source is committed and rerun cleanly. - -This lane compares Imp and Python DSPy without provider latency. It runs local -runtime benchmarks for: - -- signature parsing -- adapter message formatting -- adapter response parsing -- schema validation -- evaluation loop throughput -- metric normalization -- optimizer trial scheduling -- trace redaction and JSON serialization -- cache hits and misses -- concurrent orchestration - -The artifact reports per-case median, mean, p95, min, max, and a measured -`median_ratio_imp_over_dspy`. Every operation has its own absolute Imp-median -budget, reference-relative median budget, operation contract, and rationale. -The cache hit, cache miss, schema-validation, and BootstrapFewShot cases execute -the same logical operation and matched configuration in both runtimes. The -artifact records BEAM, Python, OS, architecture, dependency, and clean-source -identity in a verified run envelope. - -These budgets are regression alarms, not parity, superiority, or speed claims. -Ratios are measurements only. No path-specific speed claim is authorized by a -passing ceiling; such a claim would require a separately declared and powered -comparison. - -## Run Shared Inference-Time Search Evidence - -```sh -mix benchmark.search.check -``` - -This provider-free source-checkout lane runs the same natural answer-candidate -fixture through sequential and bounded-concurrent `Imp.Predict.Search`. Its -deterministic checks cover selected answer and quality, candidate-order -provenance, admitted projected budget, executed outcomes' projected budget, -and the observed concurrency bound. The artifact labels projected cost units -separately from actual provider cost; no provider is called, so billed usage is -unavailable rather than inferred. - -The artifact also records latency sample distributions and the observed median -ratio. Those fields are measurements for the checkout, runtime, scheduler, and -configured synthetic work only. A concurrent speedup is not required and does -not participate in artifact pass/fail status. This lane therefore complements -the focused runtime tests without making a flaky wall-time release assertion or -a live model-quality, provider-latency, or provider-cost claim. - -## Run Optimizer Lift Parity - -```sh -mix benchmark.optimizer_lift.check -mix benchmark.instruction_optimizer.contract.check -mix benchmark.gepa.contract.check -``` - -This provider-free lane uses a deterministic task with known baseline and -optimum scores. The current artifact directly compares Imp and DSPy -`LabeledFewShot`, `BootstrapFewShot`, `RandomSearch`, `COPRO`, `MIPROv2`, and -`SIMBA` and `GEPA` lift when the installed DSPy sidecar exposes them. It records -documented Imp-only or intentional-deviation evidence for Elixir-native -`InstructionSearch` and provider-side trainer workflows such as finetuning and -mmGRPO. Imp's GRPO implementation authority is pinned DSPy 3.2.1 source; -DeepSeekMath is background rather than an implementation-parity authority. The -artifact records the installed Python `dspy` package version and -detected optimizer capabilities so the lane stays honest as the upstream runtime -changes. The same artifact includes natural Imp user-story lanes for -classification, QA, retrieval/KNN few-shot, and instruction following, with -baseline score, optimized score, lift, call counts, cost estimate, and selected -demos or instructions. - -The second command runs the separate T1 structural differential against pinned -DSPy `3.3.0b1`. It validates exact source hashes and compares MIPROv2 budgets, -demo topology, proposal rotation, search-space shape, and full-evaluation cadence -plus SIMBA bucket, finalist, rollout, tied-rule, and eviction invariants. - -The COPRO row in the lift artifact also carries a separate pinned, provider-free -process differential against the stable authority, DSPy `3.2.1` at commit -`29448ae12756abdd14bd8796c819247ebb83673c`. Prepare an absent environment with: - -```sh -scripts/setup_dspy_stable_source.sh -IMP_DSPY_VENV=tmp/dspy-parity-venv scripts/setup_dspy_parity_env.sh -``` - -Then capture the clean, source-bound C1 receipt with: - -```sh -mix imp.benchmark.copro_isolation --require-clean --out tmp/copro-isolation -``` - -BootstrapFewShot and RandomSearch have separate provider-free, clean-source C1 -protocols against the same stable DSPy authority: - -```sh -mix imp.benchmark.bootstrap_few_shot_differential --require-clean --out tmp/bootstrap-few-shot-differential -mix imp.benchmark.random_search_differential --require-clean --out tmp/random-search-differential -``` - -The canonical BootstrapFewShot (`9b89dac9…`) and RandomSearch (`f7e49685…`) -receipts were captured from clean Imp commit `fd48e27`. COPRO (`4f2d959d…`) -was recaptured from clean commit `f9ee18c` after its runtime safety boundary -changed. Their validators recompute the exact source bindings and retained -scopes. The protocols exclude -exact Python RNG, provider behavior/effectiveness, and full optimizer parity; -BootstrapFewShot also excludes repeated-call sampling parity, and RandomSearch -excludes shuffled-row-order parity. - -The canonical validator is receipt-only and provider-free by default; fresh Python -replay is an explicit additional operation. The fixture starts COPRO in a fresh worker process after installing mutable parent -LM state. It verifies the canonical authority ledger, clean release commit/tag, -all 296 source-manifest files, distribution version, COPRO source, and upstream -test before binding fixture/script hashes. Proposal fan-out and order come from -the isolated DSPy LM's actual call history and parsed response choices, independent -of the expected-order assertion. The artifact records the tagged source's known -`dspy.__version__ == "3.2.0"` metadata anomaly separately from the authoritative -3.2.1 distribution/git identity. It also directly observes evaluation order and -equal-score duplicate removal. First-record retention is separately source-supported -by the pinned COPRO implementation's greater-than-or-equal score guard rather than -claimed as an independently observable artifact result. The fixture also covers -pinned `results_latest`/`results_best` statistics. This is narrow C1 behavioral -evidence only: it does not claim exact Python RNG parity, provider behavior, -effectiveness, or full optimizer parity. - -Run the resumable, paid one-seed AIME preflight from the shared Imp/DSPy -manifest with: - -```sh -mix imp.benchmark.instruction_optimizer_experiment \ - --manifest benchmarks/config/instruction-optimizer-aime-economical-preflight-haiku45-v1.json \ - --runtime both \ - --python tmp/dspy-parity-venv/bin/python \ - --dspy-pythonpath tmp/dspy-current-target \ - --out benchmarks/runs/instruction-optimizer-experiment -``` - -This command pins DSPy and Optuna, verifies immutable split hashes, maps the -same logical model to each runtime's provider identifier, and enforces the same -per-arm request/input/output/USD ceilings before merging results. It reports -frozen-test deltas for every arm. It does not choose a global winner from dev, -and its one seed is explicitly research preflight rather than T3 evidence. -The admitted Haiku 4.5 run completed baseline, MIPROv2, and SIMBA in both -runtimes without failures. All six runtime/arm rows scored `2/3` on frozen -test, which supports T2 live sampled behavior but neither optimizer lift nor -full parity. - -The third command runs a T1 structural differential against standalone GEPA -`v0.1.4` at commit `8b0ce6cd99a234f6b74daf37558a2ac0ce18f975`. -Set `IMP_GEPA_V014_ROOT` to the exact checkout and, when needed, -`IMP_GEPA_V014_PYTHON` to its Python environment. The task validates the tag, -commit, tagged project-version anomaly, and source hashes before comparing -provider-free acceptance, parallel proposal selection, Pareto, component -rotation, merge, frontier, budget, JSON resume/RNG, and named-program mutation -semantics. It explicitly does not establish paper reproduction, effectiveness, -or full optimizer parity. The older v0.1.1 artifact remains immutable history, -not the selected current contract. - -Optimizer lift is outcome evidence, not full optimizer parity. A missing, -stale, authority-mismatched, or failing structural artifact leaves parity -unestablished. Even a passing T1 artifact does not -replace held-out multi-seed T3 effectiveness evidence. Imp-only rows and equal -scores under unmatched internal decision paths cannot satisfy that stronger -claim. - -## Run Optimize Anything Replication - -Run the deterministic campaign contract and evaluator smoke with: - -```sh -mix benchmark.optimize_anything.check -``` - -Run the live non-prompt effectiveness campaign with a pinned provider model: - -```sh -mix imp.benchmark.optimize_anything \ - --live \ - --env-file .env \ - --provider openai \ - --model gpt-5.4-2026-03-05 \ - --pricing-profile openai-gpt-5.4-standard-2026-03-05 \ - --seeds 17,23,31 \ - --max-proposals 5 \ - --max-cost-usd 0.50 \ - --max-requests 45 \ - --max-input-tokens 100000 \ - --max-output-tokens 20000 \ - --max-output-tokens-per-request 1000 \ - --out benchmarks/runs/optimize-anything -``` - -The full lane optimizes three executable artifact classes: an Elixir retry -controller, a support-routing agent configuration, and a scheduling heuristic. -Each family has deterministic, pairwise-distinct train, selection, and untouched -test sets, a baseline, and an authored reference comparator. The optimizer sees -only train and selection data. Final scores and the multi-seed policy use the -test set, while the displayed representative is chosen by selection score so -test outcomes cannot choose the candidate. The comparator is a positive control -for evaluator headroom; it is not an upstream Optimize Anything parity result. - -Schema-v2 full evidence requires at least three distinct seeds, positive mean held-out -lift, a strict majority of improving seeds for every family, positive live -provider token and cost accounting, and per-run checkpoints. All seed -outcomes remain in the artifact, including ties and regressions. `--smoke` -proves campaign wiring and artifact validation only and never authorizes the -effectiveness claim. Full artifacts are validated against the -`optimize_anything` protocol. - -Live execution has no implicit spend allowance. It requires positive finite -ceilings for requests, input tokens, output tokens, per-request output tokens, -and dollars. Before each request, the campaign atomically reserves a -conservative input estimate plus the full per-request output allowance at the -declared prices; a reservation that could cross any ceiling rejects the call -before provider code runs. The evidence lane disables ReqLLM response caching -and transport retries, so one reservation owns one provider attempt. Provider -telemetry settles observed usage exactly once in the campaign ledger. Missing, -zero, non-finite, or non-one-to-one cost telemetry fails closed. If unexpected -provider accounting nevertheless reports usage above a declared bound on the -final call, that completed call remains in the checkpoint but the campaign -emits no full evidence. - -Each ledger transition sync-writes a temporary file and renames it over the -latest checkpoint. The final checksummed envelope is embedded in the run -artifact; admission recomputes its digest and exact equality with the seed-row -aggregate without reading the local path. The path is informational and may be -nonportable. This is neither an append-only transition log nor a restart, -power-loss, or directory-fsync guarantee. Existing run ids are refused, and a -terminated run must be reviewed before starting a new run id with a newly -declared ceiling. The telemetry handlers accept only events emitted by the -campaign owner process, preventing unrelated concurrent ReqLLM calls from -contaminating cost evidence. - -An OpenRouter engineering run on 2026-08-20 exposed that the formerly -documented 20-request ceiling could not cover the declared opportunity: three -artifact classes times three seeds times five proposals requires up to 45 -provider calls. The run stopped at its request ceiling after 20 calls, 34,706 -input tokens, 3,698 output tokens, and $0.042672, before it could emit a full -artifact; it is a treatment failure, not an effectiveness result. Its complete -checkpoint tree is retained as -`benchmarks/evidence/archive/optimize_anything/c08da2848e912487484e13cb83c09219134e15d595b3c9d050da69f611d46591.tar.zst`, -whose filename is its SHA-256. Campaign admission now rejects an insufficient -request ceiling before the first provider call. - -The pinned standard profile uses the official OpenAI API prices of $2.50 per -million input tokens and $15.00 per million output tokens from -. The documented $0.50 ceiling -is deliberately above the roughly $0.217 observed by the prior nine-run -campaign while remaining the configured pre-dispatch bound under the declared -prices, not a spending target. An unexpected provider accounting overrun is -retained in the checkpoint and invalidates evidence as described above. The -separately tracked $15 maximum belongs to the broader matched-upstream research -portfolio; it is not a spend allowance for this narrow three-class rerun and -does not add an asserted product claim. The -optional `openai-gpt-5.4-mini-standard-2026-03-17` profile uses $0.75/$4.50; -it is a cost-appropriate engineering option but has no retained effectiveness -claim until the unchanged three-class, three-seed policy passes. Other models -must supply explicit positive `--input-price-per-million`, -`--output-price-per-million`, and `--pricing-source-url` values instead of a -profile. Pricing-source URLs must be ordinary credential-free HTTP(S) -documentation URLs. Userinfo, credential or secret markers in recursively -decoded hosts, paths, queries, or fragments, excessive encoding, and -secret-shaped values are rejected rather than redacted because the URL is part -of source identity. Known profiles bind the exact provider, model, rates, and -authority URL at the CLI, campaign, and pure admission layers. - -The immutable, archived pre-v2 artifact `58ff84ac…` remains valid T2 evidence that a -live provider-backed campaign executed with recorded costs and checkpoints. -It does not establish C3 effectiveness because it selected and finally scored -candidates on the same development set. A later provider-backed retry-policy -schema-v2 condition used distinct train, selection, and untouched test rows: -two of three seeds selected proposer-generated values and improved exact -executable behavior from `3/6` to `5/6`, one retained baseline, and all three -loaded fresh. That is scoped evidence for one non-prompt task, not admission of -the declared three-class portfolio. - -The 2026-08-20 OpenRouter successor at Imp commit `7d8c5e23` completed the -schema-v2 three-class portfolio with `openai/gpt-5.4-mini`. Mean untouched-test -lift was `+0.254759` for the retry controller, `+0.518609` for the agent -configuration, and `+0.059774` for the scheduling heuristic. Improving-seed -counts were respectively `3/3`, `3/3`, and `2/3`; the remaining scheduling seed -tied baseline. The full artifact reports 44 single-attempt provider calls, -83,219 input tokens, 6,059 output tokens, and $0.082421 under the declared -$0.50 ceiling. The artifact and complete checkpoint tree are retained in -`benchmarks/evidence/archive/optimize_anything/a4181fed7bfffd3be7299f75923ec96e8aafa843f7f121804718c857fe44fc40.tar.zst`, -whose filename is its SHA-256. This satisfies the declared Imp-native -three-class effectiveness policy under that model and treatment; it does not -establish paper reproduction, equality with an upstream implementation, or -effectiveness outside those executable artifact families. - -## Run GEPA Paper Replication - -```sh -mix benchmark.gepa_replication.check -``` - -This source-checkout lane runs a deterministic smoke campaign by default and -validates GEPA paper-family artifact shape. It does not turn provider-free -optimizer lift or smoke rows into a paper claim. A full artifact must cover -`AIMEBench`, `HotpotQABench`, `hoverBench`, `IFBench`, -`LiveBenchMathBench`, and `Papillon`; for each row it must report baseline, -DSPy GEPA, Imp GEPA, MIPROv2, configured metric-call budget, observed metric -calls with enforced limits, token/cost, wall-clock, seed variance, seed-selection -provenance, and train/dev/test gap. Full rows must also carry a campaign id, -dataset source, dataset scope, split counts, split checksums, source commits -for DSPy, Imp, and the GEPA artifact, concrete non-placeholder comparator -sources, distinct train/dev/test split digests, and positive live token/cost -accounting. The full-evidence contract requires `dataset.scope == "full"` and -rejects capped `--max-per-split` dataset roots; capped roots are useful for -engineering proof runs only. SIMBA can appear as an extra comparator when a -campaign includes it, but it is not part of the upstream GEPA artifact's -required optimizer list. - -The scalar `metric_calls` and `optimizer_budgets` fields describe configuration; -they are not proof that the optimizer observed or enforced those limits. Every -full row must additionally include `metric_call_evidence` with basis -`observed_and_enforced`, a concrete counter/export source, observed counts for -all four required optimizer rows, and an affirmed enforced limit for each count. -Copying configured budgets into an "actual" field, omitting runtime provenance, -using a configured-only basis, or reporting an observed count above its limit -fails the GEPA result contract. Smoke evidence remains valid only at its lower -tier. - -Full rows must also include per-optimizer `seed_selection`. Accepted selection -methods are a predeclared seed, dev-only best-seed selection, or an aggregate -over declared seeds, all with concrete provenance and `test_scores_used: false`. -Choosing the reported best seed from test scores is test-set leakage and cannot -support parity or source-fidelity claims, even when seed variance is reported. -The Imp campaign passes every `--seeds` value and each family’s declared -`metric_calls` limit into `Imp.Optimizer.GEPA`, selects the reported seed by -dev score, and exports the optimizer report’s observed metric calls plus the -enforced limit for each seed. Comparator-side evidence must be added by the -upstream artifact converter before strict full-artifact validation. - -### Source-Shaped Program and Feedback Contract - -A full campaign conversion accepts exactly these six upstream family/program -shapes. The two `CoT` rows share a program name but retain their own source -signature, instructions, metric, splits, and budget; they are not -interchangeable rows. - -| Family | Program | Source signature | Scored output | -| --- | --- | --- | --- | -| `AIMEBench` | `CoT` | `problem -> answer` | integer exact match | -| `HotpotQABench` | `HotpotMultiHop` | `question -> answer` | answer exact match | -| `hoverBench` | `HoverMultiHop` | `claim -> retrieved_docs` | supporting-title retrieval | -| `IFBench` | `IFBenchCoT2StageProgram` | `prompt -> response` | instruction constraints | -| `LiveBenchMathBench` | `CoT` | `question -> answer` | task-specific math score | -| `Papillon` | `PAPILLON` | `user_query -> llm_request, llm_response, response` | quality/leakage judge | - -`HotpotMultiHop`, `HoverMultiHop`, and `IFBenchCoT2StageProgram` install -strict, named component-feedback maps. HotPot covers `summarize1`, -`create_query_hop2`, `summarize2`, and `final_answer`; HoVer covers -`summarize1`, `create_query_hop2`, `summarize2`, and `create_query_hop3`; and -IFBench covers `generate_response_module` and `ensure_correct_response_module`. -The map must cover the program graph exactly. Each callback receives the named -predictor input/output, full example, program output, metric result, and trace, -and must yield non-empty feedback text. Invalid callback output, a callback -failure, or a graph mismatch stops the optimization run. `AIMEBench`, -`LiveBenchMathBench`, and `Papillon` currently use metric-level feedback rather -than a custom component map. Campaign rows record the component-feedback -identity so reviewers can distinguish these contracts. - -When upstream GEPA artifact experiments have been run, convert their -`experiment_runs_data` output into Imp result rows with: - -```sh -mix imp.benchmark.gepa_dataset \ - --gepa-root path/to/gepa-artifact \ - --out benchmarks/data/gepa-campaign - -IMP_HOVER_UPSTREAM_BM25=1 \ -IMP_IFBENCH_UPSTREAM_DESCRIPTIONS=1 \ -IMP_GEPA_PYTHON=path/to/pinned/python \ -IMP_GEPA_ROOT=path/to/gepa-artifact \ -mix imp.benchmark.gepa_campaign \ - --manifest benchmarks/config/gepa-paper-campaign-v2.json - -mix imp.benchmark.gepa_replication \ - --from-gepa-artifact path/to/gepa-artifact/experiment_runs_data \ - --upstream-evidence benchmarks/runs/gepa-replication/gepa-upstream-evidence.json \ - --imp-input benchmarks/runs/gepa-campaign/imp-gepa-rows-*.json \ - --campaign-id gepa-full-YYYYMMDD \ - --artifact-model gpt-41-mini \ - --protocol-classification exact_paper_replication -``` - -That classification is an explicit C4 assertion for this paper-authority -protocol. Do not pass it to the adapted current-model no-merge table; complete -rows without it remain non-full and cannot authorize paper-replication claims. - -The canonical manifest independently binds the task, reflection, and Papillon -judge model roles, even when they share the same dated model identifier. This -matches the pinned paper artifact: GEPA leaves `teacher_lm` unset, so reflection -uses the configured task LM, while Papillon separately fixes its judge to -GPT-4.1-mini. The manifest also binds the six families, full dataset hash, -seeds, metric-call budgets, source commits, request policy, output paths, and -required source-exact environment. It also binds a checkpointed semantic -sentinel: five consecutive proposal errors abort without producing a result -artifact, while valid non-improving candidates remain ordinary GEPA evidence. -It rejects every CLI override. Direct CLI -mode remains available for partial operator runs, but it is not the canonical -paper-reproduction contract. - -For long full-scope runs, execute one or more families at a time with -`--families AIMEBench,HotpotQABench`. These partial campaign artifacts are -resumable operator evidence; before conversion, merge the six family rows into -one Imp input artifact so the replication contract can verify the complete -paper-family set. - -The Imp campaign producer expects a `families.json` file plus one directory -per GEPA family, each with `train.jsonl`, `dev.jsonl`, and `test.jsonl`. -`families.json` declares each family’s signature, instructions, input keys, -output key, program name, metric-call budget, upstream metric name, source -commit, dataset scope, optional max-per-split cap, split counts, and split -checksums. The dataset exporter imports the upstream GEPA artifact benchmark -classes and preserves their split construction. Passing `--max-per-split` -marks the root as `capped`, and those rows cannot satisfy a full GEPA research -claim. The converter then reads upstream `Baseline`, `GEPA`, and `MIPROv2-Heavy` -`evaluation_result.txt` files and merges them with Imp-produced `imp_gepa` -rows. It refuses missing families, missing comparator outputs, ambiguous -artifact models, and rows that do not satisfy the full-evidence contract after -merge. - -The upstream evidence sidecar is mandatory for conversion. Generate it from -the immutable upstream experiment archive and the matching upstream checkout: - -```sh -python3 scripts/extract_gepa_upstream_evidence.py \ - path/to/experiment_runs_data.tar.gz \ - --upstream-repo path/to/gepa-artifact \ - --model gpt-41-mini \ - --out benchmarks/runs/gepa-replication/gepa-upstream-evidence.json -``` - -The extractor requires the six family/program pairs above and the `Baseline`, -`GEPA`, and `MIPROv2-Heavy` seed-0 runs. It records the archive SHA-256 and -upstream commit; reads upstream `config.json`, metric JSONL, and -`evaluation_result.txt`; derives observed optimizer callbacks by subtracting -the matching Baseline final-test callbacks; and reads the configured comparator -budget from the upstream source. The replication task accepts only a sidecar -whose exact family/program/optimizer keys match the required comparator set, -whose reported test score and result SHA-256 match the archive result, and -whose evidence proves observed, enforced, within-budget calls and non-test seed -selection. It will not infer this evidence from `evaluation_result.txt` or -configured budgets alone. - -Campaign or converted rows that still expose only configured call budgets or -select their reported seed by test score are useful operator artifacts, but they -do not satisfy the full GEPA contract. They must remain red until the producer -emits the observed/enforced call evidence and non-test seed-selection provenance -described above. - -The exported `families.json` records upstream metric names. Imp currently -ports deterministic metric adapters for AIME integer exact match, HotPotQA -answer exact match, HoVer supporting-title retrieval, IFBench -IFEval-style instruction constraints, Papillon LLM-judge quality/leakage -scoring, the deterministic LiveBenchMath AMC/AIME parser paths, and -LiveBenchMath `imo`/`usamo` proof-rearrangement edit-distance scoring. GEPA -HoVer uses the upstream `HoverMultiHop` output contract (`claim -> -retrieved_docs`); the metric scores retrieved document titles against -`supporting_facts`, not the entailment label. Both `HotpotQABench` and -`hoverBench` require `dataset.retrieval` provenance for the same upstream -`wiki.abstracts.2017` BM25 corpus and index, including corpus and index -checksums. For either family, the campaign command requires -`IMP_HOVER_UPSTREAM_BM25=1`; it then executes retrieval through the pinned -upstream Python BM25S index. The native Elixir BM25 retriever is an explicitly -labeled approximation, not a source-exact campaign substitute: it does not -reproduce the upstream English stopword tokenizer or PyStemmer stemming. The -pinned Python adapter uses upstream commit -`cbefbc1aa0f43dd39874ec4bf42211365dbda42e`, `bm25s==0.2.12`, and -`pystemmer==2.2.0.3`; its fixed top-k title order is validated with -`IMP_HOVER_UPSTREAM_PARITY=1 mix test test/hover_bm25_parity_test.exs`. For -campaign rows over the full upstream corpus, set `IMP_HOVER_UPSTREAM_BM25=1`, -`IMP_GEPA_ROOT`, and `IMP_GEPA_PYTHON` for both families. Their multi-hop -queries are generated by the LM and the resulting rows report ReqLLM usage -telemetry. These rows do not support a full GEPA research claim until uncapped -results are merged with matching upstream comparator outputs and sidecar -evidence. -IFBench imports the larger AllenAI `instructions_registry`; Imp ports the -registry in Elixir and keeps unknown ids fail-closed rather than silently -scoring as false. Five upstream IFBench checks depend on Python NLP packages -(`langdetect`, `nltk` stopwords/POS data, `emoji`, and `syllapy`). Imp ships native fallback -checks for normal deterministic evidence and a source-exact bridge for research -campaigns: set `IMP_IFBENCH_NLP_BRIDGE=scripts/ifbench_nlp_check.py` and, when -needed, `IMP_IFBENCH_NLP_PYTHON` to a Python with those packages and corpora. -The registry differential covers all 83 active merged-registry instruction ids -and matches the pinned GEPA artifact fixtures, including language detection and -the four NLP-backed checks. Reproduce it with: - -Full IFBench GEPA campaigns also set -`IMP_IFBENCH_UPSTREAM_DESCRIPTIONS=1`, `IMP_GEPA_ROOT`, and -`IMP_GEPA_PYTHON`. Scoring remains in the Elixir registry port; reflective -feedback renders the corresponding human instruction descriptions through -`scripts/ifbench_upstream_describe.py` from the pinned upstream registry. The -campaign records that description source and fails closed if the bridge is -missing or returns an invalid description set. - -```sh -python3 -m venv tmp/ifbench-parity-venv -tmp/ifbench-parity-venv/bin/python -m pip install \ - -r benchmarks/requirements-ifbench-parity.txt -tmp/ifbench-parity-venv/bin/python -m nltk.downloader \ - -d tmp/ifbench-parity-venv/nltk_data \ - stopwords averaged_perceptron_tagger_eng punkt_tab -NLTK_DATA="$PWD/tmp/ifbench-parity-venv/nltk_data" \ -IMP_IFBENCH_UPSTREAM_PARITY=1 \ -IMP_IFBENCH_UPSTREAM_PYTHON="$PWD/tmp/ifbench-parity-venv/bin/python" \ - mix test test/gepa_metrics_test.exs -``` - -The absolute interpreter path is intentional because the test runner may -change its working directory while spawning the upstream evaluator. -Papillon campaigns must pass a judge LM and emitted research rows must include -`metric_judge` metadata naming the judge model plus quality/leakage judge -semantics; the full GEPA replication contract rejects Papillon rows without -that provenance. Unknown LiveBenchMath task branches now fail closed. -LiveBenchMath `amps_hard` remains guarded because upstream uses SymPy/Lark -symbolic equivalence; install and validate the symbolic Python bridge before -claiming AMPS_Hard parity. The default bridge is -`scripts/livebench_math_score.py`; pin `IMP_LIVEBENCH_MATH_PYTHON` and, when -needed, `IMP_LIVEBENCH_MATH_BRIDGE` for research campaigns. - -## Run RAG And Tool Differentials - -The former aggregate RAG/tool/agent evaluator was removed in 0.3 with the -disconnected `Imp.Agent` runtime. Ordinary product behavior is tested at its -canonical boundaries: ReAct/ReActV2, RLM, `Imp.Tool`, tool policy, MCP, -retrieval, streaming, persistence, and supervised tasks. The remaining -source-bound runners below exist only where an external differential adds -information that those direct product tests cannot. - -The provider-free HotPotQA retrieval differential uses the pinned -first ten `fullwiki` validation rows and materializes one shared corpus of 100 -uniquely titled passages. It binds the dataset, split manifest, scorer/config, -Imp task, Python sidecar, and DSPy 3.2.1 authority by SHA-256. Both runtimes use -the same stable token-overlap ranking, document IDs, top five, and context -ordering, then report supporting-title recall plus extractive answer-availability -EM/F1: - -```sh -mix imp.benchmark.hotpot_retrieval \ - --require-clean \ - --out benchmarks/runs/hotpot-retrieval -``` - -The bounded current result matches 10/10 rows and both aggregate summaries: -0.35 supporting-fact recall and 0.30 answer-availability EM/F1. The answer -scorer emits the gold answer only when it is present in retrieved context, so -this is retrieval/answer-availability evidence rather than language-model -generation quality. It does not close the broader HotPotQA effectiveness or -BFCL portfolio; the separate failure differential below is operational rather -than effectiveness evidence. - -The BFCL-shaped provider-free lane is intentionally C1/T1 fixture-scorer -conformance, not an official BFCL sample, DSPy differential, or operational -benchmark. Its twelve positive rows and nine adversarial mutations are -original CC0 Imp-authored cases. The mutation corpus covers wrong, missing, -extra, and reordered calls; scalar types; array order; malformed JSON; invalid -terminals; and wrong valid terminals. The provenance block explicitly records -that no upstream BFCL prompts, answers, schemas, or dataset rows were copied. -Elixir and an independent Python stdlib implementation normalize JSON -string/map arguments, recursively canonicalize object keys while preserving -arrays and scalar types, and score exact tool names, call order, arguments, -terminal states, and fail-closed invalid input: - -```sh -mix imp.benchmark.bfcl_adapted \ - --require-clean \ - --out benchmarks/runs/bfcl-adapted -``` - -The artifact must match 12/12 positives and 9/9 preregistered mutations with -1.0 scorer and mutation-detection agreement. It binds BFCL repository revision -`6ea57973c7a6097fd7c5915698c54c17c5b1b6c8` as protocol provenance only and -enumerates every adaptation from the official scorer. Neither official BFCL -nor DSPy scorer code executes. This establishes scorer-fixture agreement only; -it does not measure a model choosing calls and must not be reported as -operational evidence, DSPy parity, official BFCL accuracy, or tool-use -effectiveness. - -Canonical admission is pure: it verifies the run envelope and pinned source -bindings, reconstructs the expected fixture scores, and requires both stored -implementations to equal those scores without executing Python. Maintainers may -request an explicit Python replay during a local audit, but replay is never part -of registry admission. - -The provider-free RAG/tool failure differential preregisters six scenarios and -executes the identical queued action sequence through actual -`Imp.Predict.ReAct` in DSPy-3.2.1 mode and actual pinned DSPy 3.2.1 `ReAct`: - -```sh -mix imp.benchmark.rag_tool_failure_differential \ - --require-clean \ - --python tmp/dspy-parity-venv/bin/python \ - --out benchmarks/runs/rag-tool-failure-differential -``` - -It compares every normalized action observation and terminal state exactly. -The schedule covers a transient failure followed by retry, a retriever-tool -timeout exception, duplicate idempotency-key replay, a tool removed from the -runtime registry, a permanent tool failure, `finish`/`submit` normalization, -and iteration-budget exhaustion. Before importing DSPy, the Python sidecar -requires a clean git checkout at tag `3.2.1` and commit `29448ae…`, verifies all -296 canonical manifest files, then confirms that ReAct and Tool resolve from -that checkout. It binds the installed distribution version separately from -DSPy's historical `3.2.0` module version, so a fake package or matching version -string cannot pass. The Elixir launcher removes credential-bearing environment -variables before process start; Python scrubs again before import, disables -dotenv, and checks a dummy canary is absent during every queued LM call. - -This is C2 operational evidence. The deterministic LM supplies every action, -the retry and idempotency state machines belong to fixture tools, and the -timeout is an injected exception rather than a wall-clock cancellation test. -Therefore the artifact is not evidence for model recovery quality, retrieval -quality, native retry/idempotency features, latency, transport timeouts, or -research effectiveness parity. Its validator recomputes rows, summaries, -limitations, source hashes, and exact scenario order instead of trusting pass -booleans. - -## Run RLM Benchmark Parity - -```sh -mix benchmark.rlm.check -``` - -This command runs T0 deterministic contract replay over two hand-authored, -HotPotQA-shaped rows. It verifies that the local Imp and Python DSPy harnesses -execute their scripted paths and records traces for inspection. Gold-derived -outputs, tiny contexts, and intentionally different traces mean this artifact -does not prove effectiveness, long-context behavior, operational parity, or -statistical uncertainty. The release RLM lane requires a separate T3 -paper-protocol artifact. - -For matched provider-free operational semantics against the current DSPy RLM: - -```sh -scripts/setup_dspy_parity_env.sh -uv pip install --target tmp/dspy-current-target --no-deps 'dspy==3.3.0b1' -mix benchmark.rlm.contract.check -``` - -This T1 suite executes twelve required cases in both Imp and DSPy 3.3.0b1: -persistent state, typed submission, safe transformations, single and -programmatic-loop subqueries, ordered batches, exact and atomic call accounting, -submit repair, extraction fallback, and trajectory retention. The artifact pins -the installed upstream source SHA256 and declares Imp's symbolic `recurse/2` -helper as an extension. T1 proves matched execution semantics only; it does not -measure long-context effectiveness and cannot satisfy the T3 release lane. - -## Run Live Benchmark Smoke - -```sh -OPENAI_API_KEY=... OPENAI_MODEL=... mix benchmark.live.check -``` - -This fetches two fresh rows from GSM8K and HotPotQA, runs Imp programs against -a live provider, and writes a run artifact under `benchmarks/runs/benchmark/`. - -## Run Imp vs DSPy Parity - -Install Python DSPy in the local parity environment: - -```sh -scripts/setup_dspy_parity_env.sh -``` - -The setup script chooses `python3.13`, `python3.12`, `python3.11`, or -`python3.10`, then installs current stable DSPy with the `optuna` extra needed -by MIPROv2: - -```sh -# Equivalent manual setup: -python3.12 -m venv tmp/dspy-parity-venv -. tmp/dspy-parity-venv/bin/activate -python -m pip install -U pip setuptools wheel "dspy[optuna]>=3.2.1,<3.3" openai -``` - -Then run: - -```sh -OPENAI_API_KEY=... mix benchmark.parity.check -``` - -This runs Imp and the real Python `dspy` package over the same fetched GSM8K -and HotPotQA rows, using the same OpenAI-compatible model. For reproducible -evidence, set `OPENAI_MODEL` or pass `--model` with a provider model id you have -verified in the current account. If neither is set, Imp queries the -OpenAI-compatible `/models` endpoint and auto-selects only when exactly one -text-generation-looking candidate is visible. If discovery fails, returns no -candidate, or returns multiple candidates, the task stops and asks for an -explicit `--model`; it does not invent a fallback model or choose among paid -models on the operator's behalf. - -The parity report records: - -- Imp and DSPy versions/runtime metadata -- benchmark prompt/signature contract identity for each runtime -- requested and effective generation settings, including endpoint route - evidence -- task scores and aggregate score delta -- task latency and Imp/DSPy latency ratio -- error counts -- row-level pass/fail agreement and answers -- bounded disagreement examples and per-task disagreement direction counts -- evidence scale: `smoke`, `research_sample`, or `full` - -This is the required lane for parity claims. Imp-only benchmark truth proves -Imp behavior; parity requires the Python DSPy sidecar. - -The Python runner is a direct OTP Port executable in its own session and -process group. If the campaign caller exits or `--dspy-timeout-ms` expires, the -Port owner sends checked TERM to the complete group, probes group liveness -through a bounded grace period even if the Python leader has already exited, -and sends checked KILL to any surviving descendants. Combined stdout/stderr is -kept as a 256 KiB tail with explicit byte-count and truncation metadata, which -preserves the final `DSPY_REPORT_PATH` sentinel without allowing output floods -to grow BEAM memory without bound. Diagnostics are redacted with the configured -`--api-key-env` value before task errors are logged. Python provider exceptions -are likewise persisted as bounded, redacted error records rather than raw -exception representations. A completed report is atomically promoted from its -`.partial` path; cancellation cannot promote a partial DSPy report. - -Two rows are a smoke test, not a leaderboard. They prove only that both sides -can run against the same data and endpoint. Use research samples or the full -lane before making quality/efficiency claims: - -```sh -mix imp.benchmark.fetch --tasks gsm8k,hotpotqa --length 200 --out benchmarks/data -mix imp.benchmark.parity \ - --gsm8k benchmarks/data/gsm8k-test-0-200.jsonl \ - --hotpotqa benchmarks/data/hotpotqa-validation-0-200.jsonl \ - --max-examples 200 \ - --models "$CURRENT_LOW_COST_MODEL,$FRONTIER_SANITY_MODEL" -``` - -OpenAI is the default parity provider. For another provider, make both sides -explicit so the artifact proves a matched operational path instead of an -accidental OpenAI-shaped comparison: - -```sh -PROVIDER_API_KEY=... mix imp.benchmark.parity \ - --gsm8k benchmarks/data/gsm8k-test-0-200.jsonl \ - --hotpotqa benchmarks/data/hotpotqa-validation-0-200.jsonl \ - --max-examples 200 \ - --model "$IMP_PROVIDER_MODEL" \ - --dspy-model "$IMP_DSPY_MODEL" \ - --api-key-env PROVIDER_API_KEY -``` - -The Imp side takes a ReqLLM model spec such as `anthropic:...` or -`google:...`; the DSPy side takes the matching LiteLLM/DSPy model name such as -`anthropic/...` or `gemini/...`. The artifact records both wire API families so -the live matrix can reject endpoint mismatches. - -The intentionally expensive full live row lane is: - -```sh -OPENAI_API_KEY=... mix benchmark.parity.full -``` - -That command fetches GSM8K test and HotPotQA distractor validation in full, then -runs Imp and Python DSPy over the same rows. It can take a long time and spend -real provider money. Its artifacts can support the live matched-model part of a -full parity claim, but not the entire claim by themselves. Use -`PARITY_VALIDATION_PROGRAM.md` for the complete standard. - -## Aggregate Live Model Matrix - -```sh -mix benchmark.live_matrix -``` - -This consumes existing `imp-dspy-parity-campaign-*.json` artifacts and writes -`live-matched-model-matrix-*.json`. It is the canonical answer to "which -provider/model lanes have actually been proven?" It groups by provider and -model, skips malformed historical artifacts, tags current low-cost, frontier, -and historical/research-style lanes, and reports whether each lane satisfies -its release-evidence policy. - -Lane tags are model-family evidence buckets, not vendor commitments. Current -low-cost includes small/mini/nano/Haiku/Flash/Lite-style models; frontier sanity -includes current flagship-style GPT, Claude Sonnet/Opus, and Gemini Pro models; -historical/research-style includes legacy GPT-3.5/Davinci, Claude 3-era, Gemini -1.x, or explicitly research/legacy-labeled models. - -The matrix is intentionally an evidence index, not a benchmark runner. -`current_low_cost` requires one full accepted canonical campaign. Frontier and -historical/research lanes require fresh matched research samples: enough rows -to expose provider drift without pretending every flagship or legacy model must -pay the full canonical cost. If the matrix reports smoke coverage, stale prompt -contracts, incomplete current low-cost coverage, or unsatisfied research-sample -lanes, Imp has not yet proven live matched model parity. Each model row and -live-lane blocker reports covered rows, -remaining rows, percent coverage, and estimated remaining/full Imp-plus-DSPy -tokens so staged campaigns can be planned from the result instead of hand -calculated. Cost is token-only by default; set -`IMP_BENCH_INPUT_USD_PER_1M` and `IMP_BENCH_OUTPUT_USD_PER_1M` when you want -the matrix to include USD estimates from current provider pricing. - -When a lane has multiple candidate models, the lane-level `coverage` and `cost` -headline the strongest candidate because one satisfying model is sufficient for -that lane. The same objects retain a nested `cumulative` summary so operator -tools can still see total evidence and spend across all candidates. - -The selected artifact for a model must also carry the current Imp benchmark -prompt contract compiled into the benchmark truth runner. Older artifacts -remain valuable history, but they are not release evidence after the task prompt -or signature contract changes. The matrix exposes this as -`summary.prompt_contract.complete`; a non-current selected lane cannot support -a current parity statement. - -When a dataset contract changes or a fresh full campaign supersedes older -smoke evidence, filter the matrix to the intended lineage: - -```sh -IMP_BENCH_CAMPAIGN_ID=req-llm-current-low-cost-full-YYYYMMDD \ - mix benchmark.live_matrix -``` - -The lower-level task also accepts `--campaign-id`. This prevents invalidated -artifacts from winning matrix selection merely because they contain more rows -from an older benchmark contract. - -For operationally safer full runs, execute fixed-size chunks with `--offset` -and `--max-examples`, then preserve every emitted artifact: - -```sh -mix imp.benchmark.parity \ - --gsm8k benchmarks/data/gsm8k-test-0-1319.jsonl \ - --hotpotqa benchmarks/data/hotpotqa-validation-0-7405.jsonl \ - --campaign-id req-llm-current-low-cost-full-YYYYMMDD \ - --env-file .env \ - --offset 0 \ - --max-examples 100 \ - --max-concurrency 8 \ - --model "$CURRENT_LOW_COST_MODEL" -``` - -Chunked runs avoid losing an entire benchmark to one network interruption. A -full parity claim still requires covering the complete row range. Use one -stable `--campaign-id` for all chunks in a fresh run; aggregation can then -exclude older smoke artifacts instead of mixing them into the full-campaign -claim. - -To advance a campaign without babysitting each offset: - -```sh -mix imp.benchmark.parity.campaign \ - --model "$CURRENT_LOW_COST_MODEL" \ - --dspy-model "$CURRENT_LOW_COST_DSPY_MODEL" \ - --campaign-id req-llm-current-low-cost-full-YYYYMMDD \ - --env-file .env \ - --gsm8k benchmarks/data/gsm8k-test-0-1319.jsonl \ - --hotpotqa benchmarks/data/hotpotqa-validation-0-7405.jsonl \ - --chunk-size 100 \ - --chunks 5 \ - --target-coverage 1000 \ - --max-concurrency 8 \ - --reasoning-effort low -``` - -For non-OpenAI campaign lanes, use the provider-qualified ReqLLM model as -`--model`, the matching DSPy/LiteLLM model as `--dspy-model`, and the relevant -`--api-key-env`. The campaign driver forwards those settings to every chunk. -For example, Anthropic uses `--model anthropic:claude-haiku-4-5` on the Imp -side and `--dspy-model anthropic/claude-haiku-4-5` on the Python DSPy side. -Do not pass the ReqLLM colon form as `--dspy-model`; omit the flag when the -default Imp mapper can derive the matching LiteLLM id. - -For measured transport A/B checks, configure ReqLLM's Finch pool before startup -through the same runner: - -```sh -mix imp.benchmark.parity.campaign \ - --model "$IMP_PROVIDER_MODEL" \ - --dspy-model "$IMP_DSPY_MODEL" \ - --api-key-env PROVIDER_API_KEY \ - --env-file .env \ - --req-llm-pool-protocols http1 \ - --req-llm-pool-count 16 \ - --chunk-size 100 \ - --chunks 1 -``` - -Use this for measured transport experiments, not as a hidden release-policy -escape hatch. Release evidence should record the campaign id, model, generation -settings, and pool settings whenever they change. - -The parity runner records its effective pool topology even when no pool flags -are passed. Its HTTP/1 default is one shard with `size` equal to -`--max-concurrency`. This preserves the requested parallel capacity without -randomly queueing colliding requests behind one-connection shards. Explicit -pool flags remain authoritative. The campaign driver applies this configuration -before starting ReqLLM; applying it after application startup does not rebuild -the already-running Finch pool. - -Each new Imp row also records ReqLLM request time and Finch request, queue, -connect, send, and receive counts and durations. These fields are process-local -and contain no headers, URLs, request bodies, or credentials. Use them to decide -whether a latency difference is provider time, pool contention, connection -setup, or retries before paying for a larger campaign. - -### ReqLLM HTTP/1 latency root cause (2026-07-14) - -The completed 8,724-row current-low-cost campaign established close quality -parity but reported an Imp/DSPy latency ratio of `1.765`. Runner-order splits -were similar, so order bias did not explain the gap. Its ReqLLM pool used the -upstream default of eight HTTP/1 shards with one connection per shard. - -A provider-free 80-request test with eight concurrent delayed responses held -total connection capacity constant. The `8 x 1` topology took `1,624.0 ms` -with `48.095 ms` mean queue time; `1 x 8` took `1,014.8 ms` with `0.101 ms` -mean queue time. Finch documents that multiple HTTP/1 shards can scatter work -and reduce connection reuse. - -A paid A/B/B/A crossover then ran the same 16 HotPotQA rows and matched model -under both runner orders: - -| Pool | Runner order | Imp ms | DSPy ms | Ratio | Mean Finch queue ms | -| --- | --- | ---: | ---: | ---: | ---: | -| `8 x 1` | Imp first | 7,789.403 | 3,583.527 | 2.174 | 1,392.365 | -| `1 x 8` | DSPy first | 4,474.305 | 5,069.463 | 0.883 | 1.384 | -| `1 x 8` | Imp first | 4,244.587 | 4,943.526 | 0.859 | 1.547 | -| `8 x 1` | DSPy first | 4,545.958 | 3,014.296 | 1.508 | 540.644 | - -Every trial recorded 16 ReqLLM lifecycles, 16 Finch requests, and zero runner -errors, ruling out hidden retries. The four paired trials cost `$0.144541` in -provider-reported usage. Their raw artifacts live under -`benchmarks/results/latency-root-cause/`. This diagnoses and fixes the harness -defect; it does not retroactively turn the historical full campaign's latency -outcome green. A future full claim must use a fresh campaign id and the recorded -effective `1 x concurrency` topology. - -Historical-metric annotation (2026-07-19, dee-c2ur): the per-row -`official_hotpotqa_f1`/`official_hotpotqa_em` values inside these four artifacts -were computed with the PRE-parity-port normalization (punctuation→space, no NFD) -on BOTH arms. They remain valid as latency diagnostics — the artifacts' purpose — -but their metric columns are not comparable to post-dee-c2ur scores. They are -deliberately left unedited (historical evidence is immutable); any future metric -claim must recompute from a fresh campaign. - -When resuming a release campaign after concurrency experiments, keep passing the -release `--max-concurrency` value. The campaign runner forwards that value to -aggregation, so coverage and next offsets are computed from the comparable -execution slice instead of mixing older serial/concurrent chunks into one -latency claim. - -`--chunks` limits how many new chunks this invocation may run. -`--target-coverage` limits the total campaign coverage to reach before -stopping. When both are present, the runner aggregates current evidence before -each chunk, chooses the next canonical missing offset, shrinks `--max-examples` -for the chunk when the target is near, and stops as soon as the aggregate has -reached the requested paired-row coverage. This is the preferred way to run -staged live campaigns because the stopping condition is evidence coverage, not a -hand-counted number of offsets. - -Current parity rows retain provider-reported input tokens, output tokens, and -USD cost independently for Imp and DSPy. Imp attributes ReqLLM telemetry in -the row process; the Python side attributes DSPy LM history by canonical row -input. Aggregation marks usage complete only when both runtimes have numeric -usage for every accepted row. The live matrix then projects remaining and full -campaign spend from the observed per-row averages. Its environment-based token -model remains an explicit fallback for legacy artifacts. Run a bounded -current-model tranche and inspect this projection before approving a full paid -campaign. - -The live matrix separates evidence completion from experimental outcome. The -`frontier_sanity` lane is complete when a fresh, error-free, matched sample has -at least 200 accepted rows and complete score, latency, prompt-contract, and -generation evidence. Its `parity_outcome` remains `parity_not_established` when -the predeclared score or latency threshold misses. This does not weaken the -threshold or create a parity claim; it prevents repeated paid sampling from -being used to shop for a passing result. Full current-low-cost parity still -requires full accepted coverage and all strict parity checks. - -If a live chunk produces runner/API errors, the campaign runner halts after that -chunk instead of continuing to spend provider calls. Any rows with complete -Imp/DSPy evidence are preserved, but quota/rate-limit failures remain -incomplete evidence, not negative benchmark rows. Fix provider -quota/credentials or switch to a matched provider/model lane, then rerun the -same campaign id to continue from the earliest missing accepted row. - -Use `--dspy-model responses/` when the matching Python DSPy/LiteLLM path -must force OpenAI Responses endpoint semantics for the selected model. The -runner normalizes that shorthand to LiteLLM's provider-qualified -`openai/responses/` identity. Imp reaches the provider through ReqLLM; -the explicit DSPy model route prevents comparing Responses semantics against -Chat Completions semantics by accident. - -For reasoning models, add `--reasoning-effort low` when the parity question is -throughput and answer-quality parity under a bounded reasoning budget. The -campaign artifact records requested and effective reasoning effort, and -aggregation treats different reasoning-effort settings as different generation -contracts so latency evidence cannot be mixed accidentally. - -If `--campaign-id` is omitted, the campaign task creates a unique id for that -invocation. Reuse an explicit id when resuming a long full campaign later. - -`mix benchmark.live_matrix` also reads `benchmarks/model_availability.json` by -default. Use that file for documented external model unavailability, not for -convenience skips. For example, historical GPT-3.5 snapshots that are no longer -stable API baselines can satisfy the `historical_research` lane only when the -file names the unavailable lane, explains the limitation, and links to provider -deprecation evidence. Current-model full coverage remains required for the -`current_low_cost` lane. - -Concurrency improves wall-clock time by issuing independent row calls in -parallel on both the Imp and Python DSPy sides. It does not reduce the number -of benchmark rows or provider calls, and reports record `max_concurrency` so -serial and concurrent artifacts are auditable. Campaign aggregates require one -consistent `max_concurrency` value before `full_parity` can be true; the live -matrix surfaces mixed or missing concurrency evidence because latency and -throughput claims are not comparable otherwise. - -Aggregate chunk artifacts into a campaign report: - -```sh -mix imp.benchmark.parity.aggregate \ - --provider req_llm \ - --model "$CURRENT_LOW_COST_MODEL" \ - --in "benchmarks/runs/parity/imp-dspy-parity-${CURRENT_LOW_COST_MODEL}-*.json" \ - --max-concurrency 8 -``` - -The aggregator counts each `(task, absolute_index)` once, so overlapping smoke -or retry chunks cannot inflate coverage. It also scopes reports by Imp provider -and model, so historical direct-client artifacts cannot be mixed into ReqLLM -campaigns. Runner/API error rows are incomplete evidence: they are not included -in coverage or scores, and a newer incomplete row cannot replace an older -complete row for the same canonical index. This matters for quota/rate-limit -failures, where an attempted chunk may produce row shells without valid model -answers. Those rows stay rerunnable and appear as `runner_error_rows` and -missing ranges instead of being treated as both-failed parity rows. It reports: - -- total covered rows versus canonical expected rows -- per-task covered rows, missing ranges, incomplete rows, and runner-error rows -- weighted Imp/DSPy scores from row-level pass/fail outcomes -- aggregate and task score gaps -- latency ratio from covered chunk artifacts -- runtime instrumentation summaries: Imp LM call counts, LM-duration share, - local overhead, fallback/retry counts, prompt size, raw output size, and - DSPy-side input/message/raw-size diagnostics. The benchmark-local DSPy LM - captures the exact history entry in worker-local storage before each call - returns, so concurrent usage and cost do not depend on shared-list append - order or prompt-text matching. `history_attribution` records - `thread_local_lm`; `shared_history_match` remains a compatibility fallback. - `message_chars` uses `row_estimate` only when no attributable entry exists. -- explicit `full_parity: true/false` - -`full_parity` is false unless every canonical row is covered, the prompt -contract and effective generation settings are consistent, complete, and -matched, `max_concurrency` is consistent across the campaign evidence, aggregate -and per-task score gaps are within the configured strict thresholds, and the -Imp/DSPy latency ratio is within the configured `--max-latency-ratio` threshold -(`1.5` by default). Latency is part of the decision because parity is about -operational behavior, not only answer quality. - -Historical checked-in campaign artifacts may be useful diagnostics, but they are -not release proof unless the live matrix selects them under the current prompt -contract, model-lane policy, effective generation settings, and concurrency -requirements. Treat stale named-model campaigns as prior evidence, not as a -template for new operator commands. - -Use the `imp_instrumentation`, `dspy_instrumentation`, and `runtime_shape` -summaries before optimizing runtime code. When Imp `lm_duration_share` is close -to `1.0`, the observed live latency is dominated by the provider/model call -rather than Imp adapter parsing or metric evaluation. Large Imp-vs-DSPy -`message_chars` or `raw_chars` ratios point toward prompt/output shape work; -nonzero `json_fallbacks` or `parse_retries` point toward adapter recovery work. -`runtime_shape.coverage.complete` must be true before treating shape ratios as a -full-campaign comparison; otherwise they are partial diagnostics from the rows -where both runtimes exposed comparable instrumentation. The live matrix ranks -complete instrumentation/runtime-shape evidence ahead of larger nominal coverage -when selecting the representative artifact for a model lane, because inflated -coverage from quota-tainted or otherwise incomplete chunks is not release proof. -Review `dspy_instrumentation.message_chars_sources` before using shape ratios -for fine-grained prompt work: `lm_history` is exact sidecar evidence, while -`row_estimate` is deterministic diagnostic evidence for rows whose DSPy history -was unavailable. New live evidence should also report -`history_attribution=thread_local_lm`; the shared-history fallback is not -sufficient for a complete concurrent cost claim. - -## Evidence Standard - -### Test-only operations diagnostic - -`mix benchmark.operations_stress.check` is deliberately outside the evidence -and claim system. It runs ten useful single-process deterministic assertions, -but its timestamped JSON does not bind a git tree, RunContext, environment, or -tamper checksum. The artifact declares `test_only_diagnostic` and -`claim_eligible: false`; it must not be admitted or cited at any C0-C5 level. -The same behaviors remain mechanically covered by ExUnit. Operational evidence -must come from a source-bound lane such as failure recovery or overhead. - -A credible Imp benchmark report must include: - -- dataset manifest SHA256 digests -- train/dev/test or offset/length split description -- model/provider/version metadata -- prompt/signature contract identity -- requested and effective generation settings -- Imp git SHA -- baseline score -- optimized score -- optimizer settings -- per-example scores or enough row detail to audit failures -- a clear statement of whether the run used fixture, local, or live provider - mode -- a clear statement of whether the evidence scale is smoke, research sample, or - full - -The current benchmark truth runner establishes the data/result substrate, live -smoke path, and optimizer comparison shape across the implemented prompt -optimizers. Full benchmark parity requires the broader validation program: -golden trace replay, live matched-model lanes, optimizer lift comparisons, -production-semantics tests, and provider-free performance reports. Tiny smoke -samples are useful release evidence, not leaderboard claims. diff --git a/docs/internal/COVERAGE_MATRIX.md b/docs/internal/COVERAGE_MATRIX.md deleted file mode 100644 index f2228b32..00000000 --- a/docs/internal/COVERAGE_MATRIX.md +++ /dev/null @@ -1,145 +0,0 @@ -# Imp Coverage Matrix - -This matrix is the production release truth table for Imp. It maps the concepts from -DSPy, Ax, and optimize_anything into the Elixir-native Imp surface. - -This is a maintainer document for an Imp source checkout. Its Mix commands and -repository paths are not package-consumer APIs. - -Status values: - -- **Implemented**: production surface exists and is covered by deterministic - tests. -- **Imp-native**: Imp intentionally uses a BEAM-shaped design instead of a - direct Python-shaped API. -- **Release blocker**: the feature is not acceptable for production without the linked - ticket. -- **Intentional omission**: Imp does not claim this production surface. - -## Core Programming Model - -| Concept | Imp status | Imp surface | Deterministic tests | Integration/live proof | Docs | Release decision | -| --- | --- | --- | --- | --- | --- | --- | -| Signatures as semantic contracts | Implemented | `Imp.Signature`, `Imp.Signature.Field`, and parser internals | `test/imp_test.exs`, `test/schema_constraints_test.exs`, `test/production_hardening_test.exs` | Covered through live `Predict`, JSON, CoT, ReAct, and PoT tests | `docs/LEARNING_PATH.md`, `README.md` | Keep as canonical contract model | -| Typed fields, descriptions, constraints, JSON schema | Implemented | `Imp.Signature.Field`, `Imp.Schema` | `test/schema_constraints_test.exs`, `test/production_adapter_persistence_test.exs` | Live JSON gate exercises generic JSON and provider-native JSON schema output | `docs/LEARNING_PATH.md`, `README.md` | Keep and extend only when provider needs require it | -| Examples and train/dev/test rows | Implemented | `Imp.Example`, `Imp.Datasets` | `test/imp_test.exs`, `test/datasets_contract_test.exs`, `test/completion_surface_test.exs` | `mix integration.check` covers file-backed dataset loading through a local RAG evaluation and save/load workflow | `docs/LEARNING_PATH.md`, `livebooks/03_evaluate_and_optimize.livemd` | Keep examples as plain data with explicit input fields | -| Predictions and metadata | Implemented | `Imp.Prediction` | `test/imp_facade_test.exs`, `test/public_surface_test.exs` | Covered indirectly in live provider tests | `docs/LEARNING_PATH.md`, `README.md` | Keep as the common output envelope | -| Conversation history | Implemented | `Imp.History`, `Imp.history/1`, `Imp.append_history/2` | `test/history_test.exs` covers multi-turn adapter rendering, redaction, JSON-safe dump/load, streaming composition, and provider-message boundary | Covered through normal Predict/Streaming paths; live provider examples can use the same public API | `docs/LEARNING_PATH.md`, `livebooks/01_real_lm_front_door.livemd` | Keep as signature-shaped task history, not raw provider chat logs | -| Process-local and global settings | Imp-native | `Imp.Settings`, `Imp.configure/1`, `Imp.context/2` | `test/imp_test.exs`, `test/imp_facade_test.exs` | Covered indirectly through live provider configuration | `docs/LEARNING_PATH.md`, `README.md` | Keep process-local design as the Elixir equivalent of dynamic DSP settings | -| Public facade | Implemented | `Imp` | `test/imp_facade_test.exs`, `test/public_surface_test.exs` | Live provider tests use facade constructors for core flows | `README.md`, `docs/LEARNING_PATH.md` | Keep facade narrow and canonical | - -## Model, Adapter, and Provider Layer - -| Concept | Imp status | Imp surface | Deterministic tests | Integration/live proof | Docs | Release decision | -| --- | --- | --- | --- | --- | --- | --- | -| LM behaviour and injectable clients | Implemented | `Imp.LM`, `Imp.req_llm/2`, `Imp.Clients.ReqLLM` | `test/req_llm_client_test.exs`, `test/completion_surface_test.exs`, `test/production_hardening_test.exs` | `LIVE_PROVIDER=1 mix live.check` covers ReqLLM-backed provider paths | `docs/LEARNING_PATH.md`, `docs/PRODUCTION_OPERATIONS.md` | Keep behaviour-first design with ReqLLM as the single product provider boundary | -| ReqLLM ecosystem-backed providers | Implemented | `Imp.req_llm/2`, `Imp.Clients.ReqLLM` | `test/req_llm_client_test.exs` covers message translation, schema/provider options, tool calls, streaming, and save/load | Live prediction, JSON, native schema JSON, CoT, streaming, ReAct, orchestration, and PoT through ReqLLM | `README.md`, `docs/LEARNING_PATH.md`, `README.md` | Use ReqLLM for production provider/model transport; do not expand Imp-owned provider clients | -| Adapters: chat, JSON, XML, two-step | Implemented | `Imp.Adapter.*` | `test/production_adapter_persistence_test.exs`, `test/schema_constraints_test.exs` | Live JSON and CoT use adapter paths | `docs/LEARNING_PATH.md`, `README.md` | Keep adapters as parse/format behaviours | -| Native structured-output negotiation | Implemented | `Imp.Adapter.JSON.lm_opts/2`, `Imp.Schema` | `test/production_adapter_persistence_test.exs`, `test/schema_constraints_test.exs` | Live JSON and CoT with JSON adapter | `docs/LEARNING_PATH.md`, `docs/PRODUCTION_OPERATIONS.md` | Keep and validate against provider drift through live gate | -| Streaming | Implemented | `Imp.Streaming`, ReqLLM-backed provider streaming, optional LM streaming callback | `test/req_llm_client_test.exs`, `test/completion_surface_test.exs` | Live provider streaming test | `docs/LEARNING_PATH.md`, `docs/PRODUCTION_OPERATIONS.md`, `README.md` | Keep enumerable interface; use ReqLLM streaming where supported | -| Provider-native reasoning | Implemented | `Imp.Predict.Predict` plans typed `:reasoning` outputs against LM capability; `Imp.Clients.ReqLLM` preserves `ReqLLM.Response.thinking/1`, reasoning details, outbound `Imp.Adapter.Types.Reasoning`, and thinking stream chunks | `test/native_reasoning_contract_test.exs` covers capable, incapable, explicit-opt-out, and configured-effort paths; `test/req_llm_client_test.exs` covers transport normalization and chunks | Live CoT should exercise the same typed field through a capable provider; deterministic fixtures prove the request/response seam without spend | `docs/LEARNING_PATH.md` | Keep one typed reasoning result shape; capability must come from the LM registry/client, never model-name guesses | -| Async/concurrency | Imp-native | `Imp.Clients.ReqLLM.generate_async/3`, `Imp.Predict.Parallel` | `test/production_hardening_test.exs` | Live orchestration test covers `Parallel` | `docs/LEARNING_PATH.md`, `README.md` | Keep Task-based interface and supervised runtime boundaries | -| Cache | Implemented | `Imp.Cache`, ReqLLM-backed cache path | `test/production_hardening_test.exs`, `test/public_surface_test.exs` | Live cache behaviour is not required for the current production claim | `docs/LEARNING_PATH.md`, `README.md` | Keep cache key redaction and telemetry covered | -| Multimodal primitives and quality lane | Implemented with narrow live image/native-PDF proof | `Imp.Adapter.Types`, `Imp.BenchmarkTruth.MultimodalRunner`, `mix imp.benchmark.multimodal_quality` | `test/multimodal_adapter_test.exs`, `test/multimodal_quality_benchmark_test.exs` cover the real ReqLLM Responses serializer, exact checkpoint schemas, HMAC tamper rejection, duplicate/cross-manifest/wrong-row attacks, cached-token accounting, resume, and redaction | OpenAI Responses proof `multimodal-quality-live-20260713T224355Z.json` passed 6/6 with six fresh dispatches, zero resumed rows, serialized image/native-PDF hashes, unique ReqLLM/OpenAI request and response IDs, provider-reported cache classification, and exact cost; `20260713T215119Z` is invalidated and removed | `docs/internal/MULTIMODAL_FIDELITY.md` | Keep image and native-document claims jointly gated; pre-dispatch shapes never authorize claims, missing cache classification prevents exact cost, rendered pages never imply native file support, and audio remains unsupported/unproven | -| Option validation and runtime dependencies | Implemented | option validation helper, network-facing constructors, runtime deps in `mix.exs` | `test/req_llm_client_test.exs`, `test/production_hardening_test.exs` | Covered by deterministic gate and exercised by integration/live tests | `README.md`, `docs/maintainers/RELEASE.md` | Keep dependency set justified by ecosystem leverage | - -## Program Modules - -| Concept | Imp status | Imp surface | Deterministic tests | Integration/live proof | Docs | Release decision | -| --- | --- | --- | --- | --- | --- | --- | -| Basic prediction | Implemented | `Imp.Predict.Predict`, `Imp.predict/2` | `test/imp_test.exs`, `test/public_surface_test.exs` | Live basic prediction and JSON prediction | `README.md`, `docs/LEARNING_PATH.md` | Keep | -| Chain-of-thought style reasoning field | Implemented | `Imp.Predict.ChainOfThought`, `Imp.chain_of_thought/2`; string by default, `%Imp.Adapter.Types.Reasoning{}` via `rationale_field_type: :reasoning` | `test/imp_test.exs`, `test/native_reasoning_contract_test.exs` | Live CoT with required reasoning | `docs/LEARNING_PATH.md`, `livebooks/02_programming_not_prompting.livemd` | Preserve DSPy 3.3.1's string default and explicit typed/native opt-in | -| ReAct/provider tool loop | Implemented | `Imp.Predict.ReAct` | `test/react_contract_test.exs`, `test/req_llm_client_test.exs`, `test/golden_trace_test.exs` covers normalized DSPy/Imp tool trajectory parity | Live ReAct function tools and reserved `submit`; golden trace covers lookup, multi-tool, and tool-error trajectories | `docs/LEARNING_PATH.md`, `livebooks/04_tools_agents_mcp_rlm.livemd` | Keep `ReAct` as canonical | -| Program of Thought | Implemented | `Imp.Predict.ProgramOfThought`, `Imp.Sandbox` | `test/completion_surface_test.exs` | Live PoT drives sandbox execution | `docs/LEARNING_PATH.md`, `README.md` | Keep BEAM-safe expression model | -| CodeAct | Implemented | `Imp.Predict.CodeAct`, `Imp.Tool`, `Imp.Sandbox` | `test/completion_surface_test.exs`, `test/public_surface_test.exs` | `mix integration.check` covers provider-style tool arguments across ReAct, RLM, and CodeAct through local MCP HTTP tools | `docs/LEARNING_PATH.md`, `livebooks/04_tools_agents_mcp_rlm.livemd` | Keep as explicit tool/sandbox loop | -| Recursive language model loop | Implemented | `Imp.Predict.RLM`, `Imp.Predict.RLM.SandboxSerializable` | `test/rlm_test.exs`, `test/imp_test.exs` | Deterministic action-control tests cover lazy loading, batched subqueries, budget accounting, invalid-submit retry, extraction fallback, and optimizer-visible internal predictors; long-context benchmark comparison remains tracked separately | `docs/LEARNING_PATH.md`, `README.md`, `livebooks/04_tools_agents_mcp_rlm.livemd` | Keep as RLM, not RAG; docs must stay precise | -| Multi-chain comparison | Implemented | `Imp.multi_chain_comparison/2`, backed by `Imp.Predict.MultiChainComparison` | Covered through public-surface and optimizer suites | No live proof required for the current production claim | `docs/LEARNING_PATH.md`, `README.md` | Keep facade-first as composition primitive | -| Shared inference-time Search, Best-of-N, and Refine | Implemented | Advanced request-local `Imp.Predict.Search.run/3`; facade-first `Imp.best_of_n/3` and `Imp.refine/3` delegate through `Imp.Predict.BestOfN` and `Imp.Predict.Refine` | `test/predict_search_test.exs` covers identities, ordered-prefix multidimensional budgets, sequential context, deterministic ties, bounded concurrency, failure isolation, cancellation, speculative accounting, and telemetry; `test/refine_feedback_test.exs` covers BestOfN/Refine delegation semantics; `test/search_benchmark_artifact_test.exs` covers the source-checkout artifact contract | `mix benchmark.search.check` reports deterministic quality, projected/executed-projection cost, and observed bounded concurrency; local latency is measurement only and actual provider cost is unavailable; live orchestration covers BestOfN and Refine provider calls | `docs/LEARNING_PATH.md`, `README.md`, `docs/internal/BENCHMARK_TRUTH.md` | Keep Search request-local and advanced; preserve sequential BestOfN/Refine facade semantics; never gate release on a local wall-time speedup | -| Assertions and self-refinement | Implemented | `Imp.Assertion`, `Imp.assertion/3`, `Imp.assert/3`, backed by `Imp.Predict.Assertions` | `test/assertions_test.exs` covers feedback injection, strict failure, best failed prediction metadata, metric-result feedback, predicate errors, and streaming composition | Live refinement remains covered by the broader live orchestration gate; assertion semantics are deterministic program-control behavior | `docs/LEARNING_PATH.md` | Keep as named Elixir predicates plus refinement hints, not the removed experimental Python Assert/Suggest API | -| Parallel map | Implemented | `Imp.parallel/3`, backed by `Imp.Predict.Parallel` | `test/production_hardening_test.exs`, `test/public_surface_test.exs` | Live orchestration test covers real concurrent calls | `docs/LEARNING_PATH.md`, `README.md` | Keep facade-first for batch calls | -| KNN prediction | Implemented | `Imp.knn/3`, `Imp.nearest/2`, backed by `Imp.Predict.KNN` (faithful embedding-based DSPy KNN port; `Imp.Retrievers.KNN` remains the honest token-overlap helper) | `test/imp_test.exs`, `test/knn_few_shot_test.exs`, `test/knn_dspy_differential_test.exs`, `test/public_surface_test.exs` | Retrieval composition is covered by local RAG integration and the separate HotPot retrieval differential | `docs/LEARNING_PATH.md` | Keep facade-first as retrieval composition primitive | - -## Tools, Agents, MCP, and Retrieval - -| Concept | Imp status | Imp surface | Deterministic tests | Integration/live proof | Docs | Release decision | -| --- | --- | --- | --- | --- | --- | --- | -| Tools with schema validation | Implemented | `Imp.Tool.call/2` is the single pre-handler JSON Schema boundary for ordinary and MCP-imported tools; ReAct, ReActV2, Avatar, CodeAct, and RLM all delegate through it | `test/tool_schema_runtime_test.exs`, `test/mcp_import_test.exs` cover required/property validation, no-schema preservation, handler non-invocation, and each runtime's established tool-error surface | Live ReAct tool calls exercise valid provider arguments; schema rejection is deterministic and requires no provider proof | `docs/LEARNING_PATH.md`, `README.md` | Keep the validator centralized in `Imp.Tool`; do not wrap individual runtimes or MCP runners | -| Tool-call primitives and provider-native function calls | Implemented | `Imp.Adapter.Types.ToolCall`, `ToolCalls`, `ToolResult`, `ToolCallResults`, ReqLLM assistant/tool message serialization, provider stream chunks | `test/multimodal_adapter_test.exs`, `test/req_llm_client_test.exs`, `test/react_contract_test.exs` cover multiple calls, ids, nested OpenAI function calls, tool results, adapter/provider serialization, streaming tool-call chunks, malformed calls, and composition outside ReAct | Live ReAct function tools remain covered by `mix live.check`; deterministic ReqLLM protocol tests cover primitive serialization without paid provider dependence | `docs/LEARNING_PATH.md` | Keep primitives separate from ReAct so provider-native tool calls can compose in lower-level workflows | -| Agentic programs and supervision | Implemented without a generic agent runtime | ReAct, ReActV2, Avatar, CodeAct, RLM, `Imp.Tool`, `Imp.ToolPolicy`, and ordinary OTP supervision | React-family, tool-policy, tool-schema, RLM, task-admission, and telemetry-lineage suites | Bounded live ReAct/RLM checks exercise the model-facing paths; local integration covers MCP tools | `docs/LEARNING_PATH.md`, `README.md`, `livebooks/04_tools_agents_mcp_rlm.livemd` | `Imp.Agent` and its disconnected event vocabulary were removed in 0.3; add no replacement abstraction until a real external consumer requires one | -| MCP in-process import | Implemented | `Imp.MCP.Catalog`, `Imp.MCP.import_tools/1` | `test/mcp_import_test.exs` | `mix integration.check` covers HTTP and trusted stdio MCP lifecycle; protocol MCP gate covers JSON-RPC HTTP and Streamable HTTP SSE decoding | `docs/LEARNING_PATH.md`, `README.md` | Keep | -| MCP HTTP, stdio, Streamable HTTP transports | Implemented | `Imp.MCP.HTTPClient`, `Imp.MCP.StdioClient`, `Imp.MCP.StreamableHTTPClient` | `test/mcp_import_test.exs` | `mix integration.check` proves HTTP and stdio local E2E; Streamable HTTP remains deterministic contract coverage | `docs/LEARNING_PATH.md`, `README.md` | Keep | -| Memory retrieval | Implemented | `Imp.memory/2`, `Imp.retrieve/3`, backed by `Imp.Retrieve.Memory` and `Imp.Retrieve` | `test/imp_test.exs`, `test/imp_facade_test.exs` | Local RAG integration exercises memory retrieval through evaluation, optimization, and save/load | `docs/LEARNING_PATH.md` | Keep facade-first for local workflows | -| RAG and multi-hop RAG | Implemented | `Imp.rag/3`, backed by `Imp.Predict.RAG` with `hops:` | `test/public_surface_test.exs`, `test/production_adapter_persistence_test.exs` | Local RAG E2E covers HTTP retrieval; multi-hop deterministic test proves bridge-passage expansion and answer-passage retrieval | `docs/LEARNING_PATH.md` | Keep as one canonical RAG wrapper; `hops: 1` is one-shot RAG, higher values are iterative retrieval | -| External HTTP retrievers | Implemented | `Imp.Retrievers.HTTP`, `Weaviate`, `Databricks` | `test/external_retriever_test.exs` | `mix integration.check` proves generic HTTP retriever local E2E; `mix protocol.retriever.check` proves Weaviate-compatible and Databricks-compatible HTTP payloads | `docs/LEARNING_PATH.md`, `README.md` | Keep as payload-contract clients; paid external retriever services require separate operator-supplied endpoints | -| ColBERTv2-style retrievers | Intentional omission | Use `Imp.Retrievers.HTTP` or a custom `Imp.Retrieve` implementation pointed at a ColBERTv2 service | Covered by generic/external retriever contract tests rather than an embedded ColBERT server | `mix protocol.retriever.check` proves the external-store protocol boundary Imp owns | `docs/LEARNING_PATH.md`, `README.md` | Do not ship an Imp-owned ColBERT runtime; delegate retrieval engines to production search/vector services | -| Embeddings and local vector baselines | Implemented | `Imp.Embeddings`, `Imp.Embeddings.BagOfWords` | `test/completion_surface_test.exs`, `test/public_surface_test.exs` | Live embedding provider proof is not required for the current production claim | `docs/LEARNING_PATH.md` | Keep bag-of-words embedder as deterministic local baseline; provider boundary requires one numeric vector per input text | - -## Evaluation and Optimization - -Current GRPO boundary (2026-07-26): the bundled TRL/MPS backend has now -completed one frozen 33-step, 66-group, ordinary model-generated treatment on -a source-disjoint TREC slice. Validation selected a changed step-five LoRA -artifact (`0.21875 -> 0.3125` accuracy), which saved, rebound, and reproduced -byte-identically in a fresh OS BEAM. Held-out accuracy regressed `0.25 -> -0.225` and macro-F1 regressed `0.10204 -> 0.09783`, so this closes the prior -multi-group/multi-step engine and lifecycle gap but is negative evidence for -that exact task/model treatment, not general usefulness or parity. The -immutable summary is -`examples/local_grpo_opaque_banking77/exercised-trec-source-guided-v1-result.json`. - -Current product-lifecycle update (2026-08-22): the clean retained runners under -`examples/optimizer_lifecycles/` now add natural useful held-out and fresh-OS -stories for BootstrapFewShot, RandomSearch, KNNFewShot, SignatureOptimizer, -InferRules, SIMBA, and Ensemble. In particular, SIMBA selected a real reflected -rule and improved the shipped support-routing task's untouched test score from -`0.35` to `0.80`; this supersedes older row text that says SIMBA has no useful -selected mutation. These remain one-task product acceptance results, not new -matched C3, parity, or general-effectiveness claims. - -| Concept | Imp status | Imp surface | Deterministic tests | Integration/live proof | Docs | Release decision | -| --- | --- | --- | --- | --- | --- | --- | -| Evaluation loop | Implemented | `Imp.Evaluate` with sequential and bounded concurrent row execution | `test/imp_test.exs`, `test/public_surface_test.exs`, `test/metric_contract_test.exs` cover normalized score/feedback rows, trace-aware arity-3 metrics, program/metric errors, `max_errors`, row order, and process-local settings across concurrent tasks | Local integration file-backed dataset proof exercises RAG evaluation and optimization; no paid live proof required for evaluator mechanics | `docs/LEARNING_PATH.md`, `livebooks/03_evaluate_and_optimize.livemd` | Keep normalized score/feedback rows and explicit concurrency via `max_concurrency:` | -| Metrics | Implemented | `Imp.exact_match/1`, `answer_passage_match/2`, `extractive_qa/3`, `classification/3`, `classification_report/2`, `retrieval_recall/3`, backed by `Imp.Metrics` and `Imp.Metrics.Result` | `test/imp_facade_test.exs`, `test/metric_contract_test.exs`, optimizer suites | Benchmark truth tests cover GSM8K/HotPotQA-style QA, local classification, retrieval QA, and claim verification metric aggregation | `docs/LEARNING_PATH.md`, `README.md` | Keep facade-first for common metrics; support boolean, numeric, map, prediction, feedback, trace-aware returns, QA F1/EM, classification F1, and retrieval recall | -| Auto-evaluation helpers | Implemented; narrow C1 semantic-conformance claim asserted from admitted T1 evidence; natural-judge quality open | `Imp.Evaluate.SemanticF1`, `CompleteAndGrounded` | `test/public_surface_test.exs`, `test/auto_evaluation_contract_test.exs`, `test/auto_evaluation_fidelity_test.exs` | Canonical source-bound T1 artifact `9f7210…` covers only the declared deterministic normalization/aggregation differential and is revalidated by the current pure protocol validator. It does not prove natural judge calibration, bias, task quality, Refine advice quality, or Best-of-N lift. | `docs/LEARNING_PATH.md`, `docs/internal/BENCHMARK_TRUTH.md` | Keep semantic conformance, natural-judge effectiveness, and Refine/Best-of-N effectiveness as separate claims; all quality claims remain red. | -| LabeledFewShot and KNNFewShot | Implemented mechanics; LabeledFewShot selection semantics repaired; KNN source differential admitted separately | `Imp.Optimizer.LabeledFewShot`, `Imp.Optimizer.KNNFewShot` | `test/labeled_few_shot_selection_test.exs` covers DSPy defaults, sampled/ordered modes, explicit seeds, empty replacement, facade equivalence, and per-predictor RNG traversal; KNN has its own direct differential suite | LabeledFewShot defaults to sampled `k: 16` with seed zero, exposes `sample: false`, and uses explicit serializable BEAM RNG state rather than Python's exact `random.Random` subset order. No LabeledFewShot family effectiveness or exact RNG-sequence parity is claimed. | `docs/LEARNING_PATH.md` | Treat explicit RNG as the native equivalent; keep family effectiveness separate from selection mechanics. | -| BootstrapFewShot | Implemented; narrow source-bound family-specific C1 conformance admitted; C3 effectiveness remains open | `Imp.Optimizer.BootstrapFewShot` | `test/bootstrap_few_shot_trajectory_test.exs` covers teacher preparation, thresholds, rounds, trace acceptance, limits, errors, and compiled-program assembly; `test/classical_optimizer_differential_test.exs` and `test/python_dspy_classical_optimizer_differential_test.py` validate the pinned receipt and direct observations | Canonical artifact `9b89dac9…` binds clean Imp commit `bfda991`, authenticates DSPy 3.2.1, and matches traversal, acceptance, and ordered augmented demonstrations. Exact Python RNG, repeated-call sampling, provider behavior/effectiveness, and full parity remain excluded. | `docs/LEARNING_PATH.md`, `docs/internal/BENCHMARK_TRUTH.md` | Track semantic conformance and effectiveness independently. | -| RandomSearch and InstructionSearch | RandomSearch implemented with narrow source-bound family-specific C1 conformance admitted; InstructionSearch remains distinct; family C3 effectiveness remains open | `Imp.Optimizer.RandomSearch`, `InstructionSearch`, `InstructionProposer` | `test/random_search_policy_test.exs` covers candidate families, restrict/stop/error behavior, tie ordering, and aggregate score scaling; `test/classical_optimizer_differential_test.exs` and `test/python_dspy_classical_optimizer_differential_test.py` validate the pinned receipt | Canonical artifact `f7e49685…` binds clean Imp commit `0e96d62`, authenticates DSPy 3.2.1, and matches the candidate schedule/baselines, equal-score stable ranking, scores, subscores, and demo counts. Exact Python RNG, shuffled row order, provider behavior/effectiveness, and full parity remain excluded. | `docs/LEARNING_PATH.md`, `README.md` | Keep RandomSearch semantics, RandomSearch effectiveness, and InstructionSearch behavior distinct. | -| COPRO | Implemented; narrow source-bound C1 conformance admitted; one task/model positive held-out result retained; C3 effectiveness open | `Imp.Optimizer.COPRO` | `test/copro_isolation_artifact_test.exs`, `test/optimizer_behavioral_corpus_test.exs`, `test/local_copro_banking77_example_test.exs` | Canonical artifact `4f2d959d…` binds clean Imp commit `f9ee18c`, authenticates DSPy 3.2.1, and verifies five deterministic observations. After two duplicate/no-mutation stops, the frozen objective-correct V2 produced a genuine rendered instruction mutation, improved sixteen-row train selection `56.25% -> 62.5%`, and improved the separate forty-row optimizer-held-out set from `47.5% -> 55.0%` accuracy and `0.3333 -> 0.4651` macro-F1 with zero parse errors. Its exact parameter artifact loaded and served the verified fused model in a fresh OS BEAM, but one ordered prediction differed, so byte-identical restart acceptance remains unproved for that run. Current MLX adoption now preserves an explicit generation seed and no-model counting-server tests prove byte-identical request bodies across save/load; the immutable V2 did not retain request bytes or set a seed and is not reinterpreted. Exact Python RNG parity, general provider effectiveness, multi-seed C3, production restart determinism, and full optimizer parity remain excluded. | `docs/LEARNING_PATH.md`, `docs/LEARNING_PATH.md`, `examples/local_copro_banking77/README.md`, `docs/internal/BENCHMARK_TRUTH.md` | Publish the five-observation C1 scope and the task-specific V2 result separately; do not promote the stopped restart lifecycle or one positive run into general effectiveness. | -| MIPROv2, SIMBA, and GEPA-style reflection | Implemented mechanics and separate structural/replication lanes. Custom multi-predictor programs retain consistent reports and every report-bearing program optimizer can emit a safe checksummed selected-parameter artifact. MIPROv2 and SIMBA resume bind the task LM, adapter, demos, config, and dynamic binding policy that own admitted observations. | `Imp.Optimizer.MIPROv2`, `SIMBA`, `GEPA`, `Imp.Optimizer.Report`, `Imp.Optimizer.Artifact` | `test/optimizer_behavioral_corpus_test.exs`, `test/benchmark_test.exs`, `test/optimizer_custom_program_report_test.exs`, `test/mipro_v2_resume_test.exs`, `test/simba_resume_test.exs`, `test/gepa_parameter_artifact_lifecycle_test.exs`, and the local Banking77 example tests cover custom-program report retention, heterogeneous runtime use, shared artifact capture, selection, task-runtime drift refusal, and parameter application into freshly reconstructed trusted programs with runtime bindings retained across an OS process. | Local Banking77 runs cover operational lifecycles for all three families. Separately, the sealed three-seed strong-model TREC comparison used identical task messages, disjoint 20/40/80 train/selection/test rows, GPT-5.4 Mini task calls, Claude Sonnet 4.6 optimizer calls, and pinned DSPy 3.2.1/GEPA 0.1.4 semantic opportunity. Imp GEPA improved mean untouched accuracy over its own baseline by `+0.4000` (95% clustered interval `[0.2958, 0.5042]`, Holm-adjusted `p=0.00020`) and differed from DSPy GEPA by `-0.0083` (interval `[-0.0458, 0.0292]`), clearing the preregistered `-0.05` noninferiority margin. Imp MIPROv2 improved its own baseline by `+0.1458` (interval `[0.0458, 0.2458]`, Holm-adjusted `p=0.00270`). This is C3 evidence only for that frozen task/model/budget contract; it is not general effectiveness, paper-family replication, superiority, SIMBA evidence, or whole-family parity. The earlier Banking77 GEPA run rejected a worse two-predictor proposal; MIPROv2 rejected a worse instruction and reproduced selected output. SIMBA's genuine successful-trajectory mutation regressed validation and retained baseline, with neutral untouched behavior; a separate semantic-feedback task admitted no mutation. | `docs/LEARNING_PATH.md`, `docs/LEARNING_PATH.md`, `examples/local_gepa_banking77/README.md`, `examples/local_mipro_banking77/README.md`, `examples/local_simba_banking77/README.md`, `examples/local_simba_feedback_trec/README.md`, `examples/matched_instruction_optimizers_trec/README.md`, `benchmarks/evidence/archive/matched_experiments/trec/matched-instruction-optimizers-trec-20260726.json`, `docs/internal/INSTRUCTION_OPTIMIZER_FIDELITY.md` | Keep the matched C3 result scoped to GEPA/MIPROv2 on this contract; preserve negative SIMBA and local proposal outcomes without borrowing the flagship result. | -| Avatar, AvatarOptimizer, BetterTogether, and Ensemble | Four independently authoritative implemented families with narrow source-bound C1 semantics admitted; all C3 effectiveness claims remain open | `Imp.avatar/3`, `Imp.Predict.Avatar`, `Imp.Optimizer.Avatar`, `Imp.Optimizer.BetterTogether`, `Imp.Optimizer.Ensemble` | Purpose-built Elixir/Python differential tests validate each pinned receipt; existing actor, optimizer, composition, replay, and identity-based mixed-phase ledger suites cover broader native behavior | Current registry C1 artifacts `190b4002…`, `97dbf8ef…`, `36898d22…`, and `df73a1f7…` remain limited to their declared shared observations. Separately, `local-better-together-mlx-banking77-20260725.json` records one real retained-MLX lifecycle: prompt candidates ran through the fused task LM, validation tied `w` and `w -> p` at `0.50` and retained `w`, and the selected program scored `0.55` accuracy / `0.4561` macro-F1 / zero parse errors on 40 untouched rows before byte-identical fresh-OS reproduction. This is product-lifecycle evidence, not general BetterTogether or prompt-optimizer effectiveness. Provider generalization, exact RNG/sample parity, full parity, and BEAM superiority remain excluded. | `docs/internal/AVATAR_FIDELITY.md`, `docs/internal/WEIGHT_COMPOSITION_C1.md`, `docs/LEARNING_PATH.md`, `docs/LEARNING_PATH.md` | Preserve four authority families and twelve distinct C0/C1/C3 claims; never use one family's evidence to satisfy another or promote the scoped BetterTogether lifecycle into C3. | -| BootstrapFinetune and GRPO provider jobs | BootstrapFinetune has a supported local MLX SFT artifact/deployment path and one narrow task/model effectiveness result. GRPO has a versioned content-bound Imp↔TRL lifecycle, a pinned local TRL/PyTorch worker, controlled and ordinary model-generated MPS/LoRA weight changes, a general contract for arbitrary Imp prompts and finite external rewards, an executable verified-artifact deployment/rebind path, and an ordinary local Banking77 front door. The bundled backend executes source-bound multi-group batches and durable multi-step official-TRL continuation. One frozen JSON-adapter run produced genuine semantic signal and changed tensors at step one, then suffered sampled-format collapse at step two and tied base on selection and untouched test. Useful ordinary model-sampled improvement, crash recovery during an accepted mutation, GRPO parity, and provider/mmGRPO C3 targets remain open. | `Imp.Optimizer.BootstrapFinetune`, `Imp.Optimizer.GRPO`, `Imp.Clients.Trainer`, `Imp.Clients.TrainingJob`, `Imp.Clients.TRLProtocol`, `Imp.Clients.TRLArtifact`, `Imp.Clients.TRLTrainer`, `Imp.Clients.TRLDeployment`, `Imp.Clients.TRLLM`, `Imp.Clients.MLXLMTrainer`, `Imp.Clients.MLXLMDeployment` | `test/trl_protocol_test.exs` rejects missing/reordered/tampered protocol fields and pins CPython-owned finite-float spelling; `test/trl_protocol_grpo_lifecycle_test.exs` covers a long-lived no-model session, accepted-then-disconnected reconciliation, exact idempotency, content verification, fresh-process executable deployment/rebind, and tamper refusal. `test/trl_worker_test.exs` covers 2,048 generated finite doubles, IEEE edges, retained rollout logprobs, fail-before-mutation guards, arbitrary prompts/reward scales, uniform-reward admission, pre-model budget refusal, raw model-output adapter parsing, deployed response-identity drift refusal, no implicit second worker, plain JSON projection, and pre-validation durability. The completed controlled result is `benchmarks/results/local-trl-grpo-controlled-update-canonical-current-beam-20260726.json`. MLX tests cover real SFT fusion/deployment; pinned weight-composition and mmGRPO differentials remain separate. | One pinned Qwen2.5-0.5B MLX SFT artifact improved the frozen four-intent Banking77 subset's untouched 40-row accuracy `0.125 -> 0.55` and macro-F1 `0.0610 -> 0.4561`; save/load and fresh-process serving reproduced ordered fused predictions/errors byte-for-byte. Separately, the controlled TRL result used four externally supplied completions with semantic rewards `[1,0,0,0]`, computed non-uniform group-relative advantages, performed one optimizer step, changed the LoRA tensor digest, and created a content-verified artifact. A later fresh OS BEAM revalidated those retained bytes, loaded the adapter into the exact Qwen base through PEFT/MPS, matched the post-training LoRA tensor digest, executed an ordinary typed Imp call through the artifact-bound LM, and cleaned the worker. This establishes controlled external-rollout MPS/TRL weight-update and reusable-artifact feasibility only—not ordinary sampling, learning, effectiveness, parity, production reliability, or BEAM superiority. An earlier ordinary model-sampled general-contract run completed four diverse rollouts and a no-op step, but its scores are invalid because it exposed the then-broken raw-output adapter boundary. A separate corrected ordinary Banking77 run used the strict typed adapter and frozen 16/8/40 split: all four sampled completions were malformed, rewards were uniformly `-1`, advantages/loss were zero, tensors did not change, and selection retained base. Base and trained both scored `0.025` accuracy / `0.04545` macro-F1 with 39 parse errors on untouched test; a fresh OS BEAM reproduced the selected base outputs byte-for-byte. This is an honest task/model-specific no-signal result, not evidence against GRPO learning. A separate no-model public lifecycle now predeclares validation-only selection, retains the earliest best trained checkpoint on ties, content-verifies that historical artifact, and rebinds it while preserving the final trainer state separately. A separately named documented-defaults treatment applied its sealed `1e-6`/DAPO configuration and again changed tensors with semantic signal at step one, but its in-session checkpoint scores are invalid because generation ran while the post-Trainer policy remained in training mode; external deployment of the same selected step-one artifact tied base on selection/test. The worker now isolates all generation in eval mode and restores trainer mode, and the completed result was not rerun. A separately named post-fix treatment then produced non-uniform semantic rewards and group-relative advantages and changed LoRA tensors in both official steps under inference-mode validation. Both trained checkpoints tied base on frozen selection, the trained and base arms tied at `0.725` accuracy / `0.712885` macro-F1 / zero errors on untouched test, and stable selection retained base for byte-identical fresh-OS reproduction. This is corrected multi-group/multi-step semantic-signal and lifecycle evidence, but still a neutral one-task/model usefulness result. The immutable JSON multi-step result in `examples/local_grpo_banking77/exercised-json-multistep-result.json` additionally records non-uniform semantic rewards and advantages plus changed trainable tensors at step one, followed by step-two format collapse and a neutral final held-out result; it establishes ordinary semantic-signal weight-change mechanics, not useful learning. A separate two-group/two-step acceptance advanced official Trainer state `0 -> 1 -> 2`, emitted a standalone full-chain artifact, and reproduced the trained step-two predictions byte-for-byte in a fresh OS BEAM; all four prompt groups again had uniform `-1` rewards, so it proves lifecycle continuation rather than useful weight change. Canonical C1 artifacts `fb7212…` and `eef3e7…` remain limited to their declared topology/trace and cycling/group/reward observations. | `docs/internal/WEIGHT_COMPOSITION_C1.md`, `docs/internal/MMGRPO_C1.md`, `docs/LEARNING_PATH.md`, `docs/LEARNING_PATH.md`, `docs/PRODUCTION_OPERATIONS.md` | Keep the one-task SFT result, controlled GRPO engine feasibility, ordinary GRPO training, and GRPO effectiveness claims separate. | -| Fast-Slow Training | Paper-ordered provider-neutral orchestration implemented through the slow-update handoff; no bundled or verified CISPO optimizer | `Imp.Training.FastSlow.Runner`, `Backend`, durable state/checkpoint modules, and token-aligned `Imp.Clients.Trainer` groups | Fast-Slow state, checkpoint, reuse-cache, advantage-group, runner, and trainer tests cover exact `T/K/G` semantics, fixed prompt populations, mixed cache/live trajectories, operation budgets/events, content-bound resume, replay refusal, final stream progress, and reuse-policy enforcement | Deterministic campaign proves orchestration and trainer-option forwarding only; the official Fast-Slow code page still says code coming soon, and CISPO execution, model artifacts, provider behavior, and effectiveness remain unproved | `docs/PRODUCTION_OPERATIONS.md`, `README.md`, `docs/internal/RESEARCH_LANDSCAPE.md` | Keep orchestration claims green; do not describe the external trainer handoff as CISPO execution | -| Arbitrary artifact optimization | Production runtime implemented; scoped three-class C3 effectiveness admitted | `Imp.Optimize.Anything`, nested `Config` and `Result`, shared GEPA engine | Runner, structured artifact, batch evaluator, custom batch sampler, proposal sampling/selection/acceptance, released v0.1.4 text reflection strategies, durable-cache, best-output, tracking, and checkpoint/resume suites plus schema-v2 artifact `0aa498b5…`; public strategy tests prove multi-proposal execution, filtering/admission callbacks, native complete-artifact proposal generation without a reflection LM, JSON resume binding, selector-identity drift refusal, and explicit application of the selected artifact from an unpacked consumer | The older 2026-07-16 artifact remains T2 only because it reused development data for selection and final scoring. The 2026-08-20 GPT-5.4-mini successor used distinct train, selection, and untouched test cases for executable retry-controller code, agent configuration, and scheduling. Mean test lifts were `+0.254759`, `+0.518609`, and `+0.059774`; improving seeds were `3/3`, `3/3`, and `2/3`; 44 single-attempt calls cost `$0.082421` under a `$0.50` ceiling. Imp-native typed structured artifacts use the public `StructuredStrategy` protocol; released text-reflection strategies remain text-map compatibility only. This is scoped Imp-native evidence, not upstream parity or general artifact effectiveness. | `docs/LEARNING_PATH.md`, `docs/internal/BENCHMARK_TRUTH.md`, `docs/internal/RESEARCH_LANDSCAPE.md` | Preserve the immutable pre-v2 artifact as execution evidence only; retain the current three-class C3 claim at its exact model/evaluator scope; keep paper-scale upstream comparison open. | - -## Persistence, Operations, and Release Gates - -| Concept | Imp status | Imp surface | Deterministic tests | Integration/live proof | Docs | Release decision | -| --- | --- | --- | --- | --- | --- | --- | -| Save/load | Implemented | `Imp.dump/1`, `Imp.load/1`, `Imp.save!/2`, `Imp.load!/1`, backed by `Imp.Saving` for `Predict`, `ChainOfThought`, `ProgramOfThought`, and memory-backed `RAG` | `test/production_adapter_persistence_test.exs`, `test/production_hardening_test.exs`, `test/imp_facade_test.exs` | Live/provider save-load proof is covered through ReqLLM-backed live gates where needed; clean-room package acceptance uses the facade path | `docs/LEARNING_PATH.md`, `README.md` | Keep with explicit credential rebinding; rebuild function-tool programs with explicit tool catalogs | -| Secret redaction | Implemented | `Imp.Redaction`, trace and telemetry call sites | `test/production_hardening_test.exs`, `test/adversarial_security_stress_test.exs` | Telemetry redaction covered in production hardening tests | `docs/PRODUCTION_OPERATIONS.md` | Keep as non-negotiable security invariant | -| Telemetry and observability | Implemented | `Imp.Telemetry`, runtime call sites | `test/production_hardening_test.exs`, `test/req_llm_client_test.exs`, `test/mcp_import_test.exs`, `test/external_retriever_test.exs`, `test/provider_training_lifecycle_test.exs`, `test/optimizer_report_test.exs` | Local integration and live provider gates exercise instrumented paths | `docs/PRODUCTION_OPERATIONS.md`, `README.md` | Keep event families stable and redacted | -| Ordinary source check | Implemented | `mix check` | Format, warnings-as-errors compile, and deterministic behavioral tests | Runs locally | `docs/maintainers/GATES.md` | Keep environment-specific package, protocol, integration, live, docs, and static checks separate | -| Benchmark truth lane | Implemented | `mix benchmark.truth.check`, `mix benchmark.live.check`, internal benchmark modules | `test/benchmark_truth_test.exs` | Live benchmark smoke fetches GSM8K/HotPotQA rows and uses provider credentials when run | `docs/internal/BENCHMARK_TRUTH.md` | Keep fixture proof separate from research/model-quality claims and outside the application API | -| Golden trace parity lane | BYTE-parity implemented, measured, and enforced (dee-8zev, 2026-07-18) | `mix imp.benchmark.trace`, `mix parity.check`, `scripts/dspy_golden_trace_runner.py`, `test/fixtures/golden_trace/cases.json` (35 cases) | `test/golden_trace_test.exs` | Provider-free Imp vs pinned `dspy==3.2.1` replay compares rendered messages AND per-call request envelope byte-for-byte: **32/35 cases fully byte-identical** (predict, CoT, typed/enum/list/dict, few-shot demos, conversation history, multi-line/CRLF/unicode instructions, RAG list inputs, DSPy-faithful ReAct `:dspy_3_2_1`, capability-gated response_format). The instrument compares the full request per call (dee-idig) so it cannot mask envelope divergence. Enforced per-PR via `mix parity.check` in the heavy CI job (dee-3e4v) | `docs/internal/BENCHMARK_TRUTH.md`, `docs/internal/PARITY_VALIDATION_PROGRAM.md` | The 3 non-byte-identical cases are Imp's DEFAULT provider-native ReAct (intentional; faithful `:dspy_3_2_1` mode ships alongside). Open coverage: xml/two_step adapters (dee-1gb9); real-model response_format registry accuracy (dee-7r2t); parse leniency (dee-q2w2); multi-key dict ordering (dee-1fd0) | -| Provider-free overhead regression lane | Implemented | `mix benchmark.overhead.check`, local-only Mix task, `scripts/dspy_overhead_benchmark.py` | `test/overhead_policy_test.exs` | Named paired operations use per-case absolute and reference-relative budgets with rationale and environment/source identity; ratios remain measurements and authorize no speed claim | `docs/internal/BENCHMARK_TRUTH.md`, `docs/internal/PARITY_VALIDATION_PROGRAM.md` | Admit a clean integrated artifact and track trends for slower local paths such as cache misses | -| Live matched-model matrix lane | Implemented; measurement completion and parity outcomes are reported separately | `mix benchmark.live_matrix`, local-only Mix task | `test/benchmark_truth_test.exs` | Aggregates Imp-vs-DSPy campaign artifacts by provider/model, skips malformed historical artifacts, tags current low-cost, frontier, and historical/research-style lanes, excludes runner/API error rows from accepted coverage, and exposes token/USD cost status. | `docs/internal/BENCHMARK_TRUTH.md`, `docs/internal/PARITY_VALIDATION_PROGRAM.md` | Preserve completed artifacts, keep threshold misses visible, and do not rerun paid samples merely to obtain a favorable outcome. | -| Optimizer lift parity lane | Provider-free lift and pinned T1 structural checks implemented; fresh execution and T3 remain blocked | `mix benchmark.optimizer_lift.check`, `mix benchmark.instruction_optimizer.contract.check`, local-only Mix tasks, Python sidecars | Optimizer behavior tests cover Imp internals; artifact tests cover natural user stories and 32 pinned MIPROv2/SIMBA structural cases | Deterministic lift rows prove selected outcomes. The structural sidecar proves only the matched control-flow cases it executes and records RNG, sampler, bootstrap-call selection, and grounded-proposer deviations explicitly. Neither lane proves T3 effectiveness. | `docs/internal/PARITY_VALIDATION_PROGRAM.md`, `docs/internal/BENCHMARK_TRUTH.md`, `docs/internal/INSTRUCTION_OPTIMIZER_FIDELITY.md` | Run the fresh T1 artifact and held-out multi-seed T3 campaigns before claiming full optimizer parity | -| GEPA paper-replication lane | Implemented as strict artifact contract; real campaigns must supply fresh rows | `mix benchmark.gepa_replication.check`, local-only Mix task | `test/gepa_replication_artifact_test.exs` | The validator requires AIMEBench, HotpotQABench, hoverBench, IFBench, LiveBenchMathBench, and Papillon rows with baseline, DSPy GEPA, Imp GEPA, MIPROv2, metric-call budget, token/cost, wall-clock, seed variance, and train/dev/test gap | `docs/internal/PARITY_VALIDATION_PROGRAM.md`, `docs/internal/BENCHMARK_TRUTH.md` | Keep separate from optimizer lift; do not claim GEPA paper dominance from fixture/provider-free optimizer rows | -| RAG/tool differentials | Three narrow external-comparison rows remain; the redundant aggregate product evaluator was removed | `mix imp.benchmark.hotpot_retrieval`, `mix imp.benchmark.bfcl_adapted`, `mix imp.benchmark.rag_tool_failure_differential` | Direct product tests own ReAct, ReActV2, RLM, MCP, tool policy/schema, streaming, task supervision, persistence, and redaction. The retained runners add pinned HotPot retrieval, independent BFCL-shaped scorer agreement, or a source-authenticated queued-action differential. | None establishes answer generation, model tool selection, official BFCL performance, model-selected recovery, or wall-clock timeout parity. | `docs/internal/PARITY_VALIDATION_PROGRAM.md`, `docs/internal/BENCHMARK_TRUTH.md` | Do not rebuild a product-test dashboard; keep only differentials that add independent information and keep HotPot quality, BFCL selection, and model-selected recovery as separate red C3 claims. | -| BEAM-native RLM runtime conformance | Implemented; canonical source-bound artifact admitted | `Imp.Predict.RLM`, constrained interpreter, persistent session, compaction, shared budget coordinator, `mix imp.benchmark.rlm_runtime_differential` | `test/rlm_standalone_runtime_test.exs` and `test/rlm_runtime_differential_test.exs` cover the runtime plus nine pinned standalone-reference cases across six semantic categories | Canonical admitted artifact `aa28970c…` binds committed Imp source and the pinned `alexzhang13/rlm@72d6940` authority. Eight observations match and one preregistered deviation covers Python imports. This is provider-free C1/C2 evidence only. | `docs/internal/RLM_FIDELITY.md`, `docs/maintainers/REPRODUCTIONS.md` | Retain the admitted source-bound runtime artifact; do not promote it to live quality, DSPy adapter, or paper-effectiveness evidence. | -| RLM effectiveness evidence lane | Deterministic bounded planning and one-sample live T2 operational matrix implemented; effectiveness and exact T3 remain red | `mix imp.benchmark.rlm_campaign`, `mix benchmark.rlm.check`, `mix benchmark.rlm.contract.check`, Python sidecars | `test/rlm_campaign_test.exs`, `test/benchmark_truth_test.exs`, `test/rlm_benchmark_artifact_test.exs`, `test/rlm_contract_artifact_test.exs` | Exact plans filter pinned family, approach, runtime, and frozen row order; all filters bind checkpoint identity and force subset artifacts to T2. The 2026-07-13 OOLONG preflight completed all six Imp/DSPy direct, simple-retrieval, and RLM jobs with valid scores, positive usage, reconciled costs, latency, and bounded traces. All six scores were zero, the sample count was one, retrieval retained effectively the full context, and cost authorities differed, so this proves operation rather than effectiveness or parity and was not expanded. | `docs/internal/RLM_FIDELITY.md`, `docs/internal/PARITY_VALIDATION_PROGRAM.md`, `docs/internal/BENCHMARK_TRUTH.md`, `docs/internal/BENCHMARK_CATALOG.md` | Run a bounded multi-sample T2 campaign on a pinned row set where retrieval reduces context before evaluating effectiveness. Keep unavailable families and exact T3 red. | -| DSPy parity lane | Implemented | `mix benchmark.parity.check`, `mix benchmark.parity.full`, `mix imp.benchmark.parity.aggregate`, `scripts/dspy_parity_runner.py` | Alias and aggregate contract tests | Smoke lane proves wiring; full lane runs Imp and Python DSPy over full canonical GSM8K/HotPotQA splits; campaign aggregate proves row coverage and weighted scores | `docs/internal/BENCHMARK_TRUTH.md` | Full campaign aggregate required before claiming original-library parity | -| Live inference gate | Implemented | `mix live.check` | `test/live_provider_test.exs`, `test/live_provider_e2e_test.exs` | `LIVE_PROVIDER=1 mix live.check` | `README.md`, `docs/PRODUCTION_OPERATIONS.md` | Keep as paid live inference gate | -| Local integration gate | Implemented | `mix integration.check`, `test/integration` | `test/integration/gate_contract_test.exs`, `test/integration/local_service_e2e_test.exs` | Local service E2E covers retriever HTTP, MCP HTTP, and MCP stdio | `docs/PRODUCTION_OPERATIONS.md`, `docs/maintainers/RELEASE.md` | Keep and expand only when the production surface grows | -| Provider-compatible protocol gates | Implemented | `mix protocol.training.check`, `mix protocol.retriever.check`, `mix protocol.mcp.check`, `mix protocol.check` | Gate tests under `test/protocol_*` | Training proves provider-compatible submit/refresh over real HTTP transport; retriever proves Weaviate/Databricks-compatible HTTP retrieval; MCP proves JSON-RPC HTTP, Streamable HTTP SSE decoding, and trusted stdio clients | `docs/PRODUCTION_OPERATIONS.md`, `docs/maintainers/RELEASE.md` | Do not claim paid/external-service proof without dedicated external-service tests | -| Documentation and Livebooks | Implemented | `docs/*`, `livebooks/*`, ExDoc extras | `mix docs.check`, `mix livebook.execute.check`, `test/livebook_contract_test.exs`, `test/documentation_contract_test.exs` | `livebook.check` validates the actual `livebooks/` directory; documentation contracts execute API-guide RAG, ReAct, save/load, optimization, MCP, and streaming workflows | `README.md` | Keep contract-tested | - -## Intentional Deviations From Python DSPy - -| Upstream concept | Imp decision | Rationale | Guardrail | -| --- | --- | --- | --- | -| Python object mutation and dynamic globals | Use structs plus explicit process-local settings | Elixir code should be inspectable, immutable by default, and process-safe | `test/imp_test.exs`, `test/imp_facade_test.exs` | -| Python callbacks with loosely shaped kwargs | Use functions, behaviours, structs, and maps/keywords | Elixir callers expect explicit data shapes and pattern matching | Public constructors and behaviours use explicit validation and contract tests | -| Python-side code execution for PoT/CodeAct | Use `Imp.Sandbox` with an allowlisted expression evaluator | BEAM-safe execution is part of production trust | `test/completion_surface_test.exs`, `test/production_hardening_test.exs` | -| Provider-specific magic as default behaviour | Use explicit provider structs and injected transports | Production Elixir systems should make network and credentials visible | `test/production_hardening_test.exs`, `docs/PRODUCTION_OPERATIONS.md` | -| Treating all external workflows as one live test | Split deterministic, local integration, live inference, and stateful paid gates | Different proof levels have different cost, flake, and safety profiles | `check`, `integration.check`, `protocol.check`, and `live.check` remain distinct | diff --git a/docs/internal/IDENTITY_COMPATIBILITY.md b/docs/internal/IDENTITY_COMPATIBILITY.md deleted file mode 100644 index 87a2e0e5..00000000 --- a/docs/internal/IDENTITY_COMPATIBILITY.md +++ /dev/null @@ -1,25 +0,0 @@ -# Identity Cutover - -Imp is a greenfield product with one current identity. Its package and active -runtime paths accept only canonical Imp namespaces, configuration prefixes, -wire tags, and artifact envelopes. - -There is no compatibility reader for pre-cutover persisted programs, optimizer -artifacts, manifests, or fixtures. Readers verify canonical envelopes and -checksums, and unknown or non-canonical forms fail through the current strict -validators. Writers emit only canonical Imp data. - -Historical evidence is not rewritten merely to make a current-name audit pass, -because that would invalidate its provenance or checksum. Generic historical -schema migrations remain supported where they are part of a current Imp -contract; they do not translate the old project identity. - -This cutover is intentionally final for the greenfield package. It must not -expand into aliases for current source code, persisted data, or public APIs. - -## Maintainer Check - -The clean-room package contract proves that a consumer sees `app: :imp` and -loads the expected OTP application callback tuple. Ordinary code review and -search are sufficient for historical names; a 445-entry token allowlist was -more likely to bless residue than catch a user-visible regression. diff --git a/docs/internal/OPTIMIZER_TRAJECTORY_CONTRACT.md b/docs/internal/OPTIMIZER_TRAJECTORY_CONTRACT.md deleted file mode 100644 index 7419280b..00000000 --- a/docs/internal/OPTIMIZER_TRAJECTORY_CONTRACT.md +++ /dev/null @@ -1,30 +0,0 @@ -# Optimizer Trajectory Contract - -`Imp.Optimizer.Trajectory` is the canonical execution envelope shared by -GEPA, MIPROv2, SIMBA, RLM, ReAct, optimize-anything, and evaluation adapters. -It preserves each runtime's native `trace`, feedback, metadata, and named -parameter values while also projecting ordered provider-neutral events. - -The version 1 envelope includes: - -- text or typed multimodal examples and predictions; -- reasoning, module calls, tool calls/results, partial errors, and evaluator feedback; -- token/request/cost usage and microsecond timing; -- cache key/hit identity, program and rollout identity, and named parameters; -- optimizer-specific metadata that is intentionally not flattened. - -Use `Imp.Optimizer.Trajectory.project/3` at runtime boundaries. A trajectory -batch can be checked with `validate_aligned!/1`; events must be contiguous and -tool results must follow a unique matching call. `dump/1` emits the only -supported cross-runtime JSON representation and redacts credentials in -structured fields before they cross that boundary. Opaque image, audio, and -file bytes remain byte-for-byte intact; callers must not place credentials in -attachment payloads. `Imp.dump/1` and `Imp.load/1` use this same codec. -`load/1` accepts only the exact versioned schema, -known typed values, valid accounting, ordered events, and aligned tool calls. -It returns `{:error, %Imp.Optimizer.Trajectory.DecodeError{}}` for malformed or -future-version data rather than partially restoring it. - -The native `trace` remains available because reflection semantics differ by -optimizer. Consumers should use `events` for cross-runtime inspection and the -native fields when implementing optimizer-specific reflection or mutation. diff --git a/docs/internal/PARITY_VALIDATION_PROGRAM.md b/docs/internal/PARITY_VALIDATION_PROGRAM.md deleted file mode 100644 index 8ee08d3d..00000000 --- a/docs/internal/PARITY_VALIDATION_PROGRAM.md +++ /dev/null @@ -1,524 +0,0 @@ -# Imp Parity Validation Program - -This document defines the evidence required before Imp can honestly claim -full philosophical, operational, and performance parity with DSPy. - -This is a maintainer document for an Imp source checkout. Its Mix commands and -artifact paths are not package-consumer APIs. - -The old question, "Can Imp answer the same benchmark rows as DSPy with one -live model?", is useful but insufficient. It mostly measures provider behavior -and prompt/adapter compatibility. DSPy's real thesis is broader: programs are -declared with signatures, evaluated by metrics, compiled by optimizers, traced, -saved, served, and improved. Imp parity must be measured at those same joints. - -## Ground Truth - -The validation standard is grounded in current DSPy behavior and docs: - -- DSPy's core product claim is metric-driven compilation of programs, not - hand-written prompts. -- Built-in dataset lineage includes GSM8K, HotPotQA, and Color-style tasks. -- Metrics can return booleans, numbers, or prediction-like values carrying - feedback; optimizers may pass trace context to metrics. -- `Evaluate` runs a program over examples, handles failures with a failure - score, and supports parallel execution. -- Optimizers include few-shot bootstrapping, instruction/demo search, - MIPROv2, SIMBA, COPRO, GEPA, and finetuning-oriented workflows. -- Production behavior includes caching, saving/loading, streaming, async - execution, tools, retrieval, observability, and deployment concerns. - -Imp intentionally implements these ideas in Elixir terms: behaviours, structs, -supervised concurrency, explicit telemetry, and ReqLLM as the provider boundary. -Parity does not mean copying Python internals. It means matching the semantic -contract and proving any intentional deviation is better, not accidental. - -## Lane 1: Golden Trace Parity - -Provider-free replay is the foundation. It removes model nondeterminism and -provider latency so we can prove library semantics directly. - -Required coverage: - -- signatures and field metadata -- `Predict` -- `ChainOfThought` -- JSON/schema/chat adapters -- ReAct/tool-call normalization -- streaming chunk vocabulary -- cache hits and misses -- save/load round trips -- metric return normalization -- trace redaction and serialization -- error/failure scoring - -Required artifacts: - -- a shared fixture corpus consumed by both Python DSPy and Imp -- normalized message/prompt traces from both sides -- normalized predictions and traces -- exact-match report for required fields -- intentional-deviation table for non-identical but accepted behavior - -Pass condition: - -- all required fixtures pass exact normalized parity, or each difference is - recorded as an intentional Elixir-native deviation with a test and rationale. - -Initial executable command: - -```sh -mix benchmark.trace.check -``` - -The current checked-in fixture corpus proves normalized prediction parity for -`Predict`, `ChainOfThought`, typed fields, JSON adapter output, DSPy-style -ReAct lookup, multi-tool ReAct, and ReAct tool-error status normalized against -Imp's provider-tool loop. It also proves shared missing-field error status -parity and Imp semantic checks for incremental field streaming, save/load -credential redaction, ReqLLM cache hits, and provider text/tool-call stream -chunk replay. Byte-identical prompt/message templates are not a release -criterion: the lane records both message histories and treats Imp's -Elixir-native provider-tool prompt shape as an intentional deviation unless a -normalized semantic invariant fails. - -## Lane 2: Live Matched-Model Parity - -Live parity tests whether Imp and DSPy behave comparably when they talk to the -same real provider/model under the same constraints. - -This lane must not be confused with provider-free correctness. It measures the -full operational path: Imp through ReqLLM, DSPy through `dspy.LM`, both over -the same rows. - -Model lanes: - -- low-cost current DSPy-doc lane: used for full-row coverage when practical -- frontier sanity lane: a smaller current flagship slice to detect provider - drift, schema issues, and tool/stream behavior -- historical/research-style lane: where still available, a model family close - to models used in DSPy examples/papers, or an explicit unavailable note - -These lanes are intentionally cross-provider. Small/mini/nano/Haiku/Flash/Lite -models can satisfy the current low-cost bucket; current flagship GPT, -Claude Sonnet/Opus, and Gemini Pro models can satisfy frontier sanity; legacy -GPT-3.5/Davinci, Claude 3-era, Gemini 1.x, or explicitly research/legacy -models can satisfy the historical/research-style bucket. - -Dataset/task lanes: - -- GSM8K math with ChainOfThought -- HotPotQA / multi-hop QA with provided context -- Color/classification smoke -- at least one RAG/retrieval task -- at least one tool/ReAct task - -Required controls: - -- same input rows, offsets, and digests -- same model id and provider -- provider-explicit configuration when the lane is not OpenAI: Imp uses the - ReqLLM model spec, DSPy uses the matching LiteLLM/DSPy model name, and the - artifact records matched wire API families -- current Imp benchmark prompt/signature contract for every selected model - lane -- same temperature and token limits, or explicit documented provider limits -- same concurrency level -- one explicit campaign id for every chunk in a fresh full run -- cache disabled unless the lane explicitly tests cache -- row-level answers, pass/fail, errors, and latency recorded -- provider and model identity scoped in aggregation - -Initial executable matrix: - -```sh -mix benchmark.live_matrix -``` - -This command does not call providers. It aggregates existing -`imp-dspy-parity-campaign-*.json` artifacts into a -`live-matched-model-matrix-*.json` report, grouped by provider/model identity. -The matrix marks whether Imp has release-quality evidence for: - -- a current low-cost lane with full accepted canonical coverage -- a frontier sanity lane with a fresh matched research sample -- a historical/research-style lane with a fresh matched research sample, or an - explicit missing/unavailable note - -The matrix is a research result. A smoke matrix is useful wiring evidence, but -it is not live parity. A lane passes only when its policy is met with -current prompt contracts, matched effective generation, score/error parity, -latency ratios, and cost reporting. -When several models are present in one lane, the lane summary reports the -strongest candidate as the headline `coverage`/`cost` path and keeps -cross-candidate totals under `cumulative`; release blockers should point at the -candidate most likely to close the lane. -If legacy/research endpoints are no longer available, an operator may record -that decision explicitly: - -```sh -mix benchmark.live_matrix \ - --historical-unavailable-note "GPT-3.5 quota exhausted; Gemini 1.x credential invalid; Claude 3 endpoints unavailable on this account." -``` - -That note satisfies only the historical/research availability requirement. It -does not create `full_evidence`, does not count zero-row failed endpoint probes -as parity evidence, and does not satisfy the current low-cost or frontier -evidence lanes. - -Pass condition: - -- the current low-cost lane covers every canonical row with accepted row - evidence, not only attempted provider calls -- frontier and historical/research-style lanes reach the configured research - sample size with matched generation; measurement completion is reported - separately from the strict score/latency parity outcome -- runner/API error rows, including quota/rate-limit rows with null answers, are - counted as incomplete evidence and remain rerunnable -- selected live campaign artifacts record a single consistent `max_concurrency` - setting; mixed serial/concurrent chunks are diagnostic evidence only until - rerun or reaggregated into a comparable campaign lineage -- aggregate and per-task score gaps are within configured thresholds -- error-rate deltas are within threshold -- latency ratio is reported and meets the threshold for operational parity -- cost estimate is reported -- prompt/signature contract identity is current for every selected model lane -- effective generation settings are complete and matched, not merely requested - settings -- campaign artifacts include Imp and DSPy instrumentation summaries sufficient - to tell whether latency gaps come from provider/model time, prompt/output - shape, or local Imp overhead and adapter recovery. DSPy prompt-shape - summaries must preserve `message_chars_sources` so exact LM-history evidence - can be separated from deterministic row-shape estimates. The matrix selects a - complete-instrumentation/runtime-shape artifact over a larger incomplete - artifact for the same provider/model identity; nominal coverage from - quota-tainted chunks is not release proof, and deterministic row-estimated - fallback evidence must remain distinguishable from exact LM-history evidence. - -## Lane 3: Optimizer Lift Parity - -This is the heart of DSP-style programming. A baseline model-quality benchmark -does not prove optimizer parity. The question is whether Imp can compile a -program against a metric and improve it with comparable or better efficiency. - -Required optimizers: - -- `LabeledFewShot` -- `BootstrapFewShot` -- `KNNFewShot` -- `RandomSearch` -- `InstructionSearch` -- `COPRO` -- `MIPROv2` -- `SIMBA` -- `GEPA`-style reflection -- `BootstrapFinetune` / `GRPO` trainer workflow where the provider surface is - claimed - -Required tasks: - -- deterministic reward-coded task where the true optimum is known -- GSM8K subset -- HotPotQA/RAG subset -- tool/ReAct task -- structured extraction or classification task - -Required artifacts: - -- baseline score -- optimized score -- lift over baseline -- number of trials/candidates -- number of LM calls -- token/cost estimate where live -- wall-clock time -- optimizer trace -- selected instructions/demos -- failure and convergence information - -Initial executable command: - -```sh -scripts/setup_dspy_parity_env.sh -mix benchmark.optimizer_lift.check -``` - -The current artifact runs a deterministic provider-free matched-mechanism -lift task: the winning instruction/demo is planted in both sides' candidate -pools and scoring reuses the selection devset, so it demonstrates that Imp -optimizers select and apply an injected winner identically to DSPy — not -held-out lift (dee-5y5u). It -directly compares Imp and DSPy `LabeledFewShot`, `BootstrapFewShot`, -`RandomSearch`, `COPRO`, `MIPROv2`, `SIMBA`, and GEPA-style optimizer rows when -the installed DSPy sidecar exposes a compatible GEPA path. It records Imp -lift/non-regression plus explicit deviation notes for `InstructionSearch`, -finetuning, and GRPO where the installed DSPy sidecar lacks a stable -provider-free equivalent. The artifact includes Python package versions and -detected DSPy optimizer capabilities so stale assumptions become visible. It -also includes Imp natural user-story lanes for classification, QA, -retrieval/KNN few-shot, and instruction following. Each natural lane reports -baseline score, optimized score, lift, LM calls, estimated fixture cost, and -selected demos or instructions. - -Research-scale GEPA claims use a separate lane: - -```sh -mix benchmark.gepa_replication.check -``` - -That lane runs deterministic smoke rows and validates GEPA paper-family -artifact shape rather than generic optimizer lift. Full evidence requires -non-smoke campaign rows for AIME, HotpotQA, HoVer, IFBench, -LiveBench-Math, and Papillon/privacy delegation, with baseline, DSPy GEPA, -Imp GEPA, MIPROv2, metric-call budget, token/cost, wall-clock, seed variance, -and train/dev/test gap. Full rows must include campaign provenance, dataset -source and split checksums, DSPy/Imp/GEPA-artifact commits, concrete -non-placeholder comparator sources, distinct train/dev/test split digests, and -positive live token/cost accounting. Full rows must come from dataset roots -exported with `dataset.scope == "full"`; capped `--max-per-split` roots are -accepted only as engineering proof runs and are rejected for full research -claims. SIMBA is optional extra comparator evidence, not a required optimizer -in the upstream GEPA artifact. - -A full campaign conversion is source-shaped, not a generic six-task harness. -Required rows are `AIMEBench/CoT` (`problem -> answer`), -`HotpotQABench/HotpotMultiHop` (`question -> answer`), -`hoverBench/HoverMultiHop` (`claim -> retrieved_docs`), -`IFBench/IFBenchCoT2StageProgram` (`prompt -> response`), -`LiveBenchMathBench/CoT` (`question -> answer`), and -`Papillon/PAPILLON` (`user_query -> llm_request, llm_response, response`). The two `CoT` -rows remain separate family contracts with their own metrics, splits, -instructions, and budgets. - -HotPot, HoVer, and IFBench use strict component-specific GEPA feedback. -HotPot's map covers `summarize1`, `create_query_hop2`, `summarize2`, and -`final_answer`; HoVer's covers `summarize1`, `create_query_hop2`, -`summarize2`, and `create_query_hop3`; IFBench's covers -`generate_response_module` and `ensure_correct_response_module`. Each map must -exactly match the program predictor graph. A callback receives the selected -predictor invocation plus example, program result, metric result, and trace, -and must return non-empty feedback text; a mismatch or callback failure stops -optimization. The AIME, LiveBenchMath, and Papillon rows currently rely on -metric-level feedback rather than a custom component map. Campaign metadata -records the installed component-feedback identity. - -For source-checkout campaigns, use `mix imp.benchmark.gepa_replication ---from-gepa-artifact ... --upstream-evidence ... --imp-input ... ---protocol-classification exact_paper_replication` to convert upstream GEPA artifact -`Baseline`, `GEPA`, and `MIPROv2-Heavy` outputs into retained result rows. The -`--imp-input` file must come from Imp's own GEPA run and provide the -`imp_gepa` result plus provenance fields; the converter does not synthesize -Imp scores. -The explicit classification is reserved for a separately frozen -paper-authority protocol. The adapted current-model no-merge table must omit it -and cannot enter the exact C4 lane. - -Produce canonical Imp input with `mix imp.benchmark.gepa_campaign --manifest -benchmarks/config/gepa-paper-campaign-v2.json`. The immutable manifest binds the -full dataset hash, model roles, families, seeds, budgets, source commits, -request policy, source-exact environment, semantic-progress threshold, and -output/checkpoint paths. Five consecutive reflection proposal errors stop the -run with a checkpointed machine-readable reason before any claim artifact is -written. The v1 manifest remains available for exact reproduction of campaigns -started before this fail-fast contract. Legacy -partial runs remain path-driven: the dataset root must include a `families.json` contract and -`train.jsonl` / `dev.jsonl` / `test.jsonl` files for every GEPA family. The -runner records Imp GEPA candidate/frontier metadata, seed variance, split -digests, dataset scope, split counts, source commits, and explicit provider -token/cost accounting. It writes partial `imp-gepa-rows-*.json` artifacts; -only the replication converter can turn full-scope partial rows plus upstream -comparator outputs into a full public claim artifact. - -Build the dataset root with `mix imp.benchmark.gepa_dataset --gepa-root -path/to/gepa-artifact --out benchmarks/data/gepa-campaign`. This imports the -upstream GEPA artifact benchmark classes and writes source-derived split JSONL -plus a `families.json` manifest. The manifest records upstream metric names, -dataset scope, optional max-per-split cap, split counts, and split checksums. -Imp ports AIME integer exact match, HotPotQA answer exact match, HoVer -supporting-title retrieval, IFBench IFEval-style constraints, Papillon LLM-judge -quality/leakage scoring, and the deterministic LiveBenchMath AMC/AIME parser -paths plus `imo`/`usamo` proof-rearrangement edit-distance scoring. GEPA -HotPot and HoVer rows must carry provenance for the same upstream -`wiki.abstracts.2017` BM25 corpus/index checksums. The campaign task requires -`IMP_HOVER_UPSTREAM_BM25=1` for either family and executes both through the -pinned upstream Python BM25S index; generic retrieved-document outputs or the -native Elixir BM25 approximation cannot satisfy source-exact campaign evidence. -The native implementation does not reproduce the upstream English stopword -tokenizer or PyStemmer stemming. Validate fixed top-k title fixtures in a GEPA -source checkout at commit -`cbefbc1aa0f43dd39874ec4bf42211365dbda42e` with -`IMP_HOVER_UPSTREAM_PARITY=1 mix test test/hover_bm25_parity_test.exs` after -setting `IMP_GEPA_PYTHON` to an environment with `bm25s==0.2.12` and -`pystemmer==2.2.0.3`. -IFBench imports the larger AllenAI extended registry; Imp ports those registry -checks in Elixir and keeps unknown ids fail-closed. Four IFBench NLP-dependent -checks have native deterministic fallbacks plus a source-exact Python bridge for -research campaigns. The source-checkout differential covers all 83 active -merged-registry ids and passes against the pinned GEPA artifact, including -language detection and NLP-backed checks. Full IFBench GEPA runs additionally -require `IMP_IFBENCH_UPSTREAM_DESCRIPTIONS=1`, `IMP_GEPA_ROOT`, and -`IMP_GEPA_PYTHON`; the Elixir scorer remains native while reflective feedback -uses the pinned upstream registry's exact human descriptions. Papillon campaigns -must pass a judge LM and include `metric_judge` provenance in research rows. -LiveBenchMath `amps_hard` is guarded by the SymPy/Lark symbolic bridge and must -be validated in the research campaign Python environment before claiming -AMPS_Hard parity. - -Upstream comparator evidence is an archive-derived sidecar, not a manually -completed result field. Run `scripts/extract_gepa_upstream_evidence.py` with -the immutable `experiment_runs_data` archive, the matching GEPA artifact -checkout, selected model, and an output path. The extractor requires all six -family/program pairs and `Baseline`, `GEPA`, and `MIPROv2-Heavy` seed-0 runs; -records archive and source identities; reads run configuration, metric JSONL, -and `evaluation_result.txt`; and derives observed optimizer callbacks by -subtracting Baseline final-test callbacks. The replication converter requires -that sidecar, validates its exact key set and the result-file score/SHA-256, and -rejects configured-only budgets, missing enforcement, and test-selected seeds. - -Pass condition: - -- Imp achieves non-regression versus baseline on every optimizer lane -- Imp matches DSPy lift within threshold for equivalent optimizers, or an - intentional algorithmic deviation is documented -- Imp reports enough trace/evidence to debug every optimizer decision - -## Lane 4: RAG, Tools, Agentic Programs, and Production Semantics - -DSPy parity includes program composition, retrieval, tools, tracing, and -production behavior. These paths need their own evidence because they fail in -different ways than simple QA. - -Required coverage: - -- retriever contract and document normalization -- RAG answer quality with deterministic and live retrievers -- ReAct tool call shape, observation handling, and final-answer handling -- MCP/tool adapter behavior -- ProgramOfThought and CodeAct execution/error policy -- streaming output and incremental field parsing -- async/concurrent execution -- save/load/rebind of programs -- observability and redaction - -Executable checks: - -```sh -mix benchmark.rag_tool_failure.check -mix benchmark.rlm.check -mix test test/react_contract_test.exs test/rlm_test.exs \ - test/tool_schema_runtime_test.exs test/mcp_import_test.exs \ - test/task_supervision_test.exs test/telemetry_lineage_contract_test.exs -``` - -The aggregate RAG/tool/agent evaluator and disconnected generic agent runtime -were removed for 0.3. Product semantics now live in direct tests of the -canonical ReAct/RLM/tool/MCP/task/persistence boundaries. Retained differentials -must add an external comparison, not rebundle already-tested behavior into a -second pass/fail dashboard. - -The separate `mix imp.benchmark.rag_tool_failure_differential` lane runs one -six-scenario queued-action schedule through actual Imp ReAct and -source-authenticated DSPy 3.2.1 ReAct. Exact normalized traces cover a -transient retry, injected retriever timeout, fixture idempotency replay, -unknown/failing tools, finish/submit, and max-iteration terminals. This closes -the provider-free matched-schedule mechanics gap at C2 only. Because the LM -does not select actions and fixture tools own retry/idempotency behavior, it is -not recovery effectiveness, wall-clock timeout parity, or native policy parity. - -The RLM command produces T0 deterministic contract replay over hand-authored -fixture rows. It is useful for checking harness wiring and inspecting traces, -but gold-derived outputs and non-equivalent scripted executions make it -ineligible for effectiveness, long-context, operational-parity, or uncertainty -claims. T3 paper-protocol evidence is required for the release lane. - -`mix benchmark.rlm.contract.check` is the stronger T1 lane. It gates twelve -matched execution contracts against DSPy 3.3.0b1 and records the exact upstream -source hash. T1 remains operational evidence, not model-quality or paper-scale -evidence. - -Pass condition: - -- deterministic replay proves trace/tool semantics -- live matched slices prove provider-facing paths -- all production traces redact secrets and include enough metadata for audit - -## Lane 5: Provider-Free Performance - -Provider latency can hide language-runtime differences. To claim Elixir -performance improvements, Imp must measure work that actually happens inside -Imp and DSPy. - -Required benchmarks: - -- signature parsing/loading -- adapter message formatting -- adapter response parsing -- JSON/schema validation -- evaluation loop throughput -- metric normalization -- optimizer trial scheduling -- trace construction/redaction/serialization -- cache hit and miss overhead -- concurrent orchestration overhead - -Required controls: - -- no live provider calls -- fixed fixture corpus -- warmup and repeated measurements -- CPU/runtime metadata -- memory where practical -- Python and Elixir versions recorded - -Initial executable command: - -```sh -mix benchmark.overhead.check -``` - -The provider-free lane emits an Imp/DSPy overhead artifact for signature -parsing, adapter format/parse, schema validation, evaluation loop throughput, -metric normalization, optimizer trial scheduling, trace redaction/serialization, -cache hit/miss overhead, and concurrent orchestration. -Each named operation has an absolute Imp-median guard and a reference-relative -guard with explicit rationale. Cache, schema, and optimizer operations use -matched logical work and configuration. Runtime/environment identity is bound -into the verified artifact. Ratios are diagnostic measurements, not speed or -parity claims. - -Pass condition: - -- Imp performance claims name the benchmark they come from -- every named operation remains within its declared regression budgets -- no speed claim is inferred from a budget or ratio -- regressions have tracked remediation before release - -## Reading Results - -There is no aggregate parity or release score. Each lane retains the inputs, -runtime identities, data digests, costs, raw outcomes, and interpretation needed -for its own question. Full parity cannot be claimed unless every named surface -has appropriate evidence; a green product check is not parity evidence. - -Performance statements must point to the benchmark that supports them. Live -latency interpretations must distinguish provider/model time, prompt and output -shape, and Imp-local overhead. - -## What Counts As Done - -Imp has full parity evidence only when: - -1. Golden trace parity is complete. -2. Live matched-model parity has at least one full current low-cost lane, one - matched research-sample frontier sanity lane, and one matched - research-sample historical/research-style lane. -3. Optimizer lift parity passes for every optimizer Imp exposes as production - surface. -4. RAG/tool/agent production semantics pass deterministic and live slices. -5. Provider-free performance benchmarks support any speed claims. -6. Every named lane meets its own protocol and the cross-lane interpretation - survives review of the underlying results. - -Until then, honest language is narrower: Imp may have a passing smoke lane, -deterministic parity for specific surfaces, or performance wins on specific -provider-free paths. diff --git a/docs/internal/README.md b/docs/internal/README.md deleted file mode 100644 index c85ad068..00000000 --- a/docs/internal/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Internal Docs - -These maintainer-facing protocols and reports explain how Imp's claims are -tested against upstream systems, papers, and pinned datasets. They are not a -second user manual or roadmap, and they are excluded from the package. - -Start with the question you are trying to answer: - -- **What does parity require?** Read the - [Parity Validation Program](PARITY_VALIDATION_PROGRAM.md) for the distinct - semantic, live-model, optimizer, production, and performance evidence lanes. -- **How strong is a piece of evidence?** Read the - [evidence handbook](../maintainers/EVIDENCE.md) before treating product, - compatibility, operational, effectiveness, or paper evidence as equivalent. -- **Why does one implementation differ from upstream?** Read the source-bound - fidelity note for that surface, such as - [adapters](ADAPTER_FIDELITY.md), - [instruction optimizers](INSTRUCTION_OPTIMIZER_FIDELITY.md), or - [RLM](RLM_FIDELITY.md). - -The repository [README](../../README.md) owns the user-facing purpose and path. -The [release procedure](../maintainers/RELEASE.md) owns the current product -finish line; machine-readable pins and artifact coordinates live under -`benchmarks/`; unfinished work lives in pull requests. Dated reports and historical -results remain useful at their recorded scope, but do not silently become -current decisions. diff --git a/docs/internal/TRUST_AUDIT_2026-08-07.md b/docs/internal/TRUST_AUDIT_2026-08-07.md deleted file mode 100644 index 2a0367c0..00000000 --- a/docs/internal/TRUST_AUDIT_2026-08-07.md +++ /dev/null @@ -1,128 +0,0 @@ -# Imp Trust Audit — Final Report - -**Date:** 2026-08-07 · **Repo:** /Users/deepfates/Hacking/github/deepfates/imp · **HEAD:** e1392a17 · **Refuted findings during adversarial verification:** 0 - ---- - -## 1. Verdict: **GO-WITH-CAVEATS — but a hard NO-GO for GEPA runs as currently configured** - -The suite, docs, and harness are real. Nothing is theater, nothing is stubbed, the ground truth verifies (clean compile under `--warnings-as-errors`; 2821 tests / 0 failures in 188.5s with transparent, env-gated exclusions), and the project is conspicuously honest about its own gaps. You can trust the *machinery*. - -What you cannot trust yet is **any new GEPA/optimizer number produced with default settings**, because of a composed defect chain in core evaluation: - -> **30s default per-row kill timeout → killed rows silently scored 0.0 → those 0.0s durably written to a disk cache keyed only on (candidate text, example) — no model, no metric, no config — and replayed on resume.** - -On any slow model, this chain manufactures fake score deflation and then makes it *permanent and invisible*. Both links are confirmed against code (`lib/imp/optimizer/gepa.ex:161`, `lib/imp/optimizer/trajectory.ex:1300-1306`, `lib/imp/optimizer/gepa/evaluation_cache/disk.ex:242-255`). - -**Trust:** the offline suite, the docs (two isolated snippet bugs aside), the livebooks, the completed TREC matched result (recomputes byte-for-byte, hashes verify), and the matched IFBench *baselines*. - -**Distrust until fixed:** any fresh GEPA benchmark with default timeout + disk cache; any IFBench *optimizer-lift* number (zero completed runs exist — every attempt across four run series ended `status=stopped`); any Heavy result, which will be first-of-kind on a pipeline whose last two fidelity repairs landed within ~36 hours of HEAD. - ---- - -## 2. Per-Dimension Trust Grades - -| Dimension | Grade | One-liner | -|---|---|---| -| Ground truth (compile/test) | **solid** | Clean compile, 2821 tests / 0 failures, exclusions deliberate and documented | -| Test-theater | **solid** | Prompt-sensitive Static LM proves optimizer lift causally; real HTTP servers; byte-level DSPy tape parity | -| Docs-vs-code | **mostly-solid** | ~15 snippets executed as documented; TREC recomputation reproduces exactly; 2 broken snippets in ADVANCED.md only | -| Livebooks | **solid** | All 5 execute end-to-end offline; live cells self-gate correctly | -| Bench-harness | **mostly-solid** | TREC contract rigorous and verified; IFBench matched pipeline has never completed a run | -| Core-correctness | **shaky** | Timeout→0.0→cache poisoning chain; deadline semantics bugs; unkeyed disk cache | -| Tickets-process | **mostly-solid** | Scrupulously honest; but flagship Heavy cell has zero outcomes and HEAD sits mid-repair-churn | -| DSPy-parity | **solid** | Every claimed surface substantially implemented; effectiveness gaps self-declared in CONFORMANCE.md | - ---- - -## 3. Blockers (all adversarially confirmed) - -### B1. GEPA silently scores timed-out rows as 0.0, with a 30-second default -`lib/imp/optimizer/gepa.ex:161` defaults `timeout: 30_000`, wired through ProgramAdapter into the trajectory runner (whose own bare default is **5s**, `trajectory.ex:1080`). Rows exceeding it are killed via `on_timeout: :kill_task` and become failed trajectories hard-coded to `score: 0.0` (`trajectory.ex:1300-1306`). The module has **zero logging** — while `Imp.Evaluate` warns loudly for the identical event (`evaluate.ex:~350`), proving the hazard is known elsewhere. On a slow model, every candidate's score is silently deflated and the deflation is indistinguishable from real failure. - -### B2. Timeout kills and unclassified errors are cached as *complete* evaluations -Narrowed by verification, but real: the `:complete?` completeness guard (`engine.ex:4470-4473`) is live and correctly paired with Optimize Anything's batch evaluator (`adapter.ex:573`) — but the convention was **never extended to GEPA's ProgramAdapter** (`program_adapter.ex:88-92` sets only `%{metric_calls, failures}`) or the per-example path (`adapter.ex:238` passes `record_completeness?: false`). Classified transient failures (transport/budget/cost/cancellation) fail-closed via `Imp.OperationalSafetyError` and never reach the cache — good. But **timeout kills from B1 are unclassified**, so their 0.0s are durably cached and replayed on resume. B1 + B2 compose into permanent score corruption. - -### B3. The flagship matched IFBench optimizer comparison has never completed -Zero completed optimizer outcomes across **four** run series (`tmp/matched_gepa_mipro_ifbench{,_v2,_v3,_gepa014}`), all `status=stopped`, with four distinct unresolved launch failures: (1) v1 upstream `AttributeError` (program lacks `forward`), (2) v3 GEPA API drift (`acceptance_criterion` TypeError), (3) gepa014 Imp-side `Req.TransportError :ssl_not_started` in `verify_models!`, (4) gepa014 version-drift guard tripped because the gepa v0.1.4 git tag ships `pyproject.toml version="0.1.3"` (unreplaced release marker). Both Heavy launches also aborted on **Imp product bugs** (dispatch defect at 6cc f37e8; `max_depth`-into-ReqLLM fidelity leak at 9a8154d5, killing 18 proposal slots pre-transport). Tickets imp-yme4 (in_progress) and imp-88sn (open) state plainly: no effectiveness result has been earned. README IFBench numbers (0.7619 vs 0.7874) are baseline-denominator only — the docs say so honestly. - ---- - -## 4. Majors - -1. **Disk cache identity omits everything but candidate text + example** (`disk.ex:242-255`). No LM model, temperature, program structure, demos, adapter, or metric in the key; no config fingerprint on the run_dir; and `:auto` silently enables the disk cache whenever `run_dir` is set (`config.ex:183-184`). Reusing a run_dir after changing model or metric replays stale scores — the seed candidate *always* collides — and self-checksums make contamination undetectable. -2. **`:deadline` silently discards per-row `:timeout`** in both `Imp.Evaluate` (`evaluate.ex:374-387`) and the trajectory runner (`trajectory.ex:1112-1148`, which never even receives the timeout variable). One hung row can consume the entire remaining deadline and starve all later rows into `{:exit, :timeout}` — degrades kill granularity on any deadline-bound run. -3. **Harness stability is ~36 hours old.** The last 60 commits are a repair/record/refreeze churn loop on the matched-benchmark subsystem; three fidelity defects in the matched-treatment path itself (dispatch, GEPA treatment integrity, MIPRO proposal option leak) were each discovered only when live paid runs stopped. The repo's own tickets treat pre-repair numbers as inadmissible. (Note: verification softened the dispatch bug — it was fail-loud, raising `ArgumentError` before any provider call, so no wrong-arm scores ever existed.) -4. **Effectiveness is a self-declared open question.** CONFORMANCE.md marks `optimization.instructions` and `optimization.gepa` as "gap": TREC is task-specific C3 evidence; GEPA HotPotQA three-seed mean F1 lift was **-0.015** with zero positive seeds; Banking77 missed its preregistered bar; IFBench was downgraded after a scorer defect. This is disclosure, not concealment — but it bounds what any single new benchmark can claim. -5. **Two broken snippets in docs/LEARNING_PATH.md** (both fail loudly at construction): the HTTP retriever `response_mapper` example is arity-2 where the schema requires `{:fun, 1}` (`http.ex:45`), and `Imp.Retrievers.Databricks.new/3` is documented but only `new/2` exists (`http.ex:449`). Neither touches benchmark paths. -6. **Sealed raw evidence lives only in gitignored `tmp/`** — the recomputation hashes verify, but the underlying artifacts are one `rm -rf` from gone. - ---- - -## 5. Genuinely Solid (credit where due) - -- **The test suite is the opposite of theater.** `Imp.LM.Static` handlers inspect the actual formatted prompt and only answer correctly when the optimizer genuinely changed instructions/demos — optimizer tests prove lift *causally*. Mox is quarantined to one contract file. The real client path is tested against a live local Bandit HTTP server including parse-failure→JSON-fallback retry with telemetry assertions. Error paths (retry/backoff/Retry-After, hung-LM cancellation, secret redaction, silent-failure regressions) are unusually deep. DSPy parity is enforced byte-for-byte against pinned DSPy 3.2.1 capture tapes. -- **Docs were verified by execution, not inspection.** ~15 representative snippets ran as documented; the CASE_STUDY_TREC recomputation command reproduced its documented output line byte-for-byte with matching SHA-256; `mix docs` builds; every module named in prose exists. -- **All 5 livebooks execute end-to-end offline**, with live cells self-gating correctly. -- **The TREC matched contract is trustworthy**: exact-provider route guard, identical models both sides, real dspy.evaluate on the DSPy side, failures scored 0 *in the denominator*, loud sentinel failure if the DSPy report is missing. -- **Radical self-honesty**: the project self-reports its invalidated IFBench scorer, preserves pre-fix stopped runs under `-pre-*-fix` suffixes, labels stops "product stop, not scientific evidence," and keeps version claims consistent (unpublished 0.3.0 candidate everywhere). -- **Fail-closed transient-failure design**: classified transport/budget/cost/cancellation errors raise `OperationalSafetyError` before any Result exists — the "LM 500s poison the cache" scenario is already defused for classified errors. - ---- - -## 6. Punch-List Before Real Runs (ordered) - -**Must-fix before any paid GEPA run:** - -1. **Neutralize B1 for benchmarks now**: pass an explicit generous `timeout:` (or `:infinity`) in every benchmark config — one-line config change, do it today. Then fix properly: add a loud `Logger.warning` on timeout-kill in `trajectory.ex` (mirror `evaluate.ex:350`), and surface killed-row counts in run reports. -2. **Close B2**: extend the `:complete?` convention to `ProgramAdapter` metadata (`program_adapter.ex:88-92`, set `complete?: failures == 0`) and flip `record_completeness?` on the per-example path (`adapter.ex:238`). Until merged, run benchmarks with the disk cache disabled. -3. **Key the disk cache**: add a config fingerprint (model id, params, metric identity, program structure) to `entry_digest/2` in `disk.ex` — or, interim, enforce a fresh `run_dir` per configuration and never resume across config changes. -4. **Fix deadline semantics**: use `min(remaining, timeout)` (with `:infinity` handling) in both wave calls (`evaluate.ex:374-387`, `trajectory.ex:1137-1148`); add a combined timeout+deadline test. - -**Must-fix before the IFBench/Heavy campaign:** - -5. Repair the four launch failures: start the `:ssl` app before `verify_models!` in `run_imp.exs`; patch or vendored-fix the gepa v0.1.4 checkout's `pyproject.toml` version marker (or relax the drift guard to accept the known tag); adapt the upstream runner to the current GEPA API (`acceptance_criterion`) and the v1 `forward` issue. -6. Launch the restart-only Heavy successor pair from a fresh private root per imp-88sn's mandate — and treat its result as **first-of-kind**, not confirmation. - -**Should-do:** - -7. Run `mix live.check` (LIVE_PROVIDER=1 with keys) once before trusting real-provider behavior — the 172 excluded live tests have never run in CI. -8. Fix the two ADVANCED.md snippets (arity-1 `response_mapper`; `Databricks.new/2` with full endpoint URL; also reconcile the module's own "new/3" error-message references). -9. Copy sealed evidence out of gitignored `tmp/` into a committed or otherwise durable location. - -**Bottom line:** items 1–4 are hours of work, not days, and items 1–2 are the difference between benchmark numbers you can defend and numbers that are quietly wrong on any slow model. Fix those, disable the disk cache for the first real run, then go. ---- - -## Addendum (round 2, same day) — corrections from adversarial re-review - -Line-level facts of the original report all survived spot-checks. Three framing corrections: - -1. **"Hard NO-GO for GEPA as currently configured" was overstated.** The flagship matched config (`examples/matched_gepa_mipro_ifbench_gepa014/run_imp.exs`) already passes `timeout: 120_000`, `cache: false`, `max_concurrency: 1` — B1's config mitigation is in place for the campaign path. The library defaults remain the hazard for any other consumer. -2. **The B1+B2 poisoning chain was routed through the wrong backend.** GEPA never uses the Disk evaluation cache (Optimize Anything only). The real carrier is the **checkpoint**: `engine.ex:3396` serializes the in-memory cache (same weak candidate+example-only identity, `memory.ex:21-25`) into every checkpoint; `load_cache` (5352) replays it on resume. Conclusion unchanged, mechanism corrected. Tickets imp-emrr (widened) and imp-g22q cover it. -3. **New matched-fairness defect (was uncovered surface):** cache hits charge zero metric calls (`engine.ex:4307`); combined with checkpoint replay, a resumed Imp arm re-scores for free while the upstream arm may pay. Ticket imp-fwfe. Conversely, failed/timeout-killed rows charge full price (`program_adapter.ex:89`), so B1's deflation compounds with budget burn. - -Surfaces examined and found solid in round 2: concurrency admission (`tasks.ex` — FIFO backpressure, lease transfer, no self-deadlock), sandbox (`sandbox.ex` — whitelist ops, atom-exhaustion guard), telemetry redaction. Fifth preflight failure mode noted on imp-nbyg (`String.to_float/1` on catalog prices). - -Round-2 cold-reader and DSPy-user findings are tracked as tickets imp-cg5h, imp-st1u, imp-k5wi, imp-nnur, imp-cczl, imp-zc94, imp-n9sj, imp-vb1d. - -## Addendum (round 3) — release-machinery findings and saturation verdict - -New first-order finding: **the audited HEAD has no CI provenance.** Local main is 583 commits ahead of origin (last origin CI run 2026-07-24) — every benchmark-fidelity repair in the last two weeks is CI-unverified. No branch protection exists; the scheduled Evidence lane failed 2026-07-27 and 2026-08-03 untriaged. Ticket imp-fkwy (P0, blocks imp-88sn). The repo is also private with tags stopped at v0.2.1, so the telos's first clause (installable) is structurally unmet — tracked on imp-qoen, widened. - -Checked clean in round 3: CI workflow design, mix task catalog (80+ tasks all real), CHANGELOG vs tags, SECURITY/CONTRIBUTING accuracy, licensing (MIT, no vendored DSPy source), identity/. examples/local_* flagged as maintainer-machine-bound evidence rigs, not runnable case studies. - -**Saturation verdict:** after three rounds (workflow audit + adversarial verify; first-hand cold-user execution + cold-Elixir-reader + cold-DSPy-migrant + adversarial re-review; release-machinery sweep), remaining unexamined corners (scripts/ one-offs, docs/maintainers prose) are minor-findings territory. The gap between current state and telos is captured in the imp-yme4 ticket graph: 22 open tickets, with imp-88sn (measured usefulness) gated on score-integrity (90uc→g22q, emrr, pk5c, fwfe), harness repair (nbyg), live verification (7aah), and CI provenance (fkwy). - -## Correction (round 4, owner pushback) - -The report's claim that "the 172 excluded live tests have never run" was **wrong twice**: live-provider runs are abundantly evidenced (admitted instruction_live and multimodal_live artifacts, paid TREC runs, committed live tutorial receipts, LiveBench baselines, live logs), and the excluded count is 195 across all exclusion tags, not 172 live. The true narrow claim: live-tagged tests are excluded from CI aliases, `live.check` covers ~15 tests in 2 files, and no gate-evidence artifact records a live smoke at the current HEAD. imp-7aah rescoped accordingly. Method lesson recorded: negative existence claims ("X never happened") require positive evidence searches; ours only code-read. Separately: 4 calibration-pilot test failures observed during this correction were caused by the audit's own uncommitted files dirtying the tree — the candidate-identity clean-tree guard working as designed, not a product defect. - -## Round 5 — campaign-path certainty pass - -Resolved the remaining ambiguity about whether the score-integrity tickets gate the benchmark campaign or only library defaults. Verdict: **they gate the campaign.** (1) `raise_on_exception: true` does not intercept timeout kills — `{:exit, :timeout}` is unclassified and becomes a silent 0.0 trajectory even in the flagship config (trajectory.ex:1160-1171); the 120s campaign timeout moderates frequency only. (2) The round-2 claim that `cache: false` neutralizes cache poisoning conflated the LM request cache (which that flag controls) with the GEPA evaluation cache, which defaults on (gepa.ex:124), is not overridden by the campaign, is serialized into checkpoints, and replays on resume — the exact mode Heavy runs (two recorded stops) operate in. imp-90uc, imp-g22q, imp-emrr, imp-fwfe all confirmed on-path. - -## Round 6 — worldview correction (with owner) and final gate - -The audit's framing of optimizer effectiveness as "an open question the benchmark exists to answer" was corrected by the owner and is wrong as stated. DSPy/GEPA/MIPRO are published, replicated results on these task families; Imp is a port of known-working software. **Parity is the null hypothesis for a faithful port**: a matched deficit indicates an implementation defect, not uncertainty about the method. The repo's historical negatives don't contradict this — they were single-arm, absolute-lift runs on small local models with no DSPy arm; the only true matched head-to-head (TREC) was favorable. The benchmark is confirmation, and readiness work should *drive the probability of parity-or-better up before spending*. - -Final campaign gate (imp-88sn deps): score integrity (90uc → g22q, emrr, pk5c, fwfe) · launch repairs (nbyg, +5th preflight mode) · live smoke recorded at HEAD (7aah) · CI provenance (fkwy) · green DSPy differentials at HEAD (sqkr) · bug-or-benign verdicts on historical misses (sa2a) · preregistered in-band pilot (u3af) before Heavy. Docs narrative reframe tracked as imp-pomi (via claims machinery, not prose edits). diff --git a/docs/internal/UPSTREAM_FIDELITY_AUDIT.md b/docs/internal/UPSTREAM_FIDELITY_AUDIT.md deleted file mode 100644 index 3eb287d6..00000000 --- a/docs/internal/UPSTREAM_FIDELITY_AUDIT.md +++ /dev/null @@ -1,155 +0,0 @@ -# Upstream Conformance - -Imp tracks upstream fidelity as executable product contracts, not as a list of -names found in source code or documentation. The authoritative ledger lives in -`Imp.UpstreamFidelity`; the generated, maintainer-readable projection is the -[Executable Upstream Conformance Map](../CONFORMANCE.md). - -## Baseline Policy - -The declared compatibility baseline is DSPy `3.3.1`, pinned to commit -`638e155cf725236fe5d01b5332394a7bc128881d` (annotated tag object -`753ab03d9ee2919159e7d9e0c9f47f753845a8ff`). The published wheel's 157-file -source tree hashes to -`b9364d08e549a01fb87b37aa41ebca24c4dda58160dda13523fbc83323862c4b`. -The executable ledger also content-binds the 73-page public API inventory. - -Historical DSPy `3.2.1`, `3.3.0b1`, and `3.3.0` differentials retain their -original authority; they are evidence about those exact treatments, not the -current baseline. The stable delta includes ReActV2, normalized LM envelopes, -resource loading, MCP and adapter corrections, and experimental Flex code -optimization. Imp implements or gives an explicit BEAM-native disposition for -the stable programming/runtime surface. Flex remains a named experimental -downstream gap: existing Optimize Anything code artifacts do not earn a -Flex-shaped public module without an ordinary sandboxed optimize/reload/serve -user story. - -Standalone GEPA is versioned independently. Its current implementation -authority is release `v0.1.4` at -`8b0ce6cd99a234f6b74daf37558a2ac0ce18f975`; the exact `v0.1.1` checkout is -retained only as a historical executable differential. - -When DSPy publishes a new stable release, updating the baseline is a reviewed -product change. The ledger must first account for every added, removed, or -changed upstream surface and give each one a semantic contract and owner. - -The inventory is completed from pinned source, public documentation, tests, -examples, and release deltas—not from exported symbol names alone. For each -material surface, the audit must answer what a user can accomplish, which -observable invariants define success, and how Imp proves them. The disposition -is either an idiomatic public Imp implementation with an executable -differential, a deliberate BEAM-native alternative with a user-value test, an -explicit downstream/experimental boundary, or an owned missing capability. -Absence cannot be accepted merely because upstream's Python shape is unfamiliar -or inconvenient. Relevant gold data and provider conditions are acquired when -needed to falsify a semantic claim rather than collected as a speculative -corpus. - -## Source Lineage - -- DSP and DSPy: `arXiv:2212.14024`, `arXiv:2310.03714`, and - -- DSPy Assertions: `arXiv:2312.13382` -- MIPROv2: `arXiv:2406.11695` -- GEPA: `arXiv:2507.19457` and -- Recursive Language Models: `arXiv:2512.24601` -- optimize_anything: `arXiv:2605.19633` -- Learning, Fast and Slow: `arXiv:2605.12484v2` - -Paper-derived features also require the canonical reproduction protocol tracked -by `de-c7ui`. Unit tests can establish control-flow and data-contract semantics; -they cannot establish paper-level effectiveness. - -## Ledger Contract - -Every stable upstream surface belongs to exactly one MECE capability row. Each -row records: - -- upstream names and pinned source locations; -- the Imp modules that own the behavior; -- semantic invariants that an Elixir implementation must preserve; -- executable test, integration, live, or benchmark evidence; -- user-facing documentation; -- one of `conformant`, `elixir_native_equivalent`, or `gap`; -- a rationale for every Elixir-native equivalent; -- an open owner ticket for every gap. - -`tracking` is reserved for experimental or research-horizon behavior and is not -release blocking until Imp publicly adopts the corresponding product claim. - -A missing evidence file, missing Imp module, unowned gap, or unexplained native -equivalent becomes `invalid_evidence`. Every gap remains visible and owned, but -only rows marked as product release blockers fail the product gate. A -claim-specific gap instead prohibits the corresponding fidelity, parity, or -effectiveness claim until its evidence passes. Symbol presence, prose, fixtures, -and smoke artifacts do not independently establish conformance. - -## Commands - -Generate JSON without asserting completion: - -```sh -mix imp.upstream_fidelity \ - --out tmp/upstream-fidelity/upstream-fidelity.json -``` - -Regenerate the checked-in readable projection: - -```sh -mix imp.upstream_fidelity \ - --format markdown \ - --out docs/internal/../CONFORMANCE.md -``` - -Run the release-blocking conformance gate: - -```sh -mix upstream_fidelity.check -``` - -That command is expected to fail while any blocking ledger row remains a gap. -Its failure lists stable capability ids, which lead directly to evidence and -owner tickets in the generated map. - -## Current Frontier - -The generated map is the current source of truth. At this writing, the core -programming contracts, basic modules, adapters, typed tools, refinement, -evaluation, runtime operations, observability, and persistence/deployment rows -are conformant. ReqLLM/process context, retrieval, and the ReAct family are -explicit Elixir-native equivalents. ReAct specifically uses provider-native -function calls, a reserved `submit` tool, and fail-fast tool errors rather than -claiming DSPy's action-field, finish-tool, observation-and-continue semantics. - -Multimodal image and native-PDF quality, the BEAM-native RLM controller, -source-faithful GEPA engine, Fast-Slow orchestration boundary, normalized LM -runtime, and learning path have executable product evidence. Their broader -paper-scale, audio, experimental Flex, external CISPO execution, and dominance -claims remain explicitly unmade where evidence is incomplete. -Instruction-optimizer matched campaigns are claim-specific gaps rather than -universal release blockers. - -The weight-optimizer row has one narrow local result: a pinned Qwen2.5-0.5B MLX -SFT artifact on the frozen four-intent Banking77 subset improved untouched -40-row accuracy from 0.125 to 0.55 and macro-F1 from 0.0610 to 0.4561, then -reproduced byte-identical ordered predictions/errors after save/load and -fresh-process serving of the exact fused artifact. -Avatar and AvatarOptimizer provide bounded typed-action execution and -feedback-driven instruction optimization. BetterTogether -implements arbitrary named and repeated optimizer sequences, evaluates the -baseline and each successful prefix, selects the best validated prefix with -stable tie handling, returns the latest successful prefix without validation, -and stops on the first failed step. - -That result does not imply general Imp or SFT effectiveness, GRPO, production -reliability, BEAM superiority, paid-provider training, or matched -Avatar/AvatarOptimizer/BetterTogether parity. Those remain explicit -claim-specific gaps until their own evidence exists. - -The bounded product gate passed on the previously frozen clean candidate after -its package, persistence, deployment, documentation, and production audits. It -is a retained predecessor result, not a verdict that the current release -objective or latest-stable migration is complete. Claim-specific rows stay red -for their narrower claims without being converted into blanket package -failures. Closing a ticket or adding a module name does not change status by -itself. diff --git a/examples/deployment/data/README.md b/examples/deployment/data/README.md new file mode 100644 index 00000000..e5627107 --- /dev/null +++ b/examples/deployment/data/README.md @@ -0,0 +1,20 @@ +# Deployment example data + +Each file here carries its own source record. This note surfaces what those +records say, so you do not have to open the JSON to find the license. + +| File | Source | License | +| --- | --- | --- | +| `banking77-mipro-stage1.json`, `banking77-mipro-confirmatory-v1.json` | `PolyAI/banking77` on Hugging Face; the receipts pin the parquet files by SHA-256 and record the exclusion, normalization and ordering rules used to derive the splits | CC-BY-4.0, as declared in the receipts | +| `hotpotqa-gepa/{train,selection,test}.jsonl` | `hotpotqa/hotpot_qa`, `distractor` config, `validation` split, revision `1908d6af`; `receipt.json` pins the parquet by SHA-256 | CC-BY-SA-4.0, as declared in `receipt.json` | + +The agent-optimization story +(`examples/deployment/agent_optimization.exs`) uses neither file. Its train, +selection and held-out requests are written inline in that script — a dozen +short support requests about accounts, billing and security, invented for the +example. They are MIT, like the rest of this repository, and they are far too +few and too tidy to support a claim about agent behavior in general. See row R6 +in [benchmarks/RESULTS.md](../../../benchmarks/RESULTS.md). + +The repository's MIT license covers our code and our derived split files. It +does not replace the licenses above for the underlying corpora. diff --git a/examples/matched_instruction_optimizers_trec/data/aggregate-recomputed.json b/examples/matched_instruction_optimizers_trec/data/aggregate-recomputed.json new file mode 100644 index 00000000..37c66f49 --- /dev/null +++ b/examples/matched_instruction_optimizers_trec/data/aggregate-recomputed.json @@ -0,0 +1,703 @@ +{ + "acceptance": { + "headline_passed": true, + "improvements": { + "gepa": { + "confidence_interval": [ + 0.29583333333333334, + 0.5041666666666668 + ], + "holm_adjusted_p": 1.9998000199980003e-4, + "mean": 0.4, + "p_one_sided": 9.999000099990002e-5 + }, + "mipro_v2": { + "confidence_interval": [ + 0.04583333333333331, + 0.2458333333333334 + ], + "holm_adjusted_p": 0.0026997300269973002, + "mean": 0.14583333333333331, + "p_one_sided": 0.0026997300269973002 + } + }, + "noninferiority_margin": -0.05, + "rule": "Holm-adjusted Imp improvement over baseline at alpha 0.05, then the same optimizer's Imp-minus-DSPy 95% lower bound must exceed -0.05", + "winning_optimizer": "gepa", + "winning_optimizer_imp_minus_upstream": { + "confidence_interval": [ + -0.04583333333333334, + 0.029166666666666667 + ], + "mean": -0.008333333333333331, + "p_one_sided": 0.6808319168083191 + } + }, + "claim_boundary": "matched system outcome comparison; no general effectiveness or BEAM-superiority claim", + "imp_minus_upstream": { + "baseline": { + "held_out": { + "accuracy": { + "exact_observed_range": [ + -0.025000000000000022, + 0.0 + ], + "mean": -0.012499999999999992, + "paired_deltas": [ + -0.025000000000000022, + -0.012499999999999956, + 0.0 + ] + }, + "macro_f1": { + "exact_observed_range": [ + -0.04253332063537296, + -0.033986071492817294 + ], + "mean": -0.03763265465862322, + "paired_deltas": [ + -0.04253332063537296, + -0.033986071492817294, + -0.0363785718476794 + ] + } + }, + "selection": { + "accuracy": { + "exact_observed_range": [ + -0.04999999999999993, + 0.025000000000000022 + ], + "mean": -0.016666666666666646, + "paired_deltas": [ + 0.025000000000000022, + -0.04999999999999993, + -0.025000000000000022 + ] + }, + "macro_f1": { + "exact_observed_range": [ + -0.050877192982456076, + 0.021669958305903014 + ], + "mean": -0.017854854423360617, + "paired_deltas": [ + 0.021669958305903014, + -0.050877192982456076, + -0.024357328593528793 + ] + } + } + }, + "gepa": { + "held_out": { + "accuracy": { + "exact_observed_range": [ + -0.03749999999999998, + 0.03749999999999998 + ], + "mean": -0.008333333333333304, + "paired_deltas": [ + 0.03749999999999998, + -0.03749999999999998, + -0.02499999999999991 + ] + }, + "macro_f1": { + "exact_observed_range": [ + -0.03842208661485769, + 0.037541044024526515 + ], + "mean": -0.00868967083838105, + "paired_deltas": [ + 0.037541044024526515, + -0.03842208661485769, + -0.02518796992481198 + ] + } + }, + "selection": { + "accuracy": { + "exact_observed_range": [ + -0.10000000000000009, + 0.025000000000000022 + ], + "mean": -0.01666666666666668, + "paired_deltas": [ + -0.10000000000000009, + 0.025000000000000022, + 0.025000000000000022 + ] + }, + "macro_f1": { + "exact_observed_range": [ + -0.09059890455239294, + 0.02545647588487243 + ], + "mean": -0.013312902138767124, + "paired_deltas": [ + -0.09059890455239294, + 0.02545647588487243, + 0.02520372225121914 + ] + } + } + }, + "mipro_v2": { + "held_out": { + "accuracy": { + "exact_observed_range": [ + -0.22499999999999998, + 0.23750000000000004 + ], + "mean": -0.033333333333333326, + "paired_deltas": [ + -0.22499999999999998, + 0.23750000000000004, + -0.11250000000000004 + ] + }, + "macro_f1": { + "exact_observed_range": [ + -0.2704052447813775, + 0.2798751044210399 + ], + "mean": -0.04359662687335928, + "paired_deltas": [ + -0.2704052447813775, + 0.2798751044210399, + -0.14025974025974025 + ] + } + }, + "selection": { + "accuracy": { + "exact_observed_range": [ + -0.125, + 0.15000000000000002 + ], + "mean": -0.00833333333333334, + "paired_deltas": [ + -0.125, + 0.15000000000000002, + -0.050000000000000044 + ] + }, + "macro_f1": { + "exact_observed_range": [ + -0.147122369819071, + 0.15122588009712246 + ], + "mean": -0.016606391741193827, + "paired_deltas": [ + -0.147122369819071, + 0.15122588009712246, + -0.05392268550163293 + ] + } + } + } + }, + "kind": "matched_strong_instruction_optimizer_aggregate", + "manifest_sha256": "0253960b8c570f0e0dd3a2a84450327f3244338fdff002f40ffed44bd9f15e94", + "metrics": { + "imp": { + "2026072602": { + "baseline": { + "held_out": { + "accuracy": 0.5375, + "count": 80, + "macro_f1": 0.44266616456411223, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.625, + "count": 40, + "macro_f1": 0.6050032916392363, + "parse_errors": 0 + } + }, + "gepa": { + "held_out": { + "accuracy": 0.9625, + "count": 80, + "macro_f1": 0.9624941397093296, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.825, + "count": 40, + "macro_f1": 0.833976833976834, + "parse_errors": 1 + } + }, + "mipro_v2": { + "held_out": { + "accuracy": 0.6125, + "count": 80, + "macro_f1": 0.5626873567272086, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.725, + "count": 40, + "macro_f1": 0.7025016903313049, + "parse_errors": 0 + } + } + }, + "2026072603": { + "baseline": { + "held_out": { + "accuracy": 0.5, + "count": 80, + "macro_f1": 0.40476190476190477, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.65, + "count": 40, + "macro_f1": 0.6491228070175439, + "parse_errors": 0 + } + }, + "gepa": { + "held_out": { + "accuracy": 0.9, + "count": 80, + "macro_f1": 0.898989898989899, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.9, + "count": 40, + "macro_f1": 0.899749373433584, + "parse_errors": 0 + } + }, + "mipro_v2": { + "held_out": { + "accuracy": 0.9, + "count": 80, + "macro_f1": 0.898989898989899, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.925, + "count": 40, + "macro_f1": 0.9249530956848031, + "parse_errors": 0 + } + } + }, + "2026072604": { + "baseline": { + "held_out": { + "accuracy": 0.55, + "count": 80, + "macro_f1": 0.45080091533180777, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.625, + "count": 40, + "macro_f1": 0.6247654784240151, + "parse_errors": 0 + } + }, + "gepa": { + "held_out": { + "accuracy": 0.925, + "count": 80, + "macro_f1": 0.924812030075188, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.925, + "count": 40, + "macro_f1": 0.9249530956848031, + "parse_errors": 0 + } + }, + "mipro_v2": { + "held_out": { + "accuracy": 0.5125, + "count": 80, + "macro_f1": 0.4597402597402597, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.5, + "count": 40, + "macro_f1": 0.4949494949494949, + "parse_errors": 0 + } + } + } + }, + "upstream": { + "2026072602": { + "baseline": { + "held_out": { + "accuracy": 0.5625, + "count": 80, + "macro_f1": 0.4851994851994852, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.6, + "count": 40, + "macro_f1": 0.5833333333333333, + "parse_errors": 0 + } + }, + "gepa": { + "held_out": { + "accuracy": 0.925, + "count": 80, + "macro_f1": 0.9249530956848031, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.925, + "count": 40, + "macro_f1": 0.9245757385292269, + "parse_errors": 0 + } + }, + "mipro_v2": { + "held_out": { + "accuracy": 0.8375, + "count": 80, + "macro_f1": 0.8330926015085861, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.85, + "count": 40, + "macro_f1": 0.849624060150376, + "parse_errors": 0 + } + } + }, + "2026072603": { + "baseline": { + "held_out": { + "accuracy": 0.5125, + "count": 80, + "macro_f1": 0.43874797625472206, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.7, + "count": 40, + "macro_f1": 0.7, + "parse_errors": 0 + } + }, + "gepa": { + "held_out": { + "accuracy": 0.9375, + "count": 80, + "macro_f1": 0.9374119856047567, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.875, + "count": 40, + "macro_f1": 0.8742928975487115, + "parse_errors": 0 + } + }, + "mipro_v2": { + "held_out": { + "accuracy": 0.6625, + "count": 80, + "macro_f1": 0.6191147945688591, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.775, + "count": 40, + "macro_f1": 0.7737272155876807, + "parse_errors": 0 + } + } + }, + "2026072604": { + "baseline": { + "held_out": { + "accuracy": 0.55, + "count": 80, + "macro_f1": 0.48717948717948717, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.65, + "count": 40, + "macro_f1": 0.6491228070175439, + "parse_errors": 0 + } + }, + "gepa": { + "held_out": { + "accuracy": 0.95, + "count": 80, + "macro_f1": 0.95, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.9, + "count": 40, + "macro_f1": 0.899749373433584, + "parse_errors": 0 + } + }, + "mipro_v2": { + "held_out": { + "accuracy": 0.625, + "count": 80, + "macro_f1": 0.6, + "parse_errors": 0 + }, + "selection": { + "accuracy": 0.55, + "count": 40, + "macro_f1": 0.5488721804511278, + "parse_errors": 0 + } + } + } + } + }, + "schema_version": 3, + "seed_count": 3, + "uncertainty": { + "method": "source_id_cluster_bootstrap_all_three_seeds", + "resamples": 10000, + "seed": 2026072605, + "warning": "Three optimizer seeds still provide limited model-sampling uncertainty." + }, + "within_runtime": { + "imp": { + "gepa": { + "held_out": { + "accuracy": { + "exact_observed_range": [ + 0.375, + 0.42500000000000004 + ], + "mean": 0.4000000000000001, + "paired_deltas": [ + 0.42500000000000004, + 0.4, + 0.375 + ] + }, + "macro_f1": { + "exact_observed_range": [ + 0.4740111147433802, + 0.5198279751452174 + ], + "mean": 0.4960223613721973, + "paired_deltas": [ + 0.5198279751452174, + 0.49422799422799424, + 0.4740111147433802 + ] + } + }, + "selection": { + "accuracy": { + "exact_observed_range": [ + 0.19999999999999996, + 0.30000000000000004 + ], + "mean": 0.25, + "paired_deltas": [ + 0.19999999999999996, + 0.25, + 0.30000000000000004 + ] + }, + "macro_f1": { + "exact_observed_range": [ + 0.2289735423375977, + 0.300187617260788 + ], + "mean": 0.2599292420048086, + "paired_deltas": [ + 0.2289735423375977, + 0.2506265664160401, + 0.300187617260788 + ] + } + } + }, + "mipro_v2": { + "held_out": { + "accuracy": { + "exact_observed_range": [ + -0.03750000000000009, + 0.4 + ], + "mean": 0.14583333333333334, + "paired_deltas": [ + 0.07500000000000007, + 0.4, + -0.03750000000000009 + ] + }, + "macro_f1": { + "exact_observed_range": [ + 0.008939344408451955, + 0.49422799422799424 + ], + "mean": 0.2077295102665142, + "paired_deltas": [ + 0.12002119216309637, + 0.49422799422799424, + 0.008939344408451955 + ] + } + }, + "selection": { + "accuracy": { + "exact_observed_range": [ + -0.125, + 0.275 + ], + "mean": 0.08333333333333333, + "paired_deltas": [ + 0.09999999999999998, + 0.275, + -0.125 + ] + }, + "macro_f1": { + "exact_observed_range": [ + -0.12981598347452017, + 0.2758302886672592 + ], + "mean": 0.08117090129493591, + "paired_deltas": [ + 0.09749839869206867, + 0.2758302886672592, + -0.12981598347452017 + ] + } + } + } + }, + "upstream": { + "gepa": { + "held_out": { + "accuracy": { + "exact_observed_range": [ + 0.36250000000000004, + 0.42500000000000004 + ], + "mean": 0.3958333333333333, + "paired_deltas": [ + 0.36250000000000004, + 0.42500000000000004, + 0.3999999999999999 + ] + }, + "macro_f1": { + "exact_observed_range": [ + 0.4397536104853179, + 0.49866400935003463 + ], + "mean": 0.4670793775519551, + "paired_deltas": [ + 0.4397536104853179, + 0.49866400935003463, + 0.4628205128205128 + ] + } + }, + "selection": { + "accuracy": { + "exact_observed_range": [ + 0.17500000000000004, + 0.32500000000000007 + ], + "mean": 0.25000000000000006, + "paired_deltas": [ + 0.32500000000000007, + 0.17500000000000004, + 0.25 + ] + }, + "macro_f1": { + "exact_observed_range": [ + 0.17429289754871158, + 0.34124240519589366 + ], + "mean": 0.2553872897202151, + "paired_deltas": [ + 0.34124240519589366, + 0.17429289754871158, + 0.2506265664160401 + ] + } + } + }, + "mipro_v2": { + "held_out": { + "accuracy": { + "exact_observed_range": [ + 0.07499999999999996, + 0.275 + ], + "mean": 0.16666666666666666, + "paired_deltas": [ + 0.275, + 0.15000000000000002, + 0.07499999999999996 + ] + }, + "macro_f1": { + "exact_observed_range": [ + 0.11282051282051281, + 0.3478931163091009 + ], + "mean": 0.21369348248125028, + "paired_deltas": [ + 0.3478931163091009, + 0.18036681831413703, + 0.11282051282051281 + ] + } + }, + "selection": { + "accuracy": { + "exact_observed_range": [ + -0.09999999999999998, + 0.25 + ], + "mean": 0.07500000000000002, + "paired_deltas": [ + 0.25, + 0.07500000000000007, + -0.09999999999999998 + ] + }, + "macro_f1": { + "exact_observed_range": [ + -0.10025062656641603, + 0.2662907268170427 + ], + "mean": 0.07992243861276911, + "paired_deltas": [ + 0.2662907268170427, + 0.0737272155876807, + -0.10025062656641603 + ] + } + } + } + } + } +} diff --git a/examples/matched_instruction_optimizers_trec/data/imp-scored-rows.json b/examples/matched_instruction_optimizers_trec/data/imp-scored-rows.json new file mode 100644 index 00000000..0782f914 --- /dev/null +++ b/examples/matched_instruction_optimizers_trec/data/imp-scored-rows.json @@ -0,0 +1,8337 @@ +{ + "call_budgets": { + "__imp_type__": "map", + "entries": [ + [ + { + "__imp_type__": "tuple", + "items": [ + 2026072602, + "gepa" + ] + }, + { + "__imp_type__": "map", + "entries": [ + [ + { + "__imp_type__": "atom", + "value": "counts" + }, + { + "optimizer_logical": 4, + "task_logical": 410, + "total_logical": 414, + "transports": 414 + } + ], + [ + { + "__imp_type__": "atom", + "value": "ceiling" + }, + { + "optimizer_logical": 48, + "task_logical": 450, + "total_logical": 498, + "transports": 498 + } + ], + [ + { + "__imp_type__": "atom", + "value": "refusals" + }, + [] + ] + ] + } + ], + [ + { + "__imp_type__": "tuple", + "items": [ + 2026072602, + "baseline" + ] + }, + { + "__imp_type__": "map", + "entries": [ + [ + { + "__imp_type__": "atom", + "value": "counts" + }, + { + "optimizer_logical": 0, + "task_logical": 120, + "total_logical": 120, + "transports": 120 + } + ], + [ + { + "__imp_type__": "atom", + "value": "ceiling" + }, + { + "optimizer_logical": 0, + "task_logical": 120, + "total_logical": 120, + "transports": 120 + } + ], + [ + { + "__imp_type__": "atom", + "value": "refusals" + }, + [] + ] + ] + } + ], + [ + { + "__imp_type__": "tuple", + "items": [ + 2026072602, + "mipro_v2" + ] + }, + { + "__imp_type__": "map", + "entries": [ + [ + { + "__imp_type__": "atom", + "value": "counts" + }, + { + "optimizer_logical": 9, + "task_logical": 541, + "total_logical": 550, + "transports": 550 + } + ], + [ + { + "__imp_type__": "atom", + "value": "ceiling" + }, + { + "optimizer_logical": 9, + "task_logical": 620, + "total_logical": 629, + "transports": 629 + } + ], + [ + { + "__imp_type__": "atom", + "value": "refusals" + }, + [] + ] + ] + } + ], + [ + { + "__imp_type__": "tuple", + "items": [ + 2026072603, + "gepa" + ] + }, + { + "__imp_type__": "map", + "entries": [ + [ + { + "__imp_type__": "atom", + "value": "counts" + }, + { + "optimizer_logical": 4, + "task_logical": 410, + "total_logical": 414, + "transports": 414 + } + ], + [ + { + "__imp_type__": "atom", + "value": "ceiling" + }, + { + "optimizer_logical": 48, + "task_logical": 450, + "total_logical": 498, + "transports": 498 + } + ], + [ + { + "__imp_type__": "atom", + "value": "refusals" + }, + [] + ] + ] + } + ], + [ + { + "__imp_type__": "tuple", + "items": [ + 2026072603, + "baseline" + ] + }, + { + "__imp_type__": "map", + "entries": [ + [ + { + "__imp_type__": "atom", + "value": "counts" + }, + { + "optimizer_logical": 0, + "task_logical": 120, + "total_logical": 120, + "transports": 120 + } + ], + [ + { + "__imp_type__": "atom", + "value": "ceiling" + }, + { + "optimizer_logical": 0, + "task_logical": 120, + "total_logical": 120, + "transports": 120 + } + ], + [ + { + "__imp_type__": "atom", + "value": "refusals" + }, + [] + ] + ] + } + ], + [ + { + "__imp_type__": "tuple", + "items": [ + 2026072603, + "mipro_v2" + ] + }, + { + "__imp_type__": "map", + "entries": [ + [ + { + "__imp_type__": "atom", + "value": "counts" + }, + { + "optimizer_logical": 9, + "task_logical": 540, + "total_logical": 549, + "transports": 549 + } + ], + [ + { + "__imp_type__": "atom", + "value": "ceiling" + }, + { + "optimizer_logical": 9, + "task_logical": 620, + "total_logical": 629, + "transports": 629 + } + ], + [ + { + "__imp_type__": "atom", + "value": "refusals" + }, + [] + ] + ] + } + ], + [ + { + "__imp_type__": "tuple", + "items": [ + 2026072604, + "gepa" + ] + }, + { + "__imp_type__": "map", + "entries": [ + [ + { + "__imp_type__": "atom", + "value": "counts" + }, + { + "optimizer_logical": 5, + "task_logical": 420, + "total_logical": 425, + "transports": 425 + } + ], + [ + { + "__imp_type__": "atom", + "value": "ceiling" + }, + { + "optimizer_logical": 48, + "task_logical": 450, + "total_logical": 498, + "transports": 498 + } + ], + [ + { + "__imp_type__": "atom", + "value": "refusals" + }, + [] + ] + ] + } + ], + [ + { + "__imp_type__": "tuple", + "items": [ + 2026072604, + "baseline" + ] + }, + { + "__imp_type__": "map", + "entries": [ + [ + { + "__imp_type__": "atom", + "value": "counts" + }, + { + "optimizer_logical": 0, + "task_logical": 120, + "total_logical": 120, + "transports": 120 + } + ], + [ + { + "__imp_type__": "atom", + "value": "ceiling" + }, + { + "optimizer_logical": 0, + "task_logical": 120, + "total_logical": 120, + "transports": 120 + } + ], + [ + { + "__imp_type__": "atom", + "value": "refusals" + }, + [] + ] + ] + } + ], + [ + { + "__imp_type__": "tuple", + "items": [ + 2026072604, + "mipro_v2" + ] + }, + { + "__imp_type__": "map", + "entries": [ + [ + { + "__imp_type__": "atom", + "value": "counts" + }, + { + "optimizer_logical": 9, + "task_logical": 533, + "total_logical": 542, + "transports": 542 + } + ], + [ + { + "__imp_type__": "atom", + "value": "ceiling" + }, + { + "optimizer_logical": 9, + "task_logical": 620, + "total_logical": 629, + "transports": 629 + } + ], + [ + { + "__imp_type__": "atom", + "value": "refusals" + }, + [] + ] + ] + } + ] + ] + }, + "manifest_sha256": "0253960b8c570f0e0dd3a2a84450327f3244338fdff002f40ffed44bd9f15e94", + "runtime": "imp", + "schema_version": 3, + "seeds": [ + { + "arms": [ + { + "arm": "baseline", + "artifact_payload_sha256": "sha256:51d77c48070dff5d94f28781ac95f4536f7616609da13257cee4ff2f4c781f01", + "artifact_sha256": "02e160fe2c0d375f24ef59c9b63c6022f74f94207e478b90ee0bfc45a09f795b", + "held_out": { + "accuracy": 0.5375, + "count": 80, + "macro_f1": 0.44266616456411223, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 0, + "task_logical": 40, + "total_logical": 40, + "transports": 40 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072602, + "selected_parameters_sha256": "f9a444d315b95b75600a7fbb09e7f3e9efdcbcf0193c51d797da260b56f0b411", + "selection": { + "accuracy": 0.625, + "count": 40, + "macro_f1": 0.6050032916392363, + "parse_errors": 0 + } + }, + { + "arm": "gepa", + "artifact_payload_sha256": "sha256:d04720a472f077ab291fa603daea6cd0f5964213eab4858e50aa6dd5e9c609bd", + "artifact_sha256": "569836fb7e724a37cc2f57c90b5c987908c93591ada8f2d0f8e7da4333a026c0", + "held_out": { + "accuracy": 0.9625, + "count": 80, + "macro_f1": 0.9624941397093296, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 4, + "task_logical": 330, + "total_logical": 334, + "transports": 334 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": false, + "error": {}, + "expected": "K11", + "parsed_route": null, + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072602, + "selected_parameters_sha256": "2a5205727ef7473d8e87a8ea0568cff895bd4617c6274e8f05342bcfb689780e", + "selection": { + "accuracy": 0.825, + "count": 40, + "macro_f1": 0.833976833976834, + "parse_errors": 1 + } + }, + { + "arm": "mipro_v2", + "artifact_payload_sha256": "sha256:e268a6156b4b017ed4deca06b650f2f85085e11987de38ce0a81f5280e918479", + "artifact_sha256": "e6ef12ccbc7d816202ecac2915b6cb5c0b92ca544a50470061fc6a4b278240c5", + "held_out": { + "accuracy": 0.6125, + "count": 80, + "macro_f1": 0.5626873567272086, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 9, + "task_logical": 461, + "total_logical": 470, + "transports": 470 + }, + "rows": { + "held_out": [ + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072602, + "selected_parameters_sha256": "f4d0a253f8b392186d432fea9c2a3d5d924787720efa269c0da682ddc18b3ed8", + "selection": { + "accuracy": 0.725, + "count": 40, + "macro_f1": 0.7025016903313049, + "parse_errors": 0 + } + } + ], + "seed": 2026072602 + }, + { + "arms": [ + { + "arm": "baseline", + "artifact_payload_sha256": "sha256:0e861d5acd1723ad8240a87f5712385fb5ceab1e126877737b98d769dd9137ea", + "artifact_sha256": "d36d03cf1c909920e592377d36bdef77068adac11492ffc22f874cf2505088b6", + "held_out": { + "accuracy": 0.5, + "count": 80, + "macro_f1": 0.40476190476190477, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 0, + "task_logical": 40, + "total_logical": 40, + "transports": 40 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072603, + "selected_parameters_sha256": "f9a444d315b95b75600a7fbb09e7f3e9efdcbcf0193c51d797da260b56f0b411", + "selection": { + "accuracy": 0.65, + "count": 40, + "macro_f1": 0.6491228070175439, + "parse_errors": 0 + } + }, + { + "arm": "gepa", + "artifact_payload_sha256": "sha256:babde9736d8797a74f8cb812bf4dc9269374f71bd76794fda0efff49ef224cff", + "artifact_sha256": "c89f032047ef4ed39c06c3d888caa38ee423c8e9c3c460558954eef02c4a4196", + "held_out": { + "accuracy": 0.9, + "count": 80, + "macro_f1": 0.898989898989899, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 4, + "task_logical": 330, + "total_logical": 334, + "transports": 334 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072603, + "selected_parameters_sha256": "a17a6ff984f6271b2c3f09e2a68998d6d80a641d753207c6b02fb8ba8cd67fc5", + "selection": { + "accuracy": 0.9, + "count": 40, + "macro_f1": 0.899749373433584, + "parse_errors": 0 + } + }, + { + "arm": "mipro_v2", + "artifact_payload_sha256": "sha256:21ea1d99d1c071150a7f3281a22910fd13ef6ee348cfee75b084041b2662d5f8", + "artifact_sha256": "9a7386d3a3d9593bac7e98f07600deeaa3ef57a61afd1243dd74cb09b1e30947", + "held_out": { + "accuracy": 0.9, + "count": 80, + "macro_f1": 0.898989898989899, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 9, + "task_logical": 460, + "total_logical": 469, + "transports": 469 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072603, + "selected_parameters_sha256": "ce575e8ceca07ad544937b21828466b12e44641b89caf4d45117d4353bef062f", + "selection": { + "accuracy": 0.925, + "count": 40, + "macro_f1": 0.9249530956848031, + "parse_errors": 0 + } + } + ], + "seed": 2026072603 + }, + { + "arms": [ + { + "arm": "baseline", + "artifact_payload_sha256": "sha256:e60316ed012aeaa8a0c38f9a264ff76049be82b5802bf4dcb4fd8deae5359a8a", + "artifact_sha256": "842158a46c6dedc3474af44ada830e648a1065520cfd456b2c086020d4a65867", + "held_out": { + "accuracy": 0.55, + "count": 80, + "macro_f1": 0.45080091533180777, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 0, + "task_logical": 40, + "total_logical": 40, + "transports": 40 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072604, + "selected_parameters_sha256": "f9a444d315b95b75600a7fbb09e7f3e9efdcbcf0193c51d797da260b56f0b411", + "selection": { + "accuracy": 0.625, + "count": 40, + "macro_f1": 0.6247654784240151, + "parse_errors": 0 + } + }, + { + "arm": "gepa", + "artifact_payload_sha256": "sha256:db6f9067d8b80ea90ae3edf6d11e41a20ba71565164657ef904a5e0025144fc6", + "artifact_sha256": "ad1b637ec3a4a870c033f9d764db2cd5fbb1b7abb50359e955eb986f25613d7e", + "held_out": { + "accuracy": 0.925, + "count": 80, + "macro_f1": 0.924812030075188, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 5, + "task_logical": 340, + "total_logical": 345, + "transports": 345 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072604, + "selected_parameters_sha256": "4a944dc0b1f4ce5b3cd985753538062ad7da672d80c7300c09c6aa261cb3b215", + "selection": { + "accuracy": 0.925, + "count": 40, + "macro_f1": 0.9249530956848031, + "parse_errors": 0 + } + }, + { + "arm": "mipro_v2", + "artifact_payload_sha256": "sha256:331b5aee02f2bc2c5050c36135da6cb6fb867ef946228e13c7289c9afe42094c", + "artifact_sha256": "2d5c07dcb1b883e144348283fdcba5b190b67499dbf2715c31b9b9d6c70d59e1", + "held_out": { + "accuracy": 0.5125, + "count": 80, + "macro_f1": 0.4597402597402597, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 9, + "task_logical": 453, + "total_logical": 462, + "transports": 462 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072604, + "selected_parameters_sha256": "f9a444d315b95b75600a7fbb09e7f3e9efdcbcf0193c51d797da260b56f0b411", + "selection": { + "accuracy": 0.5, + "count": 40, + "macro_f1": 0.4949494949494949, + "parse_errors": 0 + } + } + ], + "seed": 2026072604 + } + ], + "source_commits": { + "dspy": "29448ae12756abdd14bd8796c819247ebb83673c", + "gepa": "8b0ce6cd99a234f6b74daf37558a2ac0ce18f975", + "imp": "c2205ba1e60fef09701e0f5d7915a2b9f247d51b" + }, + "source_raw": { + "availability": "local_uncommitted_raw_trace", + "bytes": 139552428, + "path": "tmp/matched_instruction_optimizers_trec/imp-result.json", + "sha256": "fdc225948555f110f5391954611ffef83672c30835ff533dd3a5fa9d6fa41434" + }, + "status": "complete" +} diff --git a/examples/matched_instruction_optimizers_trec/data/upstream-scored-rows.json b/examples/matched_instruction_optimizers_trec/data/upstream-scored-rows.json new file mode 100644 index 00000000..a10e7fe7 --- /dev/null +++ b/examples/matched_instruction_optimizers_trec/data/upstream-scored-rows.json @@ -0,0 +1,8064 @@ +{ + "call_budgets": { + "2026072602:baseline": { + "ceiling": { + "optimizer_logical": 0, + "task_logical": 120, + "total_logical": 120, + "transports": 120 + }, + "counts": { + "optimizer_logical": 0, + "task_logical": 120, + "total_logical": 120, + "transports": 120 + }, + "refusals": [] + }, + "2026072602:gepa": { + "ceiling": { + "optimizer_logical": 48, + "task_logical": 450, + "total_logical": 498, + "transports": 498 + }, + "counts": { + "optimizer_logical": 4, + "task_logical": 410, + "total_logical": 414, + "transports": 414 + }, + "refusals": [] + }, + "2026072602:mipro_v2": { + "ceiling": { + "optimizer_logical": 9, + "task_logical": 620, + "total_logical": 629, + "transports": 629 + }, + "counts": { + "optimizer_logical": 9, + "task_logical": 543, + "total_logical": 552, + "transports": 552 + }, + "refusals": [] + }, + "2026072603:baseline": { + "ceiling": { + "optimizer_logical": 0, + "task_logical": 120, + "total_logical": 120, + "transports": 120 + }, + "counts": { + "optimizer_logical": 0, + "task_logical": 120, + "total_logical": 120, + "transports": 120 + }, + "refusals": [] + }, + "2026072603:gepa": { + "ceiling": { + "optimizer_logical": 48, + "task_logical": 450, + "total_logical": 498, + "transports": 498 + }, + "counts": { + "optimizer_logical": 4, + "task_logical": 410, + "total_logical": 414, + "transports": 414 + }, + "refusals": [] + }, + "2026072603:mipro_v2": { + "ceiling": { + "optimizer_logical": 9, + "task_logical": 620, + "total_logical": 629, + "transports": 629 + }, + "counts": { + "optimizer_logical": 9, + "task_logical": 542, + "total_logical": 551, + "transports": 551 + }, + "refusals": [] + }, + "2026072604:baseline": { + "ceiling": { + "optimizer_logical": 0, + "task_logical": 120, + "total_logical": 120, + "transports": 120 + }, + "counts": { + "optimizer_logical": 0, + "task_logical": 120, + "total_logical": 120, + "transports": 120 + }, + "refusals": [] + }, + "2026072604:gepa": { + "ceiling": { + "optimizer_logical": 48, + "task_logical": 450, + "total_logical": 498, + "transports": 498 + }, + "counts": { + "optimizer_logical": 4, + "task_logical": 400, + "total_logical": 404, + "transports": 404 + }, + "refusals": [] + }, + "2026072604:mipro_v2": { + "ceiling": { + "optimizer_logical": 9, + "task_logical": 620, + "total_logical": 629, + "transports": 629 + }, + "counts": { + "optimizer_logical": 9, + "task_logical": 533, + "total_logical": 542, + "transports": 542 + }, + "refusals": [] + } + }, + "manifest_sha256": "0253960b8c570f0e0dd3a2a84450327f3244338fdff002f40ffed44bd9f15e94", + "runtime": "upstream", + "schema_version": 3, + "seeds": [ + { + "arms": [ + { + "arm": "baseline", + "artifact_payload_sha256": "b711eb4c3c43054e88c962af8db1f61da5a5f145953a2a93031d0eb28bc72c1c", + "artifact_sha256": "10698bf74648ffce50f8d6aee873372ce0ffc9e3632135267f842e7ca8a51e45", + "held_out": { + "accuracy": 0.5625, + "count": 80, + "macro_f1": 0.4851994851994852, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 0, + "task_logical": 40, + "total_logical": 40, + "transports": 40 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072602, + "selected_parameters_sha256": "4e252b3bfb35e80c11d31acb458f3dfefb241038034ab9933fccf4ccefa94a1f", + "selection": { + "accuracy": 0.6, + "count": 40, + "macro_f1": 0.5833333333333333, + "parse_errors": 0 + } + }, + { + "arm": "gepa", + "artifact_payload_sha256": "40c9ea3683125e5ae7c2d39ceb4ead8dd5d518ed067900e72ef77808c0c5a093", + "artifact_sha256": "bc2f97f7b01f6febb8040855c503691ddfde1479dbdf9c700e48c7d94e2fa5e5", + "held_out": { + "accuracy": 0.925, + "count": 80, + "macro_f1": 0.9249530956848031, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 4, + "task_logical": 330, + "total_logical": 334, + "transports": 334 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072602, + "selected_parameters_sha256": "a5f8c4d295c0b60b8e577074c8556571ca162fe0862bf11d55f5910986342190", + "selection": { + "accuracy": 0.925, + "count": 40, + "macro_f1": 0.9245757385292269, + "parse_errors": 0 + } + }, + { + "arm": "mipro_v2", + "artifact_payload_sha256": "113238a080c542b8e913035727353e406ad3442a6c2084763226d9f22f21aee6", + "artifact_sha256": "b0507414721749543c0ee0f91a9768d4a253d8b66a40990e6ccebb9b9f60adb0", + "held_out": { + "accuracy": 0.8375, + "count": 80, + "macro_f1": 0.8330926015085861, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 9, + "task_logical": 463, + "total_logical": 472, + "transports": 472 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072602, + "selected_parameters_sha256": "47728986598fb236c2a611947a83b2d8cf9c6143ee48599816364cdd98d24934", + "selection": { + "accuracy": 0.85, + "count": 40, + "macro_f1": 0.849624060150376, + "parse_errors": 0 + } + } + ], + "seed": 2026072602 + }, + { + "arms": [ + { + "arm": "baseline", + "artifact_payload_sha256": "63da1bd7f3135a2c6b09304edb5d682bbd6d8fbf40ff8242032ce93ea282a014", + "artifact_sha256": "b3a631df4440588160afdb5d66f73f8a1322fbf8d836e1de2540eecc739013a8", + "held_out": { + "accuracy": 0.5125, + "count": 80, + "macro_f1": 0.43874797625472206, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 0, + "task_logical": 40, + "total_logical": 40, + "transports": 40 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072603, + "selected_parameters_sha256": "4e252b3bfb35e80c11d31acb458f3dfefb241038034ab9933fccf4ccefa94a1f", + "selection": { + "accuracy": 0.7, + "count": 40, + "macro_f1": 0.7, + "parse_errors": 0 + } + }, + { + "arm": "gepa", + "artifact_payload_sha256": "5879ab882858dac551e7d2201e58647a78a38f139dae5ad7d4ee03f8cfd80abc", + "artifact_sha256": "4d43c88b1d993860231fab72e3790051335d6521f4543dfde0d3d8d955b133d8", + "held_out": { + "accuracy": 0.9375, + "count": 80, + "macro_f1": 0.9374119856047567, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 4, + "task_logical": 330, + "total_logical": 334, + "transports": 334 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072603, + "selected_parameters_sha256": "8359dba35c703973e089f7184011b8c109252252673c39e326455c95fc347a55", + "selection": { + "accuracy": 0.875, + "count": 40, + "macro_f1": 0.8742928975487115, + "parse_errors": 0 + } + }, + { + "arm": "mipro_v2", + "artifact_payload_sha256": "a5edce357b0a1fe509ec69ef6e2259019932aace4dda993482b537332cfea750", + "artifact_sha256": "78612fa7b51980da246895f772f2707b3d146a9b429cdcc54d2a7e32228cf4d1", + "held_out": { + "accuracy": 0.6625, + "count": 80, + "macro_f1": 0.6191147945688591, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 9, + "task_logical": 462, + "total_logical": 471, + "transports": 471 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072603, + "selected_parameters_sha256": "ce46bba7a85b3ed17611d45c15ce5a8451bf1642af0b52b545de23bbb5e6c3e8", + "selection": { + "accuracy": 0.775, + "count": 40, + "macro_f1": 0.7737272155876807, + "parse_errors": 0 + } + } + ], + "seed": 2026072603 + }, + { + "arms": [ + { + "arm": "baseline", + "artifact_payload_sha256": "056e0623c483d61ea606118f994874d2407196b2456b106382c8466eabd1fd74", + "artifact_sha256": "258b3e0506568acd7c9a42f263a056cad78e150a7e694616148d086304db593c", + "held_out": { + "accuracy": 0.55, + "count": 80, + "macro_f1": 0.48717948717948717, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 0, + "task_logical": 40, + "total_logical": 40, + "transports": 40 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072604, + "selected_parameters_sha256": "4e252b3bfb35e80c11d31acb458f3dfefb241038034ab9933fccf4ccefa94a1f", + "selection": { + "accuracy": 0.65, + "count": 40, + "macro_f1": 0.6491228070175439, + "parse_errors": 0 + } + }, + { + "arm": "gepa", + "artifact_payload_sha256": "696bfd49a00267501f7bc71be378b79d74a52551dc4b9655090c328f9a96ca37", + "artifact_sha256": "ae9b6ab35481bbfb10a873c64d5c367ac959a22c4e6c0ac308e7d7e4189adc09", + "held_out": { + "accuracy": 0.95, + "count": 80, + "macro_f1": 0.95, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 4, + "task_logical": 320, + "total_logical": 324, + "transports": 324 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072604, + "selected_parameters_sha256": "6ed825ade210ae7739fad857b29ae57572e27be9ae6352ce782880a4b6ec27b2", + "selection": { + "accuracy": 0.9, + "count": 40, + "macro_f1": 0.899749373433584, + "parse_errors": 0 + } + }, + { + "arm": "mipro_v2", + "artifact_payload_sha256": "4949af5cd2f19ee8aae2c2aba9fe56c19fa8df894042834165588b8ade37845f", + "artifact_sha256": "5cc68aa4fbf0b184260376fd9127bcfdc867d2fe2a35f050fe12f811133c6069", + "held_out": { + "accuracy": 0.625, + "count": 80, + "macro_f1": 0.6, + "parse_errors": 0 + }, + "held_out_call_counts": { + "optimizer_logical": 0, + "task_logical": 80, + "total_logical": 80, + "transports": 80 + }, + "preheld_call_counts": { + "optimizer_logical": 9, + "task_logical": 453, + "total_logical": 462, + "transports": 462 + }, + "rows": { + "held_out": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0035" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0039" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0046" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0048" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0054" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0056" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0066" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0071" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0073" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0076" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0091" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0102" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0108" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0110" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0112" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0113" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0119" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0122" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0125" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0004" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0011" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0016" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0018" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0025" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0027" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0029" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0030" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0034" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0038" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0040" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0043" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0047" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0060" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0074" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0075" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0080" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0082" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0088" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0090" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0001" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0002" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0009" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0015" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0020" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0021" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0022" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0028" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0053" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0064" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0069" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0077" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0123" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0124" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0131" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0134" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0138" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0139" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0094" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0097" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0100" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0101" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0104" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0106" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0111" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0115" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0120" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0127" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0129" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0130" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0132" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0143" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0148" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0160" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0168" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:heldout:0169" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:heldout:0177" + } + ], + "selection": [ + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0156" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0158" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0183" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0051" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0065" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0067" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0000" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0003" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0007" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0010" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0021" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0026" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0032" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0070" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0071" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0072" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0076" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0080" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0081" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0082" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0033" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0037" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0038" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0043" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0044" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0058" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0066" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0068" + }, + { + "correct": false, + "error": null, + "expected": "K11", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0084" + }, + { + "correct": true, + "error": null, + "expected": "K11", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0105" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0092" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0096" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0102" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0103" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0108" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0110" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0113" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0114" + }, + { + "correct": false, + "error": null, + "expected": "K47", + "parsed_route": "K11", + "source_id": "trec-fine:calibration:0119" + }, + { + "correct": true, + "error": null, + "expected": "K47", + "parsed_route": "K47", + "source_id": "trec-fine:calibration:0123" + } + ] + }, + "seed": 2026072604, + "selected_parameters_sha256": "eedcd65020a358cc717e604927c8a9dcaf93eedd335e932f6e5dcf8bff67ba31", + "selection": { + "accuracy": 0.55, + "count": 40, + "macro_f1": 0.5488721804511278, + "parse_errors": 0 + } + } + ], + "seed": 2026072604 + } + ], + "source_commits": { + "dspy": "29448ae12756abdd14bd8796c819247ebb83673c", + "gepa": "8b0ce6cd99a234f6b74daf37558a2ac0ce18f975", + "imp": "c2205ba1e60fef09701e0f5d7915a2b9f247d51b" + }, + "source_raw": { + "availability": "local_uncommitted_raw_trace", + "bytes": 41797465, + "path": "tmp/matched_instruction_optimizers_trec/upstream-result.json", + "sha256": "a167f56e1f56a5da053bbbdea419a748f3b2b6bbfda41d66c4fc50481bd94974" + }, + "status": "complete" +} diff --git a/examples/matched_instruction_optimizers_trec/recompute_compact.exs b/examples/matched_instruction_optimizers_trec/recompute_compact.exs index 19182554..b3363d20 100644 --- a/examples/matched_instruction_optimizers_trec/recompute_compact.exs +++ b/examples/matched_instruction_optimizers_trec/recompute_compact.exs @@ -1,3 +1,15 @@ +# Recomputes the matched TREC aggregate from the committed scored rows and +# prints what it computed. Every number this prints is derived here, in this +# process, from the row files named on the command line. Nothing is a literal. +# +# Exits 1 if the recomputed aggregate disagrees with the committed one, naming +# the three headline quantities on both sides so the disagreement is legible +# without opening the JSON. +# +# This recomputes statistics. It does not reproduce the experiment: the raw +# provider responses behind these rows were not published. See +# docs/CASE_STUDY_TREC.md. + Code.require_file("contract.exs", __DIR__) Code.require_file("aggregate.exs", __DIR__) @@ -9,20 +21,93 @@ argv = rest -> rest end +format = fn + nil -> "n/a" + value when is_float(value) -> :erlang.float_to_binary(value, decimals: 4) + value -> to_string(value) +end + +signed = fn + nil -> "n/a" + value when is_float(value) and value >= 0 -> "+" <> :erlang.float_to_binary(value, decimals: 4) + value when is_float(value) -> :erlang.float_to_binary(value, decimals: 4) + value -> to_string(value) +end + +interval = fn + [low, high] -> "[#{format.(low)}, #{format.(high)}]" + _ -> "[n/a]" +end + +headlines = fn aggregate -> + acceptance = Map.get(aggregate, "acceptance", %{}) + improvements = Map.get(acceptance, "improvements", %{}) + gepa = Map.get(improvements, "gepa", %{}) + mipro = Map.get(improvements, "mipro_v2", %{}) + matched = Map.get(acceptance, "winning_optimizer_imp_minus_upstream", %{}) + + [ + {"GEPA over its own baseline", gepa}, + {"MIPROv2 over its own baseline", mipro}, + {"GEPA Imp minus DSPy", matched} + ] +end + +report = fn label, aggregate -> + IO.puts("#{label}:") + + for {name, stats} <- headlines.(aggregate) do + IO.puts( + " #{name}: #{signed.(Map.get(stats, "mean"))} " <> + "95% CI #{interval.(Map.get(stats, "confidence_interval"))}" <> + case Map.get(stats, "holm_adjusted_p") do + nil -> "" + p -> ", Holm-adjusted p = #{:erlang.float_to_binary(p, decimals: 5)}" + end + ) + end +end + case argv do [manifest, imp_rows, upstream_rows, expected_aggregate] -> recomputed = Aggregator.aggregate!(manifest, imp_rows, upstream_rows) expected = expected_aggregate |> File.read!() |> Jason.decode!() - unless recomputed == expected do - raise "committed compact aggregate does not match recomputed scored rows" + report.( + "Recomputed from #{Path.basename(imp_rows)} and #{Path.basename(upstream_rows)}", + recomputed + ) + + if recomputed == expected do + acceptance = Map.get(recomputed, "acceptance", %{}) + + IO.puts( + " noninferiority margin #{format.(Map.get(acceptance, "noninferiority_margin"))}, " <> + "winning optimizer #{Map.get(acceptance, "winning_optimizer", "n/a")}, " <> + "headline passed: #{Map.get(acceptance, "headline_passed", "n/a")}" + ) + + IO.puts("") + IO.puts("Recomputation agrees with #{Path.basename(expected_aggregate)} in full.") + else + IO.puts("") + report.("Committed in #{Path.basename(expected_aggregate)}", expected) + + IO.puts("") + + IO.puts( + "MISMATCH: the recomputed aggregate differs from the committed one. " <> + "Either the scored rows changed or the aggregation did; both are bugs." + ) + + System.halt(1) end + _ -> IO.puts( - "matched TREC compact recomputation passed: " <> - "GEPA +0.4000, MIPROv2 +0.1458, GEPA Imp-minus-DSPy -0.0083" + :stderr, + "usage: mix run --no-start recompute_compact.exs -- MANIFEST IMP_ROWS UPSTREAM_ROWS EXPECTED_AGGREGATE" ) - _ -> - raise "usage: mix run recompute_compact.exs -- MANIFEST IMP_ROWS UPSTREAM_ROWS EXPECTED_AGGREGATE" + System.halt(2) end diff --git a/lib/mix/tasks/imp.benchmark.optimizer_lift.ex b/lib/mix/tasks/imp.benchmark.optimizer_lift.ex index 7578157f..c2017e0d 100644 --- a/lib/mix/tasks/imp.benchmark.optimizer_lift.ex +++ b/lib/mix/tasks/imp.benchmark.optimizer_lift.ex @@ -13,7 +13,7 @@ defmodule Mix.Tasks.Imp.Benchmark.OptimizerLift do instruction/demo, Imp optimizers select and apply it identically to DSPy 3.2.1 (lift_gap <= 0.001), without provider nondeterminism. It is NOT held-out lift evidence — optimizer effectiveness on data nothing selected - for remains a separately gated C3 target (see docs/internal/BENCHMARK_TRUTH.md). + for remains a separately gated C3 target (see docs/BENCHMARKS.md). """ use Mix.Task diff --git a/mix.exs b/mix.exs index 87ce7a0f..a5484256 100644 --- a/mix.exs +++ b/mix.exs @@ -16,7 +16,8 @@ defmodule Imp.MixProject do api_reference: true, warnings_as_errors: true, extras: - ["README.md", "CHANGELOG.md", "RELEASE_NOTES.md"] ++ product_docs() ++ livebooks(), + ["README.md", "CHANGELOG.md", "RELEASE_NOTES.md"] ++ + product_docs() ++ repository_docs() ++ livebooks(), groups_for_modules: public_api_doc_groups(), filter_modules: &public_doc_module?/2, skip_undefined_reference_warnings_on: &skip_filtered_doc_reference?/1, @@ -243,6 +244,7 @@ defmodule Imp.MixProject do defp product_docs do [ "docs/LEARNING_PATH.md", + "docs/TUTORIAL_TICKET_ROUTING.md", "docs/IMP_FOR_DSPY_USERS.md", "docs/PRODUCTION_OPERATIONS.md", "docs/TRAJECTORIES.md", @@ -250,6 +252,20 @@ defmodule Imp.MixProject do ] end + # Rendered into the docs but NOT shipped in the package: they describe + # source-checkout commands a Hex consumer cannot run. + # docs/BENCHMARKS.md is deliberately NOT here. It is almost entirely bare + # `mix …` command spans, which ExDoc resolves to their task modules and then + # warns about because those modules are filtered out of the public API docs. + # It is a source-checkout document like CONTRIBUTING.md; README, EVIDENCE and + # the case study link to it by URL. + defp repository_docs do + [ + "docs/CASE_STUDY_TREC.md", + "docs/EVIDENCE.md" + ] + end + defp livebooks do [ "livebooks/01_real_lm_front_door.livemd", diff --git a/priv/tutorial/SUPPORT_TICKETS_LICENSE.md b/priv/tutorial/SUPPORT_TICKETS_LICENSE.md new file mode 100644 index 00000000..e5873be4 --- /dev/null +++ b/priv/tutorial/SUPPORT_TICKETS_LICENSE.md @@ -0,0 +1,21 @@ +# Tutorial support tickets + +`support_tickets.json` is sixty short support tickets, each labeled with one of +four invented squad names (atlas, harbor, beacon, quill), split twenty/twenty/ +twenty into train, dev and test. SHA-256 of the file: +`7ea5ae7ad724dc4311cf954bafeb48a4b7b284b619c33bc00b30d37ad6df6f87`. + +The tickets were written for this repository. They are not drawn from any +external dataset, and no real customer, company or person appears in them. The +squad names encode a routing convention that no model can guess, which is the +point of [the tutorial](../../docs/TUTORIAL_TICKET_ROUTING.md): it makes the +gap between what a model can read and what an organization means measurable. + +**License: MIT, the same as the rest of this repository.** The file ships in the +Hex package; you may copy it, change it, and replace it with your own tickets. + +Because we wrote them, they carry the usual risks of an author-written set: the +labels reflect one person's idea of the convention, the tickets are short and +unambiguous compared to real ones, and twenty held-out rows is a coarse +instrument. Results measured on this file say something about Imp's optimizer +plumbing and nothing about your ticket queue. diff --git a/test/case_study_trec_recomputation_test.exs b/test/case_study_trec_recomputation_test.exs new file mode 100644 index 00000000..2e1e2aad --- /dev/null +++ b/test/case_study_trec_recomputation_test.exs @@ -0,0 +1,111 @@ +defmodule ImpTest.CaseStudyTRECRecomputationTest do + @moduledoc """ + The README calls TREC the strongest matched result and docs/CASE_STUDY_TREC.md + publishes an exact command for recomputing it. The epic requires that an + independent consumer can recompute the strongest claims without maintainer + machinery — so that command working is itself a release-blocking property. + + It silently stopped working for two days: a shared, mutable dependency lock + was edited in place for a different campaign, invalidating this contract's + seal. Nothing caught it because nothing ran it. This does. + + Provider-free and fast: it re-aggregates the committed scored rows, makes no + network calls, and starts no models. + """ + use ExUnit.Case, async: true + + import ExUnit.Callbacks, only: [on_exit: 1] + + @root Path.expand("..", __DIR__) + @expected [ + "GEPA over its own baseline: +0.4000 95% CI [0.2958, 0.5042], Holm-adjusted p = 0.00020", + "MIPROv2 over its own baseline: +0.1458 95% CI [0.0458, 0.2458], Holm-adjusted p = 0.00270", + "GEPA Imp minus DSPy: -0.0083 95% CI [-0.0458, 0.0292]" + ] + + @doc_path "docs/CASE_STUDY_TREC.md" + @script "examples/matched_instruction_optimizers_trec/recompute_compact.exs" + @contract "examples/matched_instruction_optimizers_trec/contract.json" + @imp_rows "examples/matched_instruction_optimizers_trec/data/imp-scored-rows.json" + @upstream_rows "examples/matched_instruction_optimizers_trec/data/upstream-scored-rows.json" + @aggregate "examples/matched_instruction_optimizers_trec/data/aggregate-recomputed.json" + + test "the published TREC recomputation command still reproduces the documented result" do + {output, status} = + System.cmd( + "mix", + ["run", "--no-start", @script, "--", @contract, @imp_rows, @upstream_rows, @aggregate], + cd: @root, + stderr_to_stdout: true, + env: [{"MIX_ENV", "test"}] + ) + + assert status == 0, "published recomputation command failed:\n#{output}" + + for line <- @expected do + assert String.contains?(output, line), "recomputation output drifted:\n#{output}" + end + + assert String.contains?( + output, + "Recomputation agrees with aggregate-recomputed.json in full." + ), + "recomputation no longer reports agreement:\n#{output}" + end + + test "the recomputation reports a mismatch and fails instead of printing a stored answer" do + tampered = + Path.join( + System.tmp_dir!(), + "trec-aggregate-tampered-#{System.unique_integer([:positive])}.json" + ) + + on_exit(fn -> File.rm(tampered) end) + + aggregate = + @root + |> Path.join(@aggregate) + |> File.read!() + |> Jason.decode!() + + File.write!( + tampered, + aggregate + |> put_in(["acceptance", "improvements", "gepa", "mean"], 0.5) + |> Jason.encode!() + ) + + {output, status} = + System.cmd( + "mix", + ["run", "--no-start", @script, "--", @contract, @imp_rows, @upstream_rows, tampered], + cd: @root, + stderr_to_stdout: true, + env: [{"MIX_ENV", "test"}] + ) + + assert status != 0, "a tampered aggregate did not fail the recomputation:\n#{output}" + assert String.contains?(output, "MISMATCH") + + assert String.contains?(output, "GEPA over its own baseline: +0.4000"), + "the recomputed value was not printed, so the command is not recomputing:\n#{output}" + end + + test "the case study still documents exactly the line the command prints" do + doc = File.read!(Path.join(@root, @doc_path)) + + for line <- @expected do + assert String.contains?(doc, line), + "#{@doc_path} no longer documents a line the recomputation prints; " <> + "docs and evidence have drifted apart" + end + + assert String.contains?(doc, "recomputable, not reproducible"), + "#{@doc_path} no longer states that the result cannot be reproduced" + + for path <- [@script, @contract, @imp_rows, @upstream_rows, @aggregate] do + assert String.contains?(doc, Path.basename(path)), + "#{@doc_path} no longer references #{Path.basename(path)}" + end + end +end diff --git a/test/current_dspy_state_boundary_test.exs b/test/current_dspy_state_boundary_test.exs index 72d920f5..9d738434 100644 --- a/test/current_dspy_state_boundary_test.exs +++ b/test/current_dspy_state_boundary_test.exs @@ -12,7 +12,7 @@ defmodule Imp.CurrentDSPyStateBoundaryTest do unless File.exists?(python) and File.dir?(target) do flunk( - "install the source-verified DSPy 3.3.1 target and venv documented in docs/internal/BENCHMARK_TRUTH.md" + "install the source-verified DSPy 3.3.1 target and venv documented in docs/BENCHMARKS.md" ) end diff --git a/test/documentation_contract_test.exs b/test/documentation_contract_test.exs index 91e8e713..c869aab9 100644 --- a/test/documentation_contract_test.exs +++ b/test/documentation_contract_test.exs @@ -8,36 +8,50 @@ defmodule DocumentationContractTest do "Imp.UnlinkedTaskSupervisor" ]) - test "coverage matrix describes current evidence instead of closed planning tickets" do - body = File.read!("docs/internal/COVERAGE_MATRIX.md") - - refute_closed_ticket_refs(body) - refute body =~ "integration gate should" - refute body =~ "integration gate required" - refute body =~ "integration gate needed" - refute body =~ "Imp.Embeddings.Hash" - refute body =~ "Imp.MCP.InProcess" - refute body =~ "Imp.MCP.HTTP`" - refute body =~ "Imp.MCP.Stdio`" - refute body =~ "Imp.MCP.StreamableHTTP`" - refute body =~ "before closing" - refute body =~ "waiting on live release evidence" - - assert body =~ "mix integration.check" - assert body =~ "mix protocol.training.check" - assert body =~ "mix protocol.check" - assert body =~ "Imp.Embeddings.BagOfWords" + test "the benchmark docs name runnable commands instead of closed planning tickets" do + benchmarks = File.read!("docs/BENCHMARKS.md") + contributing = File.read!("CONTRIBUTING.md") + + refute_closed_ticket_refs(benchmarks) + refute_closed_ticket_refs(contributing) + refute benchmarks =~ "before closing" + refute benchmarks =~ "waiting on live release evidence" + + assert contributing =~ "mix integration.check" + assert contributing =~ "mix protocol.check" + assert contributing =~ "mix livebook.execute.check" + assert benchmarks =~ "mix differential.check" + assert Code.ensure_loaded?(Imp.Embeddings.BagOfWords) - assert body =~ "Imp.MCP.Catalog" - assert body =~ "Imp.MCP.HTTPClient" - assert body =~ "Imp.MCP.StdioClient" - assert body =~ "Imp.MCP.StreamableHTTPClient" assert Code.ensure_loaded?(Imp.MCP.Catalog) assert Code.ensure_loaded?(Imp.MCP.HTTPClient) assert Code.ensure_loaded?(Imp.MCP.StdioClient) assert Code.ensure_loaded?(Imp.MCP.StreamableHTTPClient) end + test "every mix command the benchmark docs publish is a task that exists" do + published = + "docs/BENCHMARKS.md" + |> File.read!() + |> then(&Regex.scan(~r/mix ([a-z][a-z_0-9.]*[a-z0-9])/, &1)) + |> Enum.map(fn [_, task] -> task end) + |> Enum.uniq() + |> Enum.reject(&(&1 in ["deps.get", "run", "test", "help"])) + + assert length(published) > 20 + + aliases = + Mix.Project.config() + |> Keyword.get(:aliases, []) + |> Keyword.keys() + |> MapSet.new(&Atom.to_string/1) + + for task <- published do + assert Mix.Task.get(task) || MapSet.member?(aliases, task), + "docs/BENCHMARKS.md publishes `mix #{task}`, which is neither a task nor an alias" + end + end + test "documented Imp module references resolve to loadable modules" do missing = documented_module_references() @@ -71,16 +85,24 @@ defmodule DocumentationContractTest do refute docs =~ "Imp.HTTP.Hackneyless" end - test "parity validation program describes evidence lanes instead of ticket bookkeeping" do - body = File.read!("docs/internal/PARITY_VALIDATION_PROGRAM.md") + test "the benchmark docs separate what can be re-measured from what cannot" do + body = File.read!("docs/BENCHMARKS.md") - refute_closed_ticket_refs(body) refute body =~ "Ticket:" refute body =~ "regressions have tickets" + + assert body =~ "## Cannot be re-measured" + assert body =~ "The split files are\nabsent" + + results = File.read!("benchmarks/RESULTS.md") + + assert results =~ "## Re-measurable" + assert results =~ "## Recomputable only" + assert results =~ "## Findings that are not results" end test "adapter fidelity audit names upstream semantics and Imp evidence" do - body = File.read!("docs/internal/ADAPTER_FIDELITY.md") + body = File.read!("docs/differentials/ADAPTER_FIDELITY.md") contributing = File.read!("CONTRIBUTING.md") assert contributing =~ "Public behavior belongs to code, tests, and user documentation" @@ -112,7 +134,10 @@ defmodule DocumentationContractTest do "examples/provider_free_ticket_router/README.md" ] ++ Path.wildcard("livebooks/*.livemd") - banned = ~r/\bimp-[a-z]*\d[a-z0-9]*\b|docs\/(?:internal|maintainers)|benchmarks\/|evidence\//i + # benchmarks/RESULTS.md is a published surface: user docs are expected to + # cite it. Everything else under benchmarks/ is still internal. + banned = + ~r/\bimp-[a-z]*\d[a-z0-9]*\b|docs\/(?:internal|maintainers)|benchmarks\/(?!RESULTS\.md)|evidence\//i offenders = for path <- user_surfaces, @@ -169,9 +194,28 @@ defmodule DocumentationContractTest do "docs/IMP_FOR_DSPY_USERS.md", "docs/LEARNING_PATH.md", "docs/PRODUCTION_OPERATIONS.md", - "docs/TRAJECTORIES.md" + "docs/TRAJECTORIES.md", + "docs/TUTORIAL_TICKET_ROUTING.md" ] + extras = + Mix.Project.config() + |> Keyword.fetch!(:docs) + |> Keyword.fetch!(:extras) + + for doc <- ["docs/CASE_STUDY_TREC.md", "docs/EVIDENCE.md"] do + assert doc in extras, "#{doc} is not rendered into the docs" + refute doc in product_docs, "#{doc} names source-checkout commands and must not ship" + end + + refute "docs/BENCHMARKS.md" in extras + refute "docs/BENCHMARKS.md" in product_docs + + for doc <- ["README.md", "docs/EVIDENCE.md", "docs/TUTORIAL_TICKET_ROUTING.md"] do + assert File.read!(doc) =~ "blob/main/docs/BENCHMARKS.md", + "#{doc} does not link to the benchmark index" + end + refute readme =~ "01_programming_not_prompting" end @@ -244,12 +288,15 @@ defmodule DocumentationContractTest do assert Imp.get(agent_prediction, :answer) == "Paris" end - test "GEPA research records distinguish program and artifact evidence" do - coverage = File.read!("docs/internal/COVERAGE_MATRIX.md") - parity = File.read!("docs/internal/PARITY_VALIDATION_PROGRAM.md") + test "GEPA records distinguish the differential that runs from the campaign that cannot" do + benchmarks = File.read!("docs/BENCHMARKS.md") + + assert benchmarks =~ "mix benchmark.gepa.contract.check" + assert benchmarks =~ "**The GEPA six-family campaign.**" + + attribution = File.read!("benchmarks/data/GEPA_SPLITS_ATTRIBUTION.md") - assert coverage =~ "GEPA-style reflection" - assert parity =~ "GEPA-style optimizer rows" + assert attribution =~ "not in this repository" end test "cold learning path distinguishes portable programs from selected parameter artifacts" do @@ -271,7 +318,7 @@ defmodule DocumentationContractTest do end test "instruction optimizer fidelity defines durable run-level resume boundaries" do - fidelity = File.read!("docs/internal/INSTRUCTION_OPTIMIZER_FIDELITY.md") + fidelity = File.read!("docs/differentials/INSTRUCTION_OPTIMIZER_FIDELITY.md") assert fidelity =~ "## Durable Run-Level Resume" assert fidelity =~ "A trial is the atomic boundary" @@ -279,11 +326,12 @@ defmodule DocumentationContractTest do assert fidelity =~ "### Rebinding And Trust Boundary" end - test "embedding evidence names the deterministic baseline and provider shape contract" do - coverage = File.read!("docs/internal/COVERAGE_MATRIX.md") + test "the deterministic embedder says in its own docs that it is not a semantic model" do + {:docs_v1, _, _, _, %{"en" => doc}, _, _} = Code.fetch_docs(Imp.Embeddings.BagOfWords) - assert coverage =~ "deterministic local baseline" - assert coverage =~ "one numeric vector per input text" + assert doc =~ "local baseline" + assert doc =~ "not a semantic embedding model" + assert doc =~ "inject a real embedding provider" end test "streaming response structs are deliberate public vocabulary" do diff --git a/test/package_contract_test.exs b/test/package_contract_test.exs index 1ebbacd8..c67759a2 100644 --- a/test/package_contract_test.exs +++ b/test/package_contract_test.exs @@ -65,10 +65,10 @@ defmodule PackageContractTest do @excluded_files [ "docs/EVIDENCE.md", - "docs/internal/BENCHMARK_CATALOG.md", - "docs/internal/BENCHMARK_TRUTH.md", - "docs/internal/COVERAGE_MATRIX.md", - "docs/internal/PARITY_VALIDATION_PROGRAM.md", + "docs/BENCHMARKS.md", + "docs/differentials/README.md", + "docs/differentials/ADAPTER_FIDELITY.md", + "docs/differentials/UPSTREAM_EXAM.md", "lib/imp/benchmarks.ex", "lib/mix/tasks/imp.public_api.ex", "lib/mix/tasks/imp.package.clean_room.ex", diff --git a/test/public_surface_test.exs b/test/public_surface_test.exs index b37d2249..7408d9d8 100644 --- a/test/public_surface_test.exs +++ b/test/public_surface_test.exs @@ -1077,8 +1077,8 @@ defmodule PublicSurfaceTest do # upstream test, including adapters we deliberately do not ship). &(&1 in [ "README.md", - "docs/internal/RESEARCH_LANDSCAPE.md", - "docs/internal/UPSTREAM_EXAM.md" + "docs/differentials/RESEARCH_LANDSCAPE.md", + "docs/differentials/UPSTREAM_EXAM.md" ]) ) |> Enum.map_join("\n", &File.read!/1) diff --git a/test/tutorial_parity_test.exs b/test/tutorial_parity_test.exs index 012c77d3..1da6bc8e 100644 --- a/test/tutorial_parity_test.exs +++ b/test/tutorial_parity_test.exs @@ -1,7 +1,7 @@ defmodule TutorialParityTest do use ExUnit.Case, async: true - @parity_path "docs/internal/TUTORIAL_EXAMPLE_PARITY.md" + @parity_path "docs/differentials/TUTORIAL_EXAMPLE_PARITY.md" @families [ "Email and entity extraction", @@ -44,8 +44,8 @@ defmodule TutorialParityTest do "livebooks/04_tools_agents_mcp_rlm.livemd", "livebooks/05_operate_and_live_checks.livemd", "docs/LEARNING_PATH.md", - "docs/internal/MULTIMODAL_FIDELITY.md", - "docs/internal/BENCHMARK_CATALOG.md", + "docs/differentials/MULTIMODAL_FIDELITY.md", + "docs/BENCHMARKS.md", "examples/deployment/README.md" ] do assert File.regular?(path), "missing parity target #{path}" diff --git a/test/upstream_exam/adapters_test.exs b/test/upstream_exam/adapters_test.exs index c4182746..c853a319 100644 --- a/test/upstream_exam/adapters_test.exs +++ b/test/upstream_exam/adapters_test.exs @@ -4,7 +4,7 @@ defmodule UpstreamExam.AdaptersTest do Tranche 1 of the upstream exam: every test here cites the upstream file and test function it translates. The complete per-test disposition map (including - the tests that were NOT portable and why) is docs/internal/UPSTREAM_EXAM.md. + the tests that were NOT portable and why) is docs/differentials/UPSTREAM_EXAM.md. Rules of this file: * assertions check the SAME behavior as upstream, not a look-alike; diff --git a/test/upstream_exam/evaluate_test.exs b/test/upstream_exam/evaluate_test.exs index 88f435d2..20ab5a93 100644 --- a/test/upstream_exam/evaluate_test.exs +++ b/test/upstream_exam/evaluate_test.exs @@ -2,7 +2,7 @@ defmodule UpstreamExam.EvaluateTest do @moduledoc """ DSPy 3.2.1's own evaluate tests (tests/evaluate/), ported to Imp. - Tranche 3 of the upstream exam. Disposition map: docs/internal/UPSTREAM_EXAM.md. + Tranche 3 of the upstream exam. Disposition map: docs/differentials/UPSTREAM_EXAM.md. Seam applied throughout: Imp.Evaluate scores are fractions (1.0), DSPy's are percentages (100.0); the assertions translate the scale and nothing else. """ diff --git a/test/upstream_exam/predict_test.exs b/test/upstream_exam/predict_test.exs index 6aea3eb8..cbaec20a 100644 --- a/test/upstream_exam/predict_test.exs +++ b/test/upstream_exam/predict_test.exs @@ -4,7 +4,7 @@ defmodule UpstreamExam.PredictTest do Tranche 2 of the upstream exam: every test here cites the upstream file and test function it translates. The complete per-test disposition map (including - the tests that were NOT portable and why) is docs/internal/UPSTREAM_EXAM.md. + the tests that were NOT portable and why) is docs/differentials/UPSTREAM_EXAM.md. Rules of this file: * assertions check the SAME behavior as upstream, not a look-alike; diff --git a/test/upstream_exam/signatures_test.exs b/test/upstream_exam/signatures_test.exs index 98910f2f..0ab6e64c 100644 --- a/test/upstream_exam/signatures_test.exs +++ b/test/upstream_exam/signatures_test.exs @@ -4,7 +4,7 @@ defmodule UpstreamExam.SignaturesTest do Tranche 1 of the upstream exam: every test cites the upstream file and test function it translates. The per-test disposition map (including everything - that was NOT portable and why) is docs/internal/UPSTREAM_EXAM.md. + that was NOT portable and why) is docs/differentials/UPSTREAM_EXAM.md. DSPy signatures are Python classes (pydantic models built by a metaclass); Imp signatures are plain structs built from string specs or maps. Ports use diff --git a/test/upstream_exam/streaming_test.exs b/test/upstream_exam/streaming_test.exs index ba29c6cb..45f82f23 100644 --- a/test/upstream_exam/streaming_test.exs +++ b/test/upstream_exam/streaming_test.exs @@ -3,7 +3,7 @@ defmodule UpstreamExam.StreamingTest do DSPy 3.2.1's own streaming tests (tests/streaming/test_streaming.py), ported to Imp. - Tranche 3 of the upstream exam. Disposition map: docs/internal/UPSTREAM_EXAM.md. + Tranche 3 of the upstream exam. Disposition map: docs/differentials/UPSTREAM_EXAM.md. Design substitution throughout: DSPy streams via asyncio generators wrapped by `streamify`; Imp streams via Enumerables (`Imp.Streaming.stream/3`) and diff --git a/test/upstream_exam/teleprompt_test.exs b/test/upstream_exam/teleprompt_test.exs index bc2f41dc..1f44cc75 100644 --- a/test/upstream_exam/teleprompt_test.exs +++ b/test/upstream_exam/teleprompt_test.exs @@ -4,7 +4,7 @@ defmodule UpstreamExam.TelepromptTest do Tranche 3 of the upstream exam: every test here cites the upstream file and test function it translates. The complete per-test disposition map (including - the tests that were NOT portable and why) is docs/internal/UPSTREAM_EXAM.md. + the tests that were NOT portable and why) is docs/differentials/UPSTREAM_EXAM.md. Rules of this file: * assertions check the SAME behavior as upstream, not a look-alike; From 7a406c5cd1a9f7349e47254872628dfa03ebcd6e Mon Sep 17 00:00:00 2001 From: deepfates Date: Thu, 17 Sep 2026 12:31:09 -0700 Subject: [PATCH 2/2] Date every historical finding; make the benchmark-command check require an invocable task --- benchmarks/RESULTS.md | 21 ++++++---- test/documentation_contract_test.exs | 58 ++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 18 deletions(-) diff --git a/benchmarks/RESULTS.md b/benchmarks/RESULTS.md index ef3ee7c8..02fea676 100644 --- a/benchmarks/RESULTS.md +++ b/benchmarks/RESULTS.md @@ -68,12 +68,19 @@ them can be re-measured from this repository: the raw artifacts that diagnosed them are not all published, and in two cases the machinery that produced them has since changed. They are dated observations, not standing claims. +The first three rows below have no recoverable run date. Their artifacts lived +under `benchmarks/results/`, which is not tracked, and no commit in this +repository's history ever contained them — so the only date that can be stated +honestly is the date the finding entered the record, `2026-08-09` in commit +`bbb2a983`, when the verdicts were written down from artifacts that were then +still on a maintainer's disk. + | Finding | Date | What was observed | | --- | --- | --- | -| HotPotQA JSON-GEPA, mean lift `-0.015` over 3 seeds (task `gpt-5.4-mini`, reflection `claude-sonnet-4.6`) | 2026 | A completed treatment that did not improve held-out performance. Its 32 semantic metric calls were tiny beside the GEPA artifact's 6,871 for HotpotQA; the 24-row test moves in 0.042 steps; six strict-adapter parse failures scored zero on one seed. | -| Banking77 modeled-MIPRO, two conditions at `+0.0417` and `+0.0208` (2 of 3 improving seeds each) | 2026 | Both missed the preregistered `≥0.05` bar. Proposals, attached demos and acquisitions were real. The 48-row test and a high baseline explain the resolution limit. | -| Grue stateful-agent GEPA, 0 of 3 seeds improved (local `llama3.2:3b`) | 2026 | Every candidate scored 0.0 on every selection row, so the optimizer had no ranking signal and retained the baseline. The treatment could not answer the question it was posed. | -| IFBench scorer defect | fixed at `8c798d2e` | The scorer represented nested rule arguments incorrectly and used a non-pinned language fallback. Optimizer results produced with the faulty scorer are invalid and were withdrawn, not rescored. | -| Matched IFBench 16k rehearsal | 2026-08-20 | Both runtimes sealed six optimization-and-selection cells at 16384-token settings for `$16.42`, then the campaign stopped in the held-out phase on a 4096-token input bound of our own. It produced no held-out verdict. An earlier draft compared imp `0.8542` against upstream `0.8698`; those are different quantities (an internal champion score versus an independent re-scoring) and the comparison is withdrawn in full. | -| ChatAdapter parse failures dominate per-cell variation | 2026-08-20 | Across takes of the above, upstream logged 0–5 parse failures per 32-row evaluation (mean 8.3%), correlating with the take's mean at `r = -0.84`. Any future outcome on this design must be reported as two numbers, parse rate and score-given-parse. | -| GEPA Pareto pruning divergence | found by the recorded-tape GEPA differential | Ties were broken by an Elixir term-printing artifact rather than upstream's stable discovery order. A score comparison at the power above could not have detected it. Caught by `mix differential.check`, which is free and has no sampling noise. | +| HotPotQA JSON-GEPA, mean lift `-0.015` over 3 seeds (task `gpt-5.4-mini`, reflection `claude-sonnet-4.6`) | run date not recorded; recorded `2026-08-09` in `bbb2a983` | A completed treatment that did not improve held-out performance. Its 32 semantic metric calls were tiny beside the GEPA artifact's 6,871 for HotpotQA; the 24-row test moves in 0.042 steps; six strict-adapter parse failures scored zero on one seed. | +| Banking77 modeled-MIPRO, two conditions at `+0.0417` and `+0.0208` (2 of 3 improving seeds each) | run date not recorded; recorded `2026-08-09` in `bbb2a983` | Both missed the preregistered `≥0.05` bar. Proposals, attached demos and acquisitions were real. The 48-row test and a high baseline explain the resolution limit. | +| Grue stateful-agent GEPA, 0 of 3 seeds improved (local `llama3.2:3b`) | run date not recorded; recorded `2026-08-09` in `bbb2a983` | Every candidate scored 0.0 on every selection row, so the optimizer had no ranking signal and retained the baseline. The treatment could not answer the question it was posed. | +| IFBench scorer defect | fixed `2026-07-31` in `8c798d2e`; recorded `2026-08-09` in `bbb2a983` | The scorer represented nested rule arguments incorrectly and used a non-pinned language fallback. Optimizer results produced with the faulty scorer are invalid and were withdrawn, not rescored. | +| Matched IFBench 16k rehearsal | recorded `2026-08-20` in `6ff77214` | Both runtimes sealed six optimization-and-selection cells at 16384-token settings for `$16.42`, then the campaign stopped in the held-out phase on a 4096-token input bound of our own. It produced no held-out verdict. An earlier draft compared imp `0.8542` against upstream `0.8698`; those are different quantities (an internal champion score versus an independent re-scoring) and the comparison is withdrawn in full. | +| ChatAdapter parse failures dominate per-cell variation | recorded `2026-08-20` in `6ff77214` | Across takes of the above, upstream logged 0–5 parse failures per 32-row evaluation (mean 8.3%), correlating with the take's mean at `r = -0.84`. Any future outcome on this design must be reported as two numbers, parse rate and score-given-parse. | +| GEPA Pareto pruning divergence | recorded `2026-08-10` in `ba3311df` | Found by the recorded-tape GEPA differential: ties were broken by an Elixir term-printing artifact rather than upstream's stable discovery order. A score comparison at the power above could not have detected it. This one is still caught on every run by `mix differential.check`, which is free and has no sampling noise. | diff --git a/test/documentation_contract_test.exs b/test/documentation_contract_test.exs index c869aab9..621cf972 100644 --- a/test/documentation_contract_test.exs +++ b/test/documentation_contract_test.exs @@ -29,26 +29,62 @@ defmodule DocumentationContractTest do assert Code.ensure_loaded?(Imp.MCP.StreamableHTTPClient) end - test "every mix command the benchmark docs publish is a task that exists" do - published = - "docs/BENCHMARKS.md" - |> File.read!() - |> then(&Regex.scan(~r/mix ([a-z][a-z_0-9.]*[a-z0-9])/, &1)) - |> Enum.map(fn [_, task] -> task end) - |> Enum.uniq() - |> Enum.reject(&(&1 in ["deps.get", "run", "test", "help"])) + # A benchmark command a reader cannot run is worse than no command. The file + # name of a Mix task module is not the task name: both + # lib/mix/tasks/imp.benchmark.classical_optimizer_differential.ex and + # lib/mix/tasks/imp.benchmark.weight_composition_differential.ex define a + # module with no run/1 plus several sibling task modules that do have one, + # so `mix imp.benchmark.classical_optimizer_differential` does not exist + # while `mix imp.benchmark.bootstrap_few_shot_differential` does. Resolving + # the module is therefore not enough; it has to be invocable. + test "every mix command the benchmark docs publish is invocable" do + published = published_mix_commands("docs/BENCHMARKS.md") assert length(published) > 20 + for task <- published do + assert invocable_mix_command?(task), + "docs/BENCHMARKS.md publishes `mix #{task}`, which is neither an " <> + "alias in mix.exs nor a Mix task module exporting run/1" + end + end + + test "the invocable-command check rejects a module that is not a runnable task" do + # Guard on the guard: these two resolve to real modules under + # lib/mix/tasks, and neither can be run. + refute invocable_mix_command?("imp.benchmark.classical_optimizer_differential") + refute invocable_mix_command?("imp.benchmark.weight_composition_differential") + refute invocable_mix_command?("imp.benchmark.no_such_task_at_all") + + assert invocable_mix_command?("imp.benchmark.bootstrap_few_shot_differential") + assert invocable_mix_command?("differential.check") + end + + defp published_mix_commands(path) do + path + |> File.read!() + |> then(&Regex.scan(~r/mix ([a-z][a-z_0-9.]*[a-z0-9])/, &1)) + |> Enum.map(fn [_, task] -> task end) + |> Enum.uniq() + |> Enum.reject(&(&1 in ["deps.get", "run", "test", "help"])) + end + + defp invocable_mix_command?(task) do aliases = Mix.Project.config() |> Keyword.get(:aliases, []) |> Keyword.keys() |> MapSet.new(&Atom.to_string/1) - for task <- published do - assert Mix.Task.get(task) || MapSet.member?(aliases, task), - "docs/BENCHMARKS.md publishes `mix #{task}`, which is neither a task nor an alias" + cond do + MapSet.member?(aliases, task) -> + true + + module = Mix.Task.get(task) -> + Code.ensure_loaded?(module) and function_exported?(module, :run, 1) + + true -> + false end end