From 67caab8ac142ac4c99c56e0516232ff9a8b0c376 Mon Sep 17 00:00:00 2001 From: deepfates Date: Tue, 15 Sep 2026 18:22:37 -0700 Subject: [PATCH] Bump req_llm to 1.18.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things in this repo pin the exact req_llm build, and both had to move. `Imp.BenchmarkTruth.MultimodalManifest` carries the campaign's ReqLLM dependency — package hash, upstream revision, version — and `runtime_dependency!/1` refuses to dispatch when the loaded version differs, so the bump failed with `ReqLLM runtime version must equal "1.17.1", got: "1.18.0"`. The pin now names 1.18.0, package SHA-256 d610d4de14c7ef697a2aa4c30eada4e1ae172bec08bb3dea3bbe25f20feb5d40, upstream revision 9aa98a4e02da5083464d2eed268f48272f0d7180 (the commit tagged v1.18.0 in agentjido/req_llm). Both provider manifests carry the same dependency block, so they were updated and their payload checksums recomputed. That changes the manifests' identity. The OpenAI manifest now hashes to 5ab0c61f... instead of 04daa155..., and the Google one to c5ca53e2... instead of 4b4c8049.... The admitted multimodal_live artifact records the 1.17.1 run and keeps its own manifest SHA, so it still validates and the claim still stands; what it no longer is, is reproducible from the manifests in the tree. Admitting a 1.18.0 re-run means a live OpenAI campaign, a new artifact, and a new `multimodal_manifest_sha256/0` in the reproduction validator. MULTIMODAL_FIDELITY.md now says so. `test/campaign_budget_test.exs` asserted that the pinned req_llm overwrites a caller's `max_retries: 0` with its default of 3. 1.18.0 honours the zero, so the assertion is inverted to match and still fails loudly if upstream changes again. The transport guard itself is untouched and the attempt count is unchanged. req stays at 0.6.3 here; dependabot PR #94 moved it to 0.7.2 in the same lock, and that bump needs its own code changes (PR #109). This supersedes dependabot PR #94. mix check: 54 doctests, 9 properties, 2861 tests, 0 failures, 13 skipped (261 excluded). mix quality.check: credo clean, no vulnerabilities found. --- bench/imp/benchmark_truth/multimodal_manifest.ex | 6 +++--- benchmarks/data/multimodal/manifest.json | 8 ++++---- .../data/multimodal/openai-responses-manifest.json | 8 ++++---- docs/internal/MULTIMODAL_FIDELITY.md | 12 ++++++++++++ mix.lock | 8 ++++---- test/campaign_budget_test.exs | 6 ++++-- 6 files changed, 31 insertions(+), 17 deletions(-) diff --git a/bench/imp/benchmark_truth/multimodal_manifest.ex b/bench/imp/benchmark_truth/multimodal_manifest.ex index 72aa497b..bf82dcc1 100644 --- a/bench/imp/benchmark_truth/multimodal_manifest.ex +++ b/bench/imp/benchmark_truth/multimodal_manifest.ex @@ -15,11 +15,11 @@ defmodule Imp.BenchmarkTruth.MultimodalManifest do @deliveries ~w(typed_image_data_uri typed_native_file) @req_llm_dependency %{ "package" => "req_llm", - "package_sha256" => "266c0e06c47b4562f243dcdf41332342cbed2ec37064750edd725fb66bb6e914", + "package_sha256" => "d610d4de14c7ef697a2aa4c30eada4e1ae172bec08bb3dea3bbe25f20feb5d40", "repository" => "https://github.com/agentjido/req_llm", "source" => "hexpm", - "source_revision" => "33840077c2f1332eb6dff2d268dff02393014da4", - "version" => "1.17.1" + "source_revision" => "9aa98a4e02da5083464d2eed268f48272f0d7180", + "version" => "1.18.0" } @provider_profiles %{ "google-gemini-2.5-flash-generate-content" => %{ diff --git a/benchmarks/data/multimodal/manifest.json b/benchmarks/data/multimodal/manifest.json index d0af3910..d97c876e 100644 --- a/benchmarks/data/multimodal/manifest.json +++ b/benchmarks/data/multimodal/manifest.json @@ -83,11 +83,11 @@ "profile": "google-gemini-2.5-flash-generate-content", "req_llm_dependency": { "package": "req_llm", - "package_sha256": "266c0e06c47b4562f243dcdf41332342cbed2ec37064750edd725fb66bb6e914", + "package_sha256": "d610d4de14c7ef697a2aa4c30eada4e1ae172bec08bb3dea3bbe25f20feb5d40", "repository": "https://github.com/agentjido/req_llm", "source": "hexpm", - "source_revision": "33840077c2f1332eb6dff2d268dff02393014da4", - "version": "1.17.1" + "source_revision": "9aa98a4e02da5083464d2eed268f48272f0d7180", + "version": "1.18.0" }, "req_llm_model": "google:gemini-2.5-flash" }, @@ -180,5 +180,5 @@ "prompt_contract": "One pinned attachment followed by one pinned question; return only a single-field JSON object named answer." } }, - "payload_sha256": "4b4c80497789d74dec60b2b26236f6eee2f1f9b29046f07ef95b3fa3faf8e50f" + "payload_sha256": "c5ca53e2fc67757f07f65ab0109d57ffef9c1d0b16ef13206a394aa5010f33ec" } diff --git a/benchmarks/data/multimodal/openai-responses-manifest.json b/benchmarks/data/multimodal/openai-responses-manifest.json index 921851e9..03efc392 100644 --- a/benchmarks/data/multimodal/openai-responses-manifest.json +++ b/benchmarks/data/multimodal/openai-responses-manifest.json @@ -81,11 +81,11 @@ "profile": "openai-gpt-4.1-mini-2025-04-14-responses", "req_llm_dependency": { "package": "req_llm", - "package_sha256": "266c0e06c47b4562f243dcdf41332342cbed2ec37064750edd725fb66bb6e914", + "package_sha256": "d610d4de14c7ef697a2aa4c30eada4e1ae172bec08bb3dea3bbe25f20feb5d40", "repository": "https://github.com/agentjido/req_llm", "source": "hexpm", - "source_revision": "33840077c2f1332eb6dff2d268dff02393014da4", - "version": "1.17.1" + "source_revision": "9aa98a4e02da5083464d2eed268f48272f0d7180", + "version": "1.18.0" }, "req_llm_model": "openai:gpt-4.1-mini-2025-04-14" }, @@ -178,5 +178,5 @@ "prompt_contract": "One pinned attachment followed by one pinned question; return only a single-field JSON object named answer." } }, - "payload_sha256": "04daa155d3f97edfff62e329dfdca1248855f22b2271f7e954228432f1ae39d8" + "payload_sha256": "5ab0c61f176ba63279ca065c170a628596d0da1834bd970c1c6b61221b4db205" } diff --git a/docs/internal/MULTIMODAL_FIDELITY.md b/docs/internal/MULTIMODAL_FIDELITY.md index 8bf44bb1..8baefb0f 100644 --- a/docs/internal/MULTIMODAL_FIDELITY.md +++ b/docs/internal/MULTIMODAL_FIDELITY.md @@ -43,6 +43,18 @@ family, delivery mode, asset byte count, MIME type, asset SHA-256, provider, endpoint, API, exact model, generation options, ReqLLM dependency, and pricing. Runtime ReqLLM version drift fails before dispatch. +Because the ReqLLM dependency is part of the payload, bumping req_llm changes +the manifest payload SHA-256. Both manifests now pin req_llm 1.18.0, so the +OpenAI manifest hashes to +`5ab0c61f176ba63279ca065c170a628596d0da1834bd970c1c6b61221b4db205` and the +Google manifest to +`c5ca53e2fc67757f07f65ab0109d57ffef9c1d0b16ef13206a394aa5010f33ec`. The +admitted artifact above remains a valid record of the 1.17.1 run it names, but +it is no longer reproducible from the manifests in the tree: a live re-run +under 1.18.0 produces a new campaign identity, and admitting it means a new +artifact plus a new `multimodal_manifest_sha256/0` in +`Imp.BenchmarkTruth.ReproductionArtifactValidator`. + All assets are synthetic and repository-owned. Images cover shape counting, spatial relation, and OCR. The native-document family sends the original two-page PDF and tests a table subtotal and a cross-page join. Rendered page diff --git a/mix.lock b/mix.lock index 3cd6491a..1454e57f 100644 --- a/mix.lock +++ b/mix.lock @@ -27,7 +27,7 @@ "jaxon": {:hex, :jaxon, "2.0.8", "00951a79d354260e28d7e36f956c3de94818124768a4b22e0fc55559d1b3bfe7", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "74532853b1126609615ea98f0ceb5009e70465ca98027afbbd8ed314d887e82d"}, "jose": {:hex, :jose, "1.11.12", "06e62b467b61d3726cbc19e9b5489f7549c37993de846dfb3ee8259f9ed208b3", [:mix, :rebar3], [], "hexpm", "31e92b653e9210b696765cdd885437457de1add2a9011d92f8cf63e4641bab7b"}, "jsv": {:hex, :jsv, "0.21.2", "fb84c2a1bb5884e2b91ae66c5ed84a4db1da2bcfc1df5175c8b33c8791d39353", [:mix], [{:abnf_parsec, "~> 2.0", [hex: :abnf_parsec, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:idna, "~> 6.0 or ~> 7.0", [hex: :idna, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:texture, "~> 1.0", [hex: :texture, repo: "hexpm", optional: false]}], "hexpm", "5963f414f9fb37c541dcebfb2df96d07ae4a40d12f52d7ac5e057969e5e7d83a"}, - "llm_db": {:hex, :llm_db, "2026.7.0", "2cd32be2907e72df3d72e57ad99b3963011ceb9418b46a95c4315ce1b0ba5429", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.7", [hex: :toml, repo: "hexpm", optional: false]}, {:zoi, "~> 0.10", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "9ffff8ccf4e3c268a72cbd19e0539cab618ebd3c4bc621a43c48dd2ee8af412f"}, + "llm_db": {:hex, :llm_db, "2026.7.5", "38e345e753b027f095e9eb01157de911c1e75d5ac7d760042f771cbdeabb08c1", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.7", [hex: :toml, repo: "hexpm", optional: false]}, {:zoi, "~> 0.10", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "93ceaf3448b8cea11358854388e83480ed4e84e39adc0af8a3629935d3d5dd7c"}, "makeup": {:hex, :makeup, "1.2.2", "882d46dc0905e9ff7abf2aab61a7e6b3dcc555533977d8a23b06019e6c89ac94", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "9a1a24e5b343b8ae16abea0822c10a6f75da27af7fa802ada5251f7579bfccfa"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, @@ -45,10 +45,10 @@ "plug_crypto": {:hex, :plug_crypto, "2.2.0", "144014737daaf485407f5ed77daeaad74d651b216a28c87543f8cc7043f8efc8", [:mix], [], "hexpm", "83a95744ab1c75876542b6fab135fcc176280e0f301a111c1f757fddcec95d2c"}, "ranch": {:hex, :ranch, "2.3.0", "7de7b041a9a6a5091a3aa5898d66c0564be671d87db4f9d63b1b5ee775b097df", [:make, :rebar3], [], "hexpm", "6168ec49409d982f7cfbd83dd083144f6cbe67caa4036551d2f0a3ad67c9d023"}, "req": {:hex, :req, "0.6.3", "7fe5e68792ff0546e45d5919104fa1764a13694cfe3e48c8a0f32ad051ae77e4", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "e85b5c6c990e6c3f52bbba68e6f099118f2b8252825f96c7c3636b97a3de307d"}, - "req_llm": {:hex, :req_llm, "1.17.1", "266c0e06c47b4562f243dcdf41332342cbed2ec37064750edd725fb66bb6e914", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:ex_aws_auth, "~> 1.4", [hex: :ex_aws_auth, repo: "hexpm", optional: true]}, {:goth, "~> 1.4", [hex: :goth, repo: "hexpm", optional: true]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:jsv, "~> 0.11", [hex: :jsv, repo: "hexpm", optional: false]}, {:llm_db, "~> 2026.7.0", [hex: :llm_db, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:server_sent_events, "~> 1.1.0", [hex: :server_sent_events, repo: "hexpm", optional: false]}, {:splode, "~> 0.3.0", [hex: :splode, repo: "hexpm", optional: false]}, {:websockex, "~> 0.5.1", [hex: :websockex, repo: "hexpm", optional: false]}, {:zoi, "~> 0.14", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "169d77a77bac5cee7655a7a9158ebea1c2a7857670f4104b29516a25a2d24f1e"}, + "req_llm": {:hex, :req_llm, "1.18.0", "d610d4de14c7ef697a2aa4c30eada4e1ae172bec08bb3dea3bbe25f20feb5d40", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:ex_aws_auth, "~> 1.4", [hex: :ex_aws_auth, repo: "hexpm", optional: true]}, {:goth, "~> 1.4", [hex: :goth, repo: "hexpm", optional: true]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:jsv, "~> 0.11", [hex: :jsv, repo: "hexpm", optional: false]}, {:llm_db, "~> 2026.7.5", [hex: :llm_db, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:server_sent_events, "~> 1.1.0", [hex: :server_sent_events, repo: "hexpm", optional: false]}, {:splode, "~> 0.3.0", [hex: :splode, repo: "hexpm", optional: false]}, {:websockex, "~> 0.5.1", [hex: :websockex, repo: "hexpm", optional: false]}, {:zoi, "~> 0.14", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "e8b9f1398b67ca7a40be8fb04bfc471b72f21e8661b345e2249505ce8d41cefa"}, "saxy": {:hex, :saxy, "1.6.1", "742eff28f553c066d0b54e84662dbf384a1d1f38595472ed15f6e0a33038bbe1", [:mix], [], "hexpm", "8989d504424ba29460a61950f8968380651413fa05e63b6118084db057da1a6b"}, "server_sent_events": {:hex, :server_sent_events, "1.1.0", "54606238b9182ba673a10ce90bd95f81a0939c1baa91cdf86b66acc04edd1832", [:mix], [], "hexpm", "8e164db8e295a2d869a8faafbf4a1eeaa749b62fe93f66271adff6394d93ce15"}, - "splode": {:hex, :splode, "0.3.1", "9843c54f84f71b7833fec3f0be06c3cfb5be6b35960ee195ea4fad84b1c25030", [:mix], [], "hexpm", "8f2309b6ec2ecbb01435656429ed1d9ed04ba28797a3280c3b0d1217018ecfbd"}, + "splode": {:hex, :splode, "0.3.2", "7716b6b2260a98a6f018c65cc0393da2cdf17314202eb351a0956e8762190dff", [:mix], [], "hexpm", "08fd658f80da7f1cd254b149164dcff8acd44b22f032001d5416b97223d32bc9"}, "stream_data": {:hex, :stream_data, "1.4.0", "026f929db613aabea6208012ae9b8970d3fd5f88b3bdf26831bc536f98c42036", [:mix], [], "hexpm", "2b0ee3a340dcce1c8cf6302a763ee757d1e01c54d6e16d9069062509d68b1dc9"}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, "texture": {:hex, :texture, "1.2.0", "aa9097e0709652044ba500de817d60ce8a48dc0b4ffd92232fd33459a20282f6", [:mix], [{:abnf_parsec, "~> 2.0", [hex: :abnf_parsec, repo: "hexpm", optional: false]}], "hexpm", "f250de468cfe35f404faf232429f40114f83f73430e0f7086a9cc251865642e2"}, @@ -58,5 +58,5 @@ "websockex": {:hex, :websockex, "0.5.1", "9de28d37bbe34f371eb46e29b79c94c94fff79f93c960d842fbf447253558eb4", [:mix], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8ef39576ed56bc3804c9cd8626f8b5d6b5721848d2726c0ccd4f05385a3c9f14"}, "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, "yaml_elixir": {:hex, :yaml_elixir, "2.12.2", "9dd1330fb4cd9a36a7b0f502e5b12486eff632792ee4a5f0eba52a4d4ec32c9c", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "e7c1b10122f973e6558462d51c39026ba0e14afbc6745318e990ea82cfe9e159"}, - "zoi": {:hex, :zoi, "0.18.5", "fc23f53d5ba6c8639fb30d9a70a80ae8fed774c180c2544bfd16f53bedf95dff", [:mix], [{:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "d69c2e6468752367941d0264e21cf229c1ba8757b2973eea17c12b3cb7d2d4c4"}, + "zoi": {:hex, :zoi, "0.18.7", "0d6b09d19fd1feff4340b7c5660bab04fbc80c1642ee1e5c75f06d527ac326db", [:mix], [{:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "5fedddd755dec84a5e78b3671070a5e595026aa3479f7fa566a42b8c4e4e5ff2"}, } diff --git a/test/campaign_budget_test.exs b/test/campaign_budget_test.exs index 04429f79..4fe3303c 100644 --- a/test/campaign_budget_test.exs +++ b/test/campaign_budget_test.exs @@ -407,8 +407,10 @@ defmodule Imp.BenchmarkTruth.CampaignBudgetTest do max_retries: 0 ) - # Pinned ReqLLM currently overwrites the caller's zero with its default. - assert prepared.options.max_retries == 3 + # req_llm 1.17.1 overwrote the caller's zero with its default of 3; + # 1.18.0 honours it. Pinned here so a dependency regression is loud, + # while the guard below keeps the attempt count correct either way. + assert prepared.options.max_retries == 0 {:ok, budget} = CampaignBudget.start_link(