diff --git a/CHANGELOG.md b/CHANGELOG.md index bf6352068..d846b3799 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Build fallback variants from `/v1/models` advertised reasoning efforts (OpenRouter, Synthetic) when no built-in or user variants exist. + ## 0.157.3 - Add built-in effort variants for Claude models on `openai-chat` providers like OpenRouter, using the `verbosity` param which maps to Anthropic's `output_config.effort`. diff --git a/docs/config/variants.md b/docs/config/variants.md index 12ef53c71..f49f2fbde 100644 --- a/docs/config/variants.md +++ b/docs/config/variants.md @@ -98,6 +98,12 @@ ECA ships with built-in variants for some known models via the `variantsByModel` | `high` | `{"reasoning_effort": "high"}` | | `max` | `{"reasoning_effort": "max"}}` | +## Discovered Variants + +Some providers (GitHub Copilot, OpenAI OAuth, and gateways like OpenRouter or Synthetic) tell ECA which reasoning effort levels a model supports, and ECA builds the variants for you. Nothing to configure: just pick an effort from the model's variant list. + +Discovered variants are only used when nothing else defines variants for the model. Your custom variants always win. + ## Custom Variants You can define your own variants per model under `providers..models..variants`. Custom variants are merged with built-in ones — if names clash, your definition wins. diff --git a/integration-test/integration/chat/commands_test.clj b/integration-test/integration/chat/commands_test.clj index 0ede5dcfd..abd6a6dbe 100644 --- a/integration-test/integration/chat/commands_test.clj +++ b/integration-test/integration/chat/commands_test.clj @@ -35,6 +35,7 @@ {:name "fork" :arguments []} {:name "btw" :arguments [{:name "prompt" :required true}]} {:name "resume" :arguments [{:name "chat-id"}]} + {:name "delete-chat" :arguments [{:name "chat-id"}]} {:name "export" :arguments [{:name "filepath"}]} {:name "import" :arguments [{:name "filepath"}]} {:name "remote" :arguments []} @@ -67,6 +68,7 @@ {:name "costs" :arguments []} {:name "context" :arguments []} {:name "compact" :arguments [{:name "additional-input"}]} + {:name "delete-chat" :arguments [{:name "chat-id"}]} {:name "remote" :arguments []} {:name "config" :arguments []} {:name "subagents" :arguments []} @@ -114,6 +116,13 @@ (is (match? {:type "mcp" :name "mcpServerSample"} (eca/client-awaits-server-notification :tool/serverUpdated)))) + (testing "Mcp prompts fetched" + ;; Prompts are listed after the server reports running, in a second + ;; notification. Await it before querying commands. + (is (match? {:type "mcp" + :name "mcpServerSample" + :prompts [{:name "my-prompt" :arguments [{:name "some-arg-1"}]}]} + (eca/client-awaits-server-notification :tool/serverUpdated)))) (testing "MCP prompts available when querying commands" (let [resp (eca/request! (fixture/chat-query-commands-request diff --git a/src/eca/config.clj b/src/eca/config.clj index 7c57e7d83..f22b34ee1 100644 --- a/src/eca/config.clj +++ b/src/eca/config.clj @@ -319,9 +319,10 @@ false))) (defn effective-model-variants - "Returns effective variants for a model. Built-in regex variants are the - fallback, discovered provider variants override them, and user variants have - final priority. A variant set to {} is removed from the result." + "Returns effective variants for a model. Discovered provider variants + override built-in regex variants, effort variants parsed from the provider + /models response are the last fallback, and user variants have final + priority. A variant set to {} is removed from the result." ([config provider model-name user-variants] (effective-model-variants config provider model-name nil user-variants)) ([config provider model-name model-capabilities user-variants] @@ -338,7 +339,9 @@ (api-match? provider-api api)) variants)) (:variantsByModel config))) - merged (merge (or (not-empty (:variants model-capabilities)) builtin) + merged (merge (or (not-empty (:variants model-capabilities)) + builtin + (not-empty (:effort-variants model-capabilities))) user-variants)] (when (seq merged) (let [filtered (into {} (remove (fn [[_ v]] (= {} v))) merged)] diff --git a/src/eca/models.clj b/src/eca/models.clj index f3301158a..212bc71fc 100644 --- a/src/eca/models.clj +++ b/src/eca/models.clj @@ -427,6 +427,26 @@ :discovered-reason? (when reason? true) :discovered-variants (not-empty variants))]))) +(def ^:private effort-list-paths + [[:reasoning :supported_efforts] ; OpenRouter + [:reasoning_parameters :efforts]]) ; Synthetic + +(defn ^:private native-reasoning-effort-variants + "Builds openai-chat variants from a /models entry's advertised reasoning + efforts. Nil for other APIs or when no efforts are advertised." + [api-type entry] + (when (= "openai-chat" api-type) + (let [advertised (some (fn [path] + (let [efforts (get-in entry path)] + (when (sequential? efforts) efforts))) + effort-list-paths)] + (not-empty + (into {} + (keep (fn [effort] + (when (string? effort) + [effort {:reasoning_effort effort}])) + advertised)))))) + (defn ^:private parse-native-model-entry "Parses a generic /models entry. OpenRouter-shaped entries expose `context_length` and `top_provider.max_completion_tokens`; llama.cpp / @@ -435,17 +455,19 @@ provider-reported data wins over models.dev catalogs. Entries without that metadata (e.g. plain OpenAI/Anthropic) keep an empty config, preserving the previous id-only behavior." - [{:keys [id context_length top_provider] entry-meta :meta}] + [{:keys [id context_length top_provider] entry-meta :meta :as entry} api-type] (when (and (string? id) (not (string/blank? id))) (let [context (or (pos-num context_length) (pos-num (:n_ctx entry-meta)) (pos-num (:n_ctx_train entry-meta))) - output (sane-output-limit context (pos-num (:max_completion_tokens top_provider)))] + output (sane-output-limit context (pos-num (:max_completion_tokens top_provider))) + effort-variants (native-reasoning-effort-variants api-type entry)] [id (assoc-some {} :discovered-limit (not-empty (assoc-some {} :context context - :output output)))]))) + :output output)) + :discovered-effort-variants effort-variants)]))) (defn ^:private fetch-provider-native-models "Fetches models from provider's native /models endpoint. @@ -481,7 +503,7 @@ (not-empty (if (= "github-copilot" provider) (into {} (keep parse-copilot-model-entry) models-data) - (into {} (keep parse-native-model-entry) models-data)))))))) + (into {} (keep #(parse-native-model-entry % api-type) models-data))))))))) (catch Exception e (logger/warn logger-tag (format "Provider '%s': Failed to fetch models from %s: %s" @@ -577,8 +599,9 @@ (defn ^:private config-overrides->capabilities "Translate per-model config and discovered provider metadata into internal capability keys. User limits, costs, and image support override catalog - values; provider discovery contributes API routing, reasoning variants, and - token limits (user config > provider-discovered > models.dev catalog)." + values; provider discovery contributes API routing, reasoning variants, + effort variants, and token limits (user config > provider-discovered > + models.dev catalog)." [model-config] (let [limit (:limit model-config) cost (:cost model-config) @@ -592,6 +615,7 @@ :reason? (:discovered-reason? model-config) :api (:discovered-api model-config) :variants (:discovered-variants model-config) + :effort-variants (:discovered-effort-variants model-config) ;; Opaque provider-specific model metadata, interpreted only by ;; the provider adapter that discovered it. :provider-data (not-empty (:discovered-provider-data model-config)) diff --git a/test/eca/config_test.clj b/test/eca/config_test.clj index c9d28b129..465b2669e 100644 --- a/test/eca/config_test.clj +++ b/test/eca/config_test.clj @@ -699,6 +699,19 @@ :variants {"low" {:source "endpoint"}}} {"high" {:source "user"}})))) + (testing "Effort variants from /models are the last fallback" + (let [effort-variants {"low" {:reasoning_effort "low"} + "high" {:reasoning_effort "high"}} + caps {:api :openai-chat :effort-variants effort-variants}] + (is (= effort-variants + (config/effective-model-variants {} "synthetic" "qwen3" caps nil))) + (is (= anthropic-variants + (config/effective-model-variants config "my-proxy" "claude-opus-4-6" + (assoc caps :api :anthropic) nil))) + (is (= {"low" {:reasoning_effort "low"}} + (config/effective-model-variants {} "synthetic" "qwen3" caps + {"high" {}}))))) + (testing "User variant set to {} removes a discovered variant" (is (= {"low" {:source "endpoint"}} (config/effective-model-variants diff --git a/test/eca/llm_api_test.clj b/test/eca/llm_api_test.clj index 859b20b35..b1b8d407d 100644 --- a/test/eca/llm_api_test.clj +++ b/test/eca/llm_api_test.clj @@ -1125,7 +1125,10 @@ final-error* (atom nil) error {:status 503 :body "{\"error\":{\"message\":\"auth_unavailable: no auth available (providers=codex, model=gpt-5.6-sol)\",\"type\":\"server_error\",\"code\":\"internal_server_error\"}}" - :message "OpenAI response status: 503 body: auth_unavailable"}] + :message "OpenAI response status: 503 body: auth_unavailable"} + delivered-error (assoc error + :message "Anthropic server_error: auth_unavailable: no auth available (providers=codex, model=gpt-5.6-sol)" + :code "server_error")] (with-redefs [eca.llm-api/prompt! (fn [_] (swap! attempt* inc) {:error error}) @@ -1154,7 +1157,7 @@ :backoff-multiplier 3.0 :max-delay-ms 250} (:policy (first @retry-events*)))) - (is (= error @final-error*))))) + (is (= delivered-error @final-error*))))) (deftest sync-retry-cancelled-test (testing "stops retrying when cancelled" diff --git a/test/eca/models_test.clj b/test/eca/models_test.clj index 9497c92fb..e4ebc5061 100644 --- a/test/eca/models_test.clj +++ b/test/eca/models_test.clj @@ -285,6 +285,66 @@ :api-key "k" :api-type "openai-chat"})))))) +(deftest fetch-provider-native-models-effort-variants-test + (testing "OpenRouter and Synthetic shaped efforts become discovered effort variants" + (with-redefs [http/get (fn [_url _opts] + {:status 200 + :body {:data + [{:id "tencent/hy4-preview" + :reasoning {:supported_efforts ["high" "low" 1]}} + {:id "syn:small:text" + :reasoning_parameters {:efforts ["none" "low"]}} + {:id "bad-efforts" + :reasoning {:supported_efforts 1}} + {:id "plain-model"}]}})] + (is (match? + (m/equals + {"tencent/hy4-preview" + (m/equals {:discovered-effort-variants + {"high" {:reasoning_effort "high"} + "low" {:reasoning_effort "low"}}}) + "syn:small:text" + (m/equals {:discovered-effort-variants + {"none" {:reasoning_effort "none"} + "low" {:reasoning_effort "low"}}}) + "bad-efforts" (m/equals {}) + "plain-model" (m/equals {})}) + (#'models/fetch-provider-native-models + {:provider "openrouter" + :api-url "https://openrouter.ai/api/v1" + :auth-type nil + :api-key "k" + :api-type "openai-chat"}))))) + + (testing "other APIs build no effort variants from the same metadata" + (with-redefs [http/get (fn [_url _opts] + {:status 200 + :body {:data [{:id "claude-x" + :reasoning {:supported_efforts ["low" "high"]}}]}})] + (is (match? + {"claude-x" {}} + (#'models/fetch-provider-native-models + {:provider "my-proxy" + :api-url "https://api.anthropic.com" + :auth-type nil + :api-key "k" + :api-type "anthropic"})))))) + +(deftest synthetic-provider-effort-variants-build-model-capabilities-test + (testing "Effort variants from a synthetic /v1/models flow into model capabilities" + (let [config {:providers {"synthetic" {:api "openai-chat" + :url "https://api.synthetic.new/v1" + :key "sk-test" + :models {"hf:Qwen/Qwen3-235B" {}}}}}] + (with-redefs [http/get (fn [_url _opts] + {:status 200 + :body {:data [{:id "hf:Qwen/Qwen3-235B" + :reasoning_parameters {:efforts ["high" "low"]}}]}})] + (is (= {"high" {:reasoning_effort "high"} + "low" {:reasoning_effort "low"}} + (get-in (build-supported-models config {} {}) + ["synthetic/hf:Qwen/Qwen3-235B" :effort-variants]))))))) + (deftest fetch-provider-native-llamacpp-models-limits-test (testing "llama.cpp/llama-swap-shaped entries keep meta n_ctx (falling back to n_ctx_train) as discovered context limit" (with-redefs [http/get (fn [_url _opts]