From af284fb7824992f79ff2cdc912906d83e687c717 Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Mon, 13 Apr 2026 11:16:09 +0900 Subject: [PATCH 01/12] add mercury-2 --- llm_configs/mercury-2.json | 22 ++++++++++++++++++++++ src/ale_bench_eval/calc_cost.py | 5 +++++ 2 files changed, 27 insertions(+) create mode 100644 llm_configs/mercury-2.json diff --git a/llm_configs/mercury-2.json b/llm_configs/mercury-2.json new file mode 100644 index 00000000..8aa34cf9 --- /dev/null +++ b/llm_configs/mercury-2.json @@ -0,0 +1,22 @@ +{ + "model_name": "inception/mercury-2", + "provider": "openrouter", + "settings": { + "temperature": 0.75, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["inception"], + "require_parameters": true + }, + "reasoning": { + "enabled": true, + "effort": "high" + }, + "usage": { + "include": true + } + } + } +} diff --git a/src/ale_bench_eval/calc_cost.py b/src/ale_bench_eval/calc_cost.py index b0094a43..03c044a2 100644 --- a/src/ale_bench_eval/calc_cost.py +++ b/src/ale_bench_eval/calc_cost.py @@ -198,6 +198,11 @@ output_mtok=Decimal(3), cache_read_mtok=Decimal(1) / Decimal(10), ), + "mercury-2": ModelPrice( + input_mtok=Decimal(25) / Decimal(100), + cache_read_mtok=Decimal(25) / Decimal(1000), + output_mtok=Decimal(75) / Decimal(100), + ), "minimax-m2.1": ModelPrice( input_mtok=Decimal(3) / Decimal(10), output_mtok=Decimal(12) / Decimal(10), From ad79ec95881d181f462cc675621a88449f7b8554 Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Fri, 17 Apr 2026 18:42:02 +0900 Subject: [PATCH 02/12] add claude 4.7, gemma 4 and qwen3.5/3.6 --- llm_configs/claude-4.7-opus-no-thinking.json | 10 ++++++++ llm_configs/gemma-4-31b-it.json | 22 +++++++++++++++++ llm_configs/qwen3.5-27b.json | 23 ++++++++++++++++++ llm_configs/qwen3.6-35b-a3b.json | 25 ++++++++++++++++++++ src/ale_bench_eval/calc_cost.py | 18 ++++++++++++++ 5 files changed, 98 insertions(+) create mode 100644 llm_configs/claude-4.7-opus-no-thinking.json create mode 100644 llm_configs/gemma-4-31b-it.json create mode 100644 llm_configs/qwen3.5-27b.json create mode 100644 llm_configs/qwen3.6-35b-a3b.json diff --git a/llm_configs/claude-4.7-opus-no-thinking.json b/llm_configs/claude-4.7-opus-no-thinking.json new file mode 100644 index 00000000..c11bacfe --- /dev/null +++ b/llm_configs/claude-4.7-opus-no-thinking.json @@ -0,0 +1,10 @@ +{ + "model_name": "claude-opus-4-7", + "provider": "anthropic", + "settings": { + "max_tokens": 128000, + "anthropic_thinking": { + "type": "disabled" + } + } +} diff --git a/llm_configs/gemma-4-31b-it.json b/llm_configs/gemma-4-31b-it.json new file mode 100644 index 00000000..3d392cfa --- /dev/null +++ b/llm_configs/gemma-4-31b-it.json @@ -0,0 +1,22 @@ +{ + "model_name": "google/gemma-4-31B-it", + "provider": "vllm-chat", + "base_url": "http://localhost:18765/v1", + "api_key": "ale-bench-eval", + "settings": { + "temperature": 1.0, + "top_p": 0.95, + "extra_body": { + "top_k": 64, + "chat_template_kwargs": { + "enable_thinking": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/qwen3.5-27b.json b/llm_configs/qwen3.5-27b.json new file mode 100644 index 00000000..fa91b86d --- /dev/null +++ b/llm_configs/qwen3.5-27b.json @@ -0,0 +1,23 @@ +{ + "model_name": "qwen/qwen3.5-27b", + "provider": "openrouter", + "settings": { + "temperature": 0.6, + "top_p": 0.95, + "extra_body": { + "presence_penalty": 0.0, + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["alibaba"], + "require_parameters": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/qwen3.6-35b-a3b.json b/llm_configs/qwen3.6-35b-a3b.json new file mode 100644 index 00000000..444742bb --- /dev/null +++ b/llm_configs/qwen3.6-35b-a3b.json @@ -0,0 +1,25 @@ +{ + "model_name": "Qwen/Qwen3.6-35B-A3B", + "provider": "vllm-chat", + "base_url": "http://localhost:18000/v1", + "api_key": "ale-bench-eval", + "settings": { + "temperature": 0.6, + "top_p": 0.95, + "extra_body": { + "top_k": 20, + "min_p": 0.0, + "presence_penalty": 0.0, + "repetition_penalty": 1.0, + "chat_template_kwargs": { + "enable_thinking": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/src/ale_bench_eval/calc_cost.py b/src/ale_bench_eval/calc_cost.py index 03c044a2..370a0cf4 100644 --- a/src/ale_bench_eval/calc_cost.py +++ b/src/ale_bench_eval/calc_cost.py @@ -58,6 +58,16 @@ cache_read_mtok=Decimal(25) / Decimal(1000), output_mtok=Decimal(15) / Decimal(10), ), + "gemma-4-31b-it": ModelPrice( + input_mtok=Decimal(14) / Decimal(100), + cache_read_mtok=Decimal(7) / Decimal(100), + output_mtok=Decimal(4) / Decimal(10), + ), + "gemma-4-31B-it": ModelPrice( + input_mtok=Decimal(14) / Decimal(100), + cache_read_mtok=Decimal(7) / Decimal(100), + output_mtok=Decimal(4) / Decimal(10), + ), "gemma-4-26b-a4b-it": ModelPrice(input_mtok=Decimal(13) / Decimal(100), output_mtok=Decimal(4) / Decimal(10)), "claude-sonnet-4": ModelPrice( input_mtok=TieredPrices(base=Decimal(3), tiers=[Tier(start=200000, price=Decimal(6))]), @@ -109,6 +119,12 @@ cache_read_mtok=Decimal(3) / Decimal(10), output_mtok=Decimal(15), ), + "claude-opus-4-7": ModelPrice( + input_mtok=Decimal(5), + cache_write_mtok=Decimal(625) / Decimal(100), + cache_read_mtok=Decimal(5) / Decimal(10), + output_mtok=Decimal(25), + ), "grok-4.1-fast": ModelPrice( input_mtok=Decimal(2) / Decimal(10), cache_read_mtok=Decimal(5) / Decimal(100), @@ -252,6 +268,7 @@ "qwen3-next-80b-a3b-thinking": ModelPrice( input_mtok=Decimal(14) / Decimal(100), output_mtok=Decimal(14) / Decimal(10) ), + "qwen3.5-27b": ModelPrice(input_mtok=Decimal(3) / Decimal(10), output_mtok=Decimal(24) / Decimal(10)), "qwen3.5-35b-a3b": ModelPrice(input_mtok=Decimal(25) / Decimal(100), output_mtok=Decimal(2)), "qwen3.5-397b-a17b": ModelPrice(input_mtok=Decimal(6) / Decimal(10), output_mtok=Decimal(36) / Decimal(10)), "qwen3.5-plus-02-15": ModelPrice( @@ -267,6 +284,7 @@ input_mtok=TieredPrices(base=Decimal(5) / Decimal(10), tiers=[Tier(start=256000, price=Decimal(2))]), output_mtok=TieredPrices(base=Decimal(3), tiers=[Tier(start=256000, price=Decimal(6))]), ), + "Qwen3.6-35B-A3B": ModelPrice(input_mtok=Decimal(25) / Decimal(100), output_mtok=Decimal(2)), } From ea130f3466b2c880b6323120ed731624fb4d9d97 Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Tue, 21 Apr 2026 01:02:52 +0900 Subject: [PATCH 03/12] add kimi-k2.6 --- llm_configs/kimi-k2.6.json | 20 ++++++++++++++++++++ src/ale_bench_eval/calc_cost.py | 5 +++++ 2 files changed, 25 insertions(+) create mode 100644 llm_configs/kimi-k2.6.json diff --git a/llm_configs/kimi-k2.6.json b/llm_configs/kimi-k2.6.json new file mode 100644 index 00000000..2c3b3d0f --- /dev/null +++ b/llm_configs/kimi-k2.6.json @@ -0,0 +1,20 @@ +{ + "model_name": "moonshotai/kimi-k2.6", + "provider": "openrouter", + "settings": { + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["moonshotai/int4"], + "require_parameters": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/src/ale_bench_eval/calc_cost.py b/src/ale_bench_eval/calc_cost.py index 370a0cf4..d4cc465c 100644 --- a/src/ale_bench_eval/calc_cost.py +++ b/src/ale_bench_eval/calc_cost.py @@ -214,6 +214,11 @@ output_mtok=Decimal(3), cache_read_mtok=Decimal(1) / Decimal(10), ), + "kimi-k2.6": ModelPrice( + input_mtok=Decimal(95) / Decimal(100), + output_mtok=Decimal(4), + cache_read_mtok=Decimal(16) / Decimal(100), + ), "mercury-2": ModelPrice( input_mtok=Decimal(25) / Decimal(100), cache_read_mtok=Decimal(25) / Decimal(1000), From b0161af56a779914fc03deb715dc52c0d16af3e9 Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Fri, 24 Apr 2026 15:20:31 +0900 Subject: [PATCH 04/12] add mimo-v2.5 and deepseek-v4 Co-authored-by: Copilot --- llm_configs/deepseek-v4-flash-high.json | 23 +++++++++++++++ llm_configs/deepseek-v4-flash-max.json | 23 +++++++++++++++ llm_configs/deepseek-v4-pro-high.json | 23 +++++++++++++++ llm_configs/deepseek-v4-pro-max.json | 23 +++++++++++++++ llm_configs/mimo-v2.5-pro.json | 22 ++++++++++++++ llm_configs/mimo-v2.5.json | 22 ++++++++++++++ src/ale_bench_eval/calc_cost.py | 38 +++++++++++++++++++++++++ 7 files changed, 174 insertions(+) create mode 100644 llm_configs/deepseek-v4-flash-high.json create mode 100644 llm_configs/deepseek-v4-flash-max.json create mode 100644 llm_configs/deepseek-v4-pro-high.json create mode 100644 llm_configs/deepseek-v4-pro-max.json create mode 100644 llm_configs/mimo-v2.5-pro.json create mode 100644 llm_configs/mimo-v2.5.json diff --git a/llm_configs/deepseek-v4-flash-high.json b/llm_configs/deepseek-v4-flash-high.json new file mode 100644 index 00000000..bd7a471a --- /dev/null +++ b/llm_configs/deepseek-v4-flash-high.json @@ -0,0 +1,23 @@ +{ + "model_name": "deepseek/deepseek-v4-flash", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "allow", + "order": ["deepseek"], + "require_parameters": true + }, + "reasoning": { + "effort": "high", + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/deepseek-v4-flash-max.json b/llm_configs/deepseek-v4-flash-max.json new file mode 100644 index 00000000..cd08abf3 --- /dev/null +++ b/llm_configs/deepseek-v4-flash-max.json @@ -0,0 +1,23 @@ +{ + "model_name": "deepseek/deepseek-v4-flash", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "allow", + "order": ["deepseek"], + "require_parameters": true + }, + "reasoning": { + "effort": "xhigh", + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/deepseek-v4-pro-high.json b/llm_configs/deepseek-v4-pro-high.json new file mode 100644 index 00000000..9fde30c4 --- /dev/null +++ b/llm_configs/deepseek-v4-pro-high.json @@ -0,0 +1,23 @@ +{ + "model_name": "deepseek/deepseek-v4-pro", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "allow", + "order": ["deepseek"], + "require_parameters": true + }, + "reasoning": { + "effort": "high", + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/deepseek-v4-pro-max.json b/llm_configs/deepseek-v4-pro-max.json new file mode 100644 index 00000000..3add1a7a --- /dev/null +++ b/llm_configs/deepseek-v4-pro-max.json @@ -0,0 +1,23 @@ +{ + "model_name": "deepseek/deepseek-v4-pro", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "allow", + "order": ["deepseek"], + "require_parameters": true + }, + "reasoning": { + "effort": "xhigh", + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/mimo-v2.5-pro.json b/llm_configs/mimo-v2.5-pro.json new file mode 100644 index 00000000..344162b0 --- /dev/null +++ b/llm_configs/mimo-v2.5-pro.json @@ -0,0 +1,22 @@ +{ + "model_name": "xiaomi/mimo-v2.5-pro", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 0.95, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["xiaomi/fp8"], + "require_parameters": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/mimo-v2.5.json b/llm_configs/mimo-v2.5.json new file mode 100644 index 00000000..90e55c3a --- /dev/null +++ b/llm_configs/mimo-v2.5.json @@ -0,0 +1,22 @@ +{ + "model_name": "xiaomi/mimo-v2.5", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 0.95, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["xiaomi/fp8"], + "require_parameters": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/src/ale_bench_eval/calc_cost.py b/src/ale_bench_eval/calc_cost.py index d4cc465c..d695f12a 100644 --- a/src/ale_bench_eval/calc_cost.py +++ b/src/ale_bench_eval/calc_cost.py @@ -145,6 +145,16 @@ "deepseek-v3.1-terminus": ModelPrice(input_mtok=Decimal(27) / Decimal(100), output_mtok=Decimal(1)), "deepseek-v3.2": ModelPrice(input_mtok=Decimal(27) / Decimal(100), output_mtok=Decimal(42) / Decimal(10)), "deepseek-r1-0528": ModelPrice(input_mtok=Decimal(79) / Decimal(100), output_mtok=Decimal(4)), + "deepseek-v4-pro": ModelPrice( + input_mtok=Decimal(174) / Decimal(100), + output_mtok=Decimal(348) / Decimal(100), + cache_read_mtok=Decimal(145) / Decimal(1000), + ), + "deepseek-v4-flash": ModelPrice( + input_mtok=Decimal(14) / Decimal(100), + output_mtok=Decimal(28) / Decimal(100), + cache_read_mtok=Decimal(28) / Decimal(1000), + ), "mimo-v2-flash:free": ModelPrice(input_mtok=Decimal(1) / Decimal(10), output_mtok=Decimal(3) / Decimal(10)), "mimo-v2-pro": ModelPrice( input_mtok=TieredPrices( @@ -160,6 +170,34 @@ tiers=[Tier(start=256000, price=Decimal(4) / Decimal(10))], ), ), + "mimo-v2.5": ModelPrice( + input_mtok=TieredPrices( + base=Decimal(4) / Decimal(10), + tiers=[Tier(start=256000, price=Decimal(8) / Decimal(10))], + ), + output_mtok=TieredPrices( + base=Decimal(2), + tiers=[Tier(start=256000, price=Decimal(4))], + ), + cache_read_mtok=TieredPrices( + base=Decimal(8) / Decimal(100), + tiers=[Tier(start=256000, price=Decimal(16) / Decimal(100))], + ), + ), + "mimo-v2.5-pro": ModelPrice( + input_mtok=TieredPrices( + base=Decimal(1), + tiers=[Tier(start=256000, price=Decimal(2))], + ), + output_mtok=TieredPrices( + base=Decimal(3), + tiers=[Tier(start=256000, price=Decimal(6))], + ), + cache_read_mtok=TieredPrices( + base=Decimal(2) / Decimal(10), + tiers=[Tier(start=256000, price=Decimal(4) / Decimal(10))], + ), + ), "glm-4.5": ModelPrice(input_mtok=Decimal(59) / Decimal(100), output_mtok=Decimal(21) / Decimal(10)), "glm-4.6": ModelPrice( input_mtok=Decimal(6) / Decimal(10), From 89fe19357d95f1718990106d9743dd57c33aaf91 Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Sun, 26 Apr 2026 01:00:00 +0900 Subject: [PATCH 05/12] add gpt-5.5 --- .../deepseek-v4-flash-no-thinking.json | 22 +++++++++++++++++++ llm_configs/deepseek-v4-pro-no-thinking.json | 22 +++++++++++++++++++ llm_configs/gpt-5.5-high.json | 8 +++++++ llm_configs/gpt-5.5-none.json | 8 +++++++ llm_configs/gpt-5.5-xhigh.json | 8 +++++++ src/ale_bench_eval/calc_cost.py | 5 +++++ 6 files changed, 73 insertions(+) create mode 100644 llm_configs/deepseek-v4-flash-no-thinking.json create mode 100644 llm_configs/deepseek-v4-pro-no-thinking.json create mode 100644 llm_configs/gpt-5.5-high.json create mode 100644 llm_configs/gpt-5.5-none.json create mode 100644 llm_configs/gpt-5.5-xhigh.json diff --git a/llm_configs/deepseek-v4-flash-no-thinking.json b/llm_configs/deepseek-v4-flash-no-thinking.json new file mode 100644 index 00000000..b83e152b --- /dev/null +++ b/llm_configs/deepseek-v4-flash-no-thinking.json @@ -0,0 +1,22 @@ +{ + "model_name": "deepseek/deepseek-v4-flash", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "allow", + "order": ["deepseek"], + "require_parameters": true + }, + "reasoning": { + "enabled": false + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/deepseek-v4-pro-no-thinking.json b/llm_configs/deepseek-v4-pro-no-thinking.json new file mode 100644 index 00000000..0d0c462e --- /dev/null +++ b/llm_configs/deepseek-v4-pro-no-thinking.json @@ -0,0 +1,22 @@ +{ + "model_name": "deepseek/deepseek-v4-pro", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "allow", + "order": ["deepseek"], + "require_parameters": true + }, + "reasoning": { + "enabled": false + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/gpt-5.5-high.json b/llm_configs/gpt-5.5-high.json new file mode 100644 index 00000000..cfdfeef2 --- /dev/null +++ b/llm_configs/gpt-5.5-high.json @@ -0,0 +1,8 @@ +{ + "model_name": "gpt-5.5-2026-04-23", + "provider": "openai", + "settings": { + "openai_reasoning_effort": "high", + "openai_reasoning_summary": "detailed" + } +} diff --git a/llm_configs/gpt-5.5-none.json b/llm_configs/gpt-5.5-none.json new file mode 100644 index 00000000..8209c0ce --- /dev/null +++ b/llm_configs/gpt-5.5-none.json @@ -0,0 +1,8 @@ +{ + "model_name": "gpt-5.5-2026-04-23", + "provider": "openai", + "settings": { + "openai_reasoning_effort": "none", + "openai_reasoning_summary": "detailed" + } +} diff --git a/llm_configs/gpt-5.5-xhigh.json b/llm_configs/gpt-5.5-xhigh.json new file mode 100644 index 00000000..1d6d93d1 --- /dev/null +++ b/llm_configs/gpt-5.5-xhigh.json @@ -0,0 +1,8 @@ +{ + "model_name": "gpt-5.5-2026-04-23", + "provider": "openai", + "settings": { + "openai_reasoning_effort": "xhigh", + "openai_reasoning_summary": "detailed" + } +} diff --git a/src/ale_bench_eval/calc_cost.py b/src/ale_bench_eval/calc_cost.py index d695f12a..665f4faf 100644 --- a/src/ale_bench_eval/calc_cost.py +++ b/src/ale_bench_eval/calc_cost.py @@ -5,6 +5,11 @@ from pydantic_ai.usage import RunUsage FALLBACK_DICT = { + "gpt-5.5-2026-04-23": ModelPrice( + input_mtok=Decimal(5), + cache_read_mtok=Decimal(5) / Decimal(10), + output_mtok=Decimal(30), + ), "gpt-5.4-nano-2026-03-17": ModelPrice( input_mtok=Decimal(2) / Decimal(10), cache_read_mtok=Decimal(2) / Decimal(100), From a731047e5e072d7d48e4d0548c309c90b890d032 Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Mon, 27 Apr 2026 17:33:51 +0900 Subject: [PATCH 06/12] add gpt-5.5-medium --- llm_configs/gpt-5.5-medium.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 llm_configs/gpt-5.5-medium.json diff --git a/llm_configs/gpt-5.5-medium.json b/llm_configs/gpt-5.5-medium.json new file mode 100644 index 00000000..db4c02c9 --- /dev/null +++ b/llm_configs/gpt-5.5-medium.json @@ -0,0 +1,8 @@ +{ + "model_name": "gpt-5.5-2026-04-23", + "provider": "openai", + "settings": { + "openai_reasoning_effort": "medium", + "openai_reasoning_summary": "detailed" + } +} From fdd4402c0bf2c4346b1a318246f7469da611c4c6 Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Tue, 28 Apr 2026 09:52:12 +0900 Subject: [PATCH 07/12] add qwen3.6-flash --- llm_configs/qwen3.6-flash.json | 21 +++++++++++++++++++++ src/ale_bench_eval/calc_cost.py | 10 ++++++++++ 2 files changed, 31 insertions(+) create mode 100644 llm_configs/qwen3.6-flash.json diff --git a/llm_configs/qwen3.6-flash.json b/llm_configs/qwen3.6-flash.json new file mode 100644 index 00000000..2a7b27a0 --- /dev/null +++ b/llm_configs/qwen3.6-flash.json @@ -0,0 +1,21 @@ +{ + "model_name": "qwen/qwen3.6-flash", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["alibaba"], + "require_parameters": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/src/ale_bench_eval/calc_cost.py b/src/ale_bench_eval/calc_cost.py index 665f4faf..f626737a 100644 --- a/src/ale_bench_eval/calc_cost.py +++ b/src/ale_bench_eval/calc_cost.py @@ -332,6 +332,16 @@ input_mtok=TieredPrices(base=Decimal(5) / Decimal(10), tiers=[Tier(start=256000, price=Decimal(2))]), output_mtok=TieredPrices(base=Decimal(3), tiers=[Tier(start=256000, price=Decimal(6))]), ), + "qwen3.6-flash": ModelPrice( + input_mtok=TieredPrices(base=Decimal(25) / Decimal(100), tiers=[Tier(start=256000, price=Decimal(1))]), + output_mtok=TieredPrices(base=Decimal(15) / Decimal(10), tiers=[Tier(start=256000, price=Decimal(4))]), + cache_read_mtok=TieredPrices( + base=Decimal(3125) / Decimal(10000), tiers=[Tier(start=256000, price=Decimal(125) / Decimal(100))] + ), + cache_write_mtok=TieredPrices( + base=Decimal(25) / Decimal(1000), tiers=[Tier(start=256000, price=Decimal(1) / Decimal(10))] + ), + ), "Qwen3.6-35B-A3B": ModelPrice(input_mtok=Decimal(25) / Decimal(100), output_mtok=Decimal(2)), } From 6c1b9b58099db9db16ec42219bfc6a8f9fa563ba Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Fri, 1 May 2026 10:44:12 +0900 Subject: [PATCH 08/12] add grok-4.3 --- llm_configs/grok-4.3.json | 15 +++++++++++++++ src/ale_bench_eval/calc_cost.py | 9 +++++++++ 2 files changed, 24 insertions(+) create mode 100644 llm_configs/grok-4.3.json diff --git a/llm_configs/grok-4.3.json b/llm_configs/grok-4.3.json new file mode 100644 index 00000000..f037a6ed --- /dev/null +++ b/llm_configs/grok-4.3.json @@ -0,0 +1,15 @@ +{ + "model_name": "x-ai/grok-4.3", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "extra_body": { + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/src/ale_bench_eval/calc_cost.py b/src/ale_bench_eval/calc_cost.py index f626737a..b247c5ec 100644 --- a/src/ale_bench_eval/calc_cost.py +++ b/src/ale_bench_eval/calc_cost.py @@ -140,6 +140,15 @@ cache_read_mtok=Decimal(2) / Decimal(10), output_mtok=TieredPrices(base=Decimal(6), tiers=[Tier(start=200000, price=Decimal(12))]), ), + "grok-4.3": ModelPrice( + input_mtok=TieredPrices( + base=Decimal(125) / Decimal(100), tiers=[Tier(start=200000, price=Decimal(25) / Decimal(10))] + ), + cache_read_mtok=TieredPrices( + base=Decimal(2) / Decimal(10), tiers=[Tier(start=200000, price=Decimal(4) / Decimal(10))] + ), + output_mtok=TieredPrices(base=Decimal(25) / Decimal(10), tiers=[Tier(start=200000, price=Decimal(5))]), + ), "nova-premier-v1": ModelPrice( input_mtok=Decimal(25) / Decimal(10), cache_read_mtok=Decimal(625) / Decimal(1000), From 7e3f222d12dc0c19e18b01471da72c2be3f705c8 Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Thu, 7 May 2026 02:50:46 +0900 Subject: [PATCH 09/12] add mistral-medium-3.5 --- llm_configs/mistral-medium-3.5.json | 23 +++++++++++++++++++++++ src/ale_bench_eval/calc_cost.py | 1 + 2 files changed, 24 insertions(+) create mode 100644 llm_configs/mistral-medium-3.5.json diff --git a/llm_configs/mistral-medium-3.5.json b/llm_configs/mistral-medium-3.5.json new file mode 100644 index 00000000..2dfe5bb4 --- /dev/null +++ b/llm_configs/mistral-medium-3.5.json @@ -0,0 +1,23 @@ +{ + "model_name": "mistralai/mistral-medium-3-5", + "provider": "openrouter", + "settings": { + "temperature": 0.7, + "top_p": 0.95, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["mistral"], + "require_parameters": true + }, + "reasoning": { + "effort": "high", + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/src/ale_bench_eval/calc_cost.py b/src/ale_bench_eval/calc_cost.py index b247c5ec..0358693a 100644 --- a/src/ale_bench_eval/calc_cost.py +++ b/src/ale_bench_eval/calc_cost.py @@ -246,6 +246,7 @@ "mistral-medium-3.1": ModelPrice(input_mtok=Decimal(4) / Decimal(10), output_mtok=Decimal(2)), "mistral-large-2512": ModelPrice(input_mtok=Decimal(5) / Decimal(10), output_mtok=Decimal(15) / Decimal(10)), "mistral-small-2603": ModelPrice(input_mtok=Decimal(15) / Decimal(100), output_mtok=Decimal(6) / Decimal(10)), + "mistral-medium-3-5": ModelPrice(input_mtok=Decimal(15) / Decimal(10), output_mtok=Decimal(75) / Decimal(10)), "kimi-k2": ModelPrice( input_mtok=Decimal(6) / Decimal(10), output_mtok=Decimal(25) / Decimal(10), From 233a3ba27dff9beed24413cff134d5ec3d80e56e Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Wed, 20 May 2026 07:08:41 +0900 Subject: [PATCH 10/12] add gemini 3.5 flash --- llm_configs/gemini-3.5-flash-high.json | 10 ++++++++++ src/ale_bench_eval/calc_cost.py | 5 +++++ 2 files changed, 15 insertions(+) create mode 100644 llm_configs/gemini-3.5-flash-high.json diff --git a/llm_configs/gemini-3.5-flash-high.json b/llm_configs/gemini-3.5-flash-high.json new file mode 100644 index 00000000..7ba223e7 --- /dev/null +++ b/llm_configs/gemini-3.5-flash-high.json @@ -0,0 +1,10 @@ +{ + "model_name": "gemini-3.5-flash", + "provider": "google", + "settings": { + "google_thinking_config": { + "include_thoughts": true, + "thinking_level": "high" + } + } +} diff --git a/src/ale_bench_eval/calc_cost.py b/src/ale_bench_eval/calc_cost.py index 0358693a..1ba4e816 100644 --- a/src/ale_bench_eval/calc_cost.py +++ b/src/ale_bench_eval/calc_cost.py @@ -45,6 +45,11 @@ cache_read_mtok=Decimal(125) / Decimal(1000), output_mtok=Decimal(10), ), + "gemini-3.5-flash": ModelPrice( + input_mtok=Decimal(15) / Decimal(10), + cache_read_mtok=Decimal(15) / Decimal(100), + output_mtok=Decimal(9), + ), "gemini-3-flash-preview": ModelPrice( input_mtok=Decimal(5) / Decimal(10), cache_read_mtok=Decimal(5) / Decimal(100), From 995ef280305ca532ca13574ee6b50029f620990d Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Mon, 25 May 2026 00:02:33 +0900 Subject: [PATCH 11/12] add qwen3.7-max --- llm_configs/qwen3.7-max.json | 21 +++++++++++++++++++++ src/ale_bench_eval/calc_cost.py | 6 ++++++ 2 files changed, 27 insertions(+) create mode 100644 llm_configs/qwen3.7-max.json diff --git a/llm_configs/qwen3.7-max.json b/llm_configs/qwen3.7-max.json new file mode 100644 index 00000000..058e329e --- /dev/null +++ b/llm_configs/qwen3.7-max.json @@ -0,0 +1,21 @@ +{ + "model_name": "qwen/qwen3.7-max", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["alibaba"], + "require_parameters": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/src/ale_bench_eval/calc_cost.py b/src/ale_bench_eval/calc_cost.py index 1ba4e816..82e383f0 100644 --- a/src/ale_bench_eval/calc_cost.py +++ b/src/ale_bench_eval/calc_cost.py @@ -358,6 +358,12 @@ ), ), "Qwen3.6-35B-A3B": ModelPrice(input_mtok=Decimal(25) / Decimal(100), output_mtok=Decimal(2)), + "qwen3.7-max": ModelPrice( + input_mtok=Decimal(25) / Decimal(10), + output_mtok=Decimal(75) / Decimal(10), + cache_read_mtok=Decimal(25) / Decimal(100), + cache_write_mtok=Decimal(3125) / Decimal(1000), + ), } From 17686a79eeba78314db1f88e6cf5be1b4fe8aa42 Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Fri, 29 May 2026 11:11:40 +0900 Subject: [PATCH 12/12] add claude 4.8 opus --- llm_configs/claude-4.8-opus-no-thinking.json | 10 ++++++++++ src/ale_bench_eval/calc_cost.py | 6 ++++++ 2 files changed, 16 insertions(+) create mode 100644 llm_configs/claude-4.8-opus-no-thinking.json diff --git a/llm_configs/claude-4.8-opus-no-thinking.json b/llm_configs/claude-4.8-opus-no-thinking.json new file mode 100644 index 00000000..30120439 --- /dev/null +++ b/llm_configs/claude-4.8-opus-no-thinking.json @@ -0,0 +1,10 @@ +{ + "model_name": "claude-opus-4-8", + "provider": "anthropic", + "settings": { + "max_tokens": 128000, + "anthropic_thinking": { + "type": "disabled" + } + } +} diff --git a/src/ale_bench_eval/calc_cost.py b/src/ale_bench_eval/calc_cost.py index 82e383f0..3b3e1ac8 100644 --- a/src/ale_bench_eval/calc_cost.py +++ b/src/ale_bench_eval/calc_cost.py @@ -135,6 +135,12 @@ cache_read_mtok=Decimal(5) / Decimal(10), output_mtok=Decimal(25), ), + "claude-opus-4-8": ModelPrice( + input_mtok=Decimal(5), + cache_write_mtok=Decimal(625) / Decimal(100), + cache_read_mtok=Decimal(5) / Decimal(10), + output_mtok=Decimal(25), + ), "grok-4.1-fast": ModelPrice( input_mtok=Decimal(2) / Decimal(10), cache_read_mtok=Decimal(5) / Decimal(100),