Add models

llm_configs/claude-4.6-opus-medium.json

@@ -0,0 +1,12 @@
+{
+    "model_name": "claude-opus-4-6",
+    "provider": "anthropic",
+    "settings": {
+        "max_tokens": 128000,
+        "anthropic_thinking": {
+            "type": "adaptive",
+            "display": "summarized"
+        },
+        "anthropic_effort": "medium"
+    }
+}

llm_configs/claude-4.6-opus-no-thinking.json

@@ -0,0 +1,10 @@
+{
+    "model_name": "claude-opus-4-6",
+    "provider": "anthropic",
+    "settings": {
+        "max_tokens": 128000,
+        "anthropic_thinking": {
+            "type": "disabled"
+        }
+    }
+}

llm_configs/claude-4.6-sonnet-high.json → llm_configs/claude-4.6-sonnet-medium.json

@@ -8,7 +8,7 @@
             "provider": {
                 "allow_fallbacks": false,
                 "data_collection": "deny",
-                "order": ["anthropic"],
+                "order": ["anthropic", "google-vertex", "google-vertex/us-east5", "amazon-bedrock", "azure"],
                 "require_parameters": true
             },
             "reasoning": {
@@ -17,7 +17,7 @@
             "usage": {
                 "include": true
             },
-            "verbosity": "high"
+            "verbosity": "medium"
         }
     }
 }

llm_configs/claude-4.6-opus-max.json → llm_configs/gemma-4-26b-a4b-it.json

@@ -1,23 +1,23 @@
 {
-    "model_name": "anthropic/claude-opus-4.6",
+    "model_name": "google/gemma-4-26b-a4b-it",
     "provider": "openrouter",
     "settings": {
         "temperature": 1.0,
-        "max_tokens": 128000,
+        "top_p": 0.95,
         "extra_body": {
+            "top_k": 64,
             "provider": {
                 "allow_fallbacks": false,
                 "data_collection": "deny",
-                "order": ["anthropic"],
+                "order": ["parasail/bf16"],
                 "require_parameters": true
             },
             "reasoning": {
                 "enabled": true
             },
             "usage": {
                 "include": true
-            },
-            "verbosity": "max"
+            }
         }
     }
 }

llm_configs/claude-4.6-opus-high.json → llm_configs/glm-5.1.json

@@ -1,23 +1,22 @@
 {
-    "model_name": "anthropic/claude-opus-4.6",
+    "model_name": "z-ai/glm-5.1",
     "provider": "openrouter",
     "settings": {
         "temperature": 1.0,
-        "max_tokens": 128000,
+        "top_p": 0.95,
         "extra_body": {
             "provider": {
                 "allow_fallbacks": false,
                 "data_collection": "deny",
-                "order": ["anthropic"],
+                "order": ["z-ai"],
                 "require_parameters": true
             },
             "reasoning": {
                 "enabled": true
             },
             "usage": {
                 "include": true
-            },
-            "verbosity": "high"
+            }
         }
     }
 }

llm_configs/gpt-5.4-medium.json

@@ -0,0 +1,8 @@
+{
+    "model_name": "gpt-5.4-2026-03-05",
+    "provider": "openai",
+    "settings": {
+        "openai_reasoning_effort": "medium",
+        "openai_reasoning_summary": "detailed"
+    }
+}

llm_configs/nemotron-3-super.json

@@ -0,0 +1,22 @@
+{
+    "model_name": "nvidia/nemotron-3-super-120b-a12b:free",
+    "provider": "openrouter",
+    "settings": {
+        "temperature": 1.0,
+        "top_p": 0.95,
+        "extra_body": {
+            "provider": {
+                "allow_fallbacks": false,
+                "data_collection": "allow",
+                "order": ["nvidia"],
+                "require_parameters": true
+            },
+            "reasoning": {
+                "enabled": true
+            },
+            "usage": {
+                "include": true
+            }
+        }
+    }
+}

llm_configs/qwen3.5-35b-a3b.json

@@ -0,0 +1,23 @@
+{
+    "model_name": "qwen/qwen3.5-35b-a3b",
+    "provider": "openrouter",
+    "settings": {
+        "temperature": 0.6,
+        "top_p": 0.95,
+        "extra_body": {
+            "presence_penalty": 0.0,
+            "provider": {
+                "allow_fallbacks": false,
+                "data_collection": "deny",
+                "order": ["alibaba"],
+                "require_parameters": true
+            },
+            "reasoning": {
+                "enabled": true
+            },
+            "usage": {
+                "include": true
+            }
+        }
+    }
+}

llm_configs/claude-4.6-sonnet-max.json → llm_configs/qwen3.6-plus.json

@@ -1,23 +1,22 @@
 {
-    "model_name": "anthropic/claude-sonnet-4.6",
+    "model_name": "qwen/qwen3.6-plus",
     "provider": "openrouter",
     "settings": {
         "temperature": 1.0,
-        "max_tokens": 128000,
+        "top_p": 0.95,
         "extra_body": {
             "provider": {
                 "allow_fallbacks": false,
                 "data_collection": "deny",
-                "order": ["anthropic"],
+                "order": ["alibaba"],
                 "require_parameters": true
             },
             "reasoning": {
                 "enabled": true
             },
             "usage": {
                 "include": true
-            },
-            "verbosity": "max"
+            }
         }
     }
 }

pyproject.toml

@@ -46,8 +46,8 @@ dependencies = [
 dev = [
     "pytest>=9.0.2",
     "pytest-mock>=3.15.1",
-    "ruff>=0.15.2",
-    "ty>=0.0.17",
+    "ruff>=0.15.9",
+    "ty>=0.0.28",
 ]
 
 [project.urls]
@@ -63,6 +63,9 @@ eval = [
     "pydantic-ai>=1.30.0",
 ]
 
+[tool.uv]
+exclude-newer = "1 week"
+
 [tool.uv-dynamic-versioning]
 fallback-version = "0.0.0"
 

src/ale_bench_eval/calc_cost.py

@@ -58,6 +58,7 @@
         cache_read_mtok=Decimal(25) / Decimal(1000),
         output_mtok=Decimal(15) / Decimal(10),
     ),
+    "gemma-4-26b-a4b-it": ModelPrice(input_mtok=Decimal(13) / Decimal(100), output_mtok=Decimal(4) / Decimal(10)),
     "claude-sonnet-4": ModelPrice(
         input_mtok=TieredPrices(base=Decimal(3), tiers=[Tier(start=200000, price=Decimal(6))]),
         cache_write_mtok=TieredPrices(
@@ -97,22 +98,16 @@
         output_mtok=Decimal(25),
     ),
     "claude-opus-4.6": ModelPrice(
-        input_mtok=TieredPrices(base=Decimal(5), tiers=[Tier(start=200000, price=Decimal(10))]),
-        cache_write_mtok=TieredPrices(
-            base=Decimal(625) / Decimal(100), tiers=[Tier(start=200000, price=Decimal(125) / Decimal(10))]
-        ),
-        cache_read_mtok=TieredPrices(base=Decimal(5) / Decimal(10), tiers=[Tier(start=200000, price=Decimal(1))]),
-        output_mtok=TieredPrices(base=Decimal(25), tiers=[Tier(start=200000, price=Decimal(375) / Decimal(10))]),
+        input_mtok=Decimal(5),
+        cache_write_mtok=Decimal(625) / Decimal(100),
+        cache_read_mtok=Decimal(5) / Decimal(10),
+        output_mtok=Decimal(25),
     ),
     "claude-sonnet-4.6": ModelPrice(
-        input_mtok=TieredPrices(base=Decimal(3), tiers=[Tier(start=200000, price=Decimal(6))]),
-        cache_write_mtok=TieredPrices(
-            base=Decimal(375) / Decimal(100), tiers=[Tier(start=200000, price=Decimal(75) / Decimal(10))]
-        ),
-        cache_read_mtok=TieredPrices(
-            base=Decimal(3) / Decimal(10), tiers=[Tier(start=200000, price=Decimal(6) / Decimal(10))]
-        ),
-        output_mtok=TieredPrices(base=Decimal(15), tiers=[Tier(start=200000, price=Decimal(225) / Decimal(10))]),
+        input_mtok=Decimal(3),
+        cache_write_mtok=Decimal(375) / Decimal(100),
+        cache_read_mtok=Decimal(3) / Decimal(10),
+        output_mtok=Decimal(15),
     ),
     "grok-4.1-fast": ModelPrice(
         input_mtok=Decimal(2) / Decimal(10),
@@ -165,6 +160,11 @@
         output_mtok=Decimal(32) / Decimal(10),
         cache_read_mtok=Decimal(2) / Decimal(10),
     ),
+    "glm-5.1": ModelPrice(
+        input_mtok=Decimal(14) / Decimal(10),
+        output_mtok=Decimal(44) / Decimal(10),
+        cache_read_mtok=Decimal(26) / Decimal(100),
+    ),
     "glm-5-turbo": ModelPrice(
         input_mtok=Decimal(12) / Decimal(10),
         output_mtok=Decimal(4),
@@ -214,6 +214,11 @@
         output_mtok=Decimal(12) / Decimal(10),
         cache_read_mtok=Decimal(6) / Decimal(100),
     ),
+    "nemotron-3-super-120b-a12b:free": ModelPrice(
+        input_mtok=Decimal(1) / Decimal(10),
+        output_mtok=Decimal(5) / Decimal(10),
+        cache_read_mtok=Decimal(1) / Decimal(10),
+    ),
     "qwen3-235b-a22b-thinking-2507": ModelPrice(
         input_mtok=Decimal(3) / Decimal(10), output_mtok=Decimal(29) / Decimal(10)
     ),
@@ -242,6 +247,7 @@
     "qwen3-next-80b-a3b-thinking": ModelPrice(
         input_mtok=Decimal(14) / Decimal(100), output_mtok=Decimal(14) / Decimal(10)
     ),
+    "qwen3.5-35b-a3b": ModelPrice(input_mtok=Decimal(25) / Decimal(100), output_mtok=Decimal(2)),
     "qwen3.5-397b-a17b": ModelPrice(input_mtok=Decimal(6) / Decimal(10), output_mtok=Decimal(36) / Decimal(10)),
     "qwen3.5-plus-02-15": ModelPrice(
         input_mtok=TieredPrices(
@@ -252,6 +258,10 @@
         ),
     ),
     "qwen3.5-flash-02-23": ModelPrice(input_mtok=Decimal(1) / Decimal(10), output_mtok=Decimal(4) / Decimal(10)),
+    "qwen3.6-plus": ModelPrice(
+        input_mtok=TieredPrices(base=Decimal(5) / Decimal(10), tiers=[Tier(start=256000, price=Decimal(2))]),
+        output_mtok=TieredPrices(base=Decimal(3), tiers=[Tier(start=256000, price=Decimal(6))]),
+    ),
 }
 
 

src/ale_bench_eval/scaffolds.py

@@ -17,7 +17,7 @@
 from ale_bench_eval.safe_generation import MaxTokenError, safe_generation
 from ale_bench_eval.selection import get_worst_score
 
-TIMEOUT_SECONDS = 1200
+TIMEOUT_SECONDS = 3600
 MAX_RETRIES = 30