{ "v1": { "models": [ "Claude Fable 5", "Claude Opus 4.8", "GPT-5.5", "Claude Opus 4.7", "Kimi K2.7", "GLM 5.2", "Kimi K2.6", "GPT-5.4-mini", "Gemini 3.1 Pro", "Claude Sonnet 4.6", "Kimi K2.5", "Qwen 3.6 Plus", "MiniMax M2.7", "MiniMax M2.5", "Gemini 3.1 Flash Lite" ], "colors": { "Claude Fable 5": "#d97757", "Claude Opus 4.8": "#d97757", "GPT-5.5": "#10a37f", "Claude Opus 4.7": "#d97757", "Kimi K2.7": "#7c3aed", "GLM 5.2": "#14b8a6", "Kimi K2.6": "#7c3aed", "GPT-5.4-mini": "#10a37f", "Gemini 3.1 Pro": "#1a73e8", "Claude Sonnet 4.6": "#d97757", "Kimi K2.5": "#7c3aed", "Qwen 3.6 Plus": "#ff8800", "MiniMax M2.7": "#e5484d", "MiniMax M2.5": "#e5484d", "Gemini 3.1 Flash Lite": "#1a73e8" }, "lab_colors": { "Anthropic": "#d97757", "OpenAI": "#10a37f", "Moonshot": "#7c3aed", "Zhipu": "#14b8a6", "Google": "#1a73e8", "Alibaba": "#ff8800", "MiniMax": "#e5484d", "NVIDIA": "#76b900" }, "harness": { "Claude Fable 5": "claude-code", "Claude Opus 4.7": "claude-code", "Claude Opus 4.8": "claude-code", "Claude Sonnet 4.6": "claude-code", "GLM 5.2": "mini-swe-agent", "GPT-5.4-mini": "codex", "GPT-5.5": "codex", "Gemini 3.1 Flash Lite": "gemini-cli", "Gemini 3.1 Pro": "gemini-cli", "Kimi K2.5": "mini-swe-agent", "Kimi K2.6": "mini-swe-agent", "Kimi K2.7": "mini-swe-agent", "MiniMax M2.5": "mini-swe-agent", "MiniMax M2.7": "mini-swe-agent", "Qwen 3.6 Plus": "mini-swe-agent" }, "efforts": { "Claude Fable 5": [ "low", "medium", "high", "xhigh", "max" ], "Claude Opus 4.8": [ "low", "medium", "high", "xhigh", "max" ], "GPT-5.5": [ "low", "medium", "high", "xhigh" ], "Claude Opus 4.7": [ "low", "medium", "high", "xhigh", "max" ], "Kimi K2.7": [ "none" ], "GLM 5.2": [ "none" ], "Kimi K2.6": [ "none" ], "GPT-5.4-mini": [ "low", "medium", "high", "xhigh" ], "Gemini 3.1 Pro": [ "low", "high" ], "Claude Sonnet 4.6": [ "low", "medium", "high", "max" ], "Kimi K2.5": [ "none" ], "Qwen 3.6 Plus": [ "none" ], "MiniMax M2.7": [ "none" ], "MiniMax M2.5": [ "none" ], "Gemini 3.1 Flash Lite": [ "minimal", "low", "medium", "high" ] }, "subsets": { "main": 100, "extended": 150 }, "data": { "Claude Fable 5": { "high": { "main": { "correct": 0.462, "new_score": 0.433, "tokens": 34776.9, "cost": 8.2, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.633, "new_score": 0.594, "tokens": 29252.9, "cost": 6.8, "tool_calls": null, "steps": null, "ote": null } }, "low": { "main": { "correct": 0.405, "new_score": 0.376, "tokens": 21262.2, "cost": 4.6, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.59, "new_score": 0.547, "tokens": 17747.4, "cost": 3.8, "tool_calls": null, "steps": null, "ote": null } }, "max": { "main": { "correct": 0.477, "new_score": 0.451, "tokens": 65654.2, "cost": 16, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.638, "new_score": 0.6, "tokens": 56060.3, "cost": 13.2, "tool_calls": null, "steps": null, "ote": null } }, "medium": { "main": { "correct": 0.443, "new_score": 0.415, "tokens": 27155.5, "cost": 6, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.614, "new_score": 0.574, "tokens": 23104.1, "cost": 5, "tool_calls": null, "steps": null, "ote": null } }, "xhigh": { "main": { "correct": 0.493, "new_score": 0.468, "tokens": 45395.6, "cost": 10.6, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.653, "new_score": 0.616, "tokens": 38379.8, "cost": 8.8, "tool_calls": null, "steps": null, "ote": null } } }, "Claude Opus 4.8": { "high": { "main": { "correct": 0.3295, "new_score": 0.3031, "tokens": 37279.9, "cost": 3.9446, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.5277, "new_score": 0.4876, "tokens": 31049.5, "cost": 3.233, "tool_calls": null, "steps": null, "ote": null } }, "low": { "main": { "correct": 0.2767, "new_score": 0.2528, "tokens": 23239, "cost": 2.6811, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.4678, "new_score": 0.4306, "tokens": 19342.3, "cost": 2.2, "tool_calls": null, "steps": null, "ote": null } }, "max": { "main": { "correct": 0.341, "new_score": 0.3129, "tokens": 88168.6, "cost": 9.0344, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.5393, "new_score": 0.4989, "tokens": 75826.1, "cost": 7.5247, "tool_calls": null, "steps": null, "ote": null } }, "medium": { "main": { "correct": 0.294, "new_score": 0.2689, "tokens": 31700.5, "cost": 3.3, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.4947, "new_score": 0.4549, "tokens": 26579.6, "cost": 2.7449, "tool_calls": null, "steps": null, "ote": null } }, "xhigh": { "main": { "correct": 0.373, "new_score": 0.3427, "tokens": 62444.8, "cost": 6.6881, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.5607, "new_score": 0.5179, "tokens": 51949.3, "cost": 5.405, "tool_calls": null, "steps": null, "ote": null } } }, "GPT-5.5": { "high": { "main": { "correct": 0.281, "new_score": 0.2529, "tokens": 19002, "cost": 4.3815, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.49, "new_score": 0.4476, "tokens": 16209.7, "cost": 3.6346, "tool_calls": null, "steps": null, "ote": null } }, "low": { "main": { "correct": 0.2391, "new_score": 0.2114, "tokens": 8481.9, "cost": 1.6113, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.4273, "new_score": 0.3873, "tokens": 7409.9, "cost": 1.4, "tool_calls": null, "steps": null, "ote": null } }, "medium": { "main": { "correct": 0.248, "new_score": 0.2242, "tokens": 13911.7, "cost": 3.2931, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.464, "new_score": 0.4229, "tokens": 11940.3, "cost": 2.77, "tool_calls": null, "steps": null, "ote": null } }, "xhigh": { "main": { "correct": 0.2825, "new_score": 0.2548, "tokens": 25575.9, "cost": 5.5911, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.4903, "new_score": 0.4467, "tokens": 21819.9, "cost": 4.6186, "tool_calls": null, "steps": null, "ote": null } } }, "Claude Opus 4.7": { "high": { "main": { "correct": 0.2435, "new_score": 0.2207, "tokens": 28180.3, "cost": 4.4297, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.4597, "new_score": 0.4225, "tokens": 23585.6, "cost": 3.6482, "tool_calls": null, "steps": null, "ote": null } }, "low": { "main": { "correct": 0.1987, "new_score": 0.1791, "tokens": 16932.1, "cost": 2.4262, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.4004, "new_score": 0.3679, "tokens": 14319.1, "cost": 2.0425, "tool_calls": null, "steps": null, "ote": null } }, "max": { "main": { "correct": 0.262, "new_score": 0.238, "tokens": 43092.1, "cost": 7.8, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.475, "new_score": 0.439, "tokens": 36919, "cost": 6.6, "tool_calls": null, "steps": null, "ote": null } }, "medium": { "main": { "correct": 0.2355, "new_score": 0.213, "tokens": 20038.7, "cost": 2.8502, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.4467, "new_score": 0.4098, "tokens": 16946.1, "cost": 2.3613, "tool_calls": null, "steps": null, "ote": null } }, "xhigh": { "main": { "correct": 0.253, "new_score": 0.2296, "tokens": 35076.8, "cost": 5.9308, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.4697, "new_score": 0.4324, "tokens": 29750.8, "cost": 4.928, "tool_calls": null, "steps": null, "ote": null } } }, "Kimi K2.7": { "none": { "main": { "correct": 0.243, "new_score": 0.22, "tokens": 46554.2, "cost": 3.1, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.437, "new_score": 0.399, "tokens": 40415.9, "cost": 2.8, "tool_calls": null, "steps": null, "ote": null } } }, "GLM 5.2": { "none": { "main": { "correct": 0.212, "new_score": 0.192, "tokens": 26171.4, "cost": 3.8, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.428, "new_score": 0.39, "tokens": 23074.7, "cost": 3.2, "tool_calls": null, "steps": null, "ote": null } } }, "Kimi K2.6": { "none": { "main": { "correct": 0.18, "new_score": 0.1604, "tokens": 41781.8, "cost": 2.2, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.4067, "new_score": 0.3701, "tokens": 35050.1, "cost": 1.8, "tool_calls": null, "steps": null, "ote": null } } }, "GPT-5.4-mini": { "high": { "main": { "correct": 0.1395, "new_score": 0.1245, "tokens": 45131.4, "cost": 1.0537, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.3447, "new_score": 0.3123, "tokens": 38178.5, "cost": 0.897, "tool_calls": null, "steps": null, "ote": null } }, "low": { "main": { "correct": 0.054, "new_score": 0.047, "tokens": 11463.8, "cost": 0.4, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.194, "new_score": 0.174, "tokens": 10242.6, "cost": 0.3, "tool_calls": null, "steps": null, "ote": null } }, "medium": { "main": { "correct": 0.137, "new_score": 0.1205, "tokens": 34719.7, "cost": 0.8851, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.3219, "new_score": 0.2889, "tokens": 29705.9, "cost": 0.7602, "tool_calls": null, "steps": null, "ote": null } }, "xhigh": { "main": { "correct": 0.2, "new_score": 0.1782, "tokens": 88550.1, "cost": 1.6706, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.3987, "new_score": 0.36, "tokens": 73573.6, "cost": 1.3899, "tool_calls": null, "steps": null, "ote": null } } }, "Gemini 3.1 Pro": { "high": { "main": { "correct": 0.185, "new_score": 0.1668, "tokens": 18997.2, "cost": 2.4112, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.3633, "new_score": 0.3307, "tokens": 17049.7, "cost": 2.2139, "tool_calls": null, "steps": null, "ote": null } }, "low": { "main": { "correct": 0.1867, "new_score": 0.165, "tokens": 19662.3, "cost": 2.5151, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.38, "new_score": 0.3423, "tokens": 17712, "cost": 2.2618, "tool_calls": null, "steps": null, "ote": null } } }, "Claude Sonnet 4.6": { "high": { "main": { "correct": 0.1685, "new_score": 0.1507, "tokens": 34333.9, "cost": 2.4053, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.3683, "new_score": 0.3356, "tokens": 29774.2, "cost": 2.04, "tool_calls": null, "steps": null, "ote": null } }, "low": { "main": { "correct": 0.104, "new_score": 0.092, "tokens": 18548.8, "cost": 1.6, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.283, "new_score": 0.257, "tokens": 15820.7, "cost": 1.4, "tool_calls": null, "steps": null, "ote": null } }, "max": { "main": { "correct": 0.15, "new_score": 0.1317, "tokens": 38003.6, "cost": 2.5742, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.35, "new_score": 0.317, "tokens": 32619.3, "cost": 2.1498, "tool_calls": null, "steps": null, "ote": null } }, "medium": { "main": { "correct": 0.155, "new_score": 0.1363, "tokens": 28176.2, "cost": 2.0873, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.3563, "new_score": 0.3222, "tokens": 24003.4, "cost": 1.775, "tool_calls": null, "steps": null, "ote": null } } }, "Kimi K2.5": { "none": { "main": { "correct": 0.075, "new_score": 0.069, "tokens": 19105.8, "cost": 2.5531, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.25, "new_score": 0.2269, "tokens": 17738.2, "cost": 2.3582, "tool_calls": null, "steps": null, "ote": null } } }, "Qwen 3.6 Plus": { "none": { "main": { "correct": 0.108, "new_score": 0.097, "tokens": 51307.9, "cost": 1.4, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.235, "new_score": 0.213, "tokens": 48885.1, "cost": 1.3, "tool_calls": null, "steps": null, "ote": null } } }, "MiniMax M2.7": { "none": { "main": { "correct": 0.067, "new_score": 0.06, "tokens": 30587.4, "cost": 0.6, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.22, "new_score": 0.1986, "tokens": 28445.8, "cost": 0.5, "tool_calls": null, "steps": null, "ote": null } } }, "MiniMax M2.5": { "none": { "main": { "correct": 0.06, "new_score": 0.0534, "tokens": 21664.7, "cost": 0.3, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.1733, "new_score": 0.1578, "tokens": 19962.4, "cost": 0.3, "tool_calls": null, "steps": null, "ote": null } } }, "Gemini 3.1 Flash Lite": { "high": { "main": { "correct": 0.05, "new_score": 0.0444, "tokens": 14704.2, "cost": 0.1792, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.1556, "new_score": 0.1389, "tokens": 13908.7, "cost": 0.1759, "tool_calls": null, "steps": null, "ote": null } }, "low": { "main": { "correct": 0.0533, "new_score": 0.0484, "tokens": 13554.7, "cost": 0.1711, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.1622, "new_score": 0.146, "tokens": 13112.1, "cost": 0.1729, "tool_calls": null, "steps": null, "ote": null } }, "medium": { "main": { "correct": 0.02, "new_score": 0.0184, "tokens": 10024.2, "cost": 0.2586, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.12, "new_score": 0.106, "tokens": 9538.7, "cost": 0.239, "tool_calls": null, "steps": null, "ote": null } }, "minimal": { "main": { "correct": 0.027, "new_score": 0.025, "tokens": 11462.6, "cost": 0.3694, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.101, "new_score": 0.0905, "tokens": 10474.9, "cost": 0.3219, "tool_calls": null, "steps": null, "ote": null } } } } }, "v1_1": { "models": [ "SWE-2", "Claude Fable 5", "Claude Fable 5.1", "GPT-5.6 Sol", "GPT-6 Astra", "Claude Opus 4.8", "GPT-5.5", "Grok 4.5", "Grok 4.6", "Claude Sonnet 5", "GPT-5.6 Terra", "GPT-5.6 Luna", "SWE-1.7", "Claude Opus 4.7", "Kimi K2.7", "Composer 2.5", "GLM 5.2", "DeepSeek V4 Pro", "MiniMax M3", "Inkling", "Qwen 3.7 Plus", "Claude Opus 5", "SWE-1.6", "Kimi K3", "Gemini 3.6 Flash", "Gemini 3.7 Flash", "GPT-5.4-mini", "Claude Opus 4.6", "Claude Sonnet 4.6", "DeepSeek V4 Flash 0731", "Mistral 3.5 Medium", "DeepSeek V4 Pro 0813", "GLM 5.3", "GLM 5.3 Flash", "Nemotron 3 Ultra", "Gemini 3.8 Flash" ], "colors": { "SWE-2": "#2200ff", "Claude Fable 5": "#d97757", "GPT-5.6 Sol": "#10a37f", "Claude Opus 4.8": "#d97757", "GPT-5.5": "#10a37f", "Grok 4.5": "#1a1a1a", "Claude Sonnet 5": "#d97757", "GPT-5.6 Terra": "#10a37f", "GPT-5.6 Luna": "#10a37f", "SWE-1.7": "#2200ff", "Claude Opus 4.7": "#d97757", "Kimi K2.7": "#7c3aed", "Composer 2.5": "#565656", "GLM 5.2": "#14b8a6", "DeepSeek V4 Pro": "#4d6bfe", "MiniMax M3": "#e5484d", "Inkling": "#e3a857", "Qwen 3.7 Plus": "#ff8800", "Claude Opus 5": "#d97757", "SWE-1.6": "#2200ff", "Kimi K3": "#7c3aed", "Gemini 3.6 Flash": "#1a73e8", "Gemini 3.7 Flash": "#1a73e8", "GPT-5.4-mini": "#10a37f", "Claude Opus 4.6": "#d97757", "Claude Sonnet 4.6": "#d97757", "DeepSeek V4 Flash 0731": "#4d6bfe", "Mistral 3.5 Medium": "#fa500f", "Grok 4.6": "#1a1a1a", "Claude Fable 5.1": "#d97757", "DeepSeek V4 Pro 0813": "#4d6bfe", "GLM 5.3": "#14b8a6", "GLM 5.3 Flash": "#14b8a6", "Nemotron 3 Ultra": "#76b900", "Gemini 3.8 Flash": "#1a73e8", "GPT-6 Astra": "#10a37f" }, "lab_colors": { "Anthropic": "#d97757", "OpenAI": "#10a37f", "xAI": "#1a1a1a", "Cognition": "#2200ff", "Moonshot": "#7c3aed", "Cursor": "#565656", "Zhipu": "#14b8a6", "DeepSeek": "#4d6bfe", "MiniMax": "#e5484d", "Thinking Machines": "#e3a857", "Alibaba": "#ff8800", "NVIDIA": "#76b900", "Google": "#1a73e8", "Mistral": "#fa500f" }, "harness": { "SWE-2": "devin", "Claude Fable 5": "claude-code", "GPT-5.6 Sol": "codex", "Claude Opus 4.8": "claude-code", "GPT-5.5": "codex", "Grok 4.5": "grok-build", "Claude Sonnet 5": "claude-code", "GPT-5.6 Terra": "codex", "GPT-5.6 Luna": "codex", "SWE-1.7": "chisel", "Claude Opus 4.7": "claude-code", "Kimi K2.7": "mini-swe-agent", "Composer 2.5": "cursor-cli", "GLM 5.2": "mini-swe-agent", "DeepSeek V4 Pro": "mini-swe-agent", "MiniMax M3": "mini-swe-agent", "Inkling": "mini-swe-agent", "Qwen 3.7 Plus": "mini-swe-agent", "Claude Opus 5": "claude-code", "SWE-1.6": "chisel", "Kimi K3": "mini-swe-agent", "Gemini 3.6 Flash": "chisel", "Gemini 3.7 Flash": "chisel", "GPT-5.4-mini": "codex", "Claude Opus 4.6": "claude-code", "Claude Sonnet 4.6": "claude-code", "DeepSeek V4 Flash 0731": "chisel", "Mistral 3.5 Medium": "chisel", "Grok 4.6": "grok-build", "Claude Fable 5.1": "claude-code", "DeepSeek V4 Pro 0813": "chisel", "GLM 5.3": "chisel", "GLM 5.3 Flash": "chisel", "Nemotron 3 Ultra": "chisel", "Gemini 3.8 Flash": "chisel", "GPT-6 Astra": "codex" }, "efforts": { "SWE-2": [ "medium", "high", "max" ], "Claude Fable 5": [ "low", "medium", "high", "xhigh", "max" ], "GPT-5.6 Sol": [ "low", "medium", "high", "xhigh", "max" ], "Claude Opus 4.8": [ "low", "medium", "high", "xhigh", "max" ], "GPT-5.5": [ "low", "medium", "high", "xhigh" ], "Grok 4.5": [ "low", "medium", "high" ], "Claude Sonnet 5": [ "low", "medium", "high", "xhigh", "max" ], "GPT-5.6 Terra": [ "low", "medium", "high", "xhigh", "max" ], "GPT-5.6 Luna": [ "low", "medium", "high", "xhigh", "max" ], "SWE-1.7": [ "none" ], "Claude Opus 4.7": [ "low", "medium", "high", "xhigh", "max" ], "Kimi K2.7": [ "none" ], "Composer 2.5": [ "none" ], "GLM 5.2": [ "none" ], "DeepSeek V4 Pro": [ "none" ], "MiniMax M3": [ "none" ], "Inkling": [ "0.99" ], "Qwen 3.7 Plus": [ "none" ], "Claude Opus 5": [ "low", "medium", "high", "xhigh", "max" ], "SWE-1.6": [ "none" ], "Kimi K3": [ "none" ], "Gemini 3.6 Flash": [ "low", "medium", "high" ], "Gemini 3.7 Flash": [ "low", "medium", "high" ], "GPT-5.4-mini": [ "low", "medium", "high", "xhigh" ], "Claude Opus 4.6": [ "low", "medium", "high" ], "Claude Sonnet 4.6": [ "low", "medium", "high", "max" ], "DeepSeek V4 Flash 0731": [ "high" ], "Mistral 3.5 Medium": [ "none" ], "Grok 4.6": [ "low", "medium", "high" ], "Claude Fable 5.1": [ "low", "medium", "high", "xhigh", "max" ], "DeepSeek V4 Pro 0813": [ "high" ], "GLM 5.3": [ "max" ], "GLM 5.3 Flash": [ "max" ], "Nemotron 3 Ultra": [ "none" ], "Gemini 3.8 Flash": [ "medium", "high" ], "GPT-6 Astra": [ "low", "medium", "high", "xhigh", "max" ] }, "subsets": { "main": 100, "extended": 150 }, "data": { "SWE-2": { "medium": { "main": { "correct": 0.4833, "new_score": 0.4309, "tokens": 25701.5, "cost": 0.3712, "duration_min": null, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.6222, "new_score": 0.5641, "tokens": 21337.6, "cost": 0.3041, "duration_min": null, "tool_calls": null, "steps": null, "ote": null } }, "high": { "main": { "correct": 0.5235, "new_score": 0.472, "tokens": 52821.9, "cost": 0.7813, "duration_min": null, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.6563, "new_score": 0.6008, "tokens": 43469.1, "cost": 0.6357, "duration_min": null, "tool_calls": null, "steps": null, "ote": null } }, "max": { "main": { "correct": 0.5547, "new_score": 0.5, "tokens": 72764.7, "cost": 1.1761, "duration_min": null, "tool_calls": null, "steps": null, "ote": null }, "extended": { "correct": 0.6844, "new_score": 0.6246, "tokens": 59918.7, "cost": 0.94, "duration_min": null, "tool_calls": null, "steps": null, "ote": null } } }, "Claude Fable 5": { "low": { "extended": { "correct": 0.663, "new_score": 0.608, "tokens": 19502.3, "cost": 4.0311, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.5282, "new_score": 0.4801, "tokens": 23609.8, "cost": 4.9232, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "medium": { "extended": { "correct": 0.6825, "new_score": 0.6279, "tokens": 27536.1, "cost": 5.8428, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.002 }, "main": { "correct": 0.546, "new_score": 0.4982, "tokens": 33229.9, "cost": 7.1539, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.003 } }, "high": { "extended": { "correct": 0.6981, "new_score": 0.6425, "tokens": 36013, "cost": 7.711, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0013 }, "main": { "correct": 0.5766, "new_score": 0.5273, "tokens": 43482.8, "cost": 9.4805, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.002 } }, "xhigh": { "extended": { "correct": 0.7086, "new_score": 0.6494, "tokens": 48419.3, "cost": 10.5282, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.002 }, "main": { "correct": 0.5885, "new_score": 0.5348, "tokens": 58557.9, "cost": 13.0938, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.003 } }, "max": { "extended": { "correct": 0.6956, "new_score": 0.6363, "tokens": 68418.6, "cost": 15.4885, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0013 }, "main": { "correct": 0.5689, "new_score": 0.5157, "tokens": 80761.1, "cost": 19.0696, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.002 } } }, "GPT-5.6 Sol": { "low": { "extended": { "correct": 0.5536, "new_score": 0.4999, "tokens": 10515.7, "cost": 1.6667, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0007 }, "main": { "correct": 0.3974, "new_score": 0.3544, "tokens": 11962.2, "cost": 1.8904, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } }, "medium": { "extended": { "correct": 0.6031, "new_score": 0.5468, "tokens": 14708.8, "cost": 2.3007, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0013 }, "main": { "correct": 0.4453, "new_score": 0.3993, "tokens": 16952.7, "cost": 2.6897, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.002 } }, "high": { "extended": { "correct": 0.6475, "new_score": 0.5874, "tokens": 18822.8, "cost": 2.9349, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0008 }, "main": { "correct": 0.5028, "new_score": 0.4506, "tokens": 21877.1, "cost": 3.4801, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0013 } }, "xhigh": { "extended": { "correct": 0.6601, "new_score": 0.5998, "tokens": 22452.5, "cost": 3.4541, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0007 }, "main": { "correct": 0.5211, "new_score": 0.4684, "tokens": 26282.9, "cost": 4.1454, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } }, "max": { "extended": { "correct": 0.6661, "new_score": 0.6055, "tokens": 28060.4, "cost": 4.253, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.5291, "new_score": 0.4749, "tokens": 33167.8, "cost": 5.1898, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } } }, "Claude Opus 4.8": { "low": { "extended": { "correct": 0.5601, "new_score": 0.5083, "tokens": 21383.6, "cost": 2.2736, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0013 }, "main": { "correct": 0.3923, "new_score": 0.3514, "tokens": 25194, "cost": 2.6827, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } }, "medium": { "extended": { "correct": 0.6076, "new_score": 0.5537, "tokens": 29302, "cost": 3.062, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.004 }, "main": { "correct": 0.4493, "new_score": 0.4046, "tokens": 34906.5, "cost": 3.7029, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.005 } }, "high": { "extended": { "correct": 0.611, "new_score": 0.5563, "tokens": 34584.2, "cost": 3.5377, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0073 }, "main": { "correct": 0.4565, "new_score": 0.4102, "tokens": 41326.4, "cost": 4.2799, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.008 } }, "xhigh": { "extended": { "correct": 0.6504, "new_score": 0.5945, "tokens": 55453.9, "cost": 5.5793, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0033 }, "main": { "correct": 0.5027, "new_score": 0.4549, "tokens": 65742.4, "cost": 6.7803, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.005 } }, "max": { "extended": { "correct": 0.6549, "new_score": 0.596, "tokens": 82896.6, "cost": 8.0501, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.004 }, "main": { "correct": 0.5165, "new_score": 0.465, "tokens": 95939.9, "cost": 9.6225, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.006 } } }, "GPT-5.5": { "low": { "extended": { "correct": 0.5039, "new_score": 0.4555, "tokens": 7726.6, "cost": 1.434, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.3439, "new_score": 0.3055, "tokens": 8855, "cost": 1.6478, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "medium": { "extended": { "correct": 0.5679, "new_score": 0.5137, "tokens": 11724.3, "cost": 1.9847, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.4108, "new_score": 0.366, "tokens": 13677.2, "cost": 2.3513, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "high": { "extended": { "correct": 0.5987, "new_score": 0.5399, "tokens": 15775, "cost": 2.6054, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.002 }, "main": { "correct": 0.4571, "new_score": 0.4056, "tokens": 18422.1, "cost": 3.1187, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.003 } }, "xhigh": { "extended": { "correct": 0.6276, "new_score": 0.5666, "tokens": 21330.9, "cost": 3.3393, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0027 }, "main": { "correct": 0.4824, "new_score": 0.4296, "tokens": 24992.4, "cost": 4.0348, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.004 } } }, "Grok 4.5": { "low": { "extended": { "correct": 0.5827, "new_score": 0.5314, "tokens": 10110.1, "cost": 0.7092, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.42, "new_score": 0.3786, "tokens": 11861.9, "cost": 0.8368, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "medium": { "extended": { "correct": 0.6157, "new_score": 0.5607, "tokens": 12590.5, "cost": 1.0238, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.4655, "new_score": 0.4193, "tokens": 14945.1, "cost": 1.2308, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "high": { "extended": { "correct": 0.6227, "new_score": 0.5655, "tokens": 12956.8, "cost": 1.0863, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.472, "new_score": 0.4244, "tokens": 15314.6, "cost": 1.3007, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } } }, "Grok 4.6": { "low": { "extended": { "correct": 0.6037, "new_score": 0.5502, "tokens": 10306.3, "cost": 0.8005, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.4455, "new_score": 0.4011, "tokens": 12084.5, "cost": 0.9474, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "medium": { "extended": { "correct": 0.6547, "new_score": 0.5962, "tokens": 23092.3, "cost": 1.6501, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.004 }, "main": { "correct": 0.512, "new_score": 0.4606, "tokens": 27065.1, "cost": 1.9762, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.006 } }, "high": { "extended": { "correct": 0.6705, "new_score": 0.6131, "tokens": 31616.5, "cost": 2.3827, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0047 }, "main": { "correct": 0.5315, "new_score": 0.4801, "tokens": 36758.3, "cost": 2.8758, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.007 } } }, "Claude Sonnet 5": { "low": { "extended": { "correct": 0.4891, "new_score": 0.4446, "tokens": 21669.8, "cost": 2.0339, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0007 }, "main": { "correct": 0.32, "new_score": 0.2871, "tokens": 24817.1, "cost": 2.3897, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } }, "medium": { "extended": { "correct": 0.5413, "new_score": 0.4928, "tokens": 32169.2, "cost": 3.1582, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.3907, "new_score": 0.352, "tokens": 38268.9, "cost": 3.8149, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "high": { "extended": { "correct": 0.5784, "new_score": 0.5271, "tokens": 48028.6, "cost": 4.974, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0013 }, "main": { "correct": 0.4366, "new_score": 0.3935, "tokens": 57709.8, "cost": 6.0977, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.002 } }, "xhigh": { "extended": { "correct": 0.6166, "new_score": 0.5618, "tokens": 73988.4, "cost": 8.231, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.006 }, "main": { "correct": 0.4759, "new_score": 0.4273, "tokens": 87937.5, "cost": 10.0666, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.005 } }, "max": { "extended": { "correct": 0.6019, "new_score": 0.5485, "tokens": 121728.1, "cost": 14.0943, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0053 }, "main": { "correct": 0.4719, "new_score": 0.4236, "tokens": 142620.6, "cost": 17.1164, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.003 } } }, "GPT-5.6 Terra": { "low": { "extended": { "correct": 0.4185, "new_score": 0.3779, "tokens": 8379, "cost": 0.4489, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.2721, "new_score": 0.2411, "tokens": 9238.1, "cost": 0.4961, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "medium": { "extended": { "correct": 0.529, "new_score": 0.4766, "tokens": 14383.9, "cost": 0.7151, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0007 }, "main": { "correct": 0.3815, "new_score": 0.3388, "tokens": 16259.9, "cost": 0.8189, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } }, "high": { "extended": { "correct": 0.5636, "new_score": 0.5086, "tokens": 22060.8, "cost": 1.0168, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0007 }, "main": { "correct": 0.4134, "new_score": 0.3689, "tokens": 25255, "cost": 1.1875, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "xhigh": { "extended": { "correct": 0.5833, "new_score": 0.5263, "tokens": 26556.4, "cost": 1.1737, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0007 }, "main": { "correct": 0.4351, "new_score": 0.3875, "tokens": 30669.6, "cost": 1.3832, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } }, "max": { "extended": { "correct": 0.6175, "new_score": 0.5584, "tokens": 35054.1, "cost": 1.5696, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0007 }, "main": { "correct": 0.4632, "new_score": 0.4131, "tokens": 40472.6, "cost": 1.8735, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } } }, "GPT-5.6 Luna": { "low": { "extended": { "correct": 0.3222, "new_score": 0.2902, "tokens": 6467.2, "cost": 0.0584, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.1733, "new_score": 0.1538, "tokens": 6822.4, "cost": 0.0594, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "medium": { "extended": { "correct": 0.4566, "new_score": 0.4117, "tokens": 11612.1, "cost": 0.115, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.2909, "new_score": 0.2574, "tokens": 12903.6, "cost": 0.1275, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "high": { "extended": { "correct": 0.5606, "new_score": 0.5052, "tokens": 20353.3, "cost": 0.2002, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.4049, "new_score": 0.3589, "tokens": 23223.3, "cost": 0.2321, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "xhigh": { "extended": { "correct": 0.6033, "new_score": 0.545, "tokens": 26956.7, "cost": 0.2622, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.438, "new_score": 0.3891, "tokens": 31189.7, "cost": 0.3093, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "max": { "extended": { "correct": 0.6089, "new_score": 0.5506, "tokens": 31859.7, "cost": 0.3074, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0013 }, "main": { "correct": 0.4474, "new_score": 0.3981, "tokens": 37418.9, "cost": 0.3704, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } } }, "SWE-1.7": { "none": { "extended": { "correct": 0.6033, "new_score": 0.5426, "tokens": 56445.2, "cost": 1.6434, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0093 }, "main": { "correct": 0.474, "new_score": 0.4199, "tokens": 64756.6, "cost": 1.9734, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.012 } } }, "Claude Opus 4.7": { "low": { "extended": { "correct": 0.4783, "new_score": 0.436, "tokens": 15368.9, "cost": 2.1079, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0007 }, "main": { "correct": 0.3094, "new_score": 0.2761, "tokens": 17852.7, "cost": 2.4668, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } }, "medium": { "extended": { "correct": 0.4902, "new_score": 0.4475, "tokens": 18778.3, "cost": 2.7316, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.3223, "new_score": 0.2891, "tokens": 21880.8, "cost": 3.2194, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "high": { "extended": { "correct": 0.5608, "new_score": 0.5112, "tokens": 25742.1, "cost": 4.2117, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0007 }, "main": { "correct": 0.3924, "new_score": 0.3517, "tokens": 30140.2, "cost": 5.0292, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } }, "xhigh": { "extended": { "correct": 0.5618, "new_score": 0.5141, "tokens": 32928.6, "cost": 5.7683, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0027 }, "main": { "correct": 0.3867, "new_score": 0.3491, "tokens": 38418.6, "cost": 6.8688, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.004 } }, "max": { "extended": { "correct": 0.5909, "new_score": 0.5393, "tokens": 42112.9, "cost": 7.6144, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0007 }, "main": { "correct": 0.4283, "new_score": 0.3854, "tokens": 49113.7, "cost": 9.0876, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } } }, "Kimi K2.7": { "none": { "extended": { "correct": 0.5, "new_score": 0.4538, "tokens": 37998.5, "cost": 2.5592, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.336, "new_score": 0.3006, "tokens": 43902.8, "cost": 3.0057, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } } }, "Composer 2.5": { "none": { "extended": { "correct": 0.4513, "new_score": 0.4075, "tokens": 13768.6, "cost": 2.5688, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0227 }, "main": { "correct": 0.2934, "new_score": 0.2564, "tokens": 16027.1, "cost": 3.0875, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.024 } } }, "GLM 5.2": { "none": { "extended": { "correct": 0.4413, "new_score": 0.4005, "tokens": 15362.5, "cost": 2.1396, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.274, "new_score": 0.245, "tokens": 17783.1, "cost": 2.4704, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } } }, "DeepSeek V4 Pro": { "none": { "extended": { "correct": 0.3449, "new_score": 0.3101, "tokens": 26171.1, "cost": 1.3693, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0067 }, "main": { "correct": 0.2, "new_score": 0.1764, "tokens": 28793, "cost": 1.5548, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.01 } } }, "MiniMax M3": { "none": { "extended": { "correct": 0.3146, "new_score": 0.2854, "tokens": 31112, "cost": 0.5922, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0107 }, "main": { "correct": 0.1664, "new_score": 0.147, "tokens": 34005.5, "cost": 0.6756, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.015 } } }, "Inkling": { "0.99": { "main": { "correct": 0.159, "new_score": 0.14, "tokens": 33610.8, "cost": 3.6, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.076 }, "extended": { "correct": 0.278, "new_score": 0.248, "tokens": 30149.8, "cost": 3.2, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0727 } } }, "Qwen 3.7 Plus": { "none": { "extended": { "correct": 0.2413, "new_score": 0.2178, "tokens": 41235.2, "cost": 0.2121, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.006 }, "main": { "correct": 0.1146, "new_score": 0.102, "tokens": 45303.8, "cost": 0.2415, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.009 } } }, "Claude Opus 5": { "low": { "extended": { "correct": 0.6108, "new_score": 0.5575, "tokens": 17947.7, "cost": 2.1469, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0041 }, "main": { "correct": 0.4663, "new_score": 0.4195, "tokens": 21925.5, "cost": 2.678, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0021 } }, "medium": { "extended": { "correct": 0.6963, "new_score": 0.6363, "tokens": 27626.1, "cost": 3.511, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.004 }, "main": { "correct": 0.5885, "new_score": 0.5338, "tokens": 33575.4, "cost": 4.3127, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.006 } }, "high": { "extended": { "correct": 0.6403, "new_score": 0.5845, "tokens": 45734, "cost": 5.8448, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0013 }, "main": { "correct": 0.5293, "new_score": 0.4799, "tokens": 55349.4, "cost": 7.2397, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.002 } }, "xhigh": { "extended": { "correct": 0.6257, "new_score": 0.5687, "tokens": 56520.9, "cost": 7.438, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0097 }, "main": { "correct": 0.484, "new_score": 0.4365, "tokens": 67084.2, "cost": 9.1423, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0144 } }, "max": { "extended": { "correct": 0.6497, "new_score": 0.5894, "tokens": 85475.5, "cost": 9.6843, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0067 }, "main": { "correct": 0.5333, "new_score": 0.4804, "tokens": 98266.8, "cost": 11.416, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0101 } } }, "SWE-1.6": { "none": { "extended": { "correct": 0.2267, "new_score": 0.205, "tokens": 18240.6, "cost": 0.4496, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.1045, "new_score": 0.0939, "tokens": 19812.5, "cost": 0.4891, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } } }, "Kimi K3": { "none": { "extended": { "correct": 0.6358, "new_score": 0.5819, "tokens": 45646.3, "cost": 3.1235, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0014 }, "main": { "correct": 0.4891, "new_score": 0.4417, "tokens": 53607.2, "cost": 3.8167, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0021 } } }, "Gemini 3.6 Flash": { "low": { "extended": { "correct": 0.3627, "new_score": 0.3277, "tokens": 20851.8, "cost": 1.9405, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.2565, "new_score": 0.2282, "tokens": 23960.4, "cost": 2.2986, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "medium": { "extended": { "correct": 0.5327, "new_score": 0.4802, "tokens": 40384.1, "cost": 3.4136, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.389, "new_score": 0.3437, "tokens": 46744.3, "cost": 4.037, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "high": { "extended": { "correct": 0.5293, "new_score": 0.4758, "tokens": 42275.4, "cost": 3.5961, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.382, "new_score": 0.3369, "tokens": 48297.8, "cost": 4.2131, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } } }, "Gemini 3.7 Flash": { "low": { "extended": { "correct": 0.574, "new_score": 0.5185, "tokens": 26681.7, "cost": 1.4335, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.415, "new_score": 0.3691, "tokens": 31587.4, "cost": 1.7538, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "medium": { "extended": { "correct": 0.6233, "new_score": 0.5627, "tokens": 44479.7, "cost": 1.5323, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.489, "new_score": 0.4359, "tokens": 51067.4, "cost": 1.8227, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "high": { "extended": { "correct": 0.606, "new_score": 0.5474, "tokens": 50487.8, "cost": 1.7948, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.473, "new_score": 0.4223, "tokens": 57506.3, "cost": 2.1184, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } } }, "GPT-5.4-mini": { "low": { "extended": { "correct": 0.2302, "new_score": 0.2058, "tokens": 10771.1, "cost": 0.3288, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.106, "new_score": 0.0925, "tokens": 12065.7, "cost": 0.3642, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "medium": { "extended": { "correct": 0.3922, "new_score": 0.3506, "tokens": 29564.4, "cost": 0.6716, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.2373, "new_score": 0.2078, "tokens": 34721.2, "cost": 0.7833, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "high": { "extended": { "correct": 0.4153, "new_score": 0.3719, "tokens": 39151, "cost": 0.8043, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0007 }, "main": { "correct": 0.263, "new_score": 0.2308, "tokens": 46438.8, "cost": 0.9537, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } }, "xhigh": { "extended": { "correct": 0.4794, "new_score": 0.4301, "tokens": 75608.7, "cost": 1.2621, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.308, "new_score": 0.2704, "tokens": 90933.3, "cost": 1.5201, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } } }, "Claude Opus 4.6": { "low": { "extended": { "correct": 0.3643, "new_score": 0.3293, "tokens": 13032, "cost": 1.8406, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.002 }, "main": { "correct": 0.2054, "new_score": 0.1831, "tokens": 14888.4, "cost": 2.101, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.003 } }, "medium": { "extended": { "correct": 0.426, "new_score": 0.3865, "tokens": 17306.8, "cost": 2.3912, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0007 }, "main": { "correct": 0.265, "new_score": 0.2367, "tokens": 19836.5, "cost": 2.7941, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } }, "high": { "extended": { "correct": 0.48, "new_score": 0.4367, "tokens": 23162.1, "cost": 3.2978, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0007 }, "main": { "correct": 0.297, "new_score": 0.2664, "tokens": 26676.8, "cost": 3.9796, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.001 } } }, "Claude Sonnet 4.6": { "low": { "extended": { "correct": 0.3365, "new_score": 0.3044, "tokens": 18901.7, "cost": 1.4261, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.1691, "new_score": 0.1503, "tokens": 21707.6, "cost": 1.6481, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "medium": { "extended": { "correct": 0.4167, "new_score": 0.3749, "tokens": 28943, "cost": 1.9375, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.239, "new_score": 0.211, "tokens": 33357.3, "cost": 2.3012, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "high": { "extended": { "correct": 0.4412, "new_score": 0.3986, "tokens": 35510.4, "cost": 2.2694, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.265, "new_score": 0.2347, "tokens": 41108.9, "cost": 2.6993, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "max": { "extended": { "correct": 0.444, "new_score": 0.4, "tokens": 38859.7, "cost": 2.4401, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0013 }, "main": { "correct": 0.275, "new_score": 0.2431, "tokens": 44191.7, "cost": 2.8979, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.002 } } }, "DeepSeek V4 Flash 0731": { "high": { "extended": { "correct": 0.3496, "new_score": 0.317, "tokens": 82141, "cost": 1.3059, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.1911 }, "main": { "correct": 0.2108, "new_score": 0.188, "tokens": 94407.7, "cost": 1.5337, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.2547 } } }, "Mistral 3.5 Medium": { "none": { "extended": { "correct": 0.206, "new_score": 0.1866, "tokens": 21012.7, "cost": 1.2366, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.004 }, "main": { "correct": 0.09, "new_score": 0.08, "tokens": 22451.9, "cost": 1.3475, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.006 } } }, "Claude Fable 5.1": { "low": { "extended": { "correct": 0.6682, "new_score": 0.6162, "tokens": 15432.8, "cost": 1.9427, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.5441, "new_score": 0.4982, "tokens": 19072.6, "cost": 2.3849, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "medium": { "extended": { "correct": 0.688, "new_score": 0.636, "tokens": 21407.7, "cost": 2.6825, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.5545, "new_score": 0.5091, "tokens": 26109.6, "cost": 3.2845, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "high": { "extended": { "correct": 0.6815, "new_score": 0.6268, "tokens": 32599.6, "cost": 4.2833, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.5512, "new_score": 0.5034, "tokens": 39918.6, "cost": 5.2741, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "xhigh": { "extended": { "correct": 0.6691, "new_score": 0.6135, "tokens": 56868.2, "cost": 7.7022, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.5369, "new_score": 0.4873, "tokens": 67707.6, "cost": 9.2738, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "max": { "extended": { "correct": 0.6786, "new_score": 0.6203, "tokens": 78159.5, "cost": 10.7206, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.5559, "new_score": 0.5028, "tokens": 91731.2, "cost": 12.8257, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } } }, "DeepSeek V4 Pro 0813": { "high": { "extended": { "correct": 0.4893, "new_score": 0.4445, "tokens": 87449.7, "cost": 1.4996, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.076 }, "main": { "correct": 0.318, "new_score": 0.2855, "tokens": 104747, "cost": 1.812, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.106 } } }, "GLM 5.3": { "max": { "extended": { "correct": 0.5773, "new_score": 0.5263, "tokens": 98340.2, "cost": 13.7873, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.447, "new_score": 0.4014, "tokens": 112175.3, "cost": 16.9089, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } } }, "GLM 5.3 Flash": { "max": { "extended": { "correct": 0.5108, "new_score": 0.4621, "tokens": 143646.7, "cost": 0.9437, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.3567, "new_score": 0.3183, "tokens": 169291.4, "cost": 1.1466, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } } }, "Nemotron 3 Ultra": { "none": { "extended": { "correct": 0.3027, "new_score": 0.2721, "tokens": 25833.2, "cost": 1.3168, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.0013 }, "main": { "correct": 0.154, "new_score": 0.1361, "tokens": 27744.3, "cost": 1.4655, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0.002 } } }, "Gemini 3.8 Flash": { "medium": { "extended": { "correct": 0.5962, "new_score": 0.5345, "tokens": 69722.9, "cost": 2.2053, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.4673, "new_score": 0.4119, "tokens": 78927.7, "cost": 2.5954, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } }, "high": { "extended": { "correct": 0.5687, "new_score": 0.5124, "tokens": 75763.6, "cost": 2.2608, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 }, "main": { "correct": 0.428, "new_score": 0.3796, "tokens": 84869.3, "cost": 2.598, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": 0 } } }, "GPT-6 Astra": { "low": { "extended": { "correct": 0.6293, "new_score": 0.574, "tokens": 5797.0, "cost": 1.4997, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": null }, "main": { "correct": 0.5, "new_score": 0.4527, "tokens": 6760.0, "cost": 1.7, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": null } }, "medium": { "extended": { "correct": 0.6573, "new_score": 0.6025, "tokens": 8960.0, "cost": 2.102, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": null }, "main": { "correct": 0.536, "new_score": 0.4883, "tokens": 10630.0, "cost": 2.4259, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": null } }, "high": { "extended": { "correct": 0.6907, "new_score": 0.6307, "tokens": 12370.0, "cost": 2.624, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": null }, "main": { "correct": 0.562, "new_score": 0.5094, "tokens": 14344.0, "cost": 3.0149, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": null } }, "xhigh": { "extended": { "correct": 0.6797, "new_score": 0.6212, "tokens": 14873.0, "cost": 2.8602, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": null }, "main": { "correct": 0.5575, "new_score": 0.5062, "tokens": 16955.0, "cost": 3.2755, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": null } }, "max": { "extended": { "correct": 0.7063, "new_score": 0.6448, "tokens": 25995.0, "cost": 3.9329, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": null }, "main": { "correct": 0.5875, "new_score": 0.5326, "tokens": 30144.0, "cost": 4.586, "tool_calls": null, "steps": null, "ote": null, "flagged_rate": null } } } } } }