{
  "v1_1": {
    "models": [
      "SWE-2",
      "SWE-1.7",
      "GPT-5.6 Sol",
      "Fable 5.1",
      "GPT-6 Astra",
      "Grok 4.6",
      "Kimi K3"
    ],
    "colors": {
      "SWE-2": "#2200ff",
      "SWE-1.7": "#2200ff",
      "GPT-5.6 Sol": "#10a37f",
      "Fable 5.1": "#d97757",
      "GPT-6 Astra": "#10a37f",
      "Grok 4.6": "#5c616a",
      "Kimi K3": "#7c3aed"
    },
    "lab_colors": {
      "Anthropic": "#d97757",
      "OpenAI": "#10a37f",
      "xAI": "#1a1a1a",
      "Cognition": "#2200ff",
      "Moonshot": "#7c3aed",
      "Cursor": "#565656",
      "Zhipu": "#14b8a6",
      "DeepSeek": "#4d6bfe",
      "MiniMax": "#e5484d",
      "Thinking Machines": "#e3a857",
      "Alibaba": "#ff8800",
      "NVIDIA": "#76b900",
      "Google": "#1a73e8",
      "Mistral": "#fa500f"
    },
    "harness": {
      "SWE-2": "devin",
      "SWE-1.7": "chisel",
      "GPT-5.6 Sol": "codex",
      "Fable 5.1": "claude-code",
      "GPT-6 Astra": "codex",
      "Grok 4.6": "grok-build",
      "Kimi K3": "mini-swe-agent"
    },
    "efforts": {
      "SWE-2": [
        "medium",
        "high",
        "max"
      ],
      "SWE-1.7": [
        "none"
      ],
      "GPT-5.6 Sol": [
        "low",
        "medium",
        "high",
        "xhigh",
        "max"
      ],
      "Fable 5.1": [
        "low",
        "medium",
        "high",
        "xhigh"
      ],
      "GPT-6 Astra": [
        "low",
        "medium",
        "high",
        "xhigh",
        "max"
      ],
      "Grok 4.6": [
        "low",
        "medium",
        "high"
      ],
      "Kimi K3": [
        "none"
      ]
    },
    "subsets": {
      "main": 100,
      "extended": 150
    },
    "data": {
      "SWE-2": {
        "medium": {
          "main": {
            "correct": 0.4833,
            "new_score": 0.4309,
            "tokens": 25701.5,
            "cost": 0.3712,
            "duration_min": null,
            "tool_calls": null,
            "steps": null,
            "ote": null
          },
          "extended": {
            "correct": 0.6222,
            "new_score": 0.5641,
            "tokens": 21337.6,
            "cost": 0.3041,
            "duration_min": null,
            "tool_calls": null,
            "steps": null,
            "ote": null
          }
        },
        "high": {
          "main": {
            "correct": 0.5235,
            "new_score": 0.472,
            "tokens": 52821.9,
            "cost": 0.7813,
            "duration_min": null,
            "tool_calls": null,
            "steps": null,
            "ote": null
          },
          "extended": {
            "correct": 0.6563,
            "new_score": 0.6008,
            "tokens": 43469.1,
            "cost": 0.6357,
            "duration_min": null,
            "tool_calls": null,
            "steps": null,
            "ote": null
          }
        },
        "max": {
          "main": {
            "correct": 0.5547,
            "new_score": 0.5,
            "tokens": 72764.7,
            "cost": 1.1761,
            "duration_min": null,
            "tool_calls": null,
            "steps": null,
            "ote": null
          },
          "extended": {
            "correct": 0.6844,
            "new_score": 0.6246,
            "tokens": 59918.7,
            "cost": 0.94,
            "duration_min": null,
            "tool_calls": null,
            "steps": null,
            "ote": null
          }
        }
      },
      "SWE-1.7": {
        "none": {
          "extended": {
            "correct": 0.6033,
            "new_score": 0.5426,
            "tokens": 56445.2,
            "cost": 1.6434,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.0093
          },
          "main": {
            "correct": 0.474,
            "new_score": 0.4199,
            "tokens": 64756.6,
            "cost": 1.9734,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.012
          }
        }
      },
      "GPT-5.6 Sol": {
        "low": {
          "extended": {
            "correct": 0.5536,
            "new_score": 0.4999,
            "tokens": 10515.7,
            "cost": 1.8121,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.0007
          },
          "main": {
            "correct": 0.3974,
            "new_score": 0.3544,
            "tokens": 11962.2,
            "cost": 2.0998,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.001
          }
        },
        "medium": {
          "extended": {
            "correct": 0.6031,
            "new_score": 0.5468,
            "tokens": 14708.8,
            "cost": 2.6148,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.0013
          },
          "main": {
            "correct": 0.4453,
            "new_score": 0.3993,
            "tokens": 16952.7,
            "cost": 3.1245,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.002
          }
        },
        "high": {
          "extended": {
            "correct": 0.6475,
            "new_score": 0.5874,
            "tokens": 18822.8,
            "cost": 3.4135,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.0008
          },
          "main": {
            "correct": 0.5028,
            "new_score": 0.4506,
            "tokens": 21877.1,
            "cost": 4.1319,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.0013
          }
        },
        "xhigh": {
          "extended": {
            "correct": 0.6601,
            "new_score": 0.5998,
            "tokens": 22452.5,
            "cost": 4.0533,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.0007
          },
          "main": {
            "correct": 0.5211,
            "new_score": 0.4684,
            "tokens": 26282.9,
            "cost": 4.9582,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.001
          }
        },
        "max": {
          "extended": {
            "correct": 0.6661,
            "new_score": 0.6055,
            "tokens": 28060.4,
            "cost": 5.0715,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0
          },
          "main": {
            "correct": 0.5291,
            "new_score": 0.4749,
            "tokens": 33167.8,
            "cost": 6.2938,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0
          }
        }
      },
      "Fable 5.1": {
        "low": {
          "extended": {
            "correct": 0.6682,
            "new_score": 0.6162,
            "tokens": 15432.8,
            "cost": 1.9427,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0
          },
          "main": {
            "correct": 0.5441,
            "new_score": 0.4982,
            "tokens": 19072.6,
            "cost": 2.3849,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0
          }
        },
        "medium": {
          "extended": {
            "correct": 0.688,
            "new_score": 0.636,
            "tokens": 21407.7,
            "cost": 2.6825,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0
          },
          "main": {
            "correct": 0.5545,
            "new_score": 0.5091,
            "tokens": 26109.6,
            "cost": 3.2845,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0
          }
        },
        "high": {
          "extended": {
            "correct": 0.6815,
            "new_score": 0.6268,
            "tokens": 32599.6,
            "cost": 4.2833,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0
          },
          "main": {
            "correct": 0.5512,
            "new_score": 0.5034,
            "tokens": 39918.6,
            "cost": 5.2741,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0
          }
        },
        "xhigh": {
          "extended": {
            "correct": 0.6691,
            "new_score": 0.6135,
            "tokens": 56868.2,
            "cost": 7.7022,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0
          },
          "main": {
            "correct": 0.5369,
            "new_score": 0.4873,
            "tokens": 67707.6,
            "cost": 9.2738,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0
          }
        }
      },
      "GPT-6 Astra": {
        "low": {
          "extended": {
            "correct": 0.6293,
            "new_score": 0.574,
            "tokens": 5797.0,
            "cost": 1.393,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": null
          },
          "main": {
            "correct": 0.5,
            "new_score": 0.4527,
            "tokens": 6760.0,
            "cost": 1.597,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": null
          }
        },
        "medium": {
          "extended": {
            "correct": 0.6573,
            "new_score": 0.6025,
            "tokens": 8960.0,
            "cost": 1.997,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": null
          },
          "main": {
            "correct": 0.536,
            "new_score": 0.4883,
            "tokens": 10630.0,
            "cost": 2.336,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": null
          }
        },
        "high": {
          "extended": {
            "correct": 0.6907,
            "new_score": 0.6307,
            "tokens": 12370.0,
            "cost": 2.514,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": null
          },
          "main": {
            "correct": 0.562,
            "new_score": 0.5094,
            "tokens": 14344.0,
            "cost": 2.929,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": null
          }
        },
        "xhigh": {
          "extended": {
            "correct": 0.6797,
            "new_score": 0.6212,
            "tokens": 14873.0,
            "cost": 2.726,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": null
          },
          "main": {
            "correct": 0.5575,
            "new_score": 0.5062,
            "tokens": 16955.0,
            "cost": 3.171,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": null
          }
        },
        "max": {
          "extended": {
            "correct": 0.7063,
            "new_score": 0.6448,
            "tokens": 25995.0,
            "cost": 3.788,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": null
          },
          "main": {
            "correct": 0.5875,
            "new_score": 0.5326,
            "tokens": 30144.0,
            "cost": 4.485,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": null
          }
        }
      },
      "Grok 4.6": {
        "low": {
          "extended": {
            "correct": 0.6037,
            "new_score": 0.5502,
            "tokens": 10306.3,
            "cost": 0.8005,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0
          },
          "main": {
            "correct": 0.4455,
            "new_score": 0.4011,
            "tokens": 12084.5,
            "cost": 0.9474,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0
          }
        },
        "medium": {
          "extended": {
            "correct": 0.6547,
            "new_score": 0.5962,
            "tokens": 23092.3,
            "cost": 1.6501,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.004
          },
          "main": {
            "correct": 0.512,
            "new_score": 0.4606,
            "tokens": 27065.1,
            "cost": 1.9762,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.006
          }
        },
        "high": {
          "extended": {
            "correct": 0.6705,
            "new_score": 0.6131,
            "tokens": 31616.5,
            "cost": 2.3827,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.0047
          },
          "main": {
            "correct": 0.5315,
            "new_score": 0.4801,
            "tokens": 36758.3,
            "cost": 2.8758,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.007
          }
        }
      },
      "Kimi K3": {
        "none": {
          "extended": {
            "correct": 0.6358,
            "new_score": 0.5819,
            "tokens": 45646.3,
            "cost": 3.1235,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.0014
          },
          "main": {
            "correct": 0.4891,
            "new_score": 0.4417,
            "tokens": 53607.2,
            "cost": 3.8167,
            "tool_calls": null,
            "steps": null,
            "ote": null,
            "flagged_rate": 0.0021
          }
        }
      }
    }
  }
}