{
 "agents": [
  {
   "agent": "INGEST",
   "at": "2026-11-20T18:55:01Z",
   "hash": "c3171e34efdb5b90",
   "ms": 39.5,
   "ok": true,
   "seq": 1
  },
  {
   "agent": "NORMALIZE",
   "at": "2026-11-20T18:55:01Z",
   "hash": "14a11e7a738bc2e6",
   "ms": 41.2,
   "ok": true,
   "seq": 2
  },
  {
   "agent": "SCORE",
   "at": "2026-11-20T18:55:01Z",
   "hash": "8d01eae77e5179da",
   "ms": 41.3,
   "ok": true,
   "seq": 3
  },
  {
   "agent": "RANK",
   "at": "2026-11-20T18:55:01Z",
   "hash": "57f97b9370a34f5f",
   "ms": 42.3,
   "ok": true,
   "seq": 4
  },
  {
   "agent": "TRAJECTORY",
   "at": "2026-11-20T18:55:01Z",
   "hash": "6e82453d6c2880eb",
   "ms": 205.5,
   "ok": true,
   "seq": 5
  },
  {
   "agent": "PUBLISH",
   "at": "2026-11-20T18:55:01Z",
   "hash": "d040350157df228b",
   "ms": 453.4,
   "ok": true,
   "seq": 6
  },
  {
   "agent": "VERIFY",
   "at": "2026-11-20T18:55:01Z",
   "hash": "52417434c405287c",
   "ms": 467.6,
   "ok": true,
   "seq": 7
  }
 ],
 "agi_status": "NOT ACHIEVED",
 "benchmark": "SWE-bench Verified",
 "campaign": {
  "hard_cap_usd": 2500,
  "name": "CAMPAIGN 20",
  "target_rank": 20,
  "target_resolved": 363
 },
 "checksum": "f90f34d321e5fd7dcf01ef5504846d8b00c7ef097c4dd290da77ca8bdda40e54",
 "competitors": [
  {
   "family": "claude-opus-4-5",
   "is_cain": false,
   "order": 0,
   "p50_minutes": 258.225,
   "rank": 1,
   "rate": 0.792,
   "resolved": 396,
   "scaffold": "livesweagent",
   "submission": "20251215_livesweagent_claude-opus-4-5"
  },
  {
   "family": "claude-opus-4-5",
   "is_cain": false,
   "order": 1,
   "p50_minutes": 249.201,
   "rank": 2,
   "rate": 0.792,
   "resolved": 396,
   "scaffold": "sonar",
   "submission": "20251205_sonar-foundation-agent_claude-opus-4-5"
  },
  {
   "family": "doubao",
   "is_cain": false,
   "order": 2,
   "p50_minutes": 246.248,
   "rank": 3,
   "rate": 0.788,
   "resolved": 394,
   "scaffold": "trae",
   "submission": "20250928_trae_doubao_seed_code"
  },
  {
   "family": "claude-opus-4-5",
   "is_cain": false,
   "order": 3,
   "p50_minutes": 197.898,
   "rank": 4,
   "rate": 0.776,
   "resolved": 388,
   "scaffold": "openhands",
   "submission": "20251127_openhands_claude-opus-4-5"
  },
  {
   "family": "gemini-3-pro",
   "is_cain": false,
   "order": 4,
   "p50_minutes": 148.989,
   "rank": 5,
   "rate": 0.774,
   "resolved": 387,
   "scaffold": "livesweagent",
   "submission": "20251120_livesweagent_gemini-3-pro-preview"
  },
  {
   "family": "atlassian",
   "is_cain": false,
   "order": 5,
   "p50_minutes": 126.526,
   "rank": 6,
   "rate": 0.768,
   "resolved": 384,
   "scaffold": "other",
   "submission": "20250902_atlassian-rovo-dev"
  },
  {
   "family": "claude-4-sonnet",
   "is_cain": false,
   "order": 6,
   "p50_minutes": 99.375,
   "rank": 7,
   "rate": 0.768,
   "resolved": 384,
   "scaffold": "other",
   "submission": "20250804_epam-ai-run-claude-4-sonnet"
  },
  {
   "family": "acoder",
   "is_cain": false,
   "order": 7,
   "p50_minutes": 111.841,
   "rank": 8,
   "rate": 0.764,
   "resolved": 382,
   "scaffold": "acoder",
   "submission": "20250819_ACoder"
  },
  {
   "family": "warp",
   "is_cain": false,
   "order": 8,
   "p50_minutes": 160.337,
   "rank": 9,
   "rate": 0.756,
   "resolved": 378,
   "scaffold": "warp",
   "submission": "20250901_warp"
  },
  {
   "family": "trae",
   "is_cain": false,
   "order": 9,
   "p50_minutes": 100.378,
   "rank": 10,
   "rate": 0.752,
   "resolved": 376,
   "scaffold": "trae",
   "submission": "20250612_trae"
  },
  {
   "family": "claude-sonnet-4-5",
   "is_cain": false,
   "order": 10,
   "p50_minutes": 154.429,
   "rank": 11,
   "rate": 0.748,
   "resolved": 374,
   "scaffold": "sonar",
   "submission": "20251103_sonar-foundation-agent_claude-sonnet-4-5"
  },
  {
   "family": "harness",
   "is_cain": false,
   "order": 11,
   "p50_minutes": 84.585,
   "rank": 12,
   "rate": 0.748,
   "resolved": 374,
   "scaffold": "harness",
   "submission": "20250731_harness_ai"
  },
  {
   "family": "joycode",
   "is_cain": false,
   "order": 12,
   "p50_minutes": 139.768,
   "rank": 13,
   "rate": 0.746,
   "resolved": 373,
   "scaffold": "joycode",
   "submission": "20250915_JoyCode"
  },
  {
   "family": "claude-4-sonnet",
   "is_cain": false,
   "order": 13,
   "p50_minutes": 104.693,
   "rank": 14,
   "rate": 0.746,
   "resolved": 373,
   "scaffold": "lingxi",
   "submission": "20250720_Lingxi-v1.5_claude-4-sonnet-20250514"
  },
  {
   "family": "gpt5",
   "is_cain": false,
   "order": 14,
   "p50_minutes": 148.09,
   "rank": 15,
   "rate": 0.744,
   "resolved": 372,
   "scaffold": "prometheus",
   "submission": "20251015_Prometheus_v1.2.1_gpt5"
  },
  {
   "family": "claude-4-sonnet",
   "is_cain": false,
   "order": 15,
   "p50_minutes": 107.374,
   "rank": 16,
   "rate": 0.744,
   "resolved": 372,
   "scaffold": "refact",
   "submission": "20250603_Refact_Agent_claude-4-sonnet"
  },
  {
   "family": "openhands",
   "is_cain": false,
   "order": 16,
   "p50_minutes": 109.957,
   "rank": 17,
   "rate": 0.738,
   "resolved": 369,
   "scaffold": "openhands",
   "submission": "20251103_SalesforceAIResearch_SAGE_OpenHands"
  },
  {
   "family": "claude-4-opus",
   "is_cain": false,
   "order": 17,
   "p50_minutes": 138.941,
   "rank": 18,
   "rate": 0.732,
   "resolved": 366,
   "scaffold": "other",
   "submission": "20250522_tools_claude-4-opus"
  },
  {
   "family": "sage",
   "is_cain": false,
   "order": 18,
   "p50_minutes": 125.929,
   "rank": 19,
   "rate": 0.73,
   "resolved": 365,
   "scaffold": "sage",
   "submission": "20251021_SalesforceAIResearch_SAGE_bash_only"
  },
  {
   "family": "claude-4-sonnet",
   "is_cain": false,
   "order": 19,
   "p50_minutes": 118.909,
   "rank": 20,
   "rate": 0.724,
   "resolved": 362,
   "scaffold": "other",
   "submission": "20250522_tools_claude-4-sonnet"
  },
  {
   "family": "gemini-3-5-flash",
   "is_cain": false,
   "order": 20,
   "p50_minutes": 178.232,
   "rank": 21,
   "rate": 0.718,
   "resolved": 359,
   "scaffold": "other",
   "submission": "20260901_mini-v2.4.2_gemini-3-5-flash"
  },
  {
   "family": "gpt5",
   "is_cain": false,
   "order": 21,
   "p50_minutes": 87.582,
   "rank": 22,
   "rate": 0.718,
   "resolved": 359,
   "scaffold": "openhands",
   "submission": "20250807_openhands_gpt5"
  },
  {
   "family": "gpt5",
   "is_cain": false,
   "order": 22,
   "p50_minutes": 104.597,
   "rank": 23,
   "rate": 0.712,
   "resolved": 356,
   "scaffold": "prometheus",
   "submission": "20250929_Prometheus_v1.2_gpt5"
  },
  {
   "family": "bloop",
   "is_cain": false,
   "order": 23,
   "p50_minutes": 81.366,
   "rank": 24,
   "rate": 0.712,
   "resolved": 356,
   "scaffold": "bloop",
   "submission": "20250710_bloop"
  },
  {
   "family": "lingxi",
   "is_cain": false,
   "order": 24,
   "p50_minutes": 74.417,
   "rank": 25,
   "rate": 0.712,
   "resolved": 356,
   "scaffold": "lingxi",
   "submission": "20251014_Lingxi_kimi_k2"
  },
  {
   "family": "qodo",
   "is_cain": false,
   "order": 25,
   "p50_minutes": 73.475,
   "rank": 26,
   "rate": 0.712,
   "resolved": 356,
   "scaffold": "qodo",
   "submission": "20250715_qodo_command"
  },
  {
   "family": "warp",
   "is_cain": false,
   "order": 26,
   "p50_minutes": 101.659,
   "rank": 27,
   "rate": 0.71,
   "resolved": 355,
   "scaffold": "warp",
   "submission": "20250623_warp"
  },
  {
   "family": "claude-4-sonnet",
   "is_cain": false,
   "order": 27,
   "p50_minutes": 79.575,
   "rank": 28,
   "rate": 0.708,
   "resolved": 354,
   "scaffold": "moatless",
   "submission": "20250611_moatless_claude-4-sonnet-20250514"
  },
  {
   "family": "trae",
   "is_cain": false,
   "order": 28,
   "p50_minutes": 73.88,
   "rank": 29,
   "rate": 0.706,
   "resolved": 353,
   "scaffold": "trae",
   "submission": "20250519_trae"
  },
  {
   "family": "augment",
   "is_cain": false,
   "order": 29,
   "p50_minutes": 84.782,
   "rank": 30,
   "rate": 0.704,
   "resolved": 352,
   "scaffold": "augment",
   "submission": "20250610_augment_agent_v1"
  },
  {
   "family": "openhands",
   "is_cain": false,
   "order": 30,
   "p50_minutes": 76.896,
   "rank": 31,
   "rate": 0.704,
   "resolved": 352,
   "scaffold": "openhands",
   "submission": "20250524_openhands_claude_4_sonnet"
  },
  {
   "family": "refact",
   "is_cain": false,
   "order": 31,
   "p50_minutes": 69.668,
   "rank": 32,
   "rate": 0.704,
   "resolved": 352,
   "scaffold": "refact",
   "submission": "20250515_Refact_Agent"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 32,
   "p50_minutes": 64.679,
   "rank": 33,
   "rate": 0.702,
   "resolved": 351,
   "scaffold": "other",
   "submission": "20250519_devlo"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 33,
   "p50_minutes": 68.248,
   "rank": 34,
   "rate": 0.7,
   "resolved": 350,
   "scaffold": "other",
   "submission": "20250430_zencoder_ai"
  },
  {
   "family": "qwen",
   "is_cain": false,
   "order": 34,
   "p50_minutes": 67.616,
   "rank": 35,
   "rate": 0.696,
   "resolved": 348,
   "scaffold": "openhands",
   "submission": "20250805_openhands-Qwen3-Coder-480B-A35B-Instruct"
  },
  {
   "family": "o3",
   "is_cain": false,
   "order": 35,
   "p50_minutes": 68.289,
   "rank": 36,
   "rate": 0.682,
   "resolved": 341,
   "scaffold": "other",
   "submission": "20250516_cortexa_o3"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 36,
   "p50_minutes": 63.73,
   "rank": 37,
   "rate": 0.682,
   "resolved": 341,
   "scaffold": "other",
   "submission": "20250930_zai_glm4-6"
  },
  {
   "family": "claude-4-sonnet",
   "is_cain": false,
   "order": 37,
   "p50_minutes": 58.288,
   "rank": 38,
   "rate": 0.666,
   "resolved": 333,
   "scaffold": "agent",
   "submission": "20250522_sweagent_claude-4-sonnet-20250514"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 38,
   "p50_minutes": 64.287,
   "rank": 39,
   "rate": 0.664,
   "resolved": 332,
   "scaffold": "other",
   "submission": "20250514_aime_coder"
  },
  {
   "family": "openhands",
   "is_cain": false,
   "order": 39,
   "p50_minutes": 54.24,
   "rank": 40,
   "rate": 0.658,
   "resolved": 329,
   "scaffold": "openhands",
   "submission": "20250415_openhands"
  },
  {
   "family": "openhands",
   "is_cain": false,
   "order": 40,
   "p50_minutes": 52.312,
   "rank": 41,
   "rate": 0.654,
   "resolved": 327,
   "scaffold": "openhands",
   "submission": "20250716_openhands_kimi_k2"
  },
  {
   "family": "augment",
   "is_cain": false,
   "order": 41,
   "p50_minutes": 49.582,
   "rank": 42,
   "rate": 0.654,
   "resolved": 327,
   "scaffold": "augment",
   "submission": "20250316_augment_agent_v0"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 42,
   "p50_minutes": 48.797,
   "rank": 43,
   "rate": 0.654,
   "resolved": 327,
   "scaffold": "agent",
   "submission": "20250405_amazon-q-developer-agent-20250405-dev"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 43,
   "p50_minutes": 55.735,
   "rank": 44,
   "rate": 0.646,
   "resolved": 323,
   "scaffold": "other",
   "submission": "20250117_wandb_programmer_o1_crosscheck5"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 44,
   "p50_minutes": 51.381,
   "rank": 45,
   "rate": 0.646,
   "resolved": 323,
   "scaffold": "other",
   "submission": "20250503_patchpilot-v1.1-o4-mini"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 45,
   "p50_minutes": 51.936,
   "rank": 46,
   "rate": 0.642,
   "resolved": 321,
   "scaffold": "other",
   "submission": "20250728_zai_glm4-5"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 46,
   "p50_minutes": 40.107,
   "rank": 47,
   "rate": 0.634,
   "resolved": 317,
   "scaffold": "agent",
   "submission": "20250206_agentscope"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 47,
   "p50_minutes": 41.992,
   "rank": 48,
   "rate": 0.632,
   "resolved": 316,
   "scaffold": "other",
   "submission": "20250224_tools_claude-3-7-sonnet"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 48,
   "p50_minutes": 39.328,
   "rank": 49,
   "rate": 0.628,
   "resolved": 314,
   "scaffold": "agent",
   "submission": "20250110_blackboxai_agent_v1.1"
  },
  {
   "family": "epam",
   "is_cain": false,
   "order": 49,
   "p50_minutes": 37.53,
   "rank": 50,
   "rate": 0.628,
   "resolved": 314,
   "scaffold": "other",
   "submission": "20250228_epam-ai-run-claude-3-5-sonnet"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 50,
   "p50_minutes": 37.005,
   "rank": 51,
   "rate": 0.624,
   "resolved": 312,
   "scaffold": "agent",
   "submission": "20250225_sweagent_claude-3-7-sonnet"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 51,
   "p50_minutes": 36.945,
   "rank": 52,
   "rate": 0.622,
   "resolved": 311,
   "scaffold": "other",
   "submission": "20241221_codestory_midwit_claude-3-5-sonnet_swe-search"
  },
  {
   "family": "openhands",
   "is_cain": false,
   "order": 52,
   "p50_minutes": 37.172,
   "rank": 53,
   "rate": 0.608,
   "resolved": 304,
   "scaffold": "openhands",
   "submission": "20250203_openhands_4x_scaled"
  },
  {
   "family": "qwen",
   "is_cain": false,
   "order": 53,
   "p50_minutes": 33.486,
   "rank": 54,
   "rate": 0.604,
   "resolved": 302,
   "scaffold": "other",
   "submission": "20250901_entroPO_R2E_QwenCoder30BA3B_tts"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 54,
   "p50_minutes": 30.878,
   "rank": 55,
   "rate": 0.602,
   "resolved": 301,
   "scaffold": "other",
   "submission": "20250110_learn_by_interact_claude3.5"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 55,
   "p50_minutes": 28.049,
   "rank": 56,
   "rate": 0.588,
   "resolved": 294,
   "scaffold": "agent",
   "submission": "20250629_deepswerl_r2eagent_tts"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 56,
   "p50_minutes": 30.749,
   "rank": 57,
   "rate": 0.582,
   "resolved": 291,
   "scaffold": "other",
   "submission": "20241213_devlo"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 57,
   "p50_minutes": 29.047,
   "rank": 58,
   "rate": 0.582,
   "resolved": 291,
   "scaffold": "other",
   "submission": "20250410_cortexa"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 58,
   "p50_minutes": 26.795,
   "rank": 59,
   "rate": 0.572,
   "resolved": 286,
   "scaffold": "other",
   "submission": "20241223_emergent"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 59,
   "p50_minutes": 27.682,
   "rank": 60,
   "rate": 0.57,
   "resolved": 285,
   "scaffold": "agent",
   "submission": "20250924_artemis_agent_v2"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 60,
   "p50_minutes": 24.772,
   "rank": 61,
   "rate": 0.57,
   "resolved": 285,
   "scaffold": "other",
   "submission": "20241208_gru"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 61,
   "p50_minutes": 25.303,
   "rank": 62,
   "rate": 0.566,
   "resolved": 283,
   "scaffold": "other",
   "submission": "20250405_swe-rizzo_claude37"
  },
  {
   "family": "epam",
   "is_cain": false,
   "order": 62,
   "p50_minutes": 23.021,
   "rank": 63,
   "rate": 0.554,
   "resolved": 277,
   "scaffold": "other",
   "submission": "20241212_epam-ai-run-claude-3-5-sonnet"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 63,
   "p50_minutes": 22.633,
   "rank": 64,
   "rate": 0.55,
   "resolved": 275,
   "scaffold": "agent",
   "submission": "20241202_amazon-q-developer-agent-20241202-dev"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 64,
   "p50_minutes": 21.812,
   "rank": 65,
   "rate": 0.542,
   "resolved": 271,
   "scaffold": "other",
   "submission": "20241108_devlo"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 65,
   "p50_minutes": 20.492,
   "rank": 66,
   "rate": 0.536,
   "resolved": 268,
   "scaffold": "other",
   "submission": "20251110_frogboss-32b"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 66,
   "p50_minutes": 19.742,
   "rank": 67,
   "rate": 0.534,
   "resolved": 267,
   "scaffold": "agent",
   "submission": "20250804_codesweep_sweagent_kimi_k2_instruct"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 67,
   "p50_minutes": 21.094,
   "rank": 68,
   "rate": 0.532,
   "resolved": 266,
   "scaffold": "other",
   "submission": "20250120_Bracket"
  },
  {
   "family": "openhands",
   "is_cain": false,
   "order": 68,
   "p50_minutes": 19.295,
   "rank": 69,
   "rate": 0.53,
   "resolved": 265,
   "scaffold": "openhands",
   "submission": "20241029_OpenHands-CodeAct-2.1-sonnet-20241022"
  },
  {
   "family": "gemini",
   "is_cain": false,
   "order": 69,
   "p50_minutes": 18.238,
   "rank": 70,
   "rate": 0.522,
   "resolved": 261,
   "scaffold": "other",
   "submission": "20241212_google_jules_gemini_2.0_flash_experimental"
  },
  {
   "family": "qwen",
   "is_cain": false,
   "order": 70,
   "p50_minutes": 18.204,
   "rank": 71,
   "rate": 0.522,
   "resolved": 261,
   "scaffold": "other",
   "submission": "20250901_entroPO_R2E_QwenCoder30BA3B"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 71,
   "p50_minutes": 18.031,
   "rank": 72,
   "rate": 0.518,
   "resolved": 259,
   "scaffold": "other",
   "submission": "20241125_enginelabs"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 72,
   "p50_minutes": 17.426,
   "rank": 73,
   "rate": 0.516,
   "resolved": 258,
   "scaffold": "other",
   "submission": "20250122_autocoderover-v2.1-claude-3-5-sonnet-20241022"
  },
  {
   "family": "qwen",
   "is_cain": false,
   "order": 73,
   "p50_minutes": 17.31,
   "rank": 74,
   "rate": 0.516,
   "resolved": 258,
   "scaffold": "openhands",
   "submission": "20250805_openhands-Qwen3-Coder-30B-A3B-Instruct"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 74,
   "p50_minutes": 16.404,
   "rank": 75,
   "rate": 0.508,
   "resolved": 254,
   "scaffold": "agent",
   "submission": "20241202_agentless-1.5_claude-3.5-sonnet-20241022"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 75,
   "p50_minutes": 15.502,
   "rank": 76,
   "rate": 0.5,
   "resolved": 250,
   "scaffold": "other",
   "submission": "20241028_solver"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 76,
   "p50_minutes": 15.498,
   "rank": 77,
   "rate": 0.5,
   "resolved": 250,
   "scaffold": "agent",
   "submission": "20241125_marscode-agent-dev"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 77,
   "p50_minutes": 14.702,
   "rank": 78,
   "rate": 0.492,
   "resolved": 246,
   "scaffold": "other",
   "submission": "20241105_nfactorial"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 78,
   "p50_minutes": 14.281,
   "rank": 79,
   "rate": 0.49,
   "resolved": 245,
   "scaffold": "other",
   "submission": "20241022_tools_claude-3-5-sonnet-updated"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 79,
   "p50_minutes": 14.093,
   "rank": 80,
   "rate": 0.486,
   "resolved": 243,
   "scaffold": "other",
   "submission": "20241025_composio_swekit"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 80,
   "p50_minutes": 12.712,
   "rank": 81,
   "rate": 0.472,
   "resolved": 236,
   "scaffold": "other",
   "submission": "20241106_navie-2-gpt4o-sonnet"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 81,
   "p50_minutes": 12.303,
   "rank": 82,
   "rate": 0.47,
   "resolved": 235,
   "scaffold": "other",
   "submission": "20250616_Skywork-SWE-32B+TTS_Bo8"
  },
  {
   "family": "openhands",
   "is_cain": false,
   "order": 82,
   "p50_minutes": 12.071,
   "rank": 83,
   "rate": 0.468,
   "resolved": 234,
   "scaffold": "openhands",
   "submission": "20250520_openhands_devstral_small"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 83,
   "p50_minutes": 12.012,
   "rank": 84,
   "rate": 0.466,
   "resolved": 233,
   "scaffold": "other",
   "submission": "20241023_emergent"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 84,
   "p50_minutes": 11.902,
   "rank": 85,
   "rate": 0.462,
   "resolved": 231,
   "scaffold": "other",
   "submission": "20241108_autocoderover-v2.0-claude-3-5-sonnet-20241022"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 85,
   "p50_minutes": 11.509,
   "rank": 86,
   "rate": 0.46,
   "resolved": 230,
   "scaffold": "other",
   "submission": "20250528_patchpilot_Co-PatcheR"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 86,
   "p50_minutes": 11.342,
   "rank": 87,
   "rate": 0.454,
   "resolved": 227,
   "scaffold": "other",
   "submission": "20240924_solver"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 87,
   "p50_minutes": 11.081,
   "rank": 88,
   "rate": 0.452,
   "resolved": 226,
   "scaffold": "other",
   "submission": "20240824_gru"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 88,
   "p50_minutes": 10.266,
   "rank": 89,
   "rate": 0.45,
   "resolved": 225,
   "scaffold": "other",
   "submission": "20251110_frogmini-14b"
  },
  {
   "family": "gemini",
   "is_cain": false,
   "order": 89,
   "p50_minutes": 10.527,
   "rank": 90,
   "rate": 0.442,
   "resolved": 221,
   "scaffold": "agent",
   "submission": "20250118_codeshellagent_gemini_2.0_flash_experimental"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 90,
   "p50_minutes": 9.641,
   "rank": 91,
   "rate": 0.436,
   "resolved": 218,
   "scaffold": "other",
   "submission": "20240920_solver"
  },
  {
   "family": "o3",
   "is_cain": false,
   "order": 91,
   "p50_minutes": 8.894,
   "rank": 92,
   "rate": 0.424,
   "resolved": 212,
   "scaffold": "agent",
   "submission": "20250214_agentless_lite_o3_mini"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 92,
   "p50_minutes": 8.428,
   "rank": 93,
   "rate": 0.424,
   "resolved": 212,
   "scaffold": "other",
   "submission": "20250527_amazon.nova-premier-v1.0"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 93,
   "p50_minutes": 8.104,
   "rank": 94,
   "rate": 0.422,
   "resolved": 211,
   "scaffold": "agent",
   "submission": "20250629_deepswerl_r2eagent"
  },
  {
   "family": "deepseek",
   "is_cain": false,
   "order": 94,
   "p50_minutes": 8.38,
   "rank": 95,
   "rate": 0.42,
   "resolved": 210,
   "scaffold": "other",
   "submission": "20250806_SWE-Exp_DeepSeek-V3"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 95,
   "p50_minutes": 8.648,
   "rank": 96,
   "rate": 0.416,
   "resolved": 208,
   "scaffold": "other",
   "submission": "20250112_ugaiforge"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 96,
   "p50_minutes": 8.401,
   "rank": 97,
   "rate": 0.416,
   "resolved": 208,
   "scaffold": "other",
   "submission": "20241030_nfactorial"
  },
  {
   "family": "llama",
   "is_cain": false,
   "order": 97,
   "p50_minutes": 7.715,
   "rank": 98,
   "rate": 0.412,
   "resolved": 206,
   "scaffold": "other",
   "submission": "20250226_swerl_llama3_70b"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 98,
   "p50_minutes": 8.264,
   "rank": 99,
   "rate": 0.406,
   "resolved": 203,
   "scaffold": "other",
   "submission": "20241113_nebius-search-open-weight-models-11-24"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 99,
   "p50_minutes": 7.99,
   "rank": 100,
   "rate": 0.406,
   "resolved": 203,
   "scaffold": "other",
   "submission": "20241016_composio_swekit"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 100,
   "p50_minutes": 7.692,
   "rank": 101,
   "rate": 0.406,
   "resolved": 203,
   "scaffold": "other",
   "submission": "20240820_honeycomb"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 101,
   "p50_minutes": 7.618,
   "rank": 102,
   "rate": 0.406,
   "resolved": 203,
   "scaffold": "other",
   "submission": "20241022_tools_claude-3-5-haiku"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 102,
   "p50_minutes": 7.712,
   "rank": 103,
   "rate": 0.402,
   "resolved": 201,
   "scaffold": "agent",
   "submission": "20250511_sweagent_lm_32b"
  },
  {
   "family": "epam",
   "is_cain": false,
   "order": 103,
   "p50_minutes": 6.457,
   "rank": 104,
   "rate": 0.396,
   "resolved": 198,
   "scaffold": "other",
   "submission": "20241029_epam-ai-run-claude-3-5-sonnet"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 104,
   "p50_minutes": 7.077,
   "rank": 105,
   "rate": 0.388,
   "resolved": 194,
   "scaffold": "agent",
   "submission": "20240721_amazon-q-developer-agent-20240719-dev"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 105,
   "p50_minutes": 6.806,
   "rank": 106,
   "rate": 0.388,
   "resolved": 194,
   "scaffold": "agent",
   "submission": "20241028_agentless-1.5_gpt4o"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 106,
   "p50_minutes": 6.397,
   "rank": 107,
   "rate": 0.384,
   "resolved": 192,
   "scaffold": "other",
   "submission": "20240628_autocoderover-v20240620"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 107,
   "p50_minutes": 6.749,
   "rank": 108,
   "rate": 0.38,
   "resolved": 190,
   "scaffold": "agent",
   "submission": "20250725_sweagent_devstral_small_2507"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 108,
   "p50_minutes": 6.682,
   "rank": 109,
   "rate": 0.38,
   "resolved": 190,
   "scaffold": "other",
   "submission": "20250616_Skywork-SWE-32B"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 109,
   "p50_minutes": 5.715,
   "rank": 110,
   "rate": 0.37,
   "resolved": 185,
   "scaffold": "other",
   "submission": "20240617_factory_code_droid"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 110,
   "p50_minutes": 3.716,
   "rank": 111,
   "rate": 0.336,
   "resolved": 168,
   "scaffold": "agent",
   "submission": "20240620_sweagent_claude3.5sonnet"
  },
  {
   "family": "qwen",
   "is_cain": false,
   "order": 111,
   "p50_minutes": 4.595,
   "rank": 112,
   "rate": 0.328,
   "resolved": 164,
   "scaffold": "other",
   "submission": "20250306_SWE-Fixer_Qwen2.5-7b-retriever_Qwen2.5-72b-editor"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 112,
   "p50_minutes": 3.662,
   "rank": 113,
   "rate": 0.326,
   "resolved": 163,
   "scaffold": "other",
   "submission": "20240612_MASAI_gpt4o"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 113,
   "p50_minutes": 2.686,
   "rank": 114,
   "rate": 0.32,
   "resolved": 160,
   "scaffold": "agent",
   "submission": "20241120_artemis_agent"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 114,
   "p50_minutes": 4.349,
   "rank": 115,
   "rate": 0.316,
   "resolved": 158,
   "scaffold": "other",
   "submission": "20241007_nfactorial"
  },
  {
   "family": "qwen",
   "is_cain": false,
   "order": 115,
   "p50_minutes": 3.855,
   "rank": 116,
   "rate": 0.302,
   "resolved": 151,
   "scaffold": "other",
   "submission": "20241128_SWE-Fixer_Qwen2.5-7b-retriever_Qwen2.5-72b-editor_20241128"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 116,
   "p50_minutes": 2.899,
   "rank": 117,
   "rate": 0.288,
   "resolved": 144,
   "scaffold": "agent",
   "submission": "20241002_lingma-agent_lingma-swe-gpt-72b"
  },
  {
   "family": "gpt-4",
   "is_cain": false,
   "order": 117,
   "p50_minutes": 3.462,
   "rank": 118,
   "rate": 0.27,
   "resolved": 135,
   "scaffold": "other",
   "submission": "20241016_epam-ai-run-gpt-4o"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 118,
   "p50_minutes": 2.376,
   "rank": 119,
   "rate": 0.262,
   "resolved": 131,
   "scaffold": "other",
   "submission": "20240615_appmap-navie_gpt4o"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 119,
   "p50_minutes": 2.112,
   "rank": 120,
   "rate": 0.258,
   "resolved": 129,
   "scaffold": "other",
   "submission": "20241001_nfactorial"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 120,
   "p50_minutes": 2.234,
   "rank": 121,
   "rate": 0.256,
   "resolved": 128,
   "scaffold": "agent",
   "submission": "20240509_amazon-q-developer-agent-20240430-dev"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 121,
   "p50_minutes": 2.275,
   "rank": 122,
   "rate": 0.25,
   "resolved": 125,
   "scaffold": "agent",
   "submission": "20240918_lingma-agent_lingma-swe-gpt-72b"
  },
  {
   "family": "gpt-4",
   "is_cain": false,
   "order": 122,
   "p50_minutes": 1.729,
   "rank": 123,
   "rate": 0.24,
   "resolved": 120,
   "scaffold": "other",
   "submission": "20240820_epam-ai-run-gpt-4o"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 123,
   "p50_minutes": 1.67,
   "rank": 124,
   "rate": 0.232,
   "resolved": 116,
   "scaffold": "agent",
   "submission": "20250627_agentless_MCTS-Refine-7B"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 124,
   "p50_minutes": 1.6,
   "rank": 125,
   "rate": 0.232,
   "resolved": 116,
   "scaffold": "agent",
   "submission": "20240728_sweagent_gpt4o"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 125,
   "p50_minutes": 1.753,
   "rank": 126,
   "rate": 0.224,
   "resolved": 112,
   "scaffold": "agent",
   "submission": "20240402_sweagent_gpt4"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 126,
   "p50_minutes": 1.34,
   "rank": 127,
   "rate": 0.182,
   "resolved": 91,
   "scaffold": "agent",
   "submission": "20241002_lingma-agent_lingma-swe-gpt-7b"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 127,
   "p50_minutes": 0.941,
   "rank": 128,
   "rate": 0.158,
   "resolved": 79,
   "scaffold": "agent",
   "submission": "20240402_sweagent_claude3opus"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 128,
   "p50_minutes": 0.492,
   "rank": 129,
   "rate": 0.102,
   "resolved": 51,
   "scaffold": "agent",
   "submission": "20240918_lingma-agent_lingma-swe-gpt-7b"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 129,
   "p50_minutes": 0.35,
   "rank": 130,
   "rate": 0.07,
   "resolved": 35,
   "scaffold": "rag",
   "submission": "20240402_rag_claude3opus"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 130,
   "p50_minutes": 0.23,
   "rank": 131,
   "rate": 0.044,
   "resolved": 22,
   "scaffold": "rag",
   "submission": "20231010_rag_claude2"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 131,
   "p50_minutes": 0.142,
   "rank": 132,
   "rate": 0.028,
   "resolved": 14,
   "scaffold": "rag",
   "submission": "20240402_rag_gpt4"
  },
  {
   "family": "llama",
   "is_cain": false,
   "order": 132,
   "p50_minutes": 0.0,
   "rank": 133,
   "rate": 0.014,
   "resolved": 7,
   "scaffold": "rag",
   "submission": "20231010_rag_swellama7b"
  },
  {
   "family": "llama",
   "is_cain": false,
   "order": 133,
   "p50_minutes": 0.0,
   "rank": 134,
   "rate": 0.012,
   "resolved": 6,
   "scaffold": "rag",
   "submission": "20231010_rag_swellama13b"
  },
  {
   "family": "other",
   "is_cain": false,
   "order": 134,
   "p50_minutes": 0.0,
   "rank": 135,
   "rate": 0.004,
   "resolved": 2,
   "scaffold": "rag",
   "submission": "20231010_rag_gpt35"
  },
  {
   "family": "other",
   "is_cain": true,
   "order": 135,
   "p50_minutes": null,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "scaffold": "agent",
   "submission": "CAIN-42 (70B tool agent, E124)"
  }
 ],
 "delta": {
  "hours_elapsed": 0.167,
  "rank_delta": 0,
  "resolved_delta": 0,
  "since": "2026-11-20T18:45:02Z"
 },
 "entity": "FIRME AI CORPORATION",
 "gaps": [
  {
   "rank": 1,
   "rate": 0.792,
   "resolved": 396,
   "submission": "20251215_livesweagent_claude-opus-4-5",
   "tasks_to_overtake": 397
  },
  {
   "rank": 2,
   "rate": 0.792,
   "resolved": 396,
   "submission": "20251205_sonar-foundation-agent_claude-opus-4-5",
   "tasks_to_overtake": 397
  },
  {
   "rank": 3,
   "rate": 0.788,
   "resolved": 394,
   "submission": "20250928_trae_doubao_seed_code",
   "tasks_to_overtake": 395
  },
  {
   "rank": 4,
   "rate": 0.776,
   "resolved": 388,
   "submission": "20251127_openhands_claude-opus-4-5",
   "tasks_to_overtake": 389
  },
  {
   "rank": 5,
   "rate": 0.774,
   "resolved": 387,
   "submission": "20251120_livesweagent_gemini-3-pro-preview",
   "tasks_to_overtake": 388
  },
  {
   "rank": 6,
   "rate": 0.768,
   "resolved": 384,
   "submission": "20250902_atlassian-rovo-dev",
   "tasks_to_overtake": 385
  },
  {
   "rank": 7,
   "rate": 0.768,
   "resolved": 384,
   "submission": "20250804_epam-ai-run-claude-4-sonnet",
   "tasks_to_overtake": 385
  },
  {
   "rank": 8,
   "rate": 0.764,
   "resolved": 382,
   "submission": "20250819_ACoder",
   "tasks_to_overtake": 383
  },
  {
   "rank": 9,
   "rate": 0.756,
   "resolved": 378,
   "submission": "20250901_warp",
   "tasks_to_overtake": 379
  },
  {
   "rank": 10,
   "rate": 0.752,
   "resolved": 376,
   "submission": "20250612_trae",
   "tasks_to_overtake": 377
  },
  {
   "rank": 11,
   "rate": 0.748,
   "resolved": 374,
   "submission": "20251103_sonar-foundation-agent_claude-sonnet-4-5",
   "tasks_to_overtake": 375
  },
  {
   "rank": 12,
   "rate": 0.748,
   "resolved": 374,
   "submission": "20250731_harness_ai",
   "tasks_to_overtake": 375
  },
  {
   "rank": 13,
   "rate": 0.746,
   "resolved": 373,
   "submission": "20250915_JoyCode",
   "tasks_to_overtake": 374
  },
  {
   "rank": 14,
   "rate": 0.746,
   "resolved": 373,
   "submission": "20250720_Lingxi-v1.5_claude-4-sonnet-20250514",
   "tasks_to_overtake": 374
  },
  {
   "rank": 15,
   "rate": 0.744,
   "resolved": 372,
   "submission": "20251015_Prometheus_v1.2.1_gpt5",
   "tasks_to_overtake": 373
  },
  {
   "rank": 16,
   "rate": 0.744,
   "resolved": 372,
   "submission": "20250603_Refact_Agent_claude-4-sonnet",
   "tasks_to_overtake": 373
  },
  {
   "rank": 17,
   "rate": 0.738,
   "resolved": 369,
   "submission": "20251103_SalesforceAIResearch_SAGE_OpenHands",
   "tasks_to_overtake": 370
  },
  {
   "rank": 18,
   "rate": 0.732,
   "resolved": 366,
   "submission": "20250522_tools_claude-4-opus",
   "tasks_to_overtake": 367
  },
  {
   "rank": 19,
   "rate": 0.73,
   "resolved": 365,
   "submission": "20251021_SalesforceAIResearch_SAGE_bash_only",
   "tasks_to_overtake": 366
  },
  {
   "rank": 20,
   "rate": 0.724,
   "resolved": 362,
   "submission": "20250522_tools_claude-4-sonnet",
   "tasks_to_overtake": 363
  },
  {
   "rank": 21,
   "rate": 0.718,
   "resolved": 359,
   "submission": "20260901_mini-v2.4.2_gemini-3-5-flash",
   "tasks_to_overtake": 360
  },
  {
   "rank": 22,
   "rate": 0.718,
   "resolved": 359,
   "submission": "20250807_openhands_gpt5",
   "tasks_to_overtake": 360
  },
  {
   "rank": 23,
   "rate": 0.712,
   "resolved": 356,
   "submission": "20250929_Prometheus_v1.2_gpt5",
   "tasks_to_overtake": 357
  },
  {
   "rank": 24,
   "rate": 0.712,
   "resolved": 356,
   "submission": "20250710_bloop",
   "tasks_to_overtake": 357
  },
  {
   "rank": 25,
   "rate": 0.712,
   "resolved": 356,
   "submission": "20251014_Lingxi_kimi_k2",
   "tasks_to_overtake": 357
  },
  {
   "rank": 26,
   "rate": 0.712,
   "resolved": 356,
   "submission": "20250715_qodo_command",
   "tasks_to_overtake": 357
  },
  {
   "rank": 27,
   "rate": 0.71,
   "resolved": 355,
   "submission": "20250623_warp",
   "tasks_to_overtake": 356
  },
  {
   "rank": 28,
   "rate": 0.708,
   "resolved": 354,
   "submission": "20250611_moatless_claude-4-sonnet-20250514",
   "tasks_to_overtake": 355
  },
  {
   "rank": 29,
   "rate": 0.706,
   "resolved": 353,
   "submission": "20250519_trae",
   "tasks_to_overtake": 354
  },
  {
   "rank": 30,
   "rate": 0.704,
   "resolved": 352,
   "submission": "20250610_augment_agent_v1",
   "tasks_to_overtake": 353
  },
  {
   "rank": 31,
   "rate": 0.704,
   "resolved": 352,
   "submission": "20250524_openhands_claude_4_sonnet",
   "tasks_to_overtake": 353
  },
  {
   "rank": 32,
   "rate": 0.704,
   "resolved": 352,
   "submission": "20250515_Refact_Agent",
   "tasks_to_overtake": 353
  },
  {
   "rank": 33,
   "rate": 0.702,
   "resolved": 351,
   "submission": "20250519_devlo",
   "tasks_to_overtake": 352
  },
  {
   "rank": 34,
   "rate": 0.7,
   "resolved": 350,
   "submission": "20250430_zencoder_ai",
   "tasks_to_overtake": 351
  },
  {
   "rank": 35,
   "rate": 0.696,
   "resolved": 348,
   "submission": "20250805_openhands-Qwen3-Coder-480B-A35B-Instruct",
   "tasks_to_overtake": 349
  },
  {
   "rank": 36,
   "rate": 0.682,
   "resolved": 341,
   "submission": "20250516_cortexa_o3",
   "tasks_to_overtake": 342
  },
  {
   "rank": 37,
   "rate": 0.682,
   "resolved": 341,
   "submission": "20250930_zai_glm4-6",
   "tasks_to_overtake": 342
  },
  {
   "rank": 38,
   "rate": 0.666,
   "resolved": 333,
   "submission": "20250522_sweagent_claude-4-sonnet-20250514",
   "tasks_to_overtake": 334
  },
  {
   "rank": 39,
   "rate": 0.664,
   "resolved": 332,
   "submission": "20250514_aime_coder",
   "tasks_to_overtake": 333
  },
  {
   "rank": 40,
   "rate": 0.658,
   "resolved": 329,
   "submission": "20250415_openhands",
   "tasks_to_overtake": 330
  },
  {
   "rank": 41,
   "rate": 0.654,
   "resolved": 327,
   "submission": "20250716_openhands_kimi_k2",
   "tasks_to_overtake": 328
  },
  {
   "rank": 42,
   "rate": 0.654,
   "resolved": 327,
   "submission": "20250316_augment_agent_v0",
   "tasks_to_overtake": 328
  },
  {
   "rank": 43,
   "rate": 0.654,
   "resolved": 327,
   "submission": "20250405_amazon-q-developer-agent-20250405-dev",
   "tasks_to_overtake": 328
  },
  {
   "rank": 44,
   "rate": 0.646,
   "resolved": 323,
   "submission": "20250117_wandb_programmer_o1_crosscheck5",
   "tasks_to_overtake": 324
  },
  {
   "rank": 45,
   "rate": 0.646,
   "resolved": 323,
   "submission": "20250503_patchpilot-v1.1-o4-mini",
   "tasks_to_overtake": 324
  },
  {
   "rank": 46,
   "rate": 0.642,
   "resolved": 321,
   "submission": "20250728_zai_glm4-5",
   "tasks_to_overtake": 322
  },
  {
   "rank": 47,
   "rate": 0.634,
   "resolved": 317,
   "submission": "20250206_agentscope",
   "tasks_to_overtake": 318
  },
  {
   "rank": 48,
   "rate": 0.632,
   "resolved": 316,
   "submission": "20250224_tools_claude-3-7-sonnet",
   "tasks_to_overtake": 317
  },
  {
   "rank": 49,
   "rate": 0.628,
   "resolved": 314,
   "submission": "20250110_blackboxai_agent_v1.1",
   "tasks_to_overtake": 315
  },
  {
   "rank": 50,
   "rate": 0.628,
   "resolved": 314,
   "submission": "20250228_epam-ai-run-claude-3-5-sonnet",
   "tasks_to_overtake": 315
  },
  {
   "rank": 51,
   "rate": 0.624,
   "resolved": 312,
   "submission": "20250225_sweagent_claude-3-7-sonnet",
   "tasks_to_overtake": 313
  },
  {
   "rank": 52,
   "rate": 0.622,
   "resolved": 311,
   "submission": "20241221_codestory_midwit_claude-3-5-sonnet_swe-search",
   "tasks_to_overtake": 312
  },
  {
   "rank": 53,
   "rate": 0.608,
   "resolved": 304,
   "submission": "20250203_openhands_4x_scaled",
   "tasks_to_overtake": 305
  },
  {
   "rank": 54,
   "rate": 0.604,
   "resolved": 302,
   "submission": "20250901_entroPO_R2E_QwenCoder30BA3B_tts",
   "tasks_to_overtake": 303
  },
  {
   "rank": 55,
   "rate": 0.602,
   "resolved": 301,
   "submission": "20250110_learn_by_interact_claude3.5",
   "tasks_to_overtake": 302
  },
  {
   "rank": 56,
   "rate": 0.588,
   "resolved": 294,
   "submission": "20250629_deepswerl_r2eagent_tts",
   "tasks_to_overtake": 295
  },
  {
   "rank": 57,
   "rate": 0.582,
   "resolved": 291,
   "submission": "20241213_devlo",
   "tasks_to_overtake": 292
  },
  {
   "rank": 58,
   "rate": 0.582,
   "resolved": 291,
   "submission": "20250410_cortexa",
   "tasks_to_overtake": 292
  },
  {
   "rank": 59,
   "rate": 0.572,
   "resolved": 286,
   "submission": "20241223_emergent",
   "tasks_to_overtake": 287
  },
  {
   "rank": 60,
   "rate": 0.57,
   "resolved": 285,
   "submission": "20250924_artemis_agent_v2",
   "tasks_to_overtake": 286
  },
  {
   "rank": 61,
   "rate": 0.57,
   "resolved": 285,
   "submission": "20241208_gru",
   "tasks_to_overtake": 286
  },
  {
   "rank": 62,
   "rate": 0.566,
   "resolved": 283,
   "submission": "20250405_swe-rizzo_claude37",
   "tasks_to_overtake": 284
  },
  {
   "rank": 63,
   "rate": 0.554,
   "resolved": 277,
   "submission": "20241212_epam-ai-run-claude-3-5-sonnet",
   "tasks_to_overtake": 278
  },
  {
   "rank": 64,
   "rate": 0.55,
   "resolved": 275,
   "submission": "20241202_amazon-q-developer-agent-20241202-dev",
   "tasks_to_overtake": 276
  },
  {
   "rank": 65,
   "rate": 0.542,
   "resolved": 271,
   "submission": "20241108_devlo",
   "tasks_to_overtake": 272
  },
  {
   "rank": 66,
   "rate": 0.536,
   "resolved": 268,
   "submission": "20251110_frogboss-32b",
   "tasks_to_overtake": 269
  },
  {
   "rank": 67,
   "rate": 0.534,
   "resolved": 267,
   "submission": "20250804_codesweep_sweagent_kimi_k2_instruct",
   "tasks_to_overtake": 268
  },
  {
   "rank": 68,
   "rate": 0.532,
   "resolved": 266,
   "submission": "20250120_Bracket",
   "tasks_to_overtake": 267
  },
  {
   "rank": 69,
   "rate": 0.53,
   "resolved": 265,
   "submission": "20241029_OpenHands-CodeAct-2.1-sonnet-20241022",
   "tasks_to_overtake": 266
  },
  {
   "rank": 70,
   "rate": 0.522,
   "resolved": 261,
   "submission": "20241212_google_jules_gemini_2.0_flash_experimental",
   "tasks_to_overtake": 262
  },
  {
   "rank": 71,
   "rate": 0.522,
   "resolved": 261,
   "submission": "20250901_entroPO_R2E_QwenCoder30BA3B",
   "tasks_to_overtake": 262
  },
  {
   "rank": 72,
   "rate": 0.518,
   "resolved": 259,
   "submission": "20241125_enginelabs",
   "tasks_to_overtake": 260
  },
  {
   "rank": 73,
   "rate": 0.516,
   "resolved": 258,
   "submission": "20250122_autocoderover-v2.1-claude-3-5-sonnet-20241022",
   "tasks_to_overtake": 259
  },
  {
   "rank": 74,
   "rate": 0.516,
   "resolved": 258,
   "submission": "20250805_openhands-Qwen3-Coder-30B-A3B-Instruct",
   "tasks_to_overtake": 259
  },
  {
   "rank": 75,
   "rate": 0.508,
   "resolved": 254,
   "submission": "20241202_agentless-1.5_claude-3.5-sonnet-20241022",
   "tasks_to_overtake": 255
  },
  {
   "rank": 76,
   "rate": 0.5,
   "resolved": 250,
   "submission": "20241028_solver",
   "tasks_to_overtake": 251
  },
  {
   "rank": 77,
   "rate": 0.5,
   "resolved": 250,
   "submission": "20241125_marscode-agent-dev",
   "tasks_to_overtake": 251
  },
  {
   "rank": 78,
   "rate": 0.492,
   "resolved": 246,
   "submission": "20241105_nfactorial",
   "tasks_to_overtake": 247
  },
  {
   "rank": 79,
   "rate": 0.49,
   "resolved": 245,
   "submission": "20241022_tools_claude-3-5-sonnet-updated",
   "tasks_to_overtake": 246
  },
  {
   "rank": 80,
   "rate": 0.486,
   "resolved": 243,
   "submission": "20241025_composio_swekit",
   "tasks_to_overtake": 244
  },
  {
   "rank": 81,
   "rate": 0.472,
   "resolved": 236,
   "submission": "20241106_navie-2-gpt4o-sonnet",
   "tasks_to_overtake": 237
  },
  {
   "rank": 82,
   "rate": 0.47,
   "resolved": 235,
   "submission": "20250616_Skywork-SWE-32B+TTS_Bo8",
   "tasks_to_overtake": 236
  },
  {
   "rank": 83,
   "rate": 0.468,
   "resolved": 234,
   "submission": "20250520_openhands_devstral_small",
   "tasks_to_overtake": 235
  },
  {
   "rank": 84,
   "rate": 0.466,
   "resolved": 233,
   "submission": "20241023_emergent",
   "tasks_to_overtake": 234
  },
  {
   "rank": 85,
   "rate": 0.462,
   "resolved": 231,
   "submission": "20241108_autocoderover-v2.0-claude-3-5-sonnet-20241022",
   "tasks_to_overtake": 232
  },
  {
   "rank": 86,
   "rate": 0.46,
   "resolved": 230,
   "submission": "20250528_patchpilot_Co-PatcheR",
   "tasks_to_overtake": 231
  },
  {
   "rank": 87,
   "rate": 0.454,
   "resolved": 227,
   "submission": "20240924_solver",
   "tasks_to_overtake": 228
  },
  {
   "rank": 88,
   "rate": 0.452,
   "resolved": 226,
   "submission": "20240824_gru",
   "tasks_to_overtake": 227
  },
  {
   "rank": 89,
   "rate": 0.45,
   "resolved": 225,
   "submission": "20251110_frogmini-14b",
   "tasks_to_overtake": 226
  },
  {
   "rank": 90,
   "rate": 0.442,
   "resolved": 221,
   "submission": "20250118_codeshellagent_gemini_2.0_flash_experimental",
   "tasks_to_overtake": 222
  },
  {
   "rank": 91,
   "rate": 0.436,
   "resolved": 218,
   "submission": "20240920_solver",
   "tasks_to_overtake": 219
  },
  {
   "rank": 92,
   "rate": 0.424,
   "resolved": 212,
   "submission": "20250214_agentless_lite_o3_mini",
   "tasks_to_overtake": 213
  },
  {
   "rank": 93,
   "rate": 0.424,
   "resolved": 212,
   "submission": "20250527_amazon.nova-premier-v1.0",
   "tasks_to_overtake": 213
  },
  {
   "rank": 94,
   "rate": 0.422,
   "resolved": 211,
   "submission": "20250629_deepswerl_r2eagent",
   "tasks_to_overtake": 212
  },
  {
   "rank": 95,
   "rate": 0.42,
   "resolved": 210,
   "submission": "20250806_SWE-Exp_DeepSeek-V3",
   "tasks_to_overtake": 211
  },
  {
   "rank": 96,
   "rate": 0.416,
   "resolved": 208,
   "submission": "20250112_ugaiforge",
   "tasks_to_overtake": 209
  },
  {
   "rank": 97,
   "rate": 0.416,
   "resolved": 208,
   "submission": "20241030_nfactorial",
   "tasks_to_overtake": 209
  },
  {
   "rank": 98,
   "rate": 0.412,
   "resolved": 206,
   "submission": "20250226_swerl_llama3_70b",
   "tasks_to_overtake": 207
  },
  {
   "rank": 99,
   "rate": 0.406,
   "resolved": 203,
   "submission": "20241113_nebius-search-open-weight-models-11-24",
   "tasks_to_overtake": 204
  },
  {
   "rank": 100,
   "rate": 0.406,
   "resolved": 203,
   "submission": "20241016_composio_swekit",
   "tasks_to_overtake": 204
  },
  {
   "rank": 101,
   "rate": 0.406,
   "resolved": 203,
   "submission": "20240820_honeycomb",
   "tasks_to_overtake": 204
  },
  {
   "rank": 102,
   "rate": 0.406,
   "resolved": 203,
   "submission": "20241022_tools_claude-3-5-haiku",
   "tasks_to_overtake": 204
  },
  {
   "rank": 103,
   "rate": 0.402,
   "resolved": 201,
   "submission": "20250511_sweagent_lm_32b",
   "tasks_to_overtake": 202
  },
  {
   "rank": 104,
   "rate": 0.396,
   "resolved": 198,
   "submission": "20241029_epam-ai-run-claude-3-5-sonnet",
   "tasks_to_overtake": 199
  },
  {
   "rank": 105,
   "rate": 0.388,
   "resolved": 194,
   "submission": "20240721_amazon-q-developer-agent-20240719-dev",
   "tasks_to_overtake": 195
  },
  {
   "rank": 106,
   "rate": 0.388,
   "resolved": 194,
   "submission": "20241028_agentless-1.5_gpt4o",
   "tasks_to_overtake": 195
  },
  {
   "rank": 107,
   "rate": 0.384,
   "resolved": 192,
   "submission": "20240628_autocoderover-v20240620",
   "tasks_to_overtake": 193
  },
  {
   "rank": 108,
   "rate": 0.38,
   "resolved": 190,
   "submission": "20250725_sweagent_devstral_small_2507",
   "tasks_to_overtake": 191
  },
  {
   "rank": 109,
   "rate": 0.38,
   "resolved": 190,
   "submission": "20250616_Skywork-SWE-32B",
   "tasks_to_overtake": 191
  },
  {
   "rank": 110,
   "rate": 0.37,
   "resolved": 185,
   "submission": "20240617_factory_code_droid",
   "tasks_to_overtake": 186
  },
  {
   "rank": 111,
   "rate": 0.336,
   "resolved": 168,
   "submission": "20240620_sweagent_claude3.5sonnet",
   "tasks_to_overtake": 169
  },
  {
   "rank": 112,
   "rate": 0.328,
   "resolved": 164,
   "submission": "20250306_SWE-Fixer_Qwen2.5-7b-retriever_Qwen2.5-72b-editor",
   "tasks_to_overtake": 165
  },
  {
   "rank": 113,
   "rate": 0.326,
   "resolved": 163,
   "submission": "20240612_MASAI_gpt4o",
   "tasks_to_overtake": 164
  },
  {
   "rank": 114,
   "rate": 0.32,
   "resolved": 160,
   "submission": "20241120_artemis_agent",
   "tasks_to_overtake": 161
  },
  {
   "rank": 115,
   "rate": 0.316,
   "resolved": 158,
   "submission": "20241007_nfactorial",
   "tasks_to_overtake": 159
  },
  {
   "rank": 116,
   "rate": 0.302,
   "resolved": 151,
   "submission": "20241128_SWE-Fixer_Qwen2.5-7b-retriever_Qwen2.5-72b-editor_20241128",
   "tasks_to_overtake": 152
  },
  {
   "rank": 117,
   "rate": 0.288,
   "resolved": 144,
   "submission": "20241002_lingma-agent_lingma-swe-gpt-72b",
   "tasks_to_overtake": 145
  },
  {
   "rank": 118,
   "rate": 0.27,
   "resolved": 135,
   "submission": "20241016_epam-ai-run-gpt-4o",
   "tasks_to_overtake": 136
  },
  {
   "rank": 119,
   "rate": 0.262,
   "resolved": 131,
   "submission": "20240615_appmap-navie_gpt4o",
   "tasks_to_overtake": 132
  },
  {
   "rank": 120,
   "rate": 0.258,
   "resolved": 129,
   "submission": "20241001_nfactorial",
   "tasks_to_overtake": 130
  },
  {
   "rank": 121,
   "rate": 0.256,
   "resolved": 128,
   "submission": "20240509_amazon-q-developer-agent-20240430-dev",
   "tasks_to_overtake": 129
  },
  {
   "rank": 122,
   "rate": 0.25,
   "resolved": 125,
   "submission": "20240918_lingma-agent_lingma-swe-gpt-72b",
   "tasks_to_overtake": 126
  },
  {
   "rank": 123,
   "rate": 0.24,
   "resolved": 120,
   "submission": "20240820_epam-ai-run-gpt-4o",
   "tasks_to_overtake": 121
  },
  {
   "rank": 124,
   "rate": 0.232,
   "resolved": 116,
   "submission": "20250627_agentless_MCTS-Refine-7B",
   "tasks_to_overtake": 117
  },
  {
   "rank": 125,
   "rate": 0.232,
   "resolved": 116,
   "submission": "20240728_sweagent_gpt4o",
   "tasks_to_overtake": 117
  },
  {
   "rank": 126,
   "rate": 0.224,
   "resolved": 112,
   "submission": "20240402_sweagent_gpt4",
   "tasks_to_overtake": 113
  },
  {
   "rank": 127,
   "rate": 0.182,
   "resolved": 91,
   "submission": "20241002_lingma-agent_lingma-swe-gpt-7b",
   "tasks_to_overtake": 92
  },
  {
   "rank": 128,
   "rate": 0.158,
   "resolved": 79,
   "submission": "20240402_sweagent_claude3opus",
   "tasks_to_overtake": 80
  },
  {
   "rank": 129,
   "rate": 0.102,
   "resolved": 51,
   "submission": "20240918_lingma-agent_lingma-swe-gpt-7b",
   "tasks_to_overtake": 52
  },
  {
   "rank": 130,
   "rate": 0.07,
   "resolved": 35,
   "submission": "20240402_rag_claude3opus",
   "tasks_to_overtake": 36
  },
  {
   "rank": 131,
   "rate": 0.044,
   "resolved": 22,
   "submission": "20231010_rag_claude2",
   "tasks_to_overtake": 23
  },
  {
   "rank": 132,
   "rate": 0.028,
   "resolved": 14,
   "submission": "20240402_rag_gpt4",
   "tasks_to_overtake": 15
  },
  {
   "rank": 133,
   "rate": 0.014,
   "resolved": 7,
   "submission": "20231010_rag_swellama7b",
   "tasks_to_overtake": 8
  },
  {
   "rank": 134,
   "rate": 0.012,
   "resolved": 6,
   "submission": "20231010_rag_swellama13b",
   "tasks_to_overtake": 7
  },
  {
   "rank": 135,
   "rate": 0.004,
   "resolved": 2,
   "submission": "20231010_rag_gpt35",
   "tasks_to_overtake": 3
  }
 ],
 "generated_at": "2026-11-20T18:55:01Z",
 "history": [
  {
   "at": "2026-11-20T16:27:21Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795192041.754
  },
  {
   "at": "2026-11-20T16:32:38Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795192358.297
  },
  {
   "at": "2026-11-20T16:40:01Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795192801.566
  },
  {
   "at": "2026-11-20T16:48:08Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795193288.608
  },
  {
   "at": "2026-11-20T16:54:38Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795193678.731
  },
  {
   "at": "2026-11-20T17:00:01Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795194001.775
  },
  {
   "at": "2026-11-20T17:10:01Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795194601.809
  },
  {
   "at": "2026-11-20T17:19:39Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795195179.9
  },
  {
   "at": "2026-11-20T17:25:02Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795195502.057
  },
  {
   "at": "2026-11-20T17:30:02Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795195802.122
  },
  {
   "at": "2026-11-20T17:35:02Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795196102.194
  },
  {
   "at": "2026-11-20T17:45:01Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795196701.407
  },
  {
   "at": "2026-11-20T17:50:01Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795197001.568
  },
  {
   "at": "2026-11-20T17:55:02Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795197302.089
  },
  {
   "at": "2026-11-20T18:05:01Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795197901.824
  },
  {
   "at": "2026-11-20T18:10:02Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795198202.064
  },
  {
   "at": "2026-11-20T18:20:02Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795198802.088
  },
  {
   "at": "2026-11-20T18:25:02Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795199102.142
  },
  {
   "at": "2026-11-20T18:30:02Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795199402.434
  },
  {
   "at": "2026-11-20T18:40:01Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795200001.983
  },
  {
   "at": "2026-11-20T18:45:02Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795200302.081
  },
  {
   "at": "2026-11-20T18:55:01Z",
   "leader_resolved": 396,
   "of": 136,
   "rank": 136,
   "rate": 0.0,
   "resolved": 0,
   "source_sha256": {
    "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
    "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
    "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
   },
   "ts": 1795200901.513
  }
 ],
 "next_refresh_seconds": 60,
 "provenance": {
  "evidence": "https://clawx.click/evidence/e124-long-horizon-agent-2026-11-20/",
  "leaderboard_note": "SWE-bench Verified official submissions; scaffolds differ; not METR HCAST",
  "leaderboard_public": "/rank-cain_rank.json",
  "method": "rank = 1 + count(submissions with resolved > CAIN resolved); rate = resolved/500",
  "pipeline": "scripts/e124/rank_agents.py (7 agents, hash-chained)",
  "raw_results_public": "/rank-full_results.json",
  "report": "https://clawx.click/evidence/e124-long-horizon-agent-2026-11-20/E124_REPORT.md",
  "source_files_sha256": {
   "leaderboard": "59e3491c9a3558fa4481cbf26fe1dc85ac30fbe967c9ed1178fdb6ae24bc920c",
   "predictions": "ee6944713e97d0723d70c58e1e26dfda7fc7354ac744ca3e64ea69f88f354bfd",
   "results": "28c148db1da284d2d3b6bfb0a6be7724e41c719f8c6153d1e2a748ac6a7c0a4b"
  }
 },
 "rank": {
  "family_standings": [
   {
    "best_rate": 0.792,
    "best_resolved": 396,
    "best_submission": "20251215_livesweagent_claude-opus-4-5",
    "cain_tasks_needed": 397,
    "family": "claude-opus-4-5",
    "submissions": 3
   },
   {
    "best_rate": 0.788,
    "best_resolved": 394,
    "best_submission": "20250928_trae_doubao_seed_code",
    "cain_tasks_needed": 395,
    "family": "doubao",
    "submissions": 1
   },
   {
    "best_rate": 0.774,
    "best_resolved": 387,
    "best_submission": "20251120_livesweagent_gemini-3-pro-preview",
    "cain_tasks_needed": 388,
    "family": "gemini-3-pro",
    "submissions": 1
   },
   {
    "best_rate": 0.768,
    "best_resolved": 384,
    "best_submission": "20250902_atlassian-rovo-dev",
    "cain_tasks_needed": 385,
    "family": "atlassian",
    "submissions": 1
   },
   {
    "best_rate": 0.768,
    "best_resolved": 384,
    "best_submission": "20250804_epam-ai-run-claude-4-sonnet",
    "cain_tasks_needed": 385,
    "family": "claude-4-sonnet",
    "submissions": 6
   },
   {
    "best_rate": 0.764,
    "best_resolved": 382,
    "best_submission": "20250819_ACoder",
    "cain_tasks_needed": 383,
    "family": "acoder",
    "submissions": 1
   },
   {
    "best_rate": 0.756,
    "best_resolved": 378,
    "best_submission": "20250901_warp",
    "cain_tasks_needed": 379,
    "family": "warp",
    "submissions": 2
   },
   {
    "best_rate": 0.752,
    "best_resolved": 376,
    "best_submission": "20250612_trae",
    "cain_tasks_needed": 377,
    "family": "trae",
    "submissions": 2
   },
   {
    "best_rate": 0.748,
    "best_resolved": 374,
    "best_submission": "20251103_sonar-foundation-agent_claude-sonnet-4-5",
    "cain_tasks_needed": 375,
    "family": "claude-sonnet-4-5",
    "submissions": 1
   },
   {
    "best_rate": 0.748,
    "best_resolved": 374,
    "best_submission": "20250731_harness_ai",
    "cain_tasks_needed": 375,
    "family": "harness",
    "submissions": 1
   },
   {
    "best_rate": 0.746,
    "best_resolved": 373,
    "best_submission": "20250915_JoyCode",
    "cain_tasks_needed": 374,
    "family": "joycode",
    "submissions": 1
   },
   {
    "best_rate": 0.744,
    "best_resolved": 372,
    "best_submission": "20251015_Prometheus_v1.2.1_gpt5",
    "cain_tasks_needed": 373,
    "family": "gpt5",
    "submissions": 3
   },
   {
    "best_rate": 0.738,
    "best_resolved": 369,
    "best_submission": "20251103_SalesforceAIResearch_SAGE_OpenHands",
    "cain_tasks_needed": 370,
    "family": "openhands",
    "submissions": 7
   },
   {
    "best_rate": 0.732,
    "best_resolved": 366,
    "best_submission": "20250522_tools_claude-4-opus",
    "cain_tasks_needed": 367,
    "family": "claude-4-opus",
    "submissions": 1
   },
   {
    "best_rate": 0.73,
    "best_resolved": 365,
    "best_submission": "20251021_SalesforceAIResearch_SAGE_bash_only",
    "cain_tasks_needed": 366,
    "family": "sage",
    "submissions": 1
   },
   {
    "best_rate": 0.718,
    "best_resolved": 359,
    "best_submission": "20260901_mini-v2.4.2_gemini-3-5-flash",
    "cain_tasks_needed": 360,
    "family": "gemini-3-5-flash",
    "submissions": 1
   },
   {
    "best_rate": 0.712,
    "best_resolved": 356,
    "best_submission": "20250710_bloop",
    "cain_tasks_needed": 357,
    "family": "bloop",
    "submissions": 1
   },
   {
    "best_rate": 0.712,
    "best_resolved": 356,
    "best_submission": "20251014_Lingxi_kimi_k2",
    "cain_tasks_needed": 357,
    "family": "lingxi",
    "submissions": 1
   },
   {
    "best_rate": 0.712,
    "best_resolved": 356,
    "best_submission": "20250715_qodo_command",
    "cain_tasks_needed": 357,
    "family": "qodo",
    "submissions": 1
   },
   {
    "best_rate": 0.704,
    "best_resolved": 352,
    "best_submission": "20250610_augment_agent_v1",
    "cain_tasks_needed": 353,
    "family": "augment",
    "submissions": 2
   },
   {
    "best_rate": 0.704,
    "best_resolved": 352,
    "best_submission": "20250515_Refact_Agent",
    "cain_tasks_needed": 353,
    "family": "refact",
    "submissions": 1
   },
   {
    "best_rate": 0.702,
    "best_resolved": 351,
    "best_submission": "20250519_devlo",
    "cain_tasks_needed": 352,
    "family": "other",
    "submissions": 77
   },
   {
    "best_rate": 0.696,
    "best_resolved": 348,
    "best_submission": "20250805_openhands-Qwen3-Coder-480B-A35B-Instruct",
    "cain_tasks_needed": 349,
    "family": "qwen",
    "submissions": 6
   },
   {
    "best_rate": 0.682,
    "best_resolved": 341,
    "best_submission": "20250516_cortexa_o3",
    "cain_tasks_needed": 342,
    "family": "o3",
    "submissions": 2
   },
   {
    "best_rate": 0.628,
    "best_resolved": 314,
    "best_submission": "20250228_epam-ai-run-claude-3-5-sonnet",
    "cain_tasks_needed": 315,
    "family": "epam",
    "submissions": 3
   },
   {
    "best_rate": 0.522,
    "best_resolved": 261,
    "best_submission": "20241212_google_jules_gemini_2.0_flash_experimental",
    "cain_tasks_needed": 262,
    "family": "gemini",
    "submissions": 2
   },
   {
    "best_rate": 0.42,
    "best_resolved": 210,
    "best_submission": "20250806_SWE-Exp_DeepSeek-V3",
    "cain_tasks_needed": 211,
    "family": "deepseek",
    "submissions": 1
   },
   {
    "best_rate": 0.412,
    "best_resolved": 206,
    "best_submission": "20250226_swerl_llama3_70b",
    "cain_tasks_needed": 207,
    "family": "llama",
    "submissions": 3
   },
   {
    "best_rate": 0.27,
    "best_resolved": 135,
    "best_submission": "20241016_epam-ai-run-gpt-4o",
    "cain_tasks_needed": 136,
    "family": "gpt-4",
    "submissions": 2
   }
  ],
  "leader": {
   "rate": 0.792,
   "resolved": 396,
   "submission": "20251215_livesweagent_claude-opus-4-5"
  },
  "milestones": [
   {
    "progress": 0.0,
    "rank": 1,
    "rate": 0.792,
    "resolved": 396,
    "submission": "20251215_livesweagent_claude-opus-4-5",
    "tasks_needed": 397
   },
   {
    "progress": 0.0,
    "rank": 20,
    "rate": 0.724,
    "resolved": 362,
    "submission": "20250522_tools_claude-4-sonnet",
    "tasks_needed": 363
   },
   {
    "progress": 0.0,
    "rank": 50,
    "rate": 0.628,
    "resolved": 314,
    "submission": "20250228_epam-ai-run-claude-3-5-sonnet",
    "tasks_needed": 315
   },
   {
    "progress": 0.0,
    "rank": 100,
    "rate": 0.406,
    "resolved": 203,
    "submission": "20241016_composio_swekit",
    "tasks_needed": 204
   }
  ],
  "neighbours": [
   {
    "is_cain": false,
    "rank": 132,
    "rate": 0.028,
    "resolved": 14,
    "submission": "20240402_rag_gpt4"
   },
   {
    "is_cain": false,
    "rank": 133,
    "rate": 0.014,
    "resolved": 7,
    "submission": "20231010_rag_swellama7b"
   },
   {
    "is_cain": false,
    "rank": 134,
    "rate": 0.012,
    "resolved": 6,
    "submission": "20231010_rag_swellama13b"
   },
   {
    "is_cain": false,
    "rank": 135,
    "rate": 0.004,
    "resolved": 2,
    "submission": "20231010_rag_gpt35"
   },
   {
    "is_cain": true,
    "rank": 136,
    "rate": 0.0,
    "resolved": 0,
    "submission": "CAIN-42 (70B tool agent, E124)"
   }
  ],
  "next_above": {
   "rate": 0.004,
   "resolved": 2,
   "submission": "20231010_rag_gpt35",
   "tasks_to_overtake": 3
  },
  "of": 136,
  "percentile": 0.0,
  "rank": 136,
  "rank_thresholds": {
   "1": {
    "rate": 0.792,
    "resolved": 396,
    "submission": "20251215_livesweagent_claude-opus-4-5",
    "tasks_to_reach": 397
   },
   "10": {
    "rate": 0.752,
    "resolved": 376,
    "submission": "20250612_trae",
    "tasks_to_reach": 377
   },
   "100": {
    "rate": 0.406,
    "resolved": 203,
    "submission": "20241016_composio_swekit",
    "tasks_to_reach": 204
   },
   "125": {
    "rate": 0.232,
    "resolved": 116,
    "submission": "20240728_sweagent_gpt4o",
    "tasks_to_reach": 117
   },
   "130": {
    "rate": 0.07,
    "resolved": 35,
    "submission": "20240402_rag_claude3opus",
    "tasks_to_reach": 36
   },
   "135": {
    "rate": 0.004,
    "resolved": 2,
    "submission": "20231010_rag_gpt35",
    "tasks_to_reach": 3
   },
   "25": {
    "rate": 0.712,
    "resolved": 356,
    "submission": "20251014_Lingxi_kimi_k2",
    "tasks_to_reach": 357
   },
   "50": {
    "rate": 0.628,
    "resolved": 314,
    "submission": "20250228_epam-ai-run-claude-3-5-sonnet",
    "tasks_to_reach": 315
   },
   "75": {
    "rate": 0.508,
    "resolved": 254,
    "submission": "20241202_agentless-1.5_claude-3.5-sonnet-20241022",
    "tasks_to_reach": 255
   }
  },
  "tasks_to_leader": 397,
  "tasks_to_rank_100": 204,
  "what_if_curve": [
   {
    "rank": 136,
    "resolved": 0
   },
   {
    "rank": 136,
    "resolved": 1
   },
   {
    "rank": 135,
    "resolved": 5
   },
   {
    "rank": 133,
    "resolved": 10
   },
   {
    "rank": 131,
    "resolved": 25
   },
   {
    "rank": 130,
    "resolved": 50
   },
   {
    "rank": 127,
    "resolved": 100
   },
   {
    "rank": 104,
    "resolved": 200
   },
   {
    "rank": 56,
    "resolved": 300
   },
   {
    "rank": 1,
    "resolved": 397
   }
  ]
 },
 "run": {
  "attempts_per_task": 1,
  "buckets": {
   "1-4 hours": {
    "n": 42,
    "resolved": 0
   },
   "15 min - 1 hour": {
    "n": 261,
    "resolved": 0
   },
   "<15 min fix": {
    "n": 194,
    "resolved": 0
   },
   ">4 hours": {
    "n": 3,
    "resolved": 0
   }
  },
  "dataset": "SWE-bench_Verified",
  "limits": [
   "one attempt per task",
   "open 70B via HF inference",
   "SWE-bench not METR HCAST"
  ],
  "model": "meta-llama/Llama-3.3-70B-Instruct",
  "n": 500,
  "note": "SWE-bench Verified; human minutes from difficulty buckets; not HCAST/RE-Bench",
  "p50_minutes": null,
  "patches": {
   "bytes": 2460656,
   "empty_patches": 447,
   "n": 500,
   "nonempty_patches": 53,
   "unique_instances": 500
  },
  "rate": 0.0,
  "resolved": 0,
  "scaffold": "cain-70b-e124",
  "wilson95": [
   0.0,
   0.0076
  ]
 },
 "run_progress": {
  "evaluated": 0,
  "resolved": 0,
  "state": "NOT_STARTED",
  "total": 500
 },
 "runner": {
  "at": "2026-11-20T18:54:14Z",
  "predicted": 8,
  "schema": "cain.e124.swebench_runner.v1",
  "spend": {
   "cache_read": 0,
   "calls": 124,
   "input": 1590499,
   "output": 101632,
   "usd": 31.48
  },
  "state": "EVALUATING"
 },
 "schema": "cain.e124.rank_live.v2",
 "verify": {
  "checksum_matches": true,
  "competitor_count_matches": true,
  "no_agi_claim": true,
  "rank_recomputed_matches": true,
  "rate_matches": true,
  "task_count_is_500": true
 },
 "what_if_label": "WHAT_IF \u2014 arithmetic on the measured leaderboard, not a forecast"
}