{"generated_at":"2026-11-11T14:54:07Z","leaderboard":{"zai-org/GLM-5.3-Flash":{"stages":{"full":{"efforts":{"high":{"harnesses":{"deepseek-harness":{"n_trials":705,"n_scored":593,"n_pass":375,"n_infra":112,"cost_usd":170.1,"cost_per_trial":0.2706,"cost_per_pass":0.4279,"tokens_per_trial":5526000.0,"calls_per_trial":68.94,"model_s_per_trial":1624.0,"agent_s_per_trial":1981.0,"efforts_requested":{"high":40318,"low":566},"versions":["0.2.2-rc.1"],"benchmarks":{"deep-swe":{"n_trials":410,"n_scored":332,"n_pass":209,"n_infra":78,"cost_usd":156.3,"cost_per_trial":0.4459,"cost_per_pass":0.7083,"tokens_per_trial":9307000.0,"calls_per_trial":106.4,"model_s_per_trial":2457.0,"agent_s_per_trial":2884.0,"efforts_requested":{"high":35012,"low":308},"versions":["0.2.2-rc.1"],"n_tasks":111,"n_tasks_expected":111,"passed":70.0,"rate":0.6306,"ci":[0.5379,0.7146],"rerun_flip":0.2402,"n_tasks3":110,"pass_all3":0.4364,"pass_mean3":0.6273,"pass_any3":0.8,"pass_all3_ci":[0.3474,0.5296]},"terminal-bench-2.1":{"n_trials":295,"n_scored":261,"n_pass":166,"n_infra":34,"cost_usd":13.89,"cost_per_trial":0.04762,"cost_per_pass":0.07487,"tokens_per_trial":715900.0,"calls_per_trial":21.32,"model_s_per_trial":563.9,"agent_s_per_trial":832.1,"efforts_requested":{"high":5306,"low":258},"versions":["0.2.2-rc.1"],"n_tasks":87,"n_tasks_expected":87,"passed":55.33,"rate":0.636,"ci":[0.5312,0.7294],"rerun_flip":0.2299,"n_tasks3":87,"pass_all3":0.4713,"pass_mean3":0.636,"pass_any3":0.8161,"pass_all3_ci":[0.3698,0.5751]}},"macro":0.6333,"ci":[0.5669,0.6998],"rerun_flip":0.2351,"n_tasks3":197,"pass_all3":0.4538,"pass_mean3":0.6316,"pass_any3":0.808,"pass_all3_ci":[0.3852,0.5224],"complete":true,"vs_leader":{"diff":-0.0233,"ci":[-0.07264,0.02315],"tied":true,"n_tasks":198}},"mini-swe-agent":{"n_trials":633,"n_scored":594,"n_pass":381,"n_infra":39,"cost_usd":142.5,"cost_per_trial":0.23,"cost_per_pass":0.3585,"tokens_per_trial":6555000.0,"calls_per_trial":82.23,"model_s_per_trial":1068.0,"agent_s_per_trial":1653.0,"efforts_requested":{"high":48842},"versions":["2.4.6"],"benchmarks":{"terminal-bench-2.1":{"n_trials":270,"n_scored":261,"n_pass":175,"n_infra":9,"cost_usd":15.73,"cost_per_trial":0.05944,"cost_per_pass":0.08865,"tokens_per_trial":1343000.0,"calls_per_trial":35.1,"model_s_per_trial":641.0,"agent_s_per_trial":974.8,"efforts_requested":{"high":9162},"versions":["2.4.6"],"n_tasks":87,"n_tasks_expected":87,"passed":58.33,"rate":0.6705,"ci":[0.5664,0.7602],"rerun_flip":0.2299,"n_tasks3":87,"pass_all3":0.4828,"pass_mean3":0.6705,"pass_any3":0.8276,"pass_all3_ci":[0.3807,0.5863]},"deep-swe":{"n_trials":363,"n_scored":333,"n_pass":206,"n_infra":30,"cost_usd":126.7,"cost_per_trial":0.3636,"cost_per_pass":0.5878,"tokens_per_trial":10640000.0,"calls_per_trial":119.2,"model_s_per_trial":1403.0,"agent_s_per_trial":2184.0,"efforts_requested":{"high":39680},"versions":["2.4.6"],"n_tasks":111,"n_tasks_expected":111,"passed":68.67,"rate":0.6186,"ci":[0.5257,0.7036],"rerun_flip":0.3363,"n_tasks3":111,"pass_all3":0.3423,"pass_mean3":0.6186,"pass_any3":0.8468,"pass_all3_ci":[0.2607,0.4346]}},"macro":0.6446,"ci":[0.5787,0.7104],"rerun_flip":0.2831,"n_tasks3":198,"pass_all3":0.4126,"pass_mean3":0.6446,"pass_any3":0.8372,"pass_all3_ci":[0.3452,0.4799],"complete":true,"vs_leader":{"diff":-0.01206,"ci":[-0.06173,0.03609],"tied":true,"n_tasks":198}},"qwen-code":{"n_trials":618,"n_scored":594,"n_pass":338,"n_infra":24,"cost_usd":127.6,"cost_per_trial":0.2077,"cost_per_pass":0.3651,"tokens_per_trial":3307000.0,"calls_per_trial":46.81,"model_s_per_trial":1081.0,"agent_s_per_trial":1401.0,"efforts_requested":{"high":27783,"unset":21},"versions":["0.25.2"],"benchmarks":{"terminal-bench-2.1":{"n_trials":265,"n_scored":261,"n_pass":156,"n_infra":4,"cost_usd":15.5,"cost_per_trial":0.05764,"cost_per_pass":0.09644,"tokens_per_trial":677400.0,"calls_per_trial":20.56,"model_s_per_trial":540.2,"agent_s_per_trial":780.2,"efforts_requested":{"high":5354,"unset":11},"versions":["0.25.2"],"n_tasks":87,"n_tasks_expected":87,"passed":52.0,"rate":0.5977,"ci":[0.4926,0.6945],"rerun_flip":0.2069,"n_tasks3":87,"pass_all3":0.4483,"pass_mean3":0.5977,"pass_any3":0.7586,"pass_all3_ci":[0.3482,0.5528]},"deep-swe":{"n_trials":353,"n_scored":333,"n_pass":182,"n_infra":20,"cost_usd":112.1,"cost_per_trial":0.3254,"cost_per_pass":0.5954,"tokens_per_trial":5367000.0,"calls_per_trial":67.38,"model_s_per_trial":1504.0,"agent_s_per_trial":1887.0,"efforts_requested":{"high":22429,"unset":10},"versions":["0.25.2"],"n_tasks":111,"n_tasks_expected":111,"passed":60.67,"rate":0.5465,"ci":[0.4539,0.6361],"rerun_flip":0.3063,"n_tasks3":111,"pass_all3":0.3243,"pass_mean3":0.5465,"pass_any3":0.7838,"pass_all3_ci":[0.2444,0.416]}},"macro":0.5721,"ci":[0.5042,0.6401],"rerun_flip":0.2566,"n_tasks3":198,"pass_all3":0.3863,"pass_mean3":0.5721,"pass_any3":0.7712,"pass_all3_ci":[0.3195,0.4531],"complete":true,"vs_leader":{"diff":-0.0845,"ci":[-0.1344,-0.03477],"tied":false,"n_tasks":198}},"claude-code":{"n_trials":628,"n_scored":594,"n_pass":367,"n_infra":34,"cost_usd":108.0,"cost_per_trial":0.1726,"cost_per_pass":0.2794,"tokens_per_trial":4063000.0,"calls_per_trial":54.6,"model_s_per_trial":1177.0,"agent_s_per_trial":1531.0,"efforts_requested":{"high":32430},"versions":["2.1.300"],"benchmarks":{"deep-swe":{"n_trials":359,"n_scored":333,"n_pass":204,"n_infra":26,"cost_usd":95.76,"cost_per_trial":0.2733,"cost_per_pass":0.4461,"tokens_per_trial":6757000.0,"calls_per_trial":82.79,"model_s_per_trial":1664.0,"agent_s_per_trial":2079.0,"efforts_requested":{"high":27570},"versions":["2.1.300"],"n_tasks":111,"n_tasks_expected":111,"passed":68.0,"rate":0.6126,"ci":[0.5197,0.698],"rerun_flip":0.3003,"n_tasks3":111,"pass_all3":0.3694,"pass_mean3":0.6126,"pass_any3":0.8198,"pass_all3_ci":[0.2854,0.4621]},"terminal-bench-2.1":{"n_trials":269,"n_scored":261,"n_pass":163,"n_infra":8,"cost_usd":12.22,"cost_per_trial":0.04417,"cost_per_pass":0.07073,"tokens_per_trial":625200.0,"calls_per_trial":18.62,"model_s_per_trial":554.5,"agent_s_per_trial":832.0,"efforts_requested":{"high":4860},"versions":["2.1.300"],"n_tasks":87,"n_tasks_expected":87,"passed":54.33,"rate":0.6245,"ci":[0.5195,0.719],"rerun_flip":0.2222,"n_tasks3":87,"pass_all3":0.4483,"pass_mean3":0.6245,"pass_any3":0.7816,"pass_all3_ci":[0.3482,0.5528]}},"macro":0.6186,"ci":[0.5516,0.6855],"rerun_flip":0.2613,"n_tasks3":198,"pass_all3":0.4088,"pass_mean3":0.6186,"pass_any3":0.8007,"pass_all3_ci":[0.3412,0.4764],"complete":true,"vs_leader":{"diff":-0.03806,"ci":[-0.0861,0.009378],"tied":true,"n_tasks":198}},"pi":{"n_trials":635,"n_scored":594,"n_pass":386,"n_infra":41,"cost_usd":75.95,"cost_per_trial":0.1206,"cost_per_pass":0.1856,"tokens_per_trial":2777000.0,"calls_per_trial":55.63,"model_s_per_trial":958.3,"agent_s_per_trial":1560.0,"efforts_requested":{"high":33043},"versions":["1.2.0"],"benchmarks":{"terminal-bench-2.1":{"n_trials":269,"n_scored":261,"n_pass":182,"n_infra":8,"cost_usd":7.719,"cost_per_trial":0.02868,"cost_per_pass":0.04113,"tokens_per_trial":363900.0,"calls_per_trial":17.84,"model_s_per_trial":481.2,"agent_s_per_trial":769.0,"efforts_requested":{"high":4655},"versions":["1.2.0"],"n_tasks":87,"n_tasks_expected":87,"passed":60.67,"rate":0.6973,"ci":[0.5941,0.7838],"rerun_flip":0.1916,"n_tasks3":87,"pass_all3":0.5517,"pass_mean3":0.6973,"pass_any3":0.8391,"pass_all3_ci":[0.4472,0.6518]},"deep-swe":{"n_trials":366,"n_scored":333,"n_pass":204,"n_infra":33,"cost_usd":68.23,"cost_per_trial":0.1927,"cost_per_pass":0.3145,"tokens_per_trial":4669000.0,"calls_per_trial":85.25,"model_s_per_trial":1332.0,"agent_s_per_trial":2180.0,"efforts_requested":{"high":28388},"versions":["1.2.0"],"n_tasks":111,"n_tasks_expected":111,"passed":68.0,"rate":0.6126,"ci":[0.5197,0.698],"rerun_flip":0.2643,"n_tasks3":111,"pass_all3":0.3964,"pass_mean3":0.6126,"pass_any3":0.7928,"pass_all3_ci":[0.3103,0.4894]}},"macro":0.655,"ci":[0.5898,0.7202],"rerun_flip":0.2279,"n_tasks3":198,"pass_all3":0.4741,"pass_mean3":0.655,"pass_any3":0.8159,"pass_all3_ci":[0.4061,0.542],"complete":true,"vs_leader":{"diff":-0.001657,"ci":[-0.0451,0.04194],"tied":true,"n_tasks":198}},"kimi-code":{"n_trials":631,"n_scored":594,"n_pass":386,"n_infra":37,"cost_usd":120.9,"cost_per_trial":0.1901,"cost_per_pass":0.2926,"tokens_per_trial":5290000.0,"calls_per_trial":57.43,"model_s_per_trial":1061.0,"agent_s_per_trial":1415.0,"efforts_requested":{"high":34113},"versions":["2.2.0"],"benchmarks":{"deep-swe":{"n_trials":363,"n_scored":333,"n_pass":209,"n_infra":30,"cost_usd":108.4,"cost_per_trial":0.3028,"cost_per_pass":0.4825,"tokens_per_trial":8641000.0,"calls_per_trial":86.02,"model_s_per_trial":1463.0,"agent_s_per_trial":1909.0,"efforts_requested":{"high":28646},"versions":["2.2.0"],"n_tasks":111,"n_tasks_expected":111,"passed":69.67,"rate":0.6276,"ci":[0.5348,0.7119],"rerun_flip":0.2883,"n_tasks3":111,"pass_all3":0.4144,"pass_mean3":0.6276,"pass_any3":0.8468,"pass_all3_ci":[0.3271,0.5074]},"terminal-bench-2.1":{"n_trials":268,"n_scored":261,"n_pass":177,"n_infra":7,"cost_usd":12.51,"cost_per_trial":0.04631,"cost_per_pass":0.06829,"tokens_per_trial":1015000.0,"calls_per_trial":20.95,"model_s_per_trial":547.4,"agent_s_per_trial":783.7,"efforts_requested":{"high":5467},"versions":["2.2.0"],"n_tasks":87,"n_tasks_expected":87,"passed":59.0,"rate":0.6782,"ci":[0.5743,0.767],"rerun_flip":0.2146,"n_tasks3":87,"pass_all3":0.5057,"pass_mean3":0.6782,"pass_any3":0.8276,"pass_all3_ci":[0.4027,0.6083]}},"macro":0.6529,"ci":[0.5874,0.7184],"rerun_flip":0.2514,"n_tasks3":198,"pass_all3":0.4601,"pass_mean3":0.6529,"pass_any3":0.8372,"pass_all3_ci":[0.3918,0.5284],"complete":true,"vs_leader":{"diff":-0.003728,"ci":[-0.04691,0.03713],"tied":true,"n_tasks":198}},"hermes":{"n_trials":622,"n_scored":594,"n_pass":362,"n_infra":28,"cost_usd":105.5,"cost_per_trial":0.17,"cost_per_pass":0.279,"tokens_per_trial":4430000.0,"calls_per_trial":52.53,"model_s_per_trial":915.7,"agent_s_per_trial":1660.0,"efforts_requested":{"high":30507,"none":530,"unset":165},"versions":["v0.22.1"],"benchmarks":{"terminal-bench-2.1":{"n_trials":265,"n_scored":261,"n_pass":169,"n_infra":4,"cost_usd":10.45,"cost_per_trial":0.03958,"cost_per_pass":0.06112,"tokens_per_trial":632000.0,"calls_per_trial":19.8,"model_s_per_trial":441.1,"agent_s_per_trial":810.3,"efforts_requested":{"high":4806,"none":198,"unset":165},"versions":["v0.22.1"],"n_tasks":87,"n_tasks_expected":87,"passed":56.33,"rate":0.6475,"ci":[0.5428,0.7397],"rerun_flip":0.2299,"n_tasks3":87,"pass_all3":0.4598,"pass_mean3":0.6475,"pass_any3":0.8046,"pass_all3_ci":[0.359,0.564]},"deep-swe":{"n_trials":357,"n_scored":333,"n_pass":193,"n_infra":24,"cost_usd":95.05,"cost_per_trial":0.2722,"cost_per_pass":0.4697,"tokens_per_trial":7406000.0,"calls_per_trial":78.18,"model_s_per_trial":1288.0,"agent_s_per_trial":2327.0,"efforts_requested":{"high":25701,"none":332},"versions":["v0.22.1"],"n_tasks":111,"n_tasks_expected":111,"passed":64.33,"rate":0.5796,"ci":[0.4866,0.6672],"rerun_flip":0.2943,"n_tasks3":111,"pass_all3":0.3604,"pass_mean3":0.5796,"pass_any3":0.8018,"pass_all3_ci":[0.2771,0.453]}},"macro":0.6135,"ci":[0.5467,0.6804],"rerun_flip":0.2621,"n_tasks3":198,"pass_all3":0.4101,"pass_mean3":0.6135,"pass_any3":0.8032,"pass_all3_ci":[0.3425,0.4776],"complete":true,"vs_leader":{"diff":-0.04308,"ci":[-0.08642,-0.0009799],"tied":false,"n_tasks":198}},"codex":{"n_trials":737,"n_scored":594,"n_pass":293,"n_infra":143,"cost_usd":103.9,"cost_per_trial":0.1686,"cost_per_pass":0.3418,"tokens_per_trial":4905000.0,"calls_per_trial":72.66,"model_s_per_trial":654.8,"agent_s_per_trial":1173.0,"efforts_requested":{"high":43158},"versions":["0.162.1"],"benchmarks":{"deep-swe":{"n_trials":356,"n_scored":333,"n_pass":117,"n_infra":23,"cost_usd":85.2,"cost_per_trial":0.2449,"cost_per_pass":0.697,"tokens_per_trial":7205000.0,"calls_per_trial":98.35,"model_s_per_trial":859.9,"agent_s_per_trial":1547.0,"efforts_requested":{"high":32752},"versions":["0.162.1"],"n_tasks":111,"n_tasks_expected":111,"passed":39.0,"rate":0.3514,"ci":[0.2689,0.4438],"rerun_flip":0.3544,"n_tasks3":111,"pass_all3":0.0991,"pass_mean3":0.3514,"pass_any3":0.6306,"pass_all3_ci":[0.05624,0.1688]},"terminal-bench-2.1":{"n_trials":381,"n_scored":261,"n_pass":176,"n_infra":120,"cost_usd":18.7,"cost_per_trial":0.0712,"cost_per_pass":0.1056,"tokens_per_trial":1970000.0,"calls_per_trial":39.87,"model_s_per_trial":393.1,"agent_s_per_trial":696.5,"efforts_requested":{"high":10406},"versions":["0.162.1"],"n_tasks":87,"n_tasks_expected":87,"passed":58.67,"rate":0.6743,"ci":[0.5703,0.7636],"rerun_flip":0.2989,"n_tasks3":87,"pass_all3":0.4368,"pass_mean3":0.6743,"pass_any3":0.8851,"pass_all3_ci":[0.3374,0.5415]}},"macro":0.5128,"ci":[0.4476,0.5781],"rerun_flip":0.3266,"n_tasks3":198,"pass_all3":0.2679,"pass_mean3":0.5128,"pass_any3":0.7578,"pass_all3_ci":[0.2093,0.3266],"complete":true,"vs_leader":{"diff":-0.1438,"ci":[-0.1881,-0.09749],"tied":false,"n_tasks":198}},"opencode":{"n_trials":599,"n_scored":594,"n_pass":386,"n_infra":5,"cost_usd":101.2,"cost_per_trial":0.1704,"cost_per_pass":0.2622,"tokens_per_trial":4693000.0,"calls_per_trial":63.55,"model_s_per_trial":757.4,"agent_s_per_trial":1392.0,"efforts_requested":{"high":37748},"versions":["2.2.6"],"benchmarks":{"terminal-bench-2.1":{"n_trials":265,"n_scored":261,"n_pass":186,"n_infra":4,"cost_usd":9.021,"cost_per_trial":0.03456,"cost_per_pass":0.0485,"tokens_per_trial":645100.0,"calls_per_trial":20.11,"model_s_per_trial":359.7,"agent_s_per_trial":703.5,"efforts_requested":{"high":5248},"versions":["2.2.6"],"n_tasks":87,"n_tasks_expected":87,"passed":62.0,"rate":0.7126,"ci":[0.6102,0.7971],"rerun_flip":0.2299,"n_tasks3":87,"pass_all3":0.5402,"pass_mean3":0.7126,"pass_any3":0.8851,"pass_all3_ci":[0.436,0.641]},"deep-swe":{"n_trials":334,"n_scored":333,"n_pass":200,"n_infra":1,"cost_usd":92.18,"cost_per_trial":0.2768,"cost_per_pass":0.4609,"tokens_per_trial":7865000.0,"calls_per_trial":97.6,"model_s_per_trial":1069.0,"agent_s_per_trial":1932.0,"efforts_requested":{"high":32500},"versions":["2.2.6"],"n_tasks":111,"n_tasks_expected":111,"passed":66.67,"rate":0.6006,"ci":[0.5076,0.6869],"rerun_flip":0.3483,"n_tasks3":111,"pass_all3":0.3243,"pass_mean3":0.6006,"pass_any3":0.8468,"pass_all3_ci":[0.2444,0.416]}},"macro":0.6566,"ci":[0.5917,0.7215],"rerun_flip":0.2891,"n_tasks3":198,"pass_all3":0.4323,"pass_mean3":0.6566,"pass_any3":0.866,"pass_all3_ci":[0.3654,0.4991],"complete":true},"openclaw":{"n_trials":647,"n_scored":590,"n_pass":246,"n_infra":57,"cost_usd":207.2,"cost_per_trial":0.3452,"cost_per_pass":0.828,"tokens_per_trial":9319000.0,"calls_per_trial":127.4,"model_s_per_trial":2285.0,"agent_s_per_trial":3195.0,"efforts_requested":{"high":74827,"low":322,"unset":23},"versions":["2026.10.4"],"benchmarks":{"terminal-bench-2.1":{"n_trials":266,"n_scored":259,"n_pass":129,"n_infra":7,"cost_usd":13.61,"cost_per_trial":0.05164,"cost_per_pass":0.1037,"tokens_per_trial":1083000.0,"calls_per_trial":31.8,"model_s_per_trial":514.9,"agent_s_per_trial":878.3,"efforts_requested":{"high":8207,"unset":23,"low":5},"versions":["2026.10.4"],"n_tasks":87,"n_tasks_expected":87,"passed":43.33,"rate":0.4981,"ci":[0.3953,0.601],"rerun_flip":0.2146,"n_tasks3":85,"pass_all3":0.3412,"pass_mean3":0.498,"pass_any3":0.6353,"pass_all3_ci":[0.2492,0.4469]},"deep-swe":{"n_trials":381,"n_scored":331,"n_pass":117,"n_infra":50,"cost_usd":193.6,"cost_per_trial":0.5749,"cost_per_pass":1.627,"tokens_per_trial":15760000.0,"calls_per_trial":202.2,"model_s_per_trial":3670.0,"agent_s_per_trial":5008.0,"efforts_requested":{"high":66620,"low":317},"versions":["2026.10.4"],"n_tasks":111,"n_tasks_expected":111,"passed":39.17,"rate":0.3529,"ci":[0.2702,0.4453],"rerun_flip":0.3634,"n_tasks3":109,"pass_all3":0.1009,"pass_mean3":0.3547,"pass_any3":0.6422,"pass_all3_ci":[0.05729,0.1717]}},"macro":0.4255,"ci":[0.358,0.493],"rerun_flip":0.289,"n_tasks3":194,"pass_all3":0.221,"pass_mean3":0.4264,"pass_any3":0.6387,"pass_all3_ci":[0.1635,0.2786],"complete":true,"vs_leader":{"diff":-0.2312,"ci":[-0.2883,-0.1757],"tied":false,"n_tasks":198}}},"ranking":["opencode","pi","kimi-code","mini-swe-agent","deepseek-harness","claude-code","hermes","qwen-code","codex","openclaw"],"noise":{"rerun_flip":0.2682,"harness_flip":0.3165,"by_benchmark":{"deep-swe":{"rerun_flip":0.3096,"harness_flip":0.3614},"terminal-bench-2.1":{"rerun_flip":0.2268,"harness_flip":0.2716}},"harness_pairs":[{"harnesses":["opencode","pi"],"flip":0.2684},{"harnesses":["opencode","kimi-code"],"flip":0.2691},{"harnesses":["opencode","mini-swe-agent"],"flip":0.3163},{"harnesses":["opencode","deepseek-harness"],"flip":0.2925},{"harnesses":["opencode","claude-code"],"flip":0.3032},{"harnesses":["opencode","hermes"],"flip":0.279},{"harnesses":["opencode","qwen-code"],"flip":0.3178},{"harnesses":["opencode","codex"],"flip":0.3508},{"harnesses":["opencode","openclaw"],"flip":0.4124},{"harnesses":["pi","kimi-code"],"flip":0.2537},{"harnesses":["pi","mini-swe-agent"],"flip":0.2877},{"harnesses":["pi","deepseek-harness"],"flip":0.2636},{"harnesses":["pi","claude-code"],"flip":0.2695},{"harnesses":["pi","hermes"],"flip":0.2542},{"harnesses":["pi","qwen-code"],"flip":0.284},{"harnesses":["pi","codex"],"flip":0.3521},{"harnesses":["pi","openclaw"],"flip":0.3837},{"harnesses":["kimi-code","mini-swe-agent"],"flip":0.2892},{"harnesses":["kimi-code","deepseek-harness"],"flip":0.274},{"harnesses":["kimi-code","claude-code"],"flip":0.2881},{"harnesses":["kimi-code","hermes"],"flip":0.2772},{"harnesses":["kimi-code","qwen-code"],"flip":0.3005},{"harnesses":["kimi-code","codex"],"flip":0.3427},{"harnesses":["kimi-code","openclaw"],"flip":0.3752},{"harnesses":["mini-swe-agent","deepseek-harness"],"flip":0.2782},{"harnesses":["mini-swe-agent","claude-code"],"flip":0.2789},{"harnesses":["mini-swe-agent","hermes"],"flip":0.2972},{"harnesses":["mini-swe-agent","qwen-code"],"flip":0.3125},{"harnesses":["mini-swe-agent","codex"],"flip":0.3598},{"harnesses":["mini-swe-agent","openclaw"],"flip":0.3858},{"harnesses":["deepseek-harness","claude-code"],"flip":0.2809},{"harnesses":["deepseek-harness","hermes"],"flip":0.2827},{"harnesses":["deepseek-harness","qwen-code"],"flip":0.2678},{"harnesses":["deepseek-harness","codex"],"flip":0.3684},{"harnesses":["deepseek-harness","openclaw"],"flip":0.3643},{"harnesses":["claude-code","hermes"],"flip":0.2669},{"harnesses":["claude-code","qwen-code"],"flip":0.2751},{"harnesses":["claude-code","codex"],"flip":0.3912},{"harnesses":["claude-code","openclaw"],"flip":0.3748},{"harnesses":["hermes","qwen-code"],"flip":0.2961},{"harnesses":["hermes","codex"],"flip":0.3721},{"harnesses":["hermes","openclaw"],"flip":0.3862},{"harnesses":["qwen-code","codex"],"flip":0.3665},{"harnesses":["qwen-code","openclaw"],"flip":0.3601},{"harnesses":["codex","openclaw"],"flip":0.3704}]},"leader":"opencode"},"low":{"harnesses":{"claude-code":{"n_trials":627,"n_scored":594,"n_pass":352,"n_infra":33,"cost_usd":72.03,"cost_per_trial":0.1151,"cost_per_pass":0.1943,"tokens_per_trial":2672000.0,"calls_per_trial":46.66,"model_s_per_trial":735.8,"agent_s_per_trial":1123.0,"efforts_requested":{"low":27718},"versions":["2.1.300"],"benchmarks":{"deep-swe":{"n_trials":359,"n_scored":333,"n_pass":186,"n_infra":26,"cost_usd":61.49,"cost_per_trial":0.1745,"cost_per_pass":0.3125,"tokens_per_trial":4291000.0,"calls_per_trial":68.59,"model_s_per_trial":997.7,"agent_s_per_trial":1464.0,"efforts_requested":{"low":22839},"versions":["2.1.300"],"n_tasks":111,"n_tasks_expected":111,"passed":62.0,"rate":0.5586,"ci":[0.4658,0.6474],"rerun_flip":0.2943,"n_tasks3":111,"pass_all3":0.3243,"pass_mean3":0.5586,"pass_any3":0.7658,"pass_all3_ci":[0.2444,0.416]},"terminal-bench-2.1":{"n_trials":268,"n_scored":261,"n_pass":166,"n_infra":7,"cost_usd":10.54,"cost_per_trial":0.03935,"cost_per_pass":0.06187,"tokens_per_trial":606700.0,"calls_per_trial":18.69,"model_s_per_trial":401.7,"agent_s_per_trial":688.3,"efforts_requested":{"low":4879},"versions":["2.1.300"],"n_tasks":87,"n_tasks_expected":87,"passed":55.33,"rate":0.636,"ci":[0.5312,0.7294],"rerun_flip":0.2759,"n_tasks3":87,"pass_all3":0.4253,"pass_mean3":0.636,"pass_any3":0.8391,"pass_all3_ci":[0.3267,0.5302]}},"macro":0.5973,"ci":[0.53,0.6645],"rerun_flip":0.2851,"n_tasks3":198,"pass_all3":0.3748,"pass_mean3":0.5973,"pass_any3":0.8024,"pass_all3_ci":[0.3082,0.4414],"complete":true,"vs_leader":{"diff":-0.001916,"ci":[-0.05121,0.04593],"tied":true,"n_tasks":198}},"mini-swe-agent":{"n_trials":628,"n_scored":594,"n_pass":353,"n_infra":34,"cost_usd":88.88,"cost_per_trial":0.1437,"cost_per_pass":0.2419,"tokens_per_trial":4034000.0,"calls_per_trial":68.1,"model_s_per_trial":725.5,"agent_s_per_trial":1219.0,"efforts_requested":{"low":40453},"versions":["2.4.6"],"benchmarks":{"deep-swe":{"n_trials":359,"n_scored":333,"n_pass":186,"n_infra":26,"cost_usd":76.53,"cost_per_trial":0.2201,"cost_per_pass":0.3941,"tokens_per_trial":6327000.0,"calls_per_trial":95.93,"model_s_per_trial":947.7,"agent_s_per_trial":1578.0,"efforts_requested":{"low":31944},"versions":["2.4.6"],"n_tasks":111,"n_tasks_expected":111,"passed":62.0,"rate":0.5586,"ci":[0.4658,0.6474],"rerun_flip":0.3363,"n_tasks3":111,"pass_all3":0.2973,"pass_mean3":0.5586,"pass_any3":0.8018,"pass_all3_ci":[0.2202,0.3879]},"terminal-bench-2.1":{"n_trials":269,"n_scored":261,"n_pass":167,"n_infra":8,"cost_usd":12.35,"cost_per_trial":0.04628,"cost_per_pass":0.07232,"tokens_per_trial":1107000.0,"calls_per_trial":32.6,"model_s_per_trial":442.1,"agent_s_per_trial":760.2,"efforts_requested":{"low":8509},"versions":["2.4.6"],"n_tasks":87,"n_tasks_expected":87,"passed":55.67,"rate":0.6398,"ci":[0.535,0.7328],"rerun_flip":0.2375,"n_tasks3":87,"pass_all3":0.4598,"pass_mean3":0.6398,"pass_any3":0.8161,"pass_all3_ci":[0.359,0.564]}},"macro":0.5992,"ci":[0.532,0.6664],"rerun_flip":0.2869,"n_tasks3":198,"pass_all3":0.3785,"pass_mean3":0.5992,"pass_any3":0.8089,"pass_all3_ci":[0.3123,0.4448],"complete":true},"qwen-code":{"n_trials":611,"n_scored":594,"n_pass":305,"n_infra":17,"cost_usd":92.25,"cost_per_trial":0.1491,"cost_per_pass":0.2904,"tokens_per_trial":2369000.0,"calls_per_trial":44.63,"model_s_per_trial":642.8,"agent_s_per_trial":951.5,"efforts_requested":{"low":26493,"unset":18},"versions":["0.25.2"],"benchmarks":{"terminal-bench-2.1":{"n_trials":262,"n_scored":261,"n_pass":150,"n_infra":1,"cost_usd":13.84,"cost_per_trial":0.05296,"cost_per_pass":0.09215,"tokens_per_trial":692300.0,"calls_per_trial":22.38,"model_s_per_trial":383.3,"agent_s_per_trial":618.6,"efforts_requested":{"low":5823,"unset":18},"versions":["0.25.2"],"n_tasks":87,"n_tasks_expected":87,"passed":50.0,"rate":0.5747,"ci":[0.4698,0.6733],"rerun_flip":0.2452,"n_tasks3":87,"pass_all3":0.3793,"pass_mean3":0.5747,"pass_any3":0.7471,"pass_all3_ci":[0.2845,0.4843]},"deep-swe":{"n_trials":349,"n_scored":333,"n_pass":155,"n_infra":16,"cost_usd":78.41,"cost_per_trial":0.2245,"cost_per_pass":0.4823,"tokens_per_trial":3683000.0,"calls_per_trial":62.07,"model_s_per_trial":846.2,"agent_s_per_trial":1212.0,"efforts_requested":{"low":20670},"versions":["0.25.2"],"n_tasks":111,"n_tasks_expected":111,"passed":51.67,"rate":0.4655,"ci":[0.3754,0.5579],"rerun_flip":0.3904,"n_tasks3":111,"pass_all3":0.1802,"pass_mean3":0.4655,"pass_any3":0.7658,"pass_all3_ci":[0.1198,0.262]}},"macro":0.5201,"ci":[0.4518,0.5884],"rerun_flip":0.3178,"n_tasks3":198,"pass_all3":0.2797,"pass_mean3":0.5201,"pass_any3":0.7564,"pass_all3_ci":[0.2182,0.3413],"complete":true,"vs_leader":{"diff":-0.07911,"ci":[-0.1294,-0.03008],"tied":false,"n_tasks":198}},"pi":{"n_trials":629,"n_scored":594,"n_pass":339,"n_infra":35,"cost_usd":47.37,"cost_per_trial":0.07699,"cost_per_pass":0.1349,"tokens_per_trial":1765000.0,"calls_per_trial":46.58,"model_s_per_trial":598.2,"agent_s_per_trial":1090.0,"efforts_requested":{"low":27669},"versions":["1.2.0"],"benchmarks":{"terminal-bench-2.1":{"n_trials":270,"n_scored":261,"n_pass":163,"n_infra":9,"cost_usd":6.174,"cost_per_trial":0.02314,"cost_per_pass":0.03705,"tokens_per_trial":347900.0,"calls_per_trial":17.54,"model_s_per_trial":324.8,"agent_s_per_trial":633.1,"efforts_requested":{"low":4577},"versions":["1.2.0"],"n_tasks":87,"n_tasks_expected":87,"passed":54.33,"rate":0.6245,"ci":[0.5195,0.719],"rerun_flip":0.2682,"n_tasks3":87,"pass_all3":0.4253,"pass_mean3":0.6245,"pass_any3":0.8276,"pass_all3_ci":[0.3267,0.5302]},"deep-swe":{"n_trials":359,"n_scored":333,"n_pass":176,"n_infra":26,"cost_usd":41.2,"cost_per_trial":0.1192,"cost_per_pass":0.2255,"tokens_per_trial":2876000.0,"calls_per_trial":69.35,"model_s_per_trial":812.5,"agent_s_per_trial":1448.0,"efforts_requested":{"low":23092},"versions":["1.2.0"],"n_tasks":111,"n_tasks_expected":111,"passed":58.67,"rate":0.5285,"ci":[0.4363,0.6189],"rerun_flip":0.3904,"n_tasks3":111,"pass_all3":0.2252,"pass_mean3":0.5285,"pass_any3":0.8108,"pass_all3_ci":[0.1575,0.3114]}},"macro":0.5765,"ci":[0.5089,0.6441],"rerun_flip":0.3293,"n_tasks3":198,"pass_all3":0.3253,"pass_mean3":0.5765,"pass_any3":0.8192,"pass_all3_ci":[0.2613,0.3892],"complete":true,"vs_leader":{"diff":-0.02268,"ci":[-0.06426,0.02159],"tied":true,"n_tasks":198}},"codex":{"n_trials":752,"n_scored":594,"n_pass":246,"n_infra":158,"cost_usd":91.74,"cost_per_trial":0.1473,"cost_per_pass":0.3556,"tokens_per_trial":4302000.0,"calls_per_trial":70.06,"model_s_per_trial":546.1,"agent_s_per_trial":1069.0,"efforts_requested":{"low":41616},"versions":["0.162.1"],"benchmarks":{"deep-swe":{"n_trials":356,"n_scored":333,"n_pass":89,"n_infra":23,"cost_usd":74.3,"cost_per_trial":0.2117,"cost_per_pass":0.7922,"tokens_per_trial":6247000.0,"calls_per_trial":93.92,"model_s_per_trial":725.3,"agent_s_per_trial":1422.0,"efforts_requested":{"low":31277},"versions":["0.162.1"],"n_tasks":111,"n_tasks_expected":111,"passed":29.67,"rate":0.2673,"ci":[0.1937,0.3564],"rerun_flip":0.2943,"n_tasks3":111,"pass_all3":0.08108,"pass_mean3":0.2673,"pass_any3":0.5225,"pass_all3_ci":[0.04324,0.1469]},"terminal-bench-2.1":{"n_trials":396,"n_scored":261,"n_pass":157,"n_infra":135,"cost_usd":17.44,"cost_per_trial":0.06506,"cost_per_pass":0.1082,"tokens_per_trial":1820000.0,"calls_per_trial":39.61,"model_s_per_trial":317.6,"agent_s_per_trial":618.8,"efforts_requested":{"low":10339},"versions":["0.162.1"],"n_tasks":87,"n_tasks_expected":87,"passed":52.33,"rate":0.6015,"ci":[0.4965,0.698],"rerun_flip":0.2759,"n_tasks3":87,"pass_all3":0.3908,"pass_mean3":0.6015,"pass_any3":0.8046,"pass_all3_ci":[0.295,0.4959]}},"macro":0.4344,"ci":[0.3696,0.4992],"rerun_flip":0.2851,"n_tasks3":198,"pass_all3":0.2359,"pass_mean3":0.4344,"pass_any3":0.6636,"pass_all3_ci":[0.179,0.2929],"complete":true,"vs_leader":{"diff":-0.1648,"ci":[-0.2138,-0.1133],"tied":false,"n_tasks":198}},"kimi-code":{"n_trials":619,"n_scored":594,"n_pass":338,"n_infra":25,"cost_usd":71.62,"cost_per_trial":0.1143,"cost_per_pass":0.2008,"tokens_per_trial":3154000.0,"calls_per_trial":48.69,"model_s_per_trial":661.4,"agent_s_per_trial":1023.0,"efforts_requested":{"low":28924},"versions":["2.2.0"],"benchmarks":{"deep-swe":{"n_trials":356,"n_scored":333,"n_pass":168,"n_infra":23,"cost_usd":62.19,"cost_per_trial":0.1755,"cost_per_pass":0.3479,"tokens_per_trial":4955000.0,"calls_per_trial":70.71,"model_s_per_trial":895.5,"agent_s_per_trial":1325.0,"efforts_requested":{"low":23545},"versions":["2.2.0"],"n_tasks":111,"n_tasks_expected":111,"passed":56.0,"rate":0.5045,"ci":[0.4129,0.5958],"rerun_flip":0.3423,"n_tasks3":111,"pass_all3":0.2342,"pass_mean3":0.5045,"pass_any3":0.7477,"pass_all3_ci":[0.1652,0.3211]},"terminal-bench-2.1":{"n_trials":263,"n_scored":261,"n_pass":170,"n_infra":2,"cost_usd":9.432,"cost_per_trial":0.03614,"cost_per_pass":0.05548,"tokens_per_trial":856400.0,"calls_per_trial":20.61,"model_s_per_trial":362.8,"agent_s_per_trial":639.1,"efforts_requested":{"low":5379},"versions":["2.2.0"],"n_tasks":87,"n_tasks_expected":87,"passed":56.67,"rate":0.6513,"ci":[0.5467,0.7431],"rerun_flip":0.1686,"n_tasks3":87,"pass_all3":0.5402,"pass_mean3":0.6513,"pass_any3":0.7931,"pass_all3_ci":[0.436,0.641]}},"macro":0.5779,"ci":[0.5108,0.645],"rerun_flip":0.2555,"n_tasks3":198,"pass_all3":0.3872,"pass_mean3":0.5779,"pass_any3":0.7704,"pass_all3_ci":[0.3227,0.4517],"complete":true,"vs_leader":{"diff":-0.02128,"ci":[-0.06871,0.02382],"tied":true,"n_tasks":198}},"hermes":{"n_trials":605,"n_scored":594,"n_pass":347,"n_infra":11,"cost_usd":66.82,"cost_per_trial":0.1098,"cost_per_pass":0.1879,"tokens_per_trial":2740000.0,"calls_per_trial":45.33,"model_s_per_trial":592.0,"agent_s_per_trial":1242.0,"efforts_requested":{"low":26187,"none":538,"unset":201},"versions":["v0.22.1"],"benchmarks":{"terminal-bench-2.1":{"n_trials":261,"n_scored":261,"n_pass":177,"n_infra":0,"cost_usd":9.803,"cost_per_trial":0.03756,"cost_per_pass":0.05538,"tokens_per_trial":678200.0,"calls_per_trial":21.51,"model_s_per_trial":349.6,"agent_s_per_trial":677.2,"efforts_requested":{"low":5207,"none":205,"unset":201},"versions":["v0.22.1"],"n_tasks":87,"n_tasks_expected":87,"passed":59.0,"rate":0.6782,"ci":[0.5743,0.767],"rerun_flip":0.2146,"n_tasks3":87,"pass_all3":0.5172,"pass_mean3":0.6782,"pass_any3":0.8391,"pass_all3_ci":[0.4137,0.6193]},"deep-swe":{"n_trials":344,"n_scored":333,"n_pass":170,"n_infra":11,"cost_usd":57.02,"cost_per_trial":0.1664,"cost_per_pass":0.3259,"tokens_per_trial":4356000.0,"calls_per_trial":64.0,"model_s_per_trial":782.0,"agent_s_per_trial":1684.0,"efforts_requested":{"low":20980,"none":333},"versions":["v0.22.1"],"n_tasks":111,"n_tasks_expected":111,"passed":56.67,"rate":0.5105,"ci":[0.4187,0.6016],"rerun_flip":0.3303,"n_tasks3":111,"pass_all3":0.2793,"pass_mean3":0.5105,"pass_any3":0.7748,"pass_all3_ci":[0.2043,0.369]}},"macro":0.5943,"ci":[0.5279,0.6608],"rerun_flip":0.2724,"n_tasks3":198,"pass_all3":0.3983,"pass_mean3":0.5943,"pass_any3":0.8069,"pass_all3_ci":[0.3323,0.4642],"complete":true,"vs_leader":{"diff":-0.004867,"ci":[-0.04846,0.04024],"tied":true,"n_tasks":198}},"deepseek-harness":{"n_trials":676,"n_scored":594,"n_pass":337,"n_infra":82,"cost_usd":93.98,"cost_per_trial":0.1468,"cost_per_pass":0.2588,"tokens_per_trial":2938000.0,"calls_per_trial":57.04,"model_s_per_trial":1070.0,"agent_s_per_trial":1396.0,"efforts_requested":{"low":33881},"versions":["0.2.2-rc.1"],"benchmarks":{"deep-swe":{"n_trials":397,"n_scored":333,"n_pass":185,"n_infra":64,"cost_usd":84.07,"cost_per_trial":0.2337,"cost_per_pass":0.4206,"tokens_per_trial":4863000.0,"calls_per_trial":86.33,"model_s_per_trial":1564.0,"agent_s_per_trial":1960.0,"efforts_requested":{"low":28748},"versions":["0.2.2-rc.1"],"n_tasks":111,"n_tasks_expected":111,"passed":61.67,"rate":0.5556,"ci":[0.4628,0.6446],"rerun_flip":0.2943,"n_tasks3":111,"pass_all3":0.3333,"pass_mean3":0.5556,"pass_any3":0.7748,"pass_all3_ci":[0.2525,0.4253]},"terminal-bench-2.1":{"n_trials":279,"n_scored":261,"n_pass":152,"n_infra":18,"cost_usd":9.907,"cost_per_trial":0.036,"cost_per_pass":0.06182,"tokens_per_trial":481400.0,"calls_per_trial":19.67,"model_s_per_trial":440.3,"agent_s_per_trial":675.6,"efforts_requested":{"low":5133},"versions":["0.2.2-rc.1"],"n_tasks":87,"n_tasks_expected":87,"passed":50.67,"rate":0.5824,"ci":[0.4774,0.6804],"rerun_flip":0.2529,"n_tasks3":87,"pass_all3":0.3908,"pass_mean3":0.5824,"pass_any3":0.7701,"pass_all3_ci":[0.295,0.4959]}},"macro":0.569,"ci":[0.5009,0.6371],"rerun_flip":0.2736,"n_tasks3":198,"pass_all3":0.3621,"pass_mean3":0.569,"pass_any3":0.7724,"pass_all3_ci":[0.2958,0.4284],"complete":true,"vs_leader":{"diff":-0.03024,"ci":[-0.07627,0.01155],"tied":true,"n_tasks":198}},"opencode":{"n_trials":608,"n_scored":594,"n_pass":348,"n_infra":14,"cost_usd":55.42,"cost_per_trial":0.09329,"cost_per_pass":0.1592,"tokens_per_trial":2505000.0,"calls_per_trial":50.89,"model_s_per_trial":465.2,"agent_s_per_trial":1034.0,"efforts_requested":{"low":30226},"versions":["2.2.6"],"benchmarks":{"deep-swe":{"n_trials":337,"n_scored":333,"n_pass":183,"n_infra":4,"cost_usd":48.02,"cost_per_trial":0.1442,"cost_per_pass":0.2624,"tokens_per_trial":3992000.0,"calls_per_trial":75.37,"model_s_per_trial":633.6,"agent_s_per_trial":1363.0,"efforts_requested":{"low":25098},"versions":["2.2.6"],"n_tasks":111,"n_tasks_expected":111,"passed":61.0,"rate":0.5495,"ci":[0.4569,0.6389],"rerun_flip":0.3303,"n_tasks3":111,"pass_all3":0.3153,"pass_mean3":0.5495,"pass_any3":0.8108,"pass_all3_ci":[0.2363,0.4067]},"terminal-bench-2.1":{"n_trials":271,"n_scored":261,"n_pass":165,"n_infra":10,"cost_usd":7.401,"cost_per_trial":0.02836,"cost_per_pass":0.04485,"tokens_per_trial":607600.0,"calls_per_trial":19.65,"model_s_per_trial":250.4,"agent_s_per_trial":614.4,"efforts_requested":{"low":5128},"versions":["2.2.6"],"n_tasks":87,"n_tasks_expected":87,"passed":55.0,"rate":0.6322,"ci":[0.5273,0.7259],"rerun_flip":0.2759,"n_tasks3":87,"pass_all3":0.4253,"pass_mean3":0.6322,"pass_any3":0.8391,"pass_all3_ci":[0.3267,0.5302]}},"macro":0.5909,"ci":[0.5235,0.6582],"rerun_flip":0.3031,"n_tasks3":198,"pass_all3":0.3703,"pass_mean3":0.5909,"pass_any3":0.8249,"pass_all3_ci":[0.3039,0.4367],"complete":true,"vs_leader":{"diff":-0.008336,"ci":[-0.05183,0.03858],"tied":true,"n_tasks":198}},"openclaw":{"n_trials":632,"n_scored":594,"n_pass":217,"n_infra":38,"cost_usd":191.1,"cost_per_trial":0.3122,"cost_per_pass":0.8545,"tokens_per_trial":9051000.0,"calls_per_trial":132.7,"model_s_per_trial":1311.0,"agent_s_per_trial":2170.0,"efforts_requested":{"low":78787,"unset":13},"versions":["2026.10.4"],"benchmarks":{"terminal-bench-2.1":{"n_trials":267,"n_scored":261,"n_pass":118,"n_infra":6,"cost_usd":22.25,"cost_per_trial":0.08391,"cost_per_pass":0.1856,"tokens_per_trial":2239000.0,"calls_per_trial":47.84,"model_s_per_trial":406.4,"agent_s_per_trial":769.5,"efforts_requested":{"low":12474,"unset":13},"versions":["2026.10.4"],"n_tasks":87,"n_tasks_expected":87,"passed":39.33,"rate":0.4521,"ci":[0.3518,0.5565],"rerun_flip":0.2299,"n_tasks3":87,"pass_all3":0.2759,"pass_mean3":0.4521,"pass_any3":0.6207,"pass_all3_ci":[0.1929,0.3777]},"deep-swe":{"n_trials":365,"n_scored":333,"n_pass":99,"n_infra":32,"cost_usd":168.8,"cost_per_trial":0.4911,"cost_per_pass":1.652,"tokens_per_trial":14390000.0,"calls_per_trial":199.1,"model_s_per_trial":2020.0,"agent_s_per_trial":3267.0,"efforts_requested":{"low":66313},"versions":["2026.10.4"],"n_tasks":111,"n_tasks_expected":111,"passed":33.0,"rate":0.2973,"ci":[0.2202,0.3879],"rerun_flip":0.3003,"n_tasks3":111,"pass_all3":0.09009,"pass_mean3":0.2973,"pass_any3":0.5405,"pass_all3_ci":[0.04967,0.1579]}},"macro":0.3747,"ci":[0.3085,0.4409],"rerun_flip":0.2651,"n_tasks3":198,"pass_all3":0.183,"pass_mean3":0.3747,"pass_any3":0.5806,"pass_all3_ci":[0.1288,0.2371],"complete":true,"vs_leader":{"diff":-0.2245,"ci":[-0.2754,-0.1742],"tied":false,"n_tasks":198}}},"ranking":["mini-swe-agent","claude-code","hermes","opencode","kimi-code","pi","deepseek-harness","qwen-code","codex","openclaw"],"noise":{"rerun_flip":0.2874,"harness_flip":0.3308,"by_benchmark":{"deep-swe":{"rerun_flip":0.3303,"harness_flip":0.3759},"terminal-bench-2.1":{"rerun_flip":0.2444,"harness_flip":0.2858}},"harness_pairs":[{"harnesses":["mini-swe-agent","claude-code"],"flip":0.3057},{"harnesses":["mini-swe-agent","hermes"],"flip":0.2887},{"harnesses":["mini-swe-agent","opencode"],"flip":0.2968},{"harnesses":["mini-swe-agent","kimi-code"],"flip":0.2968},{"harnesses":["mini-swe-agent","pi"],"flip":0.3075},{"harnesses":["mini-swe-agent","deepseek-harness"],"flip":0.2886},{"harnesses":["mini-swe-agent","qwen-code"],"flip":0.3285},{"harnesses":["mini-swe-agent","codex"],"flip":0.361},{"harnesses":["mini-swe-agent","openclaw"],"flip":0.3701},{"harnesses":["claude-code","hermes"],"flip":0.3019},{"harnesses":["claude-code","opencode"],"flip":0.3334},{"harnesses":["claude-code","kimi-code"],"flip":0.2908},{"harnesses":["claude-code","pi"],"flip":0.3192},{"harnesses":["claude-code","deepseek-harness"],"flip":0.3163},{"harnesses":["claude-code","qwen-code"],"flip":0.3371},{"harnesses":["claude-code","codex"],"flip":0.3728},{"harnesses":["claude-code","openclaw"],"flip":0.3871},{"harnesses":["hermes","opencode"],"flip":0.3123},{"harnesses":["hermes","kimi-code"],"flip":0.2813},{"harnesses":["hermes","pi"],"flip":0.3206},{"harnesses":["hermes","deepseek-harness"],"flip":0.2971},{"harnesses":["hermes","qwen-code"],"flip":0.316},{"harnesses":["hermes","codex"],"flip":0.3419},{"harnesses":["hermes","openclaw"],"flip":0.3742},{"harnesses":["opencode","kimi-code"],"flip":0.3032},{"harnesses":["opencode","pi"],"flip":0.3138},{"harnesses":["opencode","deepseek-harness"],"flip":0.3244},{"harnesses":["opencode","qwen-code"],"flip":0.3538},{"harnesses":["opencode","codex"],"flip":0.3837},{"harnesses":["opencode","openclaw"],"flip":0.4048},{"harnesses":["kimi-code","pi"],"flip":0.3058},{"harnesses":["kimi-code","deepseek-harness"],"flip":0.2872},{"harnesses":["kimi-code","qwen-code"],"flip":0.3004},{"harnesses":["kimi-code","codex"],"flip":0.3358},{"harnesses":["kimi-code","openclaw"],"flip":0.367},{"harnesses":["pi","deepseek-harness"],"flip":0.3442},{"harnesses":["pi","qwen-code"],"flip":0.336},{"harnesses":["pi","codex"],"flip":0.3704},{"harnesses":["pi","openclaw"],"flip":0.3868},{"harnesses":["deepseek-harness","qwen-code"],"flip":0.3086},{"harnesses":["deepseek-harness","codex"],"flip":0.3569},{"harnesses":["deepseek-harness","openclaw"],"flip":0.3426},{"harnesses":["qwen-code","codex"],"flip":0.3442},{"harnesses":["qwen-code","openclaw"],"flip":0.3379},{"harnesses":["codex","openclaw"],"flip":0.335}]},"leader":"mini-swe-agent"},"_all":{"harnesses":{"claude-code":{"cost_per_trial":0.1439,"cost_per_pass":0.2368,"tokens_per_trial":3367000.0,"calls_per_trial":50.63,"model_s_per_trial":956.2,"agent_s_per_trial":1327.0,"rerun_flip":0.2732,"pass_all3":0.3918,"pass_mean3":0.6079,"pass_any3":0.8016,"n_trials":1255,"n_scored":1188,"n_pass":719,"n_infra":67,"cost_usd":180.0,"n_tasks3":396,"macro":0.6079,"ci":[0.5605,0.6554],"pass_all3_ci":[0.3444,0.4392],"benchmarks":{"deep-swe":{"cost_per_trial":0.2239,"cost_per_pass":0.3793,"tokens_per_trial":5524000.0,"calls_per_trial":75.69,"model_s_per_trial":1331.0,"agent_s_per_trial":1772.0,"rerun_flip":0.2973,"pass_all3":0.3468,"pass_mean3":0.5856,"pass_any3":0.7928,"n_trials":718,"n_scored":666,"n_pass":390,"n_infra":52,"cost_usd":157.2,"n_tasks3":222,"rate":0.5856,"ci":[0.5219,0.6492],"pass_all3_ci":[0.2853,0.4084],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04176,"cost_per_pass":0.0663,"tokens_per_trial":616000.0,"calls_per_trial":18.66,"model_s_per_trial":478.1,"agent_s_per_trial":760.2,"rerun_flip":0.249,"pass_all3":0.4368,"pass_mean3":0.6303,"pass_any3":0.8103,"n_trials":537,"n_scored":522,"n_pass":329,"n_infra":15,"cost_usd":22.77,"n_tasks3":174,"rate":0.6303,"ci":[0.56,0.7006],"pass_all3_ci":[0.3647,0.5089],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.6186,"low":0.5973},"efforts_requested":{"high":32430,"low":27718},"versions":["2.1.300"],"vs_leader":{"diff":-0.01582,"ci":[-0.05833,0.02307],"tied":true,"n_tasks":198}},"codex":{"cost_per_trial":0.1579,"cost_per_pass":0.3487,"tokens_per_trial":4603000.0,"calls_per_trial":71.36,"model_s_per_trial":600.4,"agent_s_per_trial":1121.0,"rerun_flip":0.3058,"pass_all3":0.2519,"pass_mean3":0.4736,"pass_any3":0.7107,"n_trials":1489,"n_scored":1188,"n_pass":539,"n_infra":301,"cost_usd":195.6,"n_tasks3":396,"macro":0.4736,"ci":[0.4276,0.5196],"pass_all3_ci":[0.2111,0.2928],"benchmarks":{"deep-swe":{"cost_per_trial":0.2283,"cost_per_pass":0.7446,"tokens_per_trial":6726000.0,"calls_per_trial":96.14,"model_s_per_trial":792.6,"agent_s_per_trial":1485.0,"rerun_flip":0.3243,"pass_all3":0.09009,"pass_mean3":0.3093,"pass_any3":0.5766,"n_trials":712,"n_scored":666,"n_pass":206,"n_infra":46,"cost_usd":159.5,"n_tasks3":222,"rate":0.3093,"ci":[0.2496,0.369],"pass_all3_ci":[0.05183,0.1283],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.06813,"cost_per_pass":0.1069,"tokens_per_trial":1895000.0,"calls_per_trial":39.74,"model_s_per_trial":355.3,"agent_s_per_trial":657.7,"rerun_flip":0.2874,"pass_all3":0.4138,"pass_mean3":0.6379,"pass_any3":0.8448,"n_trials":777,"n_scored":522,"n_pass":333,"n_infra":255,"cost_usd":36.14,"n_tasks3":174,"rate":0.6379,"ci":[0.5681,0.7077],"pass_all3_ci":[0.3422,0.4854],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.5128,"low":0.4344},"efforts_requested":{"high":43158,"low":41616},"versions":["0.162.1"],"vs_leader":{"diff":-0.1501,"ci":[-0.1889,-0.1108],"tied":false,"n_tasks":198}},"deepseek-harness":{"cost_per_trial":0.2087,"cost_per_pass":0.3433,"tokens_per_trial":4232000.0,"calls_per_trial":62.99,"model_s_per_trial":1347.0,"agent_s_per_trial":1688.0,"rerun_flip":0.2543,"pass_all3":0.4079,"pass_mean3":0.6003,"pass_any3":0.7902,"n_trials":1381,"n_scored":1187,"n_pass":712,"n_infra":194,"cost_usd":264.1,"n_tasks3":395,"macro":0.6011,"ci":[0.5536,0.6487],"pass_all3_ci":[0.3602,0.4556],"benchmarks":{"deep-swe":{"cost_per_trial":0.3398,"cost_per_pass":0.5645,"tokens_per_trial":7085000.0,"calls_per_trial":96.36,"model_s_per_trial":2011.0,"agent_s_per_trial":2422.0,"rerun_flip":0.2673,"pass_all3":0.3848,"pass_mean3":0.5914,"pass_any3":0.7874,"n_trials":807,"n_scored":665,"n_pass":394,"n_infra":142,"cost_usd":240.3,"n_tasks3":221,"rate":0.5931,"ci":[0.5297,0.6565],"pass_all3_ci":[0.3221,0.4476],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04181,"cost_per_pass":0.06834,"tokens_per_trial":598700.0,"calls_per_trial":20.49,"model_s_per_trial":502.1,"agent_s_per_trial":753.9,"rerun_flip":0.2414,"pass_all3":0.431,"pass_mean3":0.6092,"pass_any3":0.7931,"n_trials":574,"n_scored":522,"n_pass":318,"n_infra":52,"cost_usd":23.79,"n_tasks3":174,"rate":0.6092,"ci":[0.5383,0.6801],"pass_all3_ci":[0.3592,0.5028],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.6333,"low":0.569},"efforts_requested":{"high":40318,"low":34447},"versions":["0.2.2-rc.1"],"vs_leader":{"diff":-0.0226,"ci":[-0.06351,0.01515],"tied":true,"n_tasks":198}},"hermes":{"cost_per_trial":0.1399,"cost_per_pass":0.2334,"tokens_per_trial":3585000.0,"calls_per_trial":48.93,"model_s_per_trial":753.9,"agent_s_per_trial":1451.0,"rerun_flip":0.2673,"pass_all3":0.4042,"pass_mean3":0.6039,"pass_any3":0.8051,"n_trials":1227,"n_scored":1188,"n_pass":709,"n_infra":39,"cost_usd":172.3,"n_tasks3":396,"macro":0.6039,"ci":[0.5568,0.6511],"pass_all3_ci":[0.357,0.4513],"benchmarks":{"deep-swe":{"cost_per_trial":0.2193,"cost_per_pass":0.3978,"tokens_per_trial":5881000.0,"calls_per_trial":71.09,"model_s_per_trial":1035.0,"agent_s_per_trial":2006.0,"rerun_flip":0.3123,"pass_all3":0.3198,"pass_mean3":0.545,"pass_any3":0.7883,"n_trials":701,"n_scored":666,"n_pass":363,"n_infra":35,"cost_usd":152.1,"n_tasks3":222,"rate":0.545,"ci":[0.4808,0.6093],"pass_all3_ci":[0.2596,0.3801],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.03857,"cost_per_pass":0.05825,"tokens_per_trial":655100.0,"calls_per_trial":20.66,"model_s_per_trial":395.4,"agent_s_per_trial":743.8,"rerun_flip":0.2222,"pass_all3":0.4885,"pass_mean3":0.6628,"pass_any3":0.8218,"n_trials":526,"n_scored":522,"n_pass":346,"n_infra":4,"cost_usd":20.26,"n_tasks3":174,"rate":0.6628,"ci":[0.594,0.7317],"pass_all3_ci":[0.4159,0.5611],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.6135,"low":0.5943},"efforts_requested":{"high":30507,"low":26187,"none":1068,"unset":366},"versions":["v0.22.1"],"vs_leader":{"diff":-0.0198,"ci":[-0.05615,0.01385],"tied":true,"n_tasks":198}},"kimi-code":{"cost_per_trial":0.1522,"cost_per_pass":0.2467,"tokens_per_trial":4222000.0,"calls_per_trial":53.06,"model_s_per_trial":861.2,"agent_s_per_trial":1219.0,"rerun_flip":0.2534,"pass_all3":0.4237,"pass_mean3":0.6154,"pass_any3":0.8038,"n_trials":1250,"n_scored":1188,"n_pass":724,"n_infra":62,"cost_usd":192.5,"n_tasks3":396,"macro":0.6154,"ci":[0.5685,0.6623],"pass_all3_ci":[0.3767,0.4706],"benchmarks":{"deep-swe":{"cost_per_trial":0.2392,"cost_per_pass":0.4152,"tokens_per_trial":6798000.0,"calls_per_trial":78.36,"model_s_per_trial":1179.0,"agent_s_per_trial":1617.0,"rerun_flip":0.3153,"pass_all3":0.3243,"pass_mean3":0.5661,"pass_any3":0.7973,"n_trials":719,"n_scored":666,"n_pass":377,"n_infra":53,"cost_usd":170.6,"n_tasks3":222,"rate":0.5661,"ci":[0.5024,0.6297],"pass_all3_ci":[0.2647,0.3839],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04122,"cost_per_pass":0.06188,"tokens_per_trial":935800.0,"calls_per_trial":20.78,"model_s_per_trial":455.1,"agent_s_per_trial":711.4,"rerun_flip":0.1916,"pass_all3":0.523,"pass_mean3":0.6648,"pass_any3":0.8103,"n_trials":531,"n_scored":522,"n_pass":347,"n_infra":9,"cost_usd":21.95,"n_tasks3":174,"rate":0.6648,"ci":[0.596,0.7335],"pass_all3_ci":[0.4504,0.5956],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.6529,"low":0.5779},"efforts_requested":{"high":34113,"low":28924},"versions":["2.2.0"],"vs_leader":{"diff":-0.008336,"ci":[-0.0393,0.02167],"tied":true,"n_tasks":198}},"mini-swe-agent":{"cost_per_trial":0.1868,"cost_per_pass":0.3002,"tokens_per_trial":5294000.0,"calls_per_trial":75.16,"model_s_per_trial":896.8,"agent_s_per_trial":1436.0,"rerun_flip":0.285,"pass_all3":0.3955,"pass_mean3":0.6219,"pass_any3":0.8231,"n_trials":1261,"n_scored":1188,"n_pass":734,"n_infra":73,"cost_usd":231.4,"n_tasks3":396,"macro":0.6219,"ci":[0.5749,0.6689],"pass_all3_ci":[0.3483,0.4428],"benchmarks":{"deep-swe":{"cost_per_trial":0.2919,"cost_per_pass":0.4909,"tokens_per_trial":8484000.0,"calls_per_trial":107.5,"model_s_per_trial":1175.0,"agent_s_per_trial":1881.0,"rerun_flip":0.3363,"pass_all3":0.3198,"pass_mean3":0.5886,"pass_any3":0.8243,"n_trials":722,"n_scored":666,"n_pass":392,"n_infra":56,"cost_usd":203.3,"n_tasks3":222,"rate":0.5886,"ci":[0.525,0.6521],"pass_all3_ci":[0.2594,0.3802],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.05286,"cost_per_pass":0.08049,"tokens_per_trial":1225000.0,"calls_per_trial":33.85,"model_s_per_trial":541.6,"agent_s_per_trial":867.5,"rerun_flip":0.2337,"pass_all3":0.4713,"pass_mean3":0.6552,"pass_any3":0.8218,"n_trials":539,"n_scored":522,"n_pass":342,"n_infra":17,"cost_usd":28.08,"n_tasks3":174,"rate":0.6552,"ci":[0.5859,0.7244],"pass_all3_ci":[0.3987,0.5438],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.6446,"low":0.5992},"efforts_requested":{"high":48842,"low":40453},"versions":["2.4.6"],"vs_leader":{"diff":-0.001864,"ci":[-0.03958,0.03322],"tied":true,"n_tasks":198}},"openclaw":{"cost_per_trial":0.3287,"cost_per_pass":0.8412,"tokens_per_trial":9185000.0,"calls_per_trial":130.0,"model_s_per_trial":1798.0,"agent_s_per_trial":2683.0,"rerun_flip":0.277,"pass_all3":0.202,"pass_mean3":0.4005,"pass_any3":0.6097,"n_trials":1279,"n_scored":1184,"n_pass":463,"n_infra":95,"cost_usd":398.3,"n_tasks3":392,"macro":0.4001,"ci":[0.3528,0.4474],"pass_all3_ci":[0.1625,0.2415],"benchmarks":{"deep-swe":{"cost_per_trial":0.533,"cost_per_pass":1.639,"tokens_per_trial":15080000.0,"calls_per_trial":200.7,"model_s_per_trial":2845.0,"agent_s_per_trial":4138.0,"rerun_flip":0.3318,"pass_all3":0.0955,"pass_mean3":0.326,"pass_any3":0.5914,"n_trials":746,"n_scored":664,"n_pass":216,"n_infra":82,"cost_usd":362.4,"n_tasks3":220,"rate":0.3251,"ci":[0.2645,0.3857],"pass_all3_ci":[0.05612,0.1349],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.06778,"cost_per_pass":0.1446,"tokens_per_trial":1661000.0,"calls_per_trial":39.82,"model_s_per_trial":460.6,"agent_s_per_trial":823.9,"rerun_flip":0.2222,"pass_all3":0.3085,"pass_mean3":0.4751,"pass_any3":0.628,"n_trials":533,"n_scored":520,"n_pass":247,"n_infra":13,"cost_usd":35.86,"n_tasks3":172,"rate":0.4751,"ci":[0.4026,0.5476],"pass_all3_ci":[0.2409,0.3762],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.4255,"low":0.3747},"efforts_requested":{"low":79109,"high":74827,"unset":36},"versions":["2026.10.4"],"vs_leader":{"diff":-0.2237,"ci":[-0.2723,-0.1763],"tied":false,"n_tasks":198}},"opencode":{"cost_per_trial":0.1318,"cost_per_pass":0.2107,"tokens_per_trial":3599000.0,"calls_per_trial":57.22,"model_s_per_trial":611.3,"agent_s_per_trial":1213.0,"rerun_flip":0.2961,"pass_all3":0.4013,"pass_mean3":0.6237,"pass_any3":0.8454,"n_trials":1207,"n_scored":1188,"n_pass":734,"n_infra":19,"cost_usd":156.6,"n_tasks3":396,"macro":0.6237,"ci":[0.577,0.6705],"pass_all3_ci":[0.3542,0.4484],"benchmarks":{"deep-swe":{"cost_per_trial":0.2105,"cost_per_pass":0.3616,"tokens_per_trial":5929000.0,"calls_per_trial":86.48,"model_s_per_trial":851.3,"agent_s_per_trial":1647.0,"rerun_flip":0.3393,"pass_all3":0.3198,"pass_mean3":0.5751,"pass_any3":0.8288,"n_trials":671,"n_scored":666,"n_pass":383,"n_infra":5,"cost_usd":140.2,"n_tasks3":222,"rate":0.5751,"ci":[0.5112,0.6389],"pass_all3_ci":[0.2594,0.3803],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.03146,"cost_per_pass":0.04668,"tokens_per_trial":626300.0,"calls_per_trial":19.88,"model_s_per_trial":305.0,"agent_s_per_trial":659.0,"rerun_flip":0.2529,"pass_all3":0.4828,"pass_mean3":0.6724,"pass_any3":0.8621,"n_trials":536,"n_scored":522,"n_pass":351,"n_infra":14,"cost_usd":16.42,"n_tasks3":174,"rate":0.6724,"ci":[0.6042,0.7406],"pass_all3_ci":[0.4106,0.555],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.6566,"low":0.5909},"efforts_requested":{"high":37748,"low":30226},"versions":["2.2.6"]},"pi":{"cost_per_trial":0.0988,"cost_per_pass":0.1602,"tokens_per_trial":2271000.0,"calls_per_trial":51.1,"model_s_per_trial":778.3,"agent_s_per_trial":1325.0,"rerun_flip":0.2786,"pass_all3":0.3997,"pass_mean3":0.6157,"pass_any3":0.8176,"n_trials":1264,"n_scored":1188,"n_pass":725,"n_infra":76,"cost_usd":123.3,"n_tasks3":396,"macro":0.6157,"ci":[0.5688,0.6627],"pass_all3_ci":[0.353,0.4463],"benchmarks":{"deep-swe":{"cost_per_trial":0.1559,"cost_per_pass":0.27,"tokens_per_trial":3772000.0,"calls_per_trial":77.3,"model_s_per_trial":1072.0,"agent_s_per_trial":1814.0,"rerun_flip":0.3273,"pass_all3":0.3108,"pass_mean3":0.5706,"pass_any3":0.8018,"n_trials":725,"n_scored":666,"n_pass":380,"n_infra":59,"cost_usd":109.4,"n_tasks3":222,"rate":0.5706,"ci":[0.5068,0.6344],"pass_all3_ci":[0.2518,0.3698],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.02591,"cost_per_pass":0.03909,"tokens_per_trial":355900.0,"calls_per_trial":17.69,"model_s_per_trial":403.0,"agent_s_per_trial":701.1,"rerun_flip":0.2299,"pass_all3":0.4885,"pass_mean3":0.6609,"pass_any3":0.8333,"n_trials":539,"n_scored":522,"n_pass":345,"n_infra":17,"cost_usd":13.89,"n_tasks3":174,"rate":0.6609,"ci":[0.5921,0.7297],"pass_all3_ci":[0.4164,0.5606],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.655,"low":0.5765},"efforts_requested":{"high":33043,"low":27669},"versions":["1.2.0"],"vs_leader":{"diff":-0.007999,"ci":[-0.04088,0.02265],"tied":true,"n_tasks":198}},"qwen-code":{"cost_per_trial":0.1784,"cost_per_pass":0.3278,"tokens_per_trial":2838000.0,"calls_per_trial":45.72,"model_s_per_trial":861.8,"agent_s_per_trial":1176.0,"rerun_flip":0.2872,"pass_all3":0.333,"pass_mean3":0.5461,"pass_any3":0.7638,"n_trials":1229,"n_scored":1188,"n_pass":643,"n_infra":41,"cost_usd":219.9,"n_tasks3":396,"macro":0.5461,"ci":[0.4979,0.5943],"pass_all3_ci":[0.2876,0.3784],"benchmarks":{"deep-swe":{"cost_per_trial":0.2749,"cost_per_pass":0.5388,"tokens_per_trial":4525000.0,"calls_per_trial":64.73,"model_s_per_trial":1175.0,"agent_s_per_trial":1550.0,"rerun_flip":0.3483,"pass_all3":0.2523,"pass_mean3":0.506,"pass_any3":0.7748,"n_trials":702,"n_scored":666,"n_pass":337,"n_infra":36,"cost_usd":190.5,"n_tasks3":222,"rate":0.506,"ci":[0.4416,0.5705],"pass_all3_ci":[0.1965,0.308],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.0553,"cost_per_pass":0.0943,"tokens_per_trial":684800.0,"calls_per_trial":21.47,"model_s_per_trial":461.8,"agent_s_per_trial":699.4,"rerun_flip":0.2261,"pass_all3":0.4138,"pass_mean3":0.5862,"pass_any3":0.7529,"n_trials":527,"n_scored":522,"n_pass":306,"n_infra":5,"cost_usd":29.34,"n_tasks3":174,"rate":0.5862,"ci":[0.5146,0.6579],"pass_all3_ci":[0.3423,0.4853],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.5721,"low":0.5201},"efforts_requested":{"high":27783,"low":26493,"unset":39},"versions":["0.25.2"],"vs_leader":{"diff":-0.07764,"ci":[-0.1189,-0.03663],"tied":false,"n_tasks":198}}},"ranking":["opencode","mini-swe-agent","pi","kimi-code","claude-code","hermes","deepseek-harness","qwen-code","codex","openclaw"],"efforts":["high","low"],"noise":{"rerun_flip":0.2778,"harness_flip":0.3237,"effort_flip":0.2929,"by_benchmark":{"deep-swe":{"rerun_flip":0.32,"harness_flip":0.3686,"effort_flip":0.3401},"terminal-bench-2.1":{"rerun_flip":0.2356,"harness_flip":0.2787,"effort_flip":0.2457}},"harness_pairs":[{"harnesses":["opencode","mini-swe-agent"],"flip":0.3065},{"harnesses":["opencode","pi"],"flip":0.2911},{"harnesses":["opencode","kimi-code"],"flip":0.2862},{"harnesses":["opencode","claude-code"],"flip":0.3183},{"harnesses":["opencode","hermes"],"flip":0.2956},{"harnesses":["opencode","deepseek-harness"],"flip":0.3085},{"harnesses":["opencode","qwen-code"],"flip":0.3358},{"harnesses":["opencode","codex"],"flip":0.3673},{"harnesses":["opencode","openclaw"],"flip":0.4086},{"harnesses":["mini-swe-agent","pi"],"flip":0.2976},{"harnesses":["mini-swe-agent","kimi-code"],"flip":0.293},{"harnesses":["mini-swe-agent","claude-code"],"flip":0.2923},{"harnesses":["mini-swe-agent","hermes"],"flip":0.2929},{"harnesses":["mini-swe-agent","deepseek-harness"],"flip":0.2834},{"harnesses":["mini-swe-agent","qwen-code"],"flip":0.3205},{"harnesses":["mini-swe-agent","codex"],"flip":0.3604},{"harnesses":["mini-swe-agent","openclaw"],"flip":0.3779},{"harnesses":["pi","kimi-code"],"flip":0.2798},{"harnesses":["pi","claude-code"],"flip":0.2943},{"harnesses":["pi","hermes"],"flip":0.2874},{"harnesses":["pi","deepseek-harness"],"flip":0.3039},{"harnesses":["pi","qwen-code"],"flip":0.31},{"harnesses":["pi","codex"],"flip":0.3613},{"harnesses":["pi","openclaw"],"flip":0.3853},{"harnesses":["kimi-code","claude-code"],"flip":0.2894},{"harnesses":["kimi-code","hermes"],"flip":0.2793},{"harnesses":["kimi-code","deepseek-harness"],"flip":0.2806},{"harnesses":["kimi-code","qwen-code"],"flip":0.3004},{"harnesses":["kimi-code","codex"],"flip":0.3393},{"harnesses":["kimi-code","openclaw"],"flip":0.3711},{"harnesses":["claude-code","hermes"],"flip":0.2844},{"harnesses":["claude-code","deepseek-harness"],"flip":0.2986},{"harnesses":["claude-code","qwen-code"],"flip":0.3061},{"harnesses":["claude-code","codex"],"flip":0.382},{"harnesses":["claude-code","openclaw"],"flip":0.381},{"harnesses":["hermes","deepseek-harness"],"flip":0.2899},{"harnesses":["hermes","qwen-code"],"flip":0.306},{"harnesses":["hermes","codex"],"flip":0.357},{"harnesses":["hermes","openclaw"],"flip":0.3802},{"harnesses":["deepseek-harness","qwen-code"],"flip":0.2882},{"harnesses":["deepseek-harness","codex"],"flip":0.3627},{"harnesses":["deepseek-harness","openclaw"],"flip":0.3534},{"harnesses":["qwen-code","codex"],"flip":0.3554},{"harnesses":["qwen-code","openclaw"],"flip":0.349},{"harnesses":["codex","openclaw"],"flip":0.3527}]},"leader":"opencode"}}}}},"RecursiveSI/flint":{"stages":{"full":{"efforts":{"default":{"harnesses":{"mini-swe-agent":{"n_trials":647,"n_scored":593,"n_pass":119,"n_infra":54,"cost_usd":136.0,"cost_per_trial":0.2246,"cost_per_pass":1.119,"tokens_per_trial":10180000.0,"calls_per_trial":95.05,"model_s_per_trial":3283.0,"agent_s_per_trial":3650.0,"efforts_requested":{"medium":56365},"versions":["2.4.6"],"benchmarks":{"deep-swe":{"n_trials":380,"n_scored":332,"n_pass":35,"n_infra":48,"cost_usd":128.0,"cost_per_trial":0.377,"cost_per_pass":3.576,"tokens_per_trial":16870000.0,"calls_per_trial":141.1,"model_s_per_trial":5074.0,"agent_s_per_trial":5496.0,"efforts_requested":{"medium":46852},"versions":["2.4.6"],"n_tasks":111,"n_tasks_expected":111,"passed":11.67,"rate":0.1051,"ci":[0.06069,0.1759],"rerun_flip":0.1441,"n_tasks3":110,"pass_all3":0.01818,"pass_mean3":0.1061,"pass_any3":0.2364,"pass_all3_ci":[0.005,0.06388]},"terminal-bench-2.1":{"n_trials":267,"n_scored":261,"n_pass":84,"n_infra":6,"cost_usd":8.001,"cost_per_trial":0.03066,"cost_per_pass":0.09525,"tokens_per_trial":1666000.0,"calls_per_trial":36.45,"model_s_per_trial":1004.0,"agent_s_per_trial":1302.0,"efforts_requested":{"medium":9513},"versions":["2.4.6"],"n_tasks":87,"n_tasks_expected":87,"passed":28.0,"rate":0.3218,"ci":[0.233,0.4257],"rerun_flip":0.2146,"n_tasks3":87,"pass_all3":0.1609,"pass_mean3":0.3218,"pass_any3":0.4828,"pass_all3_ci":[0.09835,0.2522]}},"macro":0.2135,"ci":[0.1568,0.2701],"rerun_flip":0.1794,"n_tasks3":197,"pass_all3":0.08955,"pass_mean3":0.2139,"pass_any3":0.3596,"pass_all3_ci":[0.04695,0.1321],"complete":true,"vs_leader":{"diff":-0.01564,"ci":[-0.04996,0.01771],"tied":true,"n_tasks":198}},"deepseek-harness":{"n_trials":723,"n_scored":594,"n_pass":114,"n_infra":129,"cost_usd":128.1,"cost_per_trial":0.2023,"cost_per_pass":1.054,"tokens_per_trial":17640000.0,"calls_per_trial":186.6,"model_s_per_trial":2316.0,"agent_s_per_trial":2541.0,"efforts_requested":{"medium":110267,"low":567},"versions":["0.2.2-rc.1"],"benchmarks":{"terminal-bench-2.1":{"n_trials":307,"n_scored":261,"n_pass":86,"n_infra":46,"cost_usd":15.91,"cost_per_trial":0.05647,"cost_per_pass":0.1714,"tokens_per_trial":5666000.0,"calls_per_trial":112.6,"model_s_per_trial":879.7,"agent_s_per_trial":1108.0,"efforts_requested":{"medium":29138,"low":258},"versions":["0.2.2-rc.1"],"n_tasks":87,"n_tasks_expected":87,"passed":28.67,"rate":0.3295,"ci":[0.2398,0.4336],"rerun_flip":0.2146,"n_tasks3":87,"pass_all3":0.1839,"pass_mean3":0.3295,"pass_any3":0.5057,"pass_all3_ci":[0.1165,0.2781]},"deep-swe":{"n_trials":416,"n_scored":333,"n_pass":28,"n_infra":83,"cost_usd":112.2,"cost_per_trial":0.3165,"cost_per_pass":3.765,"tokens_per_trial":27020000.0,"calls_per_trial":244.6,"model_s_per_trial":3442.0,"agent_s_per_trial":3664.0,"efforts_requested":{"medium":81129,"low":309},"versions":["0.2.2-rc.1"],"n_tasks":111,"n_tasks_expected":111,"passed":9.333,"rate":0.08408,"ci":[0.04537,0.1506],"rerun_flip":0.1261,"n_tasks3":111,"pass_all3":0.009009,"pass_mean3":0.08408,"pass_any3":0.1982,"pass_all3_ci":[0.001592,0.04927]}},"macro":0.2068,"ci":[0.1511,0.2625],"rerun_flip":0.1703,"n_tasks3":198,"pass_all3":0.09646,"pass_mean3":0.2068,"pass_any3":0.352,"pass_all3_ci":[0.05302,0.1399],"complete":true,"vs_leader":{"diff":-0.02232,"ci":[-0.05773,0.01217],"tied":true,"n_tasks":198}},"kimi-code":{"n_trials":659,"n_scored":593,"n_pass":121,"n_infra":66,"cost_usd":181.1,"cost_per_trial":0.2988,"cost_per_pass":1.464,"tokens_per_trial":15460000.0,"calls_per_trial":110.7,"model_s_per_trial":3568.0,"agent_s_per_trial":3898.0,"efforts_requested":{"medium":65648},"versions":["2.2.0"],"benchmarks":{"deep-swe":{"n_trials":392,"n_scored":332,"n_pass":37,"n_infra":60,"cost_usd":170.5,"cost_per_trial":0.502,"cost_per_pass":4.504,"tokens_per_trial":25840000.0,"calls_per_trial":171.6,"model_s_per_trial":5559.0,"agent_s_per_trial":5914.0,"efforts_requested":{"medium":56956},"versions":["2.2.0"],"n_tasks":111,"n_tasks_expected":111,"passed":12.33,"rate":0.1111,"ci":[0.06519,0.1831],"rerun_flip":0.1502,"n_tasks3":110,"pass_all3":0.01818,"pass_mean3":0.1121,"pass_any3":0.2455,"pass_all3_ci":[0.005,0.06388]},"terminal-bench-2.1":{"n_trials":267,"n_scored":261,"n_pass":84,"n_infra":6,"cost_usd":10.61,"cost_per_trial":0.04032,"cost_per_pass":0.1253,"tokens_per_trial":2259000.0,"calls_per_trial":33.3,"model_s_per_trial":1035.0,"agent_s_per_trial":1334.0,"efforts_requested":{"medium":8692},"versions":["2.2.0"],"n_tasks":87,"n_tasks_expected":87,"passed":28.0,"rate":0.3218,"ci":[0.233,0.4257],"rerun_flip":0.1839,"n_tasks3":87,"pass_all3":0.1724,"pass_mean3":0.3218,"pass_any3":0.4483,"pass_all3_ci":[0.1074,0.2652]}},"macro":0.2165,"ci":[0.1595,0.2734],"rerun_flip":0.167,"n_tasks3":197,"pass_all3":0.0953,"pass_mean3":0.217,"pass_any3":0.3469,"pass_all3_ci":[0.05184,0.1388],"complete":true,"vs_leader":{"diff":-0.01263,"ci":[-0.05054,0.02428],"tied":true,"n_tasks":198}},"pi":{"n_trials":619,"n_scored":594,"n_pass":99,"n_infra":25,"cost_usd":58.13,"cost_per_trial":0.09591,"cost_per_pass":0.5755,"tokens_per_trial":5264000.0,"calls_per_trial":91.23,"model_s_per_trial":2597.0,"agent_s_per_trial":2996.0,"efforts_requested":{"medium":54191},"versions":["1.2.0"],"benchmarks":{"deep-swe":{"n_trials":351,"n_scored":333,"n_pass":23,"n_infra":18,"cost_usd":52.74,"cost_per_trial":0.1549,"cost_per_pass":2.243,"tokens_per_trial":8484000.0,"calls_per_trial":137.8,"model_s_per_trial":3967.0,"agent_s_per_trial":4452.0,"efforts_requested":{"medium":45896},"versions":["1.2.0"],"n_tasks":111,"n_tasks_expected":111,"passed":7.667,"rate":0.06907,"ci":[0.03492,0.1321],"rerun_flip":0.08408,"n_tasks3":111,"pass_all3":0.01802,"pass_mean3":0.06907,"pass_any3":0.1441,"pass_all3_ci":[0.004955,0.06333]},"terminal-bench-2.1":{"n_trials":268,"n_scored":261,"n_pass":76,"n_infra":7,"cost_usd":5.386,"cost_per_trial":0.0206,"cost_per_pass":0.07075,"tokens_per_trial":1157000.0,"calls_per_trial":31.78,"model_s_per_trial":849.4,"agent_s_per_trial":1138.0,"efforts_requested":{"medium":8295},"versions":["1.2.0"],"n_tasks":87,"n_tasks_expected":87,"passed":25.33,"rate":0.2912,"ci":[0.2062,0.3939],"rerun_flip":0.1839,"n_tasks3":87,"pass_all3":0.1609,"pass_mean3":0.2912,"pass_any3":0.4368,"pass_all3_ci":[0.09835,0.2522]}},"macro":0.1801,"ci":[0.1266,0.2336],"rerun_flip":0.134,"n_tasks3":198,"pass_all3":0.08947,"pass_mean3":0.1801,"pass_any3":0.2905,"pass_all3_ci":[0.04693,0.132],"complete":true,"vs_leader":{"diff":-0.04898,"ci":[-0.09248,-0.009889],"tied":false,"n_tasks":198}},"claude-code":{"n_trials":606,"n_scored":594,"n_pass":94,"n_infra":12,"cost_usd":109.8,"cost_per_trial":0.1833,"cost_per_pass":1.158,"tokens_per_trial":9299000.0,"calls_per_trial":104.9,"model_s_per_trial":3783.0,"agent_s_per_trial":4426.0,"efforts_requested":{"medium":62284},"versions":["2.1.300"],"benchmarks":{"terminal-bench-2.1":{"n_trials":263,"n_scored":261,"n_pass":76,"n_infra":2,"cost_usd":7.554,"cost_per_trial":0.02894,"cost_per_pass":0.09937,"tokens_per_trial":1485000.0,"calls_per_trial":27.71,"model_s_per_trial":924.1,"agent_s_per_trial":1374.0,"efforts_requested":{"medium":7233},"versions":["2.1.300"],"n_tasks":87,"n_tasks_expected":87,"passed":25.33,"rate":0.2912,"ci":[0.2062,0.3939],"rerun_flip":0.2452,"n_tasks3":87,"pass_all3":0.1149,"pass_mean3":0.2912,"pass_any3":0.4828,"pass_all3_ci":[0.06364,0.1988]},"deep-swe":{"n_trials":343,"n_scored":333,"n_pass":18,"n_infra":10,"cost_usd":102.2,"cost_per_trial":0.3043,"cost_per_pass":5.629,"tokens_per_trial":15420000.0,"calls_per_trial":165.3,"model_s_per_trial":6023.0,"agent_s_per_trial":6818.0,"efforts_requested":{"medium":55051},"versions":["2.1.300"],"n_tasks":111,"n_tasks_expected":111,"passed":6.0,"rate":0.05405,"ci":[0.02501,0.1129],"rerun_flip":0.06006,"n_tasks3":111,"pass_all3":0.009009,"pass_mean3":0.05405,"pass_any3":0.0991,"pass_all3_ci":[0.001592,0.04927]}},"macro":0.1726,"ci":[0.1201,0.2251],"rerun_flip":0.1526,"n_tasks3":198,"pass_all3":0.06198,"pass_mean3":0.1726,"pass_any3":0.2909,"pass_all3_ci":[0.02429,0.09966],"complete":true,"vs_leader":{"diff":-0.05649,"ci":[-0.09424,-0.02039],"tied":false,"n_tasks":198}},"codex":{"n_trials":704,"n_scored":586,"n_pass":91,"n_infra":118,"cost_usd":161.4,"cost_per_trial":0.2534,"cost_per_pass":1.632,"tokens_per_trial":12580000.0,"calls_per_trial":117.5,"model_s_per_trial":3363.0,"agent_s_per_trial":3659.0,"efforts_requested":{"medium":68826},"versions":["0.162.1"],"benchmarks":{"deep-swe":{"n_trials":386,"n_scored":330,"n_pass":13,"n_infra":56,"cost_usd":152.8,"cost_per_trial":0.4253,"cost_per_pass":10.8,"tokens_per_trial":20730000.0,"calls_per_trial":177.5,"model_s_per_trial":5358.0,"agent_s_per_trial":5670.0,"efforts_requested":{"medium":58584},"versions":["0.162.1"],"n_tasks":111,"n_tasks_expected":111,"passed":4.333,"rate":0.03904,"ci":[0.01582,0.09309],"rerun_flip":0.06061,"n_tasks3":109,"pass_all3":0.0,"pass_mean3":0.03976,"pass_any3":0.09174,"pass_all3_ci":[0.0,0.03404]},"terminal-bench-2.1":{"n_trials":318,"n_scored":256,"n_pass":78,"n_infra":62,"cost_usd":8.606,"cost_per_trial":0.03176,"cost_per_pass":0.1042,"tokens_per_trial":2090000.0,"calls_per_trial":40.01,"model_s_per_trial":792.7,"agent_s_per_trial":1068.0,"efforts_requested":{"medium":10242},"versions":["0.162.1"],"n_tasks":87,"n_tasks_expected":87,"passed":26.5,"rate":0.3046,"ci":[0.2179,0.4079],"rerun_flip":0.2107,"n_tasks3":82,"pass_all3":0.1341,"pass_mean3":0.3049,"pass_any3":0.4512,"pass_all3_ci":[0.07659,0.2245]}},"macro":0.1718,"ci":[0.1198,0.2238],"rerun_flip":0.1357,"n_tasks3":191,"pass_all3":0.06707,"pass_mean3":0.1723,"pass_any3":0.2715,"pass_all3_ci":[0.02732,0.1068],"complete":true,"vs_leader":{"diff":-0.05729,"ci":[-0.1003,-0.0132],"tied":false,"n_tasks":198}},"opencode":{"n_trials":628,"n_scored":592,"n_pass":125,"n_infra":36,"cost_usd":128.3,"cost_per_trial":0.2093,"cost_per_pass":0.9913,"tokens_per_trial":13590000.0,"calls_per_trial":129.4,"model_s_per_trial":2799.0,"agent_s_per_trial":3326.0,"efforts_requested":{"medium":76599},"versions":["2.2.6"],"benchmarks":{"deep-swe":{"n_trials":363,"n_scored":331,"n_pass":25,"n_infra":32,"cost_usd":122.1,"cost_per_trial":0.3558,"cost_per_pass":4.71,"tokens_per_trial":22940000.0,"calls_per_trial":198.2,"model_s_per_trial":4441.0,"agent_s_per_trial":5098.0,"efforts_requested":{"medium":65589},"versions":["2.2.6"],"n_tasks":111,"n_tasks_expected":111,"passed":8.333,"rate":0.07508,"ci":[0.03904,0.1395],"rerun_flip":0.1091,"n_tasks3":110,"pass_all3":0.009091,"pass_mean3":0.07576,"pass_any3":0.1727,"pass_all3_ci":[0.001607,0.04971]},"terminal-bench-2.1":{"n_trials":265,"n_scored":261,"n_pass":100,"n_infra":4,"cost_usd":6.181,"cost_per_trial":0.02358,"cost_per_pass":0.06153,"tokens_per_trial":1730000.0,"calls_per_trial":42.18,"model_s_per_trial":716.3,"agent_s_per_trial":1078.0,"efforts_requested":{"medium":11010},"versions":["2.2.6"],"n_tasks":87,"n_tasks_expected":87,"passed":33.33,"rate":0.3831,"ci":[0.288,0.4882],"rerun_flip":0.1379,"n_tasks3":87,"pass_all3":0.2759,"pass_mean3":0.3831,"pass_any3":0.4828,"pass_all3_ci":[0.1929,0.3777]}},"macro":0.2291,"ci":[0.1726,0.2856],"rerun_flip":0.1235,"n_tasks3":197,"pass_all3":0.1425,"pass_mean3":0.2294,"pass_any3":0.3277,"pass_all3_ci":[0.0938,0.1912],"complete":true},"hermes":{"n_trials":636,"n_scored":592,"n_pass":105,"n_infra":44,"cost_usd":168.9,"cost_per_trial":0.2807,"cost_per_pass":1.583,"tokens_per_trial":14250000.0,"calls_per_trial":133.7,"model_s_per_trial":4217.0,"agent_s_per_trial":4725.0,"efforts_requested":{"medium":78600,"unset":429,"none":96},"versions":["v0.22.1"],"benchmarks":{"terminal-bench-2.1":{"n_trials":265,"n_scored":261,"n_pass":74,"n_infra":4,"cost_usd":9.309,"cost_per_trial":0.03567,"cost_per_pass":0.1258,"tokens_per_trial":1856000.0,"calls_per_trial":33.92,"model_s_per_trial":1037.0,"agent_s_per_trial":1366.0,"efforts_requested":{"medium":8785,"unset":48,"none":21},"versions":["v0.22.1"],"n_tasks":87,"n_tasks_expected":87,"passed":24.67,"rate":0.2835,"ci":[0.1995,0.3858],"rerun_flip":0.1456,"n_tasks3":87,"pass_all3":0.1724,"pass_mean3":0.2835,"pass_any3":0.3908,"pass_all3_ci":[0.1074,0.2652]},"deep-swe":{"n_trials":371,"n_scored":331,"n_pass":31,"n_infra":40,"cost_usd":159.6,"cost_per_trial":0.474,"cost_per_pass":5.061,"tokens_per_trial":24030000.0,"calls_per_trial":212.3,"model_s_per_trial":6726.0,"agent_s_per_trial":7373.0,"efforts_requested":{"medium":69815,"unset":381,"none":75},"versions":["v0.22.1"],"n_tasks":111,"n_tasks_expected":111,"passed":10.33,"rate":0.09309,"ci":[0.05185,0.1616],"rerun_flip":0.103,"n_tasks3":110,"pass_all3":0.02727,"pass_mean3":0.09394,"pass_any3":0.1818,"pass_all3_ci":[0.009318,0.07713]}},"macro":0.1883,"ci":[0.1337,0.243],"rerun_flip":0.1243,"n_tasks3":197,"pass_all3":0.09984,"pass_mean3":0.1887,"pass_any3":0.2863,"pass_all3_ci":[0.05561,0.1441],"complete":true,"vs_leader":{"diff":-0.0408,"ci":[-0.0759,-0.006213],"tied":false,"n_tasks":198}},"qwen-code":{"n_trials":606,"n_scored":594,"n_pass":67,"n_infra":12,"cost_usd":35.76,"cost_per_trial":0.0594,"cost_per_pass":0.5266,"tokens_per_trial":2866000.0,"calls_per_trial":45.91,"model_s_per_trial":1976.0,"agent_s_per_trial":2082.0,"efforts_requested":{"medium":27175,"unset":97},"versions":["0.25.2"],"benchmarks":{"terminal-bench-2.1":{"n_trials":262,"n_scored":261,"n_pass":65,"n_infra":1,"cost_usd":6.11,"cost_per_trial":0.02341,"cost_per_pass":0.094,"tokens_per_trial":1222000.0,"calls_per_trial":26.42,"model_s_per_trial":1027.0,"agent_s_per_trial":1174.0,"efforts_requested":{"medium":6889,"unset":7},"versions":["0.25.2"],"n_tasks":87,"n_tasks_expected":87,"passed":21.67,"rate":0.249,"ci":[0.1701,0.3492],"rerun_flip":0.2146,"n_tasks3":87,"pass_all3":0.1034,"pass_mean3":0.249,"pass_any3":0.4253,"pass_all3_ci":[0.05538,0.1851]},"deep-swe":{"n_trials":344,"n_scored":333,"n_pass":2,"n_infra":11,"cost_usd":29.65,"cost_per_trial":0.08761,"cost_per_pass":14.59,"tokens_per_trial":4154000.0,"calls_per_trial":61.19,"model_s_per_trial":2720.0,"agent_s_per_trial":2793.0,"efforts_requested":{"medium":20286,"unset":90},"versions":["0.25.2"],"n_tasks":111,"n_tasks_expected":111,"passed":0.6667,"rate":0.006006,"ci":[0.0007875,0.04427],"rerun_flip":0.01201,"n_tasks3":111,"pass_all3":0.0,"pass_mean3":0.006006,"pass_any3":0.01802,"pass_all3_ci":[0.0,0.03345]}},"macro":0.1275,"ci":[0.08041,0.1746],"rerun_flip":0.1133,"n_tasks3":198,"pass_all3":0.05172,"pass_mean3":0.1275,"pass_any3":0.2217,"pass_all3_ci":[0.01617,0.08728],"complete":true,"vs_leader":{"diff":-0.1016,"ci":[-0.1406,-0.06643],"tied":false,"n_tasks":198}},"openclaw":{"n_trials":606,"n_scored":594,"n_pass":29,"n_infra":12,"cost_usd":94.31,"cost_per_trial":0.1533,"cost_per_pass":3.139,"tokens_per_trial":12140000.0,"calls_per_trial":158.8,"model_s_per_trial":1518.0,"agent_s_per_trial":1655.0,"efforts_requested":{"medium":94120,"low":216,"unset":3},"versions":["2026.10.4"],"benchmarks":{"deep-swe":{"n_trials":339,"n_scored":333,"n_pass":0,"n_infra":6,"cost_usd":66.41,"cost_per_trial":0.1911,"cost_per_pass":null,"tokens_per_trial":14630000.0,"calls_per_trial":172.0,"model_s_per_trial":1864.0,"agent_s_per_trial":1978.0,"efforts_requested":{"medium":57089,"low":176},"versions":["2026.10.4"],"n_tasks":111,"n_tasks_expected":111,"passed":0.0,"rate":0.0,"ci":[0.0,0.03345],"rerun_flip":0.0,"n_tasks3":111,"pass_all3":0.0,"pass_mean3":0.0,"pass_any3":0.0,"pass_all3_ci":[0.0,0.03345]},"terminal-bench-2.1":{"n_trials":267,"n_scored":261,"n_pass":29,"n_infra":6,"cost_usd":27.9,"cost_per_trial":0.105,"cost_per_pass":0.9453,"tokens_per_trial":8969000.0,"calls_per_trial":142.0,"model_s_per_trial":1076.0,"agent_s_per_trial":1243.0,"efforts_requested":{"medium":37031,"low":40,"unset":3},"versions":["2026.10.4"],"n_tasks":87,"n_tasks_expected":87,"passed":9.667,"rate":0.1111,"ci":[0.06087,0.1942],"rerun_flip":0.0613,"n_tasks3":87,"pass_all3":0.06897,"pass_mean3":0.1111,"pass_any3":0.1609,"pass_all3_ci":[0.03199,0.1424]}},"macro":0.05556,"ci":[0.01919,0.09192],"rerun_flip":0.03065,"n_tasks3":198,"pass_all3":0.03448,"pass_mean3":0.05556,"pass_any3":0.08046,"pass_all3_ci":[0.003035,0.06593],"complete":true,"vs_leader":{"diff":-0.1736,"ci":[-0.2175,-0.1305],"tied":false,"n_tasks":198}}},"ranking":["opencode","kimi-code","mini-swe-agent","deepseek-harness","hermes","pi","claude-code","codex","qwen-code","openclaw"],"noise":{"rerun_flip":0.1331,"harness_flip":0.1584,"by_benchmark":{"deep-swe":{"rerun_flip":0.08493,"harness_flip":0.09672},"terminal-bench-2.1":{"rerun_flip":0.1812,"harness_flip":0.2201}},"harness_pairs":[{"harnesses":["opencode","kimi-code"],"flip":0.1695},{"harnesses":["opencode","mini-swe-agent"],"flip":0.1622},{"harnesses":["opencode","deepseek-harness"],"flip":0.158},{"harnesses":["opencode","hermes"],"flip":0.1479},{"harnesses":["opencode","pi"],"flip":0.1672},{"harnesses":["opencode","claude-code"],"flip":0.1671},{"harnesses":["opencode","codex"],"flip":0.184},{"harnesses":["opencode","qwen-code"],"flip":0.1559},{"harnesses":["opencode","openclaw"],"flip":0.1787},{"harnesses":["kimi-code","mini-swe-agent"],"flip":0.1579},{"harnesses":["kimi-code","deepseek-harness"],"flip":0.1836},{"harnesses":["kimi-code","hermes"],"flip":0.1489},{"harnesses":["kimi-code","pi"],"flip":0.156},{"harnesses":["kimi-code","claude-code"],"flip":0.1583},{"harnesses":["kimi-code","codex"],"flip":0.1872},{"harnesses":["kimi-code","qwen-code"],"flip":0.1551},{"harnesses":["kimi-code","openclaw"],"flip":0.1852},{"harnesses":["mini-swe-agent","deepseek-harness"],"flip":0.1765},{"harnesses":["mini-swe-agent","hermes"],"flip":0.1509},{"harnesses":["mini-swe-agent","pi"],"flip":0.1513},{"harnesses":["mini-swe-agent","claude-code"],"flip":0.1626},{"harnesses":["mini-swe-agent","codex"],"flip":0.1713},{"harnesses":["mini-swe-agent","qwen-code"],"flip":0.1534},{"harnesses":["mini-swe-agent","openclaw"],"flip":0.1694},{"harnesses":["deepseek-harness","hermes"],"flip":0.163},{"harnesses":["deepseek-harness","pi"],"flip":0.1767},{"harnesses":["deepseek-harness","claude-code"],"flip":0.1801},{"harnesses":["deepseek-harness","codex"],"flip":0.1734},{"harnesses":["deepseek-harness","qwen-code"],"flip":0.1563},{"harnesses":["deepseek-harness","openclaw"],"flip":0.1755},{"harnesses":["hermes","pi"],"flip":0.1479},{"harnesses":["hermes","claude-code"],"flip":0.1429},{"harnesses":["hermes","codex"],"flip":0.1774},{"harnesses":["hermes","qwen-code"],"flip":0.1289},{"harnesses":["hermes","openclaw"],"flip":0.1481},{"harnesses":["pi","claude-code"],"flip":0.1504},{"harnesses":["pi","codex"],"flip":0.1536},{"harnesses":["pi","qwen-code"],"flip":0.1345},{"harnesses":["pi","openclaw"],"flip":0.1322},{"harnesses":["claude-code","codex"],"flip":0.1713},{"harnesses":["claude-code","qwen-code"],"flip":0.1388},{"harnesses":["claude-code","openclaw"],"flip":0.1413},{"harnesses":["codex","qwen-code"],"flip":0.1438},{"harnesses":["codex","openclaw"],"flip":0.1341},{"harnesses":["qwen-code","openclaw"],"flip":0.09879}]},"leader":"opencode"},"_all":{"harnesses":{"claude-code":{"cost_per_trial":0.1833,"cost_per_pass":1.158,"tokens_per_trial":9299000.0,"calls_per_trial":104.9,"model_s_per_trial":3783.0,"agent_s_per_trial":4426.0,"rerun_flip":0.1526,"pass_all3":0.06198,"pass_mean3":0.1726,"pass_any3":0.2909,"n_trials":606,"n_scored":594,"n_pass":94,"n_infra":12,"cost_usd":109.8,"n_tasks3":198,"macro":0.1726,"ci":[0.1201,0.2251],"pass_all3_ci":[0.02429,0.09966],"benchmarks":{"deep-swe":{"cost_per_trial":0.3043,"cost_per_pass":5.629,"tokens_per_trial":15420000.0,"calls_per_trial":165.3,"model_s_per_trial":6023.0,"agent_s_per_trial":6818.0,"rerun_flip":0.06006,"pass_all3":0.009009,"pass_mean3":0.05405,"pass_any3":0.0991,"n_trials":343,"n_scored":333,"n_pass":18,"n_infra":10,"cost_usd":102.2,"n_tasks3":111,"rate":0.05405,"ci":[0.01009,0.09802],"pass_all3_ci":[0.0,0.03285],"n_tasks":111,"n_tasks_expected":111},"terminal-bench-2.1":{"cost_per_trial":0.02894,"cost_per_pass":0.09937,"tokens_per_trial":1485000.0,"calls_per_trial":27.71,"model_s_per_trial":924.1,"agent_s_per_trial":1374.0,"rerun_flip":0.2452,"pass_all3":0.1149,"pass_mean3":0.2912,"pass_any3":0.4828,"n_trials":263,"n_scored":261,"n_pass":76,"n_infra":2,"cost_usd":7.554,"n_tasks3":87,"rate":0.2912,"ci":[0.1973,0.385],"pass_all3_ci":[0.04736,0.1825],"n_tasks":87,"n_tasks_expected":87}},"complete":true,"per_effort":{"default":0.1726},"efforts_requested":{"medium":62284},"versions":["2.1.300"],"vs_leader":{"diff":-0.05649,"ci":[-0.09424,-0.02039],"tied":false,"n_tasks":198}},"codex":{"cost_per_trial":0.2534,"cost_per_pass":1.632,"tokens_per_trial":12580000.0,"calls_per_trial":117.5,"model_s_per_trial":3363.0,"agent_s_per_trial":3659.0,"rerun_flip":0.1357,"pass_all3":0.06707,"pass_mean3":0.1723,"pass_any3":0.2715,"n_trials":704,"n_scored":586,"n_pass":91,"n_infra":118,"cost_usd":161.4,"n_tasks3":191,"macro":0.1718,"ci":[0.1198,0.2238],"pass_all3_ci":[0.02732,0.1068],"benchmarks":{"deep-swe":{"cost_per_trial":0.4253,"cost_per_pass":10.8,"tokens_per_trial":20730000.0,"calls_per_trial":177.5,"model_s_per_trial":5358.0,"agent_s_per_trial":5670.0,"rerun_flip":0.06061,"pass_all3":0.0,"pass_mean3":0.03976,"pass_any3":0.09174,"n_trials":386,"n_scored":330,"n_pass":13,"n_infra":56,"cost_usd":152.8,"n_tasks3":109,"rate":0.03904,"ci":[0.0004036,0.07767],"pass_all3_ci":[0.0,0.01702],"n_tasks":111,"n_tasks_expected":111},"terminal-bench-2.1":{"cost_per_trial":0.03176,"cost_per_pass":0.1042,"tokens_per_trial":2090000.0,"calls_per_trial":40.01,"model_s_per_trial":792.7,"agent_s_per_trial":1068.0,"rerun_flip":0.2107,"pass_all3":0.1341,"pass_mean3":0.3049,"pass_any3":0.4512,"n_trials":318,"n_scored":256,"n_pass":78,"n_infra":62,"cost_usd":8.606,"n_tasks3":82,"rate":0.3046,"ci":[0.2096,0.3996],"pass_all3_ci":[0.06021,0.2081],"n_tasks":87,"n_tasks_expected":87}},"complete":true,"per_effort":{"default":0.1718},"efforts_requested":{"medium":68826},"versions":["0.162.1"],"vs_leader":{"diff":-0.05729,"ci":[-0.1003,-0.0132],"tied":false,"n_tasks":198}},"deepseek-harness":{"cost_per_trial":0.2023,"cost_per_pass":1.054,"tokens_per_trial":17640000.0,"calls_per_trial":186.6,"model_s_per_trial":2316.0,"agent_s_per_trial":2541.0,"rerun_flip":0.1703,"pass_all3":0.09646,"pass_mean3":0.2068,"pass_any3":0.352,"n_trials":723,"n_scored":594,"n_pass":114,"n_infra":129,"cost_usd":128.1,"n_tasks3":198,"macro":0.2068,"ci":[0.1511,0.2625],"pass_all3_ci":[0.05302,0.1399],"benchmarks":{"deep-swe":{"cost_per_trial":0.3165,"cost_per_pass":3.765,"tokens_per_trial":27020000.0,"calls_per_trial":244.6,"model_s_per_trial":3442.0,"agent_s_per_trial":3664.0,"rerun_flip":0.1261,"pass_all3":0.009009,"pass_mean3":0.08408,"pass_any3":0.1982,"n_trials":416,"n_scored":333,"n_pass":28,"n_infra":83,"cost_usd":112.2,"n_tasks3":111,"rate":0.08408,"ci":[0.03146,0.1367],"pass_all3_ci":[0.0,0.03285],"n_tasks":111,"n_tasks_expected":111},"terminal-bench-2.1":{"cost_per_trial":0.05647,"cost_per_pass":0.1714,"tokens_per_trial":5666000.0,"calls_per_trial":112.6,"model_s_per_trial":879.7,"agent_s_per_trial":1108.0,"rerun_flip":0.2146,"pass_all3":0.1839,"pass_mean3":0.3295,"pass_any3":0.5057,"n_trials":307,"n_scored":261,"n_pass":86,"n_infra":46,"cost_usd":15.91,"n_tasks3":87,"rate":0.3295,"ci":[0.2326,0.4264],"pass_all3_ci":[0.1031,0.2647],"n_tasks":87,"n_tasks_expected":87}},"complete":true,"per_effort":{"default":0.2068},"efforts_requested":{"medium":110267,"low":567},"versions":["0.2.2-rc.1"],"vs_leader":{"diff":-0.02232,"ci":[-0.05773,0.01217],"tied":true,"n_tasks":198}},"hermes":{"cost_per_trial":0.2807,"cost_per_pass":1.583,"tokens_per_trial":14250000.0,"calls_per_trial":133.7,"model_s_per_trial":4217.0,"agent_s_per_trial":4725.0,"rerun_flip":0.1243,"pass_all3":0.09984,"pass_mean3":0.1887,"pass_any3":0.2863,"n_trials":636,"n_scored":592,"n_pass":105,"n_infra":44,"cost_usd":168.9,"n_tasks3":197,"macro":0.1883,"ci":[0.1337,0.243],"pass_all3_ci":[0.05561,0.1441],"benchmarks":{"deep-swe":{"cost_per_trial":0.474,"cost_per_pass":5.061,"tokens_per_trial":24030000.0,"calls_per_trial":212.3,"model_s_per_trial":6726.0,"agent_s_per_trial":7373.0,"rerun_flip":0.103,"pass_all3":0.02727,"pass_mean3":0.09394,"pass_any3":0.1818,"n_trials":371,"n_scored":331,"n_pass":31,"n_infra":40,"cost_usd":159.6,"n_tasks3":110,"rate":0.09309,"ci":[0.03823,0.148],"pass_all3_ci":[0.0,0.06118],"n_tasks":111,"n_tasks_expected":111},"terminal-bench-2.1":{"cost_per_trial":0.03567,"cost_per_pass":0.1258,"tokens_per_trial":1856000.0,"calls_per_trial":33.92,"model_s_per_trial":1037.0,"agent_s_per_trial":1366.0,"rerun_flip":0.1456,"pass_all3":0.1724,"pass_mean3":0.2835,"pass_any3":0.3908,"n_trials":265,"n_scored":261,"n_pass":74,"n_infra":4,"cost_usd":9.309,"n_tasks3":87,"rate":0.2835,"ci":[0.1904,0.3767],"pass_all3_ci":[0.09351,0.2513],"n_tasks":87,"n_tasks_expected":87}},"complete":true,"per_effort":{"default":0.1883},"efforts_requested":{"medium":78600,"unset":429,"none":96},"versions":["v0.22.1"],"vs_leader":{"diff":-0.0408,"ci":[-0.0759,-0.006213],"tied":false,"n_tasks":198}},"kimi-code":{"cost_per_trial":0.2988,"cost_per_pass":1.464,"tokens_per_trial":15460000.0,"calls_per_trial":110.7,"model_s_per_trial":3568.0,"agent_s_per_trial":3898.0,"rerun_flip":0.167,"pass_all3":0.0953,"pass_mean3":0.217,"pass_any3":0.3469,"n_trials":659,"n_scored":593,"n_pass":121,"n_infra":66,"cost_usd":181.1,"n_tasks3":197,"macro":0.2165,"ci":[0.1595,0.2734],"pass_all3_ci":[0.05184,0.1388],"benchmarks":{"deep-swe":{"cost_per_trial":0.502,"cost_per_pass":4.504,"tokens_per_trial":25840000.0,"calls_per_trial":171.6,"model_s_per_trial":5559.0,"agent_s_per_trial":5914.0,"rerun_flip":0.1502,"pass_all3":0.01818,"pass_mean3":0.1121,"pass_any3":0.2455,"n_trials":392,"n_scored":332,"n_pass":37,"n_infra":60,"cost_usd":170.5,"n_tasks3":110,"rate":0.1111,"ci":[0.05218,0.17],"pass_all3_ci":[0.0,0.04762],"n_tasks":111,"n_tasks_expected":111},"terminal-bench-2.1":{"cost_per_trial":0.04032,"cost_per_pass":0.1253,"tokens_per_trial":2259000.0,"calls_per_trial":33.3,"model_s_per_trial":1035.0,"agent_s_per_trial":1334.0,"rerun_flip":0.1839,"pass_all3":0.1724,"pass_mean3":0.3218,"pass_any3":0.4483,"n_trials":267,"n_scored":261,"n_pass":84,"n_infra":6,"cost_usd":10.61,"n_tasks3":87,"rate":0.3218,"ci":[0.2255,0.4182],"pass_all3_ci":[0.09351,0.2513],"n_tasks":87,"n_tasks_expected":87}},"complete":true,"per_effort":{"default":0.2165},"efforts_requested":{"medium":65648},"versions":["2.2.0"],"vs_leader":{"diff":-0.01263,"ci":[-0.05054,0.02428],"tied":true,"n_tasks":198}},"mini-swe-agent":{"cost_per_trial":0.2246,"cost_per_pass":1.119,"tokens_per_trial":10180000.0,"calls_per_trial":95.05,"model_s_per_trial":3283.0,"agent_s_per_trial":3650.0,"rerun_flip":0.1794,"pass_all3":0.08955,"pass_mean3":0.2139,"pass_any3":0.3596,"n_trials":647,"n_scored":593,"n_pass":119,"n_infra":54,"cost_usd":136.0,"n_tasks3":197,"macro":0.2135,"ci":[0.1568,0.2701],"pass_all3_ci":[0.04695,0.1321],"benchmarks":{"deep-swe":{"cost_per_trial":0.377,"cost_per_pass":3.576,"tokens_per_trial":16870000.0,"calls_per_trial":141.1,"model_s_per_trial":5074.0,"agent_s_per_trial":5496.0,"rerun_flip":0.1441,"pass_all3":0.01818,"pass_mean3":0.1061,"pass_any3":0.2364,"n_trials":380,"n_scored":332,"n_pass":35,"n_infra":48,"cost_usd":128.0,"n_tasks3":110,"rate":0.1051,"ci":[0.04748,0.1627],"pass_all3_ci":[0.0,0.04762],"n_tasks":111,"n_tasks_expected":111},"terminal-bench-2.1":{"cost_per_trial":0.03066,"cost_per_pass":0.09525,"tokens_per_trial":1666000.0,"calls_per_trial":36.45,"model_s_per_trial":1004.0,"agent_s_per_trial":1302.0,"rerun_flip":0.2146,"pass_all3":0.1609,"pass_mean3":0.3218,"pass_any3":0.4828,"n_trials":267,"n_scored":261,"n_pass":84,"n_infra":6,"cost_usd":8.001,"n_tasks3":87,"rate":0.3218,"ci":[0.2255,0.4182],"pass_all3_ci":[0.08401,0.2378],"n_tasks":87,"n_tasks_expected":87}},"complete":true,"per_effort":{"default":0.2135},"efforts_requested":{"medium":56365},"versions":["2.4.6"],"vs_leader":{"diff":-0.01564,"ci":[-0.04996,0.01771],"tied":true,"n_tasks":198}},"openclaw":{"cost_per_trial":0.1533,"cost_per_pass":3.139,"tokens_per_trial":12140000.0,"calls_per_trial":158.8,"model_s_per_trial":1518.0,"agent_s_per_trial":1655.0,"rerun_flip":0.03065,"pass_all3":0.03448,"pass_mean3":0.05556,"pass_any3":0.08046,"n_trials":606,"n_scored":594,"n_pass":29,"n_infra":12,"cost_usd":94.31,"n_tasks3":198,"macro":0.05556,"ci":[0.01919,0.09192],"pass_all3_ci":[0.003035,0.06593],"benchmarks":{"deep-swe":{"cost_per_trial":0.1911,"cost_per_pass":null,"tokens_per_trial":14630000.0,"calls_per_trial":172.0,"model_s_per_trial":1864.0,"agent_s_per_trial":1978.0,"rerun_flip":0.0,"pass_all3":0.0,"pass_mean3":0.0,"pass_any3":0.0,"n_trials":339,"n_scored":333,"n_pass":0,"n_infra":6,"cost_usd":66.41,"n_tasks3":111,"rate":0.0,"ci":[0.0,0.01673],"pass_all3_ci":[0.0,0.01673],"n_tasks":111,"n_tasks_expected":111},"terminal-bench-2.1":{"cost_per_trial":0.105,"cost_per_pass":0.9453,"tokens_per_trial":8969000.0,"calls_per_trial":142.0,"model_s_per_trial":1076.0,"agent_s_per_trial":1243.0,"rerun_flip":0.0613,"pass_all3":0.06897,"pass_mean3":0.1111,"pass_any3":0.1609,"n_trials":267,"n_scored":261,"n_pass":29,"n_infra":6,"cost_usd":27.9,"n_tasks3":87,"rate":0.1111,"ci":[0.04442,0.1778],"pass_all3_ci":[0.01376,0.1242],"n_tasks":87,"n_tasks_expected":87}},"complete":true,"per_effort":{"default":0.05556},"efforts_requested":{"medium":94120,"low":216,"unset":3},"versions":["2026.10.4"],"vs_leader":{"diff":-0.1736,"ci":[-0.2175,-0.1305],"tied":false,"n_tasks":198}},"opencode":{"cost_per_trial":0.2093,"cost_per_pass":0.9913,"tokens_per_trial":13590000.0,"calls_per_trial":129.4,"model_s_per_trial":2799.0,"agent_s_per_trial":3326.0,"rerun_flip":0.1235,"pass_all3":0.1425,"pass_mean3":0.2294,"pass_any3":0.3277,"n_trials":628,"n_scored":592,"n_pass":125,"n_infra":36,"cost_usd":128.3,"n_tasks3":197,"macro":0.2291,"ci":[0.1726,0.2856],"pass_all3_ci":[0.0938,0.1912],"benchmarks":{"deep-swe":{"cost_per_trial":0.3558,"cost_per_pass":4.71,"tokens_per_trial":22940000.0,"calls_per_trial":198.2,"model_s_per_trial":4441.0,"agent_s_per_trial":5098.0,"rerun_flip":0.1091,"pass_all3":0.009091,"pass_mean3":0.07576,"pass_any3":0.1727,"n_trials":363,"n_scored":331,"n_pass":25,"n_infra":32,"cost_usd":122.1,"n_tasks3":110,"rate":0.07508,"ci":[0.02483,0.1253],"pass_all3_ci":[0.0,0.03314],"n_tasks":111,"n_tasks_expected":111},"terminal-bench-2.1":{"cost_per_trial":0.02358,"cost_per_pass":0.06153,"tokens_per_trial":1730000.0,"calls_per_trial":42.18,"model_s_per_trial":716.3,"agent_s_per_trial":1078.0,"rerun_flip":0.1379,"pass_all3":0.2759,"pass_mean3":0.3831,"pass_any3":0.4828,"n_trials":265,"n_scored":261,"n_pass":100,"n_infra":4,"cost_usd":6.181,"n_tasks3":87,"rate":0.3831,"ci":[0.283,0.4832],"pass_all3_ci":[0.1835,0.3683],"n_tasks":87,"n_tasks_expected":87}},"complete":true,"per_effort":{"default":0.2291},"efforts_requested":{"medium":76599},"versions":["2.2.6"]},"pi":{"cost_per_trial":0.09591,"cost_per_pass":0.5755,"tokens_per_trial":5264000.0,"calls_per_trial":91.23,"model_s_per_trial":2597.0,"agent_s_per_trial":2996.0,"rerun_flip":0.134,"pass_all3":0.08947,"pass_mean3":0.1801,"pass_any3":0.2905,"n_trials":619,"n_scored":594,"n_pass":99,"n_infra":25,"cost_usd":58.13,"n_tasks3":198,"macro":0.1801,"ci":[0.1266,0.2336],"pass_all3_ci":[0.04693,0.132],"benchmarks":{"deep-swe":{"cost_per_trial":0.1549,"cost_per_pass":2.243,"tokens_per_trial":8484000.0,"calls_per_trial":137.8,"model_s_per_trial":3967.0,"agent_s_per_trial":4452.0,"rerun_flip":0.08408,"pass_all3":0.01802,"pass_mean3":0.06907,"pass_any3":0.1441,"n_trials":351,"n_scored":333,"n_pass":23,"n_infra":18,"cost_usd":52.74,"n_tasks3":111,"rate":0.06907,"ci":[0.0205,0.1176],"pass_all3_ci":[0.0,0.0472],"n_tasks":111,"n_tasks_expected":111},"terminal-bench-2.1":{"cost_per_trial":0.0206,"cost_per_pass":0.07075,"tokens_per_trial":1157000.0,"calls_per_trial":31.78,"model_s_per_trial":849.4,"agent_s_per_trial":1138.0,"rerun_flip":0.1839,"pass_all3":0.1609,"pass_mean3":0.2912,"pass_any3":0.4368,"n_trials":268,"n_scored":261,"n_pass":76,"n_infra":7,"cost_usd":5.386,"n_tasks3":87,"rate":0.2912,"ci":[0.1973,0.385],"pass_all3_ci":[0.08401,0.2378],"n_tasks":87,"n_tasks_expected":87}},"complete":true,"per_effort":{"default":0.1801},"efforts_requested":{"medium":54191},"versions":["1.2.0"],"vs_leader":{"diff":-0.04898,"ci":[-0.09248,-0.009889],"tied":false,"n_tasks":198}},"qwen-code":{"cost_per_trial":0.0594,"cost_per_pass":0.5266,"tokens_per_trial":2866000.0,"calls_per_trial":45.91,"model_s_per_trial":1976.0,"agent_s_per_trial":2082.0,"rerun_flip":0.1133,"pass_all3":0.05172,"pass_mean3":0.1275,"pass_any3":0.2217,"n_trials":606,"n_scored":594,"n_pass":67,"n_infra":12,"cost_usd":35.76,"n_tasks3":198,"macro":0.1275,"ci":[0.08041,0.1746],"pass_all3_ci":[0.01617,0.08728],"benchmarks":{"deep-swe":{"cost_per_trial":0.08761,"cost_per_pass":14.59,"tokens_per_trial":4154000.0,"calls_per_trial":61.19,"model_s_per_trial":2720.0,"agent_s_per_trial":2793.0,"rerun_flip":0.01201,"pass_all3":0.0,"pass_mean3":0.006006,"pass_any3":0.01802,"n_trials":344,"n_scored":333,"n_pass":2,"n_infra":11,"cost_usd":29.65,"n_tasks3":111,"rate":0.006006,"ci":[0.0,0.02775],"pass_all3_ci":[0.0,0.01673],"n_tasks":111,"n_tasks_expected":111},"terminal-bench-2.1":{"cost_per_trial":0.02341,"cost_per_pass":0.094,"tokens_per_trial":1222000.0,"calls_per_trial":26.42,"model_s_per_trial":1027.0,"agent_s_per_trial":1174.0,"rerun_flip":0.2146,"pass_all3":0.1034,"pass_mean3":0.249,"pass_any3":0.4253,"n_trials":262,"n_scored":261,"n_pass":65,"n_infra":1,"cost_usd":6.11,"n_tasks3":87,"rate":0.249,"ci":[0.1595,0.3386],"pass_all3_ci":[0.03861,0.1683],"n_tasks":87,"n_tasks_expected":87}},"complete":true,"per_effort":{"default":0.1275},"efforts_requested":{"medium":27175,"unset":97},"versions":["0.25.2"],"vs_leader":{"diff":-0.1016,"ci":[-0.1406,-0.06643],"tied":false,"n_tasks":198}}},"ranking":["opencode","kimi-code","mini-swe-agent","deepseek-harness","hermes","pi","claude-code","codex","qwen-code","openclaw"],"efforts":["default"],"noise":{"rerun_flip":0.1331,"harness_flip":0.1584,"by_benchmark":{"deep-swe":{"rerun_flip":0.08493,"harness_flip":0.09672},"terminal-bench-2.1":{"rerun_flip":0.1812,"harness_flip":0.2201}},"harness_pairs":[{"harnesses":["opencode","kimi-code"],"flip":0.1695},{"harnesses":["opencode","mini-swe-agent"],"flip":0.1622},{"harnesses":["opencode","deepseek-harness"],"flip":0.158},{"harnesses":["opencode","hermes"],"flip":0.1479},{"harnesses":["opencode","pi"],"flip":0.1672},{"harnesses":["opencode","claude-code"],"flip":0.1671},{"harnesses":["opencode","codex"],"flip":0.184},{"harnesses":["opencode","qwen-code"],"flip":0.1559},{"harnesses":["opencode","openclaw"],"flip":0.1787},{"harnesses":["kimi-code","mini-swe-agent"],"flip":0.1579},{"harnesses":["kimi-code","deepseek-harness"],"flip":0.1836},{"harnesses":["kimi-code","hermes"],"flip":0.1489},{"harnesses":["kimi-code","pi"],"flip":0.156},{"harnesses":["kimi-code","claude-code"],"flip":0.1583},{"harnesses":["kimi-code","codex"],"flip":0.1872},{"harnesses":["kimi-code","qwen-code"],"flip":0.1551},{"harnesses":["kimi-code","openclaw"],"flip":0.1852},{"harnesses":["mini-swe-agent","deepseek-harness"],"flip":0.1765},{"harnesses":["mini-swe-agent","hermes"],"flip":0.1509},{"harnesses":["mini-swe-agent","pi"],"flip":0.1513},{"harnesses":["mini-swe-agent","claude-code"],"flip":0.1626},{"harnesses":["mini-swe-agent","codex"],"flip":0.1713},{"harnesses":["mini-swe-agent","qwen-code"],"flip":0.1534},{"harnesses":["mini-swe-agent","openclaw"],"flip":0.1694},{"harnesses":["deepseek-harness","hermes"],"flip":0.163},{"harnesses":["deepseek-harness","pi"],"flip":0.1767},{"harnesses":["deepseek-harness","claude-code"],"flip":0.1801},{"harnesses":["deepseek-harness","codex"],"flip":0.1734},{"harnesses":["deepseek-harness","qwen-code"],"flip":0.1563},{"harnesses":["deepseek-harness","openclaw"],"flip":0.1755},{"harnesses":["hermes","pi"],"flip":0.1479},{"harnesses":["hermes","claude-code"],"flip":0.1429},{"harnesses":["hermes","codex"],"flip":0.1774},{"harnesses":["hermes","qwen-code"],"flip":0.1289},{"harnesses":["hermes","openclaw"],"flip":0.1481},{"harnesses":["pi","claude-code"],"flip":0.1504},{"harnesses":["pi","codex"],"flip":0.1536},{"harnesses":["pi","qwen-code"],"flip":0.1345},{"harnesses":["pi","openclaw"],"flip":0.1322},{"harnesses":["claude-code","codex"],"flip":0.1713},{"harnesses":["claude-code","qwen-code"],"flip":0.1388},{"harnesses":["claude-code","openclaw"],"flip":0.1413},{"harnesses":["codex","qwen-code"],"flip":0.1438},{"harnesses":["codex","openclaw"],"flip":0.1341},{"harnesses":["qwen-code","openclaw"],"flip":0.09879}]},"leader":"opencode"}}}}},"deepseek-ai/DeepSeek-V4.1-Flash":{"stages":{"full":{"efforts":{"high":{"harnesses":{"deepseek-harness":{"n_trials":640,"n_scored":594,"n_pass":424,"n_infra":46,"cost_usd":82.59,"cost_per_trial":0.1363,"cost_per_pass":0.191,"tokens_per_trial":11090000.0,"calls_per_trial":92.75,"model_s_per_trial":812.4,"agent_s_per_trial":1132.0,"efforts_requested":{"high":54525,"low":567},"versions":["0.2.2-rc.1"],"benchmarks":{"deep-swe":{"n_trials":363,"n_scored":333,"n_pass":230,"n_infra":30,"cost_usd":71.69,"cost_per_trial":0.2114,"cost_per_pass":0.3061,"tokens_per_trial":18600000.0,"calls_per_trial":142.3,"model_s_per_trial":1179.0,"agent_s_per_trial":1589.0,"efforts_requested":{"high":47092,"low":309},"versions":["0.2.2-rc.1"],"n_tasks":111,"n_tasks_expected":111,"passed":76.67,"rate":0.6907,"ci":[0.5995,0.7691],"rerun_flip":0.2643,"n_tasks3":111,"pass_all3":0.4775,"pass_mean3":0.6907,"pass_any3":0.8739,"pass_all3_ci":[0.3869,0.5696]},"terminal-bench-2.1":{"n_trials":277,"n_scored":261,"n_pass":194,"n_infra":16,"cost_usd":10.9,"cost_per_trial":0.04046,"cost_per_pass":0.05444,"tokens_per_trial":1509000.0,"calls_per_trial":29.47,"model_s_per_trial":344.0,"agent_s_per_trial":548.8,"efforts_requested":{"high":7433,"low":258},"versions":["0.2.2-rc.1"],"n_tasks":87,"n_tasks_expected":87,"passed":64.67,"rate":0.7433,"ci":[0.6426,0.8234],"rerun_flip":0.1533,"n_tasks3":87,"pass_all3":0.6207,"pass_mean3":0.7433,"pass_any3":0.8506,"pass_all3_ci":[0.5157,0.7155]}},"macro":0.717,"ci":[0.6548,0.7792],"rerun_flip":0.2088,"n_tasks3":198,"pass_all3":0.5491,"pass_mean3":0.717,"pass_any3":0.8622,"pass_all3_ci":[0.4814,0.6168],"complete":true,"vs_leader":{"diff":-0.01279,"ci":[-0.06845,0.03832],"tied":true,"n_tasks":198}},"hermes":{"n_trials":637,"n_scored":594,"n_pass":435,"n_infra":43,"cost_usd":77.7,"cost_per_trial":0.1228,"cost_per_pass":0.1677,"tokens_per_trial":9387000.0,"calls_per_trial":70.31,"model_s_per_trial":921.5,"agent_s_per_trial":1482.0,"efforts_requested":{"high":41063,"none":486,"unset":216},"versions":["v0.22.1"],"benchmarks":{"terminal-bench-2.1":{"n_trials":261,"n_scored":261,"n_pass":185,"n_infra":0,"cost_usd":11.78,"cost_per_trial":0.04514,"cost_per_pass":0.06369,"tokens_per_trial":1766000.0,"calls_per_trial":29.62,"model_s_per_trial":612.0,"agent_s_per_trial":940.9,"efforts_requested":{"high":7349,"unset":214,"none":168},"versions":["v0.22.1"],"n_tasks":87,"n_tasks_expected":87,"passed":61.67,"rate":0.7088,"ci":[0.6061,0.7938],"rerun_flip":0.2069,"n_tasks3":87,"pass_all3":0.5402,"pass_mean3":0.7088,"pass_any3":0.8506,"pass_all3_ci":[0.436,0.641]},"deep-swe":{"n_trials":376,"n_scored":333,"n_pass":250,"n_infra":43,"cost_usd":65.92,"cost_per_trial":0.1836,"cost_per_pass":0.2446,"tokens_per_trial":15360000.0,"calls_per_trial":102.2,"model_s_per_trial":1164.0,"agent_s_per_trial":1906.0,"efforts_requested":{"high":33714,"none":318,"unset":2},"versions":["v0.22.1"],"n_tasks":111,"n_tasks_expected":111,"passed":83.33,"rate":0.7508,"ci":[0.6628,0.8219],"rerun_flip":0.3003,"n_tasks3":111,"pass_all3":0.4955,"pass_mean3":0.7508,"pass_any3":0.9459,"pass_all3_ci":[0.4042,0.5871]}},"macro":0.7298,"ci":[0.668,0.7916],"rerun_flip":0.2536,"n_tasks3":198,"pass_all3":0.5179,"pass_mean3":0.7298,"pass_any3":0.8983,"pass_all3_ci":[0.4492,0.5865],"complete":true},"kimi-code":{"n_trials":625,"n_scored":593,"n_pass":390,"n_infra":32,"cost_usd":76.33,"cost_per_trial":0.1241,"cost_per_pass":0.1887,"tokens_per_trial":9435000.0,"calls_per_trial":78.32,"model_s_per_trial":1024.0,"agent_s_per_trial":1271.0,"efforts_requested":{"high":46445},"versions":["2.2.0"],"benchmarks":{"deep-swe":{"n_trials":363,"n_scored":332,"n_pass":204,"n_infra":31,"cost_usd":63.75,"cost_per_trial":0.1837,"cost_per_pass":0.299,"tokens_per_trial":15300000.0,"calls_per_trial":115.9,"model_s_per_trial":1350.0,"agent_s_per_trial":1617.0,"efforts_requested":{"high":38471},"versions":["2.2.0"],"n_tasks":111,"n_tasks_expected":111,"passed":68.17,"rate":0.6141,"ci":[0.5212,0.6994],"rerun_flip":0.3393,"n_tasks3":110,"pass_all3":0.3545,"pass_mean3":0.6152,"pass_any3":0.8545,"pass_all3_ci":[0.2714,0.4475]},"terminal-bench-2.1":{"n_trials":262,"n_scored":261,"n_pass":186,"n_infra":1,"cost_usd":12.58,"cost_per_trial":0.0482,"cost_per_pass":0.06763,"tokens_per_trial":1978000.0,"calls_per_trial":30.55,"model_s_per_trial":607.8,"agent_s_per_trial":830.5,"efforts_requested":{"high":7974},"versions":["2.2.0"],"n_tasks":87,"n_tasks_expected":87,"passed":62.0,"rate":0.7126,"ci":[0.6102,0.7971],"rerun_flip":0.2069,"n_tasks3":87,"pass_all3":0.5287,"pass_mean3":0.7126,"pass_any3":0.8391,"pass_all3_ci":[0.4249,0.6302]}},"macro":0.6634,"ci":[0.5987,0.7281],"rerun_flip":0.2731,"n_tasks3":197,"pass_all3":0.4416,"pass_mean3":0.6639,"pass_any3":0.8468,"pass_all3_ci":[0.374,0.5092],"complete":true,"vs_leader":{"diff":-0.0664,"ci":[-0.1123,-0.02034],"tied":false,"n_tasks":198}},"qwen-code":{"n_trials":603,"n_scored":594,"n_pass":392,"n_infra":9,"cost_usd":75.32,"cost_per_trial":0.125,"cost_per_pass":0.1894,"tokens_per_trial":6298000.0,"calls_per_trial":74.33,"model_s_per_trial":847.7,"agent_s_per_trial":1104.0,"efforts_requested":{"high":43232,"unset":919},"versions":["0.25.2"],"benchmarks":{"terminal-bench-2.1":{"n_trials":261,"n_scored":261,"n_pass":189,"n_infra":0,"cost_usd":12.9,"cost_per_trial":0.04944,"cost_per_pass":0.06827,"tokens_per_trial":1416000.0,"calls_per_trial":27.26,"model_s_per_trial":463.0,"agent_s_per_trial":714.9,"efforts_requested":{"high":7024,"unset":92},"versions":["0.25.2"],"n_tasks":87,"n_tasks_expected":87,"passed":63.0,"rate":0.7241,"ci":[0.6223,0.8071],"rerun_flip":0.1839,"n_tasks3":87,"pass_all3":0.5747,"pass_mean3":0.7241,"pass_any3":0.8506,"pass_all3_ci":[0.4698,0.6733]},"deep-swe":{"n_trials":342,"n_scored":333,"n_pass":203,"n_infra":9,"cost_usd":62.42,"cost_per_trial":0.1842,"cost_per_pass":0.3022,"tokens_per_trial":10120000.0,"calls_per_trial":111.2,"model_s_per_trial":1149.0,"agent_s_per_trial":1409.0,"efforts_requested":{"high":36208,"unset":827},"versions":["0.25.2"],"n_tasks":111,"n_tasks_expected":111,"passed":67.67,"rate":0.6096,"ci":[0.5166,0.6952],"rerun_flip":0.2883,"n_tasks3":111,"pass_all3":0.3784,"pass_mean3":0.6096,"pass_any3":0.8108,"pass_all3_ci":[0.2937,0.4712]}},"macro":0.6669,"ci":[0.6025,0.7313],"rerun_flip":0.2361,"n_tasks3":198,"pass_all3":0.4765,"pass_mean3":0.6669,"pass_any3":0.8307,"pass_all3_ci":[0.409,0.5441],"complete":true,"vs_leader":{"diff":-0.06291,"ci":[-0.1057,-0.02081],"tied":false,"n_tasks":198}},"mini-swe-agent":{"n_trials":764,"n_scored":594,"n_pass":408,"n_infra":170,"cost_usd":108.1,"cost_per_trial":0.151,"cost_per_pass":0.2198,"tokens_per_trial":11880000.0,"calls_per_trial":100.2,"model_s_per_trial":1001.0,"agent_s_per_trial":1382.0,"efforts_requested":{"high":59486,"unset":23,"none":3,"low":2},"versions":["2.4.6"],"benchmarks":{"deep-swe":{"n_trials":502,"n_scored":333,"n_pass":219,"n_infra":169,"cost_usd":94.57,"cost_per_trial":0.2293,"cost_per_pass":0.3487,"tokens_per_trial":19520000.0,"calls_per_trial":148.8,"model_s_per_trial":1349.0,"agent_s_per_trial":1821.0,"efforts_requested":{"high":49520,"unset":23,"none":3,"low":2},"versions":["2.4.6"],"n_tasks":111,"n_tasks_expected":111,"passed":73.0,"rate":0.6577,"ci":[0.5654,0.7393],"rerun_flip":0.2282,"n_tasks3":111,"pass_all3":0.4775,"pass_mean3":0.6577,"pass_any3":0.8198,"pass_all3_ci":[0.3869,0.5696]},"terminal-bench-2.1":{"n_trials":262,"n_scored":261,"n_pass":189,"n_infra":1,"cost_usd":13.53,"cost_per_trial":0.05111,"cost_per_pass":0.07058,"tokens_per_trial":2124000.0,"calls_per_trial":38.18,"model_s_per_trial":556.3,"agent_s_per_trial":822.1,"efforts_requested":{"high":9966},"versions":["2.4.6"],"n_tasks":87,"n_tasks_expected":87,"passed":63.0,"rate":0.7241,"ci":[0.6223,0.8071],"rerun_flip":0.1916,"n_tasks3":87,"pass_all3":0.5632,"pass_mean3":0.7241,"pass_any3":0.8506,"pass_all3_ci":[0.4585,0.6626]}},"macro":0.6909,"ci":[0.6273,0.7545],"rerun_flip":0.2099,"n_tasks3":198,"pass_all3":0.5203,"pass_mean3":0.6909,"pass_any3":0.8352,"pass_all3_ci":[0.4519,0.5888],"complete":true,"vs_leader":{"diff":-0.03888,"ci":[-0.09206,0.009435],"tied":true,"n_tasks":198}},"pi":{"n_trials":687,"n_scored":594,"n_pass":393,"n_infra":93,"cost_usd":71.9,"cost_per_trial":0.1073,"cost_per_pass":0.1622,"tokens_per_trial":6179000.0,"calls_per_trial":91.87,"model_s_per_trial":856.2,"agent_s_per_trial":1368.0,"efforts_requested":{"high":54571},"versions":["1.2.0"],"benchmarks":{"deep-swe":{"n_trials":426,"n_scored":333,"n_pass":198,"n_infra":93,"cost_usd":64.0,"cost_per_trial":0.1677,"cost_per_pass":0.282,"tokens_per_trial":10330000.0,"calls_per_trial":143.7,"model_s_per_trial":1229.0,"agent_s_per_trial":1952.0,"efforts_requested":{"high":47848},"versions":["1.2.0"],"n_tasks":111,"n_tasks_expected":111,"passed":66.0,"rate":0.5946,"ci":[0.5016,0.6813],"rerun_flip":0.3303,"n_tasks3":111,"pass_all3":0.3514,"pass_mean3":0.5946,"pass_any3":0.8468,"pass_all3_ci":[0.2689,0.4438]},"terminal-bench-2.1":{"n_trials":261,"n_scored":261,"n_pass":195,"n_infra":0,"cost_usd":7.897,"cost_per_trial":0.03026,"cost_per_pass":0.0405,"tokens_per_trial":888700.0,"calls_per_trial":25.76,"model_s_per_trial":380.3,"agent_s_per_trial":622.8,"efforts_requested":{"high":6723},"versions":["1.2.0"],"n_tasks":87,"n_tasks_expected":87,"passed":65.0,"rate":0.7471,"ci":[0.6467,0.8267],"rerun_flip":0.1686,"n_tasks3":87,"pass_all3":0.6092,"pass_mean3":0.7471,"pass_any3":0.8621,"pass_all3_ci":[0.5041,0.705]}},"macro":0.6709,"ci":[0.6071,0.7346],"rerun_flip":0.2495,"n_tasks3":198,"pass_all3":0.4803,"pass_mean3":0.6709,"pass_any3":0.8545,"pass_all3_ci":[0.4136,0.5469],"complete":true,"vs_leader":{"diff":-0.05892,"ci":[-0.1073,-0.01216],"tied":false,"n_tasks":198}},"claude-code":{"n_trials":614,"n_scored":594,"n_pass":409,"n_infra":20,"cost_usd":72.87,"cost_per_trial":0.1204,"cost_per_pass":0.1748,"tokens_per_trial":7276000.0,"calls_per_trial":85.06,"model_s_per_trial":942.7,"agent_s_per_trial":1290.0,"efforts_requested":{"high":50524},"versions":["2.1.300"],"benchmarks":{"deep-swe":{"n_trials":352,"n_scored":333,"n_pass":218,"n_infra":19,"cost_usd":61.6,"cost_per_trial":0.1809,"cost_per_pass":0.2763,"tokens_per_trial":11870000.0,"calls_per_trial":130.2,"model_s_per_trial":1305.0,"agent_s_per_trial":1653.0,"efforts_requested":{"high":43344},"versions":["2.1.300"],"n_tasks":111,"n_tasks_expected":111,"passed":72.67,"rate":0.6547,"ci":[0.5624,0.7366],"rerun_flip":0.3063,"n_tasks3":111,"pass_all3":0.4144,"pass_mean3":0.6547,"pass_any3":0.8739,"pass_all3_ci":[0.3271,0.5074]},"terminal-bench-2.1":{"n_trials":262,"n_scored":261,"n_pass":191,"n_infra":1,"cost_usd":11.26,"cost_per_trial":0.04316,"cost_per_pass":0.05898,"tokens_per_trial":1410000.0,"calls_per_trial":27.51,"model_s_per_trial":481.1,"agent_s_per_trial":826.0,"efforts_requested":{"high":7180},"versions":["2.1.300"],"n_tasks":87,"n_tasks_expected":87,"passed":63.67,"rate":0.7318,"ci":[0.6304,0.8136],"rerun_flip":0.1762,"n_tasks3":87,"pass_all3":0.5977,"pass_mean3":0.7318,"pass_any3":0.8621,"pass_all3_ci":[0.4926,0.6945]}},"macro":0.6932,"ci":[0.6298,0.7566],"rerun_flip":0.2413,"n_tasks3":198,"pass_all3":0.5061,"pass_mean3":0.6932,"pass_any3":0.868,"pass_all3_ci":[0.4384,0.5737],"complete":true,"vs_leader":{"diff":-0.03655,"ci":[-0.0817,0.009792],"tied":true,"n_tasks":198}},"codex":{"n_trials":618,"n_scored":594,"n_pass":408,"n_infra":24,"cost_usd":61.07,"cost_per_trial":0.1024,"cost_per_pass":0.149,"tokens_per_trial":8233000.0,"calls_per_trial":91.98,"model_s_per_trial":613.0,"agent_s_per_trial":893.6,"efforts_requested":{"high":54636},"versions":["0.162.1"],"benchmarks":{"terminal-bench-2.1":{"n_trials":270,"n_scored":261,"n_pass":195,"n_infra":9,"cost_usd":8.413,"cost_per_trial":0.03212,"cost_per_pass":0.043,"tokens_per_trial":1353000.0,"calls_per_trial":30.62,"model_s_per_trial":305.2,"agent_s_per_trial":572.8,"efforts_requested":{"high":7993},"versions":["0.162.1"],"n_tasks":87,"n_tasks_expected":87,"passed":65.0,"rate":0.7471,"ci":[0.6467,0.8267],"rerun_flip":0.1762,"n_tasks3":87,"pass_all3":0.5977,"pass_mean3":0.7471,"pass_any3":0.8621,"pass_all3_ci":[0.4926,0.6945]},"deep-swe":{"n_trials":348,"n_scored":333,"n_pass":213,"n_infra":15,"cost_usd":52.66,"cost_per_trial":0.1574,"cost_per_pass":0.2461,"tokens_per_trial":13630000.0,"calls_per_trial":140.1,"model_s_per_trial":854.2,"agent_s_per_trial":1145.0,"efforts_requested":{"high":46643},"versions":["0.162.1"],"n_tasks":111,"n_tasks_expected":111,"passed":71.0,"rate":0.6396,"ci":[0.547,0.7229],"rerun_flip":0.3183,"n_tasks3":111,"pass_all3":0.3874,"pass_mean3":0.6396,"pass_any3":0.8649,"pass_all3_ci":[0.302,0.4803]}},"macro":0.6934,"ci":[0.6303,0.7565],"rerun_flip":0.2473,"n_tasks3":198,"pass_all3":0.4925,"pass_mean3":0.6934,"pass_any3":0.8635,"pass_all3_ci":[0.4252,0.5599],"complete":true,"vs_leader":{"diff":-0.0364,"ci":[-0.08041,0.008493],"tied":true,"n_tasks":198}},"opencode":{"n_trials":599,"n_scored":594,"n_pass":409,"n_infra":5,"cost_usd":70.13,"cost_per_trial":0.1181,"cost_per_pass":0.1715,"tokens_per_trial":9862000.0,"calls_per_trial":82.9,"model_s_per_trial":583.7,"agent_s_per_trial":865.7,"efforts_requested":{"high":49241},"versions":["2.2.6"],"benchmarks":{"deep-swe":{"n_trials":337,"n_scored":333,"n_pass":216,"n_infra":4,"cost_usd":61.55,"cost_per_trial":0.1848,"cost_per_pass":0.285,"tokens_per_trial":16590000.0,"calls_per_trial":127.0,"model_s_per_trial":856.2,"agent_s_per_trial":1155.0,"efforts_requested":{"high":42298},"versions":["2.2.6"],"n_tasks":111,"n_tasks_expected":111,"passed":72.0,"rate":0.6486,"ci":[0.5562,0.7311],"rerun_flip":0.3243,"n_tasks3":111,"pass_all3":0.3874,"pass_mean3":0.6486,"pass_any3":0.8739,"pass_all3_ci":[0.302,0.4803]},"terminal-bench-2.1":{"n_trials":262,"n_scored":261,"n_pass":193,"n_infra":1,"cost_usd":8.58,"cost_per_trial":0.03287,"cost_per_pass":0.04446,"tokens_per_trial":1283000.0,"calls_per_trial":26.6,"model_s_per_trial":236.0,"agent_s_per_trial":496.6,"efforts_requested":{"high":6943},"versions":["2.2.6"],"n_tasks":87,"n_tasks_expected":87,"passed":64.33,"rate":0.7395,"ci":[0.6385,0.8202],"rerun_flip":0.1686,"n_tasks3":87,"pass_all3":0.6207,"pass_mean3":0.7395,"pass_any3":0.8736,"pass_all3_ci":[0.5157,0.7155]}},"macro":0.6941,"ci":[0.6308,0.7573],"rerun_flip":0.2465,"n_tasks3":198,"pass_all3":0.504,"pass_mean3":0.6941,"pass_any3":0.8737,"pass_all3_ci":[0.4371,0.571],"complete":true,"vs_leader":{"diff":-0.03573,"ci":[-0.08341,0.01394],"tied":true,"n_tasks":198}},"openclaw":{"n_trials":604,"n_scored":594,"n_pass":184,"n_infra":10,"cost_usd":63.79,"cost_per_trial":0.1074,"cost_per_pass":0.3467,"tokens_per_trial":4709000.0,"calls_per_trial":64.83,"model_s_per_trial":1380.0,"agent_s_per_trial":1901.0,"efforts_requested":{"high":38276,"low":132,"unset":103},"versions":["2026.10.4"],"benchmarks":{"deep-swe":{"n_trials":340,"n_scored":333,"n_pass":61,"n_infra":7,"cost_usd":52.59,"cost_per_trial":0.1579,"cost_per_pass":0.8622,"tokens_per_trial":7256000.0,"calls_per_trial":86.77,"model_s_per_trial":1981.0,"agent_s_per_trial":2675.0,"efforts_requested":{"high":28771,"low":122},"versions":["2026.10.4"],"n_tasks":111,"n_tasks_expected":111,"passed":20.33,"rate":0.1832,"ci":[0.1222,0.2653],"rerun_flip":0.1982,"n_tasks3":111,"pass_all3":0.07207,"pass_mean3":0.1832,"pass_any3":0.3694,"pass_all3_ci":[0.03697,0.1358]},"terminal-bench-2.1":{"n_trials":264,"n_scored":261,"n_pass":123,"n_infra":3,"cost_usd":11.19,"cost_per_trial":0.04289,"cost_per_pass":0.09101,"tokens_per_trial":1460000.0,"calls_per_trial":36.85,"model_s_per_trial":613.8,"agent_s_per_trial":912.6,"efforts_requested":{"high":9505,"unset":103,"low":10},"versions":["2026.10.4"],"n_tasks":87,"n_tasks_expected":87,"passed":41.0,"rate":0.4713,"ci":[0.3698,0.5751],"rerun_flip":0.1839,"n_tasks3":87,"pass_all3":0.3563,"pass_mean3":0.4713,"pass_any3":0.6322,"pass_all3_ci":[0.2637,0.4611]}},"macro":0.3272,"ci":[0.2645,0.39],"rerun_flip":0.1911,"n_tasks3":198,"pass_all3":0.2142,"pass_mean3":0.3272,"pass_any3":0.5008,"pass_all3_ci":[0.1585,0.2699],"complete":true,"vs_leader":{"diff":-0.4026,"ci":[-0.4576,-0.353],"tied":false,"n_tasks":198}}},"ranking":["hermes","deepseek-harness","opencode","codex","claude-code","mini-swe-agent","pi","qwen-code","kimi-code","openclaw"],"noise":{"rerun_flip":0.2357,"harness_flip":0.3,"by_benchmark":{"deep-swe":{"rerun_flip":0.2898,"harness_flip":0.3644},"terminal-bench-2.1":{"rerun_flip":0.1816,"harness_flip":0.2356}},"harness_pairs":[{"harnesses":["hermes","deepseek-harness"],"flip":0.2988},{"harnesses":["hermes","opencode"],"flip":0.2962},{"harnesses":["hermes","codex"],"flip":0.2755},{"harnesses":["hermes","claude-code"],"flip":0.2776},{"harnesses":["hermes","mini-swe-agent"],"flip":0.2947},{"harnesses":["hermes","pi"],"flip":0.2931},{"harnesses":["hermes","qwen-code"],"flip":0.2758},{"harnesses":["hermes","kimi-code"],"flip":0.2941},{"harnesses":["hermes","openclaw"],"flip":0.4797},{"harnesses":["deepseek-harness","opencode"],"flip":0.2616},{"harnesses":["deepseek-harness","codex"],"flip":0.2671},{"harnesses":["deepseek-harness","claude-code"],"flip":0.2816},{"harnesses":["deepseek-harness","mini-swe-agent"],"flip":0.2455},{"harnesses":["deepseek-harness","pi"],"flip":0.2664},{"harnesses":["deepseek-harness","qwen-code"],"flip":0.2572},{"harnesses":["deepseek-harness","kimi-code"],"flip":0.2885},{"harnesses":["deepseek-harness","openclaw"],"flip":0.4533},{"harnesses":["opencode","codex"],"flip":0.2382},{"harnesses":["opencode","claude-code"],"flip":0.2558},{"harnesses":["opencode","mini-swe-agent"],"flip":0.2356},{"harnesses":["opencode","pi"],"flip":0.2315},{"harnesses":["opencode","qwen-code"],"flip":0.2436},{"harnesses":["opencode","kimi-code"],"flip":0.2685},{"harnesses":["opencode","openclaw"],"flip":0.4638},{"harnesses":["codex","claude-code"],"flip":0.265},{"harnesses":["codex","mini-swe-agent"],"flip":0.2528},{"harnesses":["codex","pi"],"flip":0.2503},{"harnesses":["codex","qwen-code"],"flip":0.2391},{"harnesses":["codex","kimi-code"],"flip":0.271},{"harnesses":["codex","openclaw"],"flip":0.4473},{"harnesses":["claude-code","mini-swe-agent"],"flip":0.2537},{"harnesses":["claude-code","pi"],"flip":0.2662},{"harnesses":["claude-code","qwen-code"],"flip":0.2557},{"harnesses":["claude-code","kimi-code"],"flip":0.2623},{"harnesses":["claude-code","openclaw"],"flip":0.4428},{"harnesses":["mini-swe-agent","pi"],"flip":0.2439},{"harnesses":["mini-swe-agent","qwen-code"],"flip":0.2592},{"harnesses":["mini-swe-agent","kimi-code"],"flip":0.2529},{"harnesses":["mini-swe-agent","openclaw"],"flip":0.4507},{"harnesses":["pi","qwen-code"],"flip":0.2403},{"harnesses":["pi","kimi-code"],"flip":0.266},{"harnesses":["pi","openclaw"],"flip":0.4385},{"harnesses":["qwen-code","kimi-code"],"flip":0.246},{"harnesses":["qwen-code","openclaw"],"flip":0.4223},{"harnesses":["kimi-code","openclaw"],"flip":0.4316}]},"leader":"hermes"},"low":{"harnesses":{"hermes":{"n_trials":675,"n_scored":594,"n_pass":411,"n_infra":81,"cost_usd":54.58,"cost_per_trial":0.08365,"cost_per_pass":0.1209,"tokens_per_trial":5848000.0,"calls_per_trial":55.96,"model_s_per_trial":729.6,"agent_s_per_trial":1186.0,"efforts_requested":{"low":32539,"none":493,"unset":210},"versions":["v0.22.1"],"benchmarks":{"terminal-bench-2.1":{"n_trials":262,"n_scored":261,"n_pass":175,"n_infra":1,"cost_usd":10.09,"cost_per_trial":0.03867,"cost_per_pass":0.05768,"tokens_per_trial":1414000.0,"calls_per_trial":27.39,"model_s_per_trial":545.2,"agent_s_per_trial":832.4,"efforts_requested":{"low":6760,"unset":210,"none":179},"versions":["v0.22.1"],"n_tasks":87,"n_tasks_expected":87,"passed":58.33,"rate":0.6705,"ci":[0.5664,0.7602],"rerun_flip":0.1839,"n_tasks3":87,"pass_all3":0.5172,"pass_mean3":0.6705,"pass_any3":0.7931,"pass_all3_ci":[0.4137,0.6193]},"deep-swe":{"n_trials":413,"n_scored":333,"n_pass":236,"n_infra":80,"cost_usd":44.49,"cost_per_trial":0.1189,"cost_per_pass":0.1678,"tokens_per_trial":9323000.0,"calls_per_trial":78.36,"model_s_per_trial":874.1,"agent_s_per_trial":1462.0,"efforts_requested":{"low":25779,"none":314},"versions":["v0.22.1"],"n_tasks":111,"n_tasks_expected":111,"passed":78.67,"rate":0.7087,"ci":[0.6183,0.7851],"rerun_flip":0.3604,"n_tasks3":111,"pass_all3":0.4054,"pass_mean3":0.7087,"pass_any3":0.9459,"pass_all3_ci":[0.3187,0.4984]}},"macro":0.6896,"ci":[0.6255,0.7537],"rerun_flip":0.2721,"n_tasks3":198,"pass_all3":0.4613,"pass_mean3":0.6896,"pass_any3":0.8695,"pass_all3_ci":[0.3931,0.5295],"complete":true,"vs_leader":{"diff":-0.02837,"ci":[-0.0756,0.01807],"tied":true,"n_tasks":198}},"kimi-code":{"n_trials":612,"n_scored":594,"n_pass":415,"n_infra":18,"cost_usd":62.36,"cost_per_trial":0.1026,"cost_per_pass":0.1468,"tokens_per_trial":7550000.0,"calls_per_trial":71.01,"model_s_per_trial":815.9,"agent_s_per_trial":1037.0,"efforts_requested":{"low":42181},"versions":["2.2.0"],"benchmarks":{"terminal-bench-2.1":{"n_trials":261,"n_scored":261,"n_pass":189,"n_infra":0,"cost_usd":9.795,"cost_per_trial":0.03753,"cost_per_pass":0.05183,"tokens_per_trial":1604000.0,"calls_per_trial":27.61,"model_s_per_trial":448.4,"agent_s_per_trial":663.1,"efforts_requested":{"low":7207},"versions":["2.2.0"],"n_tasks":87,"n_tasks_expected":87,"passed":63.0,"rate":0.7241,"ci":[0.6223,0.8071],"rerun_flip":0.1916,"n_tasks3":87,"pass_all3":0.5632,"pass_mean3":0.7241,"pass_any3":0.8506,"pass_all3_ci":[0.4585,0.6626]},"deep-swe":{"n_trials":351,"n_scored":333,"n_pass":226,"n_infra":18,"cost_usd":52.56,"cost_per_trial":0.1535,"cost_per_pass":0.2262,"tokens_per_trial":12210000.0,"calls_per_trial":105.0,"model_s_per_trial":1104.0,"agent_s_per_trial":1329.0,"efforts_requested":{"low":34974},"versions":["2.2.0"],"n_tasks":111,"n_tasks_expected":111,"passed":75.33,"rate":0.6787,"ci":[0.5871,0.7583],"rerun_flip":0.3183,"n_tasks3":111,"pass_all3":0.4234,"pass_mean3":0.6787,"pass_any3":0.9009,"pass_all3_ci":[0.3356,0.5164]}},"macro":0.7014,"ci":[0.6382,0.7646],"rerun_flip":0.2549,"n_tasks3":198,"pass_all3":0.4933,"pass_mean3":0.7014,"pass_any3":0.8757,"pass_all3_ci":[0.4252,0.5615],"complete":true,"vs_leader":{"diff":-0.01657,"ci":[-0.0582,0.02636],"tied":true,"n_tasks":198}},"qwen-code":{"n_trials":603,"n_scored":594,"n_pass":393,"n_infra":9,"cost_usd":61.06,"cost_per_trial":0.1013,"cost_per_pass":0.1531,"tokens_per_trial":5373000.0,"calls_per_trial":65.51,"model_s_per_trial":697.3,"agent_s_per_trial":930.7,"efforts_requested":{"low":38084,"unset":827},"versions":["0.25.2"],"benchmarks":{"terminal-bench-2.1":{"n_trials":262,"n_scored":261,"n_pass":191,"n_infra":1,"cost_usd":9.99,"cost_per_trial":0.03828,"cost_per_pass":0.0523,"tokens_per_trial":1216000.0,"calls_per_trial":25.66,"model_s_per_trial":353.5,"agent_s_per_trial":579.6,"efforts_requested":{"low":6613,"unset":84},"versions":["0.25.2"],"n_tasks":87,"n_tasks_expected":87,"passed":63.67,"rate":0.7318,"ci":[0.6304,0.8136],"rerun_flip":0.1762,"n_tasks3":87,"pass_all3":0.5747,"pass_mean3":0.7318,"pass_any3":0.8391,"pass_all3_ci":[0.4698,0.6733]},"deep-swe":{"n_trials":341,"n_scored":333,"n_pass":202,"n_infra":8,"cost_usd":51.07,"cost_per_trial":0.1507,"cost_per_pass":0.2484,"tokens_per_trial":8632000.0,"calls_per_trial":96.74,"model_s_per_trial":966.7,"agent_s_per_trial":1206.0,"efforts_requested":{"low":31471,"unset":743},"versions":["0.25.2"],"n_tasks":111,"n_tasks_expected":111,"passed":67.33,"rate":0.6066,"ci":[0.5136,0.6925],"rerun_flip":0.3003,"n_tasks3":111,"pass_all3":0.3694,"pass_mean3":0.6066,"pass_any3":0.8198,"pass_all3_ci":[0.2854,0.4621]}},"macro":0.6692,"ci":[0.605,0.7334],"rerun_flip":0.2383,"n_tasks3":198,"pass_all3":0.472,"pass_mean3":0.6692,"pass_any3":0.8295,"pass_all3_ci":[0.4047,0.5394],"complete":true,"vs_leader":{"diff":-0.04877,"ci":[-0.09739,-0.001345],"tied":false,"n_tasks":198}},"mini-swe-agent":{"n_trials":716,"n_scored":594,"n_pass":398,"n_infra":122,"cost_usd":72.47,"cost_per_trial":0.1039,"cost_per_pass":0.1551,"tokens_per_trial":7703000.0,"calls_per_trial":82.26,"model_s_per_trial":785.5,"agent_s_per_trial":1071.0,"efforts_requested":{"low":48860},"versions":["2.4.6"],"benchmarks":{"deep-swe":{"n_trials":455,"n_scored":333,"n_pass":199,"n_infra":122,"cost_usd":61.76,"cost_per_trial":0.1532,"cost_per_pass":0.2564,"tokens_per_trial":12310000.0,"calls_per_trial":118.2,"model_s_per_trial":1022.0,"agent_s_per_trial":1360.0,"efforts_requested":{"low":39370},"versions":["2.4.6"],"n_tasks":111,"n_tasks_expected":111,"passed":66.33,"rate":0.5976,"ci":[0.5046,0.6841],"rerun_flip":0.3303,"n_tasks3":111,"pass_all3":0.3423,"pass_mean3":0.5976,"pass_any3":0.8378,"pass_all3_ci":[0.2607,0.4346]},"terminal-bench-2.1":{"n_trials":261,"n_scored":261,"n_pass":199,"n_infra":0,"cost_usd":10.71,"cost_per_trial":0.04105,"cost_per_pass":0.05384,"tokens_per_trial":1821000.0,"calls_per_trial":36.36,"model_s_per_trial":483.6,"agent_s_per_trial":702.9,"efforts_requested":{"low":9490},"versions":["2.4.6"],"n_tasks":87,"n_tasks_expected":87,"passed":66.33,"rate":0.7625,"ci":[0.6631,0.8396],"rerun_flip":0.2222,"n_tasks3":87,"pass_all3":0.5747,"pass_mean3":0.7625,"pass_any3":0.908,"pass_all3_ci":[0.4698,0.6733]}},"macro":0.68,"ci":[0.6169,0.7432],"rerun_flip":0.2763,"n_tasks3":198,"pass_all3":0.4585,"pass_mean3":0.68,"pass_any3":0.8729,"pass_all3_ci":[0.3916,0.5255],"complete":true,"vs_leader":{"diff":-0.03795,"ci":[-0.0829,0.00798],"tied":true,"n_tasks":198}},"deepseek-harness":{"n_trials":640,"n_scored":594,"n_pass":409,"n_infra":46,"cost_usd":69.64,"cost_per_trial":0.1143,"cost_per_pass":0.1659,"tokens_per_trial":8994000.0,"calls_per_trial":85.9,"model_s_per_trial":703.7,"agent_s_per_trial":1011.0,"efforts_requested":{"low":51022},"versions":["0.2.2-rc.1"],"benchmarks":{"deep-swe":{"n_trials":365,"n_scored":333,"n_pass":220,"n_infra":32,"cost_usd":59.98,"cost_per_trial":0.1753,"cost_per_pass":0.2653,"tokens_per_trial":14910000.0,"calls_per_trial":130.1,"model_s_per_trial":1014.0,"agent_s_per_trial":1396.0,"efforts_requested":{"low":43338},"versions":["0.2.2-rc.1"],"n_tasks":111,"n_tasks_expected":111,"passed":73.33,"rate":0.6607,"ci":[0.5685,0.7421],"rerun_flip":0.2883,"n_tasks3":111,"pass_all3":0.4234,"pass_mean3":0.6607,"pass_any3":0.8559,"pass_all3_ci":[0.3356,0.5164]},"terminal-bench-2.1":{"n_trials":275,"n_scored":261,"n_pass":189,"n_infra":14,"cost_usd":9.663,"cost_per_trial":0.03643,"cost_per_pass":0.0503,"tokens_per_trial":1446000.0,"calls_per_trial":29.44,"model_s_per_trial":307.7,"agent_s_per_trial":520.5,"efforts_requested":{"low":7684},"versions":["0.2.2-rc.1"],"n_tasks":87,"n_tasks_expected":87,"passed":63.0,"rate":0.7241,"ci":[0.6223,0.8071],"rerun_flip":0.1533,"n_tasks3":87,"pass_all3":0.6092,"pass_mean3":0.7241,"pass_any3":0.8391,"pass_all3_ci":[0.5041,0.705]}},"macro":0.6924,"ci":[0.6288,0.756],"rerun_flip":0.2208,"n_tasks3":198,"pass_all3":0.5163,"pass_mean3":0.6924,"pass_any3":0.8475,"pass_all3_ci":[0.4487,0.5839],"complete":true,"vs_leader":{"diff":-0.02558,"ci":[-0.06851,0.01736],"tied":true,"n_tasks":198}},"pi":{"n_trials":676,"n_scored":594,"n_pass":387,"n_infra":82,"cost_usd":57.66,"cost_per_trial":0.08613,"cost_per_pass":0.1322,"tokens_per_trial":5098000.0,"calls_per_trial":80.9,"model_s_per_trial":770.2,"agent_s_per_trial":1283.0,"efforts_requested":{"low":48056},"versions":["1.2.0"],"benchmarks":{"deep-swe":{"n_trials":415,"n_scored":333,"n_pass":193,"n_infra":82,"cost_usd":50.99,"cost_per_trial":0.1336,"cost_per_pass":0.2305,"tokens_per_trial":8500000.0,"calls_per_trial":125.3,"model_s_per_trial":1097.0,"agent_s_per_trial":1779.0,"efforts_requested":{"low":41733},"versions":["1.2.0"],"n_tasks":111,"n_tasks_expected":111,"passed":64.33,"rate":0.5796,"ci":[0.4866,0.6672],"rerun_flip":0.2763,"n_tasks3":111,"pass_all3":0.3694,"pass_mean3":0.5796,"pass_any3":0.7838,"pass_all3_ci":[0.2854,0.4621]},"terminal-bench-2.1":{"n_trials":261,"n_scored":261,"n_pass":194,"n_infra":0,"cost_usd":6.671,"cost_per_trial":0.02556,"cost_per_pass":0.03438,"tokens_per_trial":756300.0,"calls_per_trial":24.23,"model_s_per_trial":353.1,"agent_s_per_trial":651.2,"efforts_requested":{"low":6323},"versions":["1.2.0"],"n_tasks":87,"n_tasks_expected":87,"passed":64.67,"rate":0.7433,"ci":[0.6426,0.8234],"rerun_flip":0.2222,"n_tasks3":87,"pass_all3":0.5632,"pass_mean3":0.7433,"pass_any3":0.8966,"pass_all3_ci":[0.4585,0.6626]}},"macro":0.6614,"ci":[0.5974,0.7255],"rerun_flip":0.2492,"n_tasks3":198,"pass_all3":0.4663,"pass_mean3":0.6614,"pass_any3":0.8402,"pass_all3_ci":[0.3988,0.5338],"complete":true,"vs_leader":{"diff":-0.05654,"ci":[-0.09983,-0.0117],"tied":false,"n_tasks":198}},"codex":{"n_trials":613,"n_scored":594,"n_pass":401,"n_infra":19,"cost_usd":51.28,"cost_per_trial":0.08589,"cost_per_pass":0.1272,"tokens_per_trial":6718000.0,"calls_per_trial":82.74,"model_s_per_trial":555.7,"agent_s_per_trial":809.8,"efforts_requested":{"low":49145},"versions":["0.162.1"],"benchmarks":{"terminal-bench-2.1":{"n_trials":268,"n_scored":261,"n_pass":197,"n_infra":7,"cost_usd":6.872,"cost_per_trial":0.02609,"cost_per_pass":0.03457,"tokens_per_trial":1136000.0,"calls_per_trial":27.81,"model_s_per_trial":262.8,"agent_s_per_trial":510.2,"efforts_requested":{"low":7259},"versions":["0.162.1"],"n_tasks":87,"n_tasks_expected":87,"passed":65.67,"rate":0.7548,"ci":[0.6549,0.8331],"rerun_flip":0.1609,"n_tasks3":87,"pass_all3":0.6322,"pass_mean3":0.7548,"pass_any3":0.8736,"pass_all3_ci":[0.5273,0.7259]},"deep-swe":{"n_trials":345,"n_scored":333,"n_pass":204,"n_infra":12,"cost_usd":44.41,"cost_per_trial":0.1328,"cost_per_pass":0.2167,"tokens_per_trial":11090000.0,"calls_per_trial":125.8,"model_s_per_trial":785.2,"agent_s_per_trial":1045.0,"efforts_requested":{"low":41886},"versions":["0.162.1"],"n_tasks":111,"n_tasks_expected":111,"passed":68.0,"rate":0.6126,"ci":[0.5197,0.698],"rerun_flip":0.3544,"n_tasks3":111,"pass_all3":0.3243,"pass_mean3":0.6126,"pass_any3":0.8559,"pass_all3_ci":[0.2444,0.416]}},"macro":0.6837,"ci":[0.6204,0.747],"rerun_flip":0.2576,"n_tasks3":198,"pass_all3":0.4783,"pass_mean3":0.6837,"pass_any3":0.8647,"pass_all3_ci":[0.4125,0.544],"complete":true,"vs_leader":{"diff":-0.03428,"ci":[-0.07238,0.007562],"tied":true,"n_tasks":198}},"claude-code":{"n_trials":618,"n_scored":594,"n_pass":409,"n_infra":24,"cost_usd":58.49,"cost_per_trial":0.09569,"cost_per_pass":0.139,"tokens_per_trial":6163000.0,"calls_per_trial":76.18,"model_s_per_trial":791.8,"agent_s_per_trial":1089.0,"efforts_requested":{"low":45252},"versions":["2.1.300"],"benchmarks":{"deep-swe":{"n_trials":356,"n_scored":333,"n_pass":215,"n_infra":23,"cost_usd":49.75,"cost_per_trial":0.1446,"cost_per_pass":0.224,"tokens_per_trial":10040000.0,"calls_per_trial":115.9,"model_s_per_trial":1097.0,"agent_s_per_trial":1394.0,"efforts_requested":{"low":38609},"versions":["2.1.300"],"n_tasks":111,"n_tasks_expected":111,"passed":71.67,"rate":0.6456,"ci":[0.5532,0.7284],"rerun_flip":0.2823,"n_tasks3":111,"pass_all3":0.4144,"pass_mean3":0.6456,"pass_any3":0.8378,"pass_all3_ci":[0.3271,0.5074]},"terminal-bench-2.1":{"n_trials":262,"n_scored":261,"n_pass":194,"n_infra":1,"cost_usd":8.739,"cost_per_trial":0.03325,"cost_per_pass":0.04473,"tokens_per_trial":1218000.0,"calls_per_trial":25.45,"model_s_per_trial":401.8,"agent_s_per_trial":698.4,"efforts_requested":{"low":6643},"versions":["2.1.300"],"n_tasks":87,"n_tasks_expected":87,"passed":64.67,"rate":0.7433,"ci":[0.6426,0.8234],"rerun_flip":0.1533,"n_tasks3":87,"pass_all3":0.6092,"pass_mean3":0.7433,"pass_any3":0.8391,"pass_all3_ci":[0.5041,0.705]}},"macro":0.6945,"ci":[0.6313,0.7576],"rerun_flip":0.2178,"n_tasks3":198,"pass_all3":0.5118,"pass_mean3":0.6945,"pass_any3":0.8385,"pass_all3_ci":[0.4443,0.5793],"complete":true,"vs_leader":{"diff":-0.02351,"ci":[-0.0612,0.01844],"tied":true,"n_tasks":198}},"opencode":{"n_trials":600,"n_scored":594,"n_pass":423,"n_infra":6,"cost_usd":57.13,"cost_per_trial":0.09617,"cost_per_pass":0.135,"tokens_per_trial":7789000.0,"calls_per_trial":76.58,"model_s_per_trial":492.6,"agent_s_per_trial":732.1,"efforts_requested":{"low":45491},"versions":["2.2.6"],"benchmarks":{"deep-swe":{"n_trials":339,"n_scored":333,"n_pass":223,"n_infra":6,"cost_usd":50.38,"cost_per_trial":0.1513,"cost_per_pass":0.2259,"tokens_per_trial":13120000.0,"calls_per_trial":117.2,"model_s_per_trial":729.1,"agent_s_per_trial":994.8,"efforts_requested":{"low":39039},"versions":["2.2.6"],"n_tasks":111,"n_tasks_expected":111,"passed":74.33,"rate":0.6697,"ci":[0.5778,0.7502],"rerun_flip":0.2462,"n_tasks3":111,"pass_all3":0.4505,"pass_mean3":0.6697,"pass_any3":0.8198,"pass_all3_ci":[0.3611,0.5431]},"terminal-bench-2.1":{"n_trials":261,"n_scored":261,"n_pass":200,"n_infra":0,"cost_usd":6.742,"cost_per_trial":0.02583,"cost_per_pass":0.03371,"tokens_per_trial":985200.0,"calls_per_trial":24.72,"model_s_per_trial":190.9,"agent_s_per_trial":396.9,"efforts_requested":{"low":6452},"versions":["2.2.6"],"n_tasks":87,"n_tasks_expected":87,"passed":66.67,"rate":0.7663,"ci":[0.6673,0.8428],"rerun_flip":0.1762,"n_tasks3":87,"pass_all3":0.6322,"pass_mean3":0.7663,"pass_any3":0.8966,"pass_all3_ci":[0.5273,0.7259]}},"macro":0.718,"ci":[0.6562,0.7798],"rerun_flip":0.2112,"n_tasks3":198,"pass_all3":0.5413,"pass_mean3":0.718,"pass_any3":0.8582,"pass_all3_ci":[0.4739,0.6087],"complete":true},"openclaw":{"n_trials":605,"n_scored":594,"n_pass":218,"n_infra":11,"cost_usd":61.55,"cost_per_trial":0.1032,"cost_per_pass":0.2812,"tokens_per_trial":4884000.0,"calls_per_trial":69.1,"model_s_per_trial":1135.0,"agent_s_per_trial":1618.0,"efforts_requested":{"low":40736,"unset":310,"None":1},"versions":["2026.10.4"],"benchmarks":{"deep-swe":{"n_trials":340,"n_scored":333,"n_pass":81,"n_infra":7,"cost_usd":50.65,"cost_per_trial":0.1513,"cost_per_pass":0.6222,"tokens_per_trial":7465000.0,"calls_per_trial":92.53,"model_s_per_trial":1635.0,"agent_s_per_trial":2271.0,"efforts_requested":{"low":30814},"versions":["2026.10.4"],"n_tasks":111,"n_tasks_expected":111,"passed":27.0,"rate":0.2432,"ci":[0.1729,0.3308],"rerun_flip":0.2643,"n_tasks3":111,"pass_all3":0.08108,"pass_mean3":0.2432,"pass_any3":0.4775,"pass_all3_ci":[0.04324,0.1469]},"terminal-bench-2.1":{"n_trials":265,"n_scored":261,"n_pass":137,"n_infra":4,"cost_usd":10.9,"cost_per_trial":0.04176,"cost_per_pass":0.07955,"tokens_per_trial":1590000.0,"calls_per_trial":39.21,"model_s_per_trial":496.5,"agent_s_per_trial":784.5,"efforts_requested":{"low":9922,"unset":310,"None":1},"versions":["2026.10.4"],"n_tasks":87,"n_tasks_expected":87,"passed":45.67,"rate":0.5249,"ci":[0.4212,0.6266],"rerun_flip":0.2375,"n_tasks3":87,"pass_all3":0.3448,"pass_mean3":0.5249,"pass_any3":0.7011,"pass_all3_ci":[0.2534,0.4494]}},"macro":0.3841,"ci":[0.3192,0.4489],"rerun_flip":0.2509,"n_tasks3":198,"pass_all3":0.213,"pass_mean3":0.3841,"pass_any3":0.5893,"pass_all3_ci":[0.157,0.2689],"complete":true,"vs_leader":{"diff":-0.3339,"ci":[-0.3921,-0.2717],"tied":false,"n_tasks":198}}},"ranking":["opencode","kimi-code","claude-code","deepseek-harness","hermes","codex","mini-swe-agent","qwen-code","pi","openclaw"],"noise":{"rerun_flip":0.2449,"harness_flip":0.3046,"by_benchmark":{"deep-swe":{"rerun_flip":0.3021,"harness_flip":0.3729},"terminal-bench-2.1":{"rerun_flip":0.1877,"harness_flip":0.2363}},"harness_pairs":[{"harnesses":["opencode","kimi-code"],"flip":0.2523},{"harnesses":["opencode","claude-code"],"flip":0.2245},{"harnesses":["opencode","deepseek-harness"],"flip":0.2376},{"harnesses":["opencode","hermes"],"flip":0.2845},{"harnesses":["opencode","codex"],"flip":0.244},{"harnesses":["opencode","mini-swe-agent"],"flip":0.2748},{"harnesses":["opencode","qwen-code"],"flip":0.2719},{"harnesses":["opencode","pi"],"flip":0.2555},{"harnesses":["opencode","openclaw"],"flip":0.4567},{"harnesses":["kimi-code","claude-code"],"flip":0.252},{"harnesses":["kimi-code","deepseek-harness"],"flip":0.2619},{"harnesses":["kimi-code","hermes"],"flip":0.2907},{"harnesses":["kimi-code","codex"],"flip":0.2685},{"harnesses":["kimi-code","mini-swe-agent"],"flip":0.2667},{"harnesses":["kimi-code","qwen-code"],"flip":0.2817},{"harnesses":["kimi-code","pi"],"flip":0.291},{"harnesses":["kimi-code","openclaw"],"flip":0.4718},{"harnesses":["claude-code","deepseek-harness"],"flip":0.2456},{"harnesses":["claude-code","hermes"],"flip":0.2575},{"harnesses":["claude-code","codex"],"flip":0.2561},{"harnesses":["claude-code","mini-swe-agent"],"flip":0.2573},{"harnesses":["claude-code","qwen-code"],"flip":0.249},{"harnesses":["claude-code","pi"],"flip":0.2341},{"harnesses":["claude-code","openclaw"],"flip":0.4521},{"harnesses":["deepseek-harness","hermes"],"flip":0.2977},{"harnesses":["deepseek-harness","codex"],"flip":0.2631},{"harnesses":["deepseek-harness","mini-swe-agent"],"flip":0.2709},{"harnesses":["deepseek-harness","qwen-code"],"flip":0.2701},{"harnesses":["deepseek-harness","pi"],"flip":0.2597},{"harnesses":["deepseek-harness","openclaw"],"flip":0.443},{"harnesses":["hermes","codex"],"flip":0.2887},{"harnesses":["hermes","mini-swe-agent"],"flip":0.309},{"harnesses":["hermes","qwen-code"],"flip":0.3028},{"harnesses":["hermes","pi"],"flip":0.3015},{"harnesses":["hermes","openclaw"],"flip":0.4734},{"harnesses":["codex","mini-swe-agent"],"flip":0.2766},{"harnesses":["codex","qwen-code"],"flip":0.268},{"harnesses":["codex","pi"],"flip":0.2723},{"harnesses":["codex","openclaw"],"flip":0.4406},{"harnesses":["mini-swe-agent","qwen-code"],"flip":0.2684},{"harnesses":["mini-swe-agent","pi"],"flip":0.2751},{"harnesses":["mini-swe-agent","openclaw"],"flip":0.452},{"harnesses":["qwen-code","pi"],"flip":0.2659},{"harnesses":["qwen-code","openclaw"],"flip":0.4278},{"harnesses":["pi","openclaw"],"flip":0.4419}]},"leader":"opencode"},"_all":{"harnesses":{"claude-code":{"cost_per_trial":0.108,"cost_per_pass":0.1569,"tokens_per_trial":6720000.0,"calls_per_trial":80.62,"model_s_per_trial":867.2,"agent_s_per_trial":1189.0,"rerun_flip":0.2295,"pass_all3":0.5089,"pass_mean3":0.6938,"pass_any3":0.8532,"n_trials":1232,"n_scored":1188,"n_pass":818,"n_infra":44,"cost_usd":131.4,"n_tasks3":396,"macro":0.6938,"ci":[0.6491,0.7386],"pass_all3_ci":[0.4612,0.5567],"benchmarks":{"deep-swe":{"cost_per_trial":0.1627,"cost_per_pass":0.2501,"tokens_per_trial":10960000.0,"calls_per_trial":123.1,"model_s_per_trial":1201.0,"agent_s_per_trial":1524.0,"rerun_flip":0.2943,"pass_all3":0.4144,"pass_mean3":0.6502,"pass_any3":0.8559,"n_trials":708,"n_scored":666,"n_pass":433,"n_infra":42,"cost_usd":111.4,"n_tasks3":222,"rate":0.6502,"ci":[0.5884,0.7119],"pass_all3_ci":[0.3507,0.4782],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.0382,"cost_per_pass":0.05185,"tokens_per_trial":1314000.0,"calls_per_trial":26.48,"model_s_per_trial":441.4,"agent_s_per_trial":762.2,"rerun_flip":0.1648,"pass_all3":0.6034,"pass_mean3":0.7375,"pass_any3":0.8506,"n_trials":524,"n_scored":522,"n_pass":385,"n_infra":2,"cost_usd":20.0,"n_tasks3":174,"rate":0.7375,"ci":[0.6732,0.8019],"pass_all3_ci":[0.5323,0.6746],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.6932,"low":0.6945},"efforts_requested":{"high":50524,"low":45252},"versions":["2.1.300"],"vs_leader":{"diff":-0.01584,"ci":[-0.05051,0.01748],"tied":true,"n_tasks":198}},"codex":{"cost_per_trial":0.09412,"cost_per_pass":0.1381,"tokens_per_trial":7476000.0,"calls_per_trial":87.36,"model_s_per_trial":584.3,"agent_s_per_trial":851.7,"rerun_flip":0.2525,"pass_all3":0.4854,"pass_mean3":0.6885,"pass_any3":0.8641,"n_trials":1231,"n_scored":1188,"n_pass":809,"n_infra":43,"cost_usd":112.4,"n_tasks3":396,"macro":0.6885,"ci":[0.6439,0.7332],"pass_all3_ci":[0.4384,0.5324],"benchmarks":{"deep-swe":{"cost_per_trial":0.1451,"cost_per_pass":0.2314,"tokens_per_trial":12360000.0,"calls_per_trial":132.9,"model_s_per_trial":819.7,"agent_s_per_trial":1095.0,"rerun_flip":0.3363,"pass_all3":0.3559,"pass_mean3":0.6261,"pass_any3":0.8604,"n_trials":693,"n_scored":666,"n_pass":417,"n_infra":27,"cost_usd":97.07,"n_tasks3":222,"rate":0.6261,"ci":[0.5635,0.6887],"pass_all3_ci":[0.294,0.4177],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.02911,"cost_per_pass":0.03878,"tokens_per_trial":1244000.0,"calls_per_trial":29.22,"model_s_per_trial":284.0,"agent_s_per_trial":541.5,"rerun_flip":0.1686,"pass_all3":0.6149,"pass_mean3":0.751,"pass_any3":0.8678,"n_trials":538,"n_scored":522,"n_pass":392,"n_infra":16,"cost_usd":15.29,"n_tasks3":174,"rate":0.751,"ci":[0.6876,0.8143],"pass_all3_ci":[0.5441,0.6857],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.6934,"low":0.6837},"efforts_requested":{"high":54636,"low":49145},"versions":["0.162.1"],"vs_leader":{"diff":-0.02115,"ci":[-0.05525,0.01308],"tied":true,"n_tasks":198}},"deepseek-harness":{"cost_per_trial":0.1253,"cost_per_pass":0.1785,"tokens_per_trial":10040000.0,"calls_per_trial":89.32,"model_s_per_trial":758.0,"agent_s_per_trial":1072.0,"rerun_flip":0.2148,"pass_all3":0.5327,"pass_mean3":0.7047,"pass_any3":0.8548,"n_trials":1280,"n_scored":1188,"n_pass":833,"n_infra":92,"cost_usd":152.2,"n_tasks3":396,"macro":0.7047,"ci":[0.6602,0.7492],"pass_all3_ci":[0.4849,0.5805],"benchmarks":{"deep-swe":{"cost_per_trial":0.1934,"cost_per_pass":0.2857,"tokens_per_trial":16750000.0,"calls_per_trial":136.2,"model_s_per_trial":1097.0,"agent_s_per_trial":1493.0,"rerun_flip":0.2763,"pass_all3":0.4505,"pass_mean3":0.6757,"pass_any3":0.8649,"n_trials":728,"n_scored":666,"n_pass":450,"n_infra":62,"cost_usd":131.7,"n_tasks3":222,"rate":0.6757,"ci":[0.615,0.7363],"pass_all3_ci":[0.3862,0.5147],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.03844,"cost_per_pass":0.05237,"tokens_per_trial":1477000.0,"calls_per_trial":29.45,"model_s_per_trial":325.8,"agent_s_per_trial":534.7,"rerun_flip":0.1533,"pass_all3":0.6149,"pass_mean3":0.7337,"pass_any3":0.8448,"n_trials":552,"n_scored":522,"n_pass":383,"n_infra":30,"cost_usd":20.57,"n_tasks3":174,"rate":0.7337,"ci":[0.6691,0.7984],"pass_all3_ci":[0.5441,0.6858],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.717,"low":0.6924},"efforts_requested":{"high":54525,"low":51589},"versions":["0.2.2-rc.1"],"vs_leader":{"diff":-0.004996,"ci":[-0.04937,0.03776],"tied":true,"n_tasks":198}},"hermes":{"cost_per_trial":0.1032,"cost_per_pass":0.1443,"tokens_per_trial":7618000.0,"calls_per_trial":63.14,"model_s_per_trial":825.5,"agent_s_per_trial":1334.0,"rerun_flip":0.2629,"pass_all3":0.4896,"pass_mean3":0.7097,"pass_any3":0.8839,"n_trials":1312,"n_scored":1188,"n_pass":846,"n_infra":124,"cost_usd":132.3,"n_tasks3":396,"macro":0.7097,"ci":[0.6652,0.7542],"pass_all3_ci":[0.4412,0.538],"benchmarks":{"deep-swe":{"cost_per_trial":0.1513,"cost_per_pass":0.2062,"tokens_per_trial":12340000.0,"calls_per_trial":90.28,"model_s_per_trial":1019.0,"agent_s_per_trial":1684.0,"rerun_flip":0.3303,"pass_all3":0.4505,"pass_mean3":0.7297,"pass_any3":0.9459,"n_trials":789,"n_scored":666,"n_pass":486,"n_infra":123,"cost_usd":110.4,"n_tasks3":222,"rate":0.7297,"ci":[0.6721,0.7874],"pass_all3_ci":[0.3863,0.5146],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04191,"cost_per_pass":0.06068,"tokens_per_trial":1590000.0,"calls_per_trial":28.51,"model_s_per_trial":578.6,"agent_s_per_trial":886.7,"rerun_flip":0.1954,"pass_all3":0.5287,"pass_mean3":0.6897,"pass_any3":0.8218,"n_trials":523,"n_scored":522,"n_pass":360,"n_infra":1,"cost_usd":21.88,"n_tasks3":174,"rate":0.6897,"ci":[0.6222,0.7571],"pass_all3_ci":[0.4562,0.6013],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.7298,"low":0.6896},"efforts_requested":{"high":41063,"low":32539,"none":979,"unset":426},"versions":["v0.22.1"]},"kimi-code":{"cost_per_trial":0.1133,"cost_per_pass":0.1677,"tokens_per_trial":8493000.0,"calls_per_trial":74.67,"model_s_per_trial":919.7,"agent_s_per_trial":1154.0,"rerun_flip":0.264,"pass_all3":0.4675,"pass_mean3":0.6827,"pass_any3":0.8613,"n_trials":1237,"n_scored":1187,"n_pass":805,"n_infra":50,"cost_usd":138.7,"n_tasks3":395,"macro":0.6824,"ci":[0.6372,0.7276],"pass_all3_ci":[0.4195,0.5155],"benchmarks":{"deep-swe":{"cost_per_trial":0.1686,"cost_per_pass":0.2626,"tokens_per_trial":13750000.0,"calls_per_trial":110.5,"model_s_per_trial":1227.0,"agent_s_per_trial":1473.0,"rerun_flip":0.3288,"pass_all3":0.389,"pass_mean3":0.6469,"pass_any3":0.8777,"n_trials":714,"n_scored":665,"n_pass":430,"n_infra":49,"cost_usd":116.3,"n_tasks3":221,"rate":0.6464,"ci":[0.5846,0.7082],"pass_all3_ci":[0.3259,0.4521],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04286,"cost_per_pass":0.05973,"tokens_per_trial":1791000.0,"calls_per_trial":29.08,"model_s_per_trial":528.1,"agent_s_per_trial":746.8,"rerun_flip":0.1992,"pass_all3":0.546,"pass_mean3":0.7184,"pass_any3":0.8448,"n_trials":523,"n_scored":522,"n_pass":375,"n_infra":1,"cost_usd":22.38,"n_tasks3":174,"rate":0.7184,"ci":[0.6527,0.7841],"pass_all3_ci":[0.4736,0.6183],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.6634,"low":0.7014},"efforts_requested":{"high":46445,"low":42181},"versions":["2.2.0"],"vs_leader":{"diff":-0.0273,"ci":[-0.06265,0.00882],"tied":true,"n_tasks":198}},"mini-swe-agent":{"cost_per_trial":0.1275,"cost_per_pass":0.1875,"tokens_per_trial":9790000.0,"calls_per_trial":91.22,"model_s_per_trial":893.2,"agent_s_per_trial":1227.0,"rerun_flip":0.2431,"pass_all3":0.4894,"pass_mean3":0.6855,"pass_any3":0.8541,"n_trials":1480,"n_scored":1188,"n_pass":806,"n_infra":292,"cost_usd":180.6,"n_tasks3":396,"macro":0.6855,"ci":[0.6406,0.7303],"pass_all3_ci":[0.4416,0.5373],"benchmarks":{"deep-swe":{"cost_per_trial":0.1913,"cost_per_pass":0.3026,"tokens_per_trial":15920000.0,"calls_per_trial":133.5,"model_s_per_trial":1186.0,"agent_s_per_trial":1591.0,"rerun_flip":0.2793,"pass_all3":0.4099,"pass_mean3":0.6276,"pass_any3":0.8288,"n_trials":957,"n_scored":666,"n_pass":418,"n_infra":291,"cost_usd":156.3,"n_tasks3":222,"rate":0.6276,"ci":[0.5651,0.6901],"pass_all3_ci":[0.3469,0.473],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04608,"cost_per_pass":0.06221,"tokens_per_trial":1972000.0,"calls_per_trial":37.27,"model_s_per_trial":520.0,"agent_s_per_trial":762.5,"rerun_flip":0.2069,"pass_all3":0.569,"pass_mean3":0.7433,"pass_any3":0.8793,"n_trials":523,"n_scored":522,"n_pass":388,"n_infra":1,"cost_usd":24.25,"n_tasks3":174,"rate":0.7433,"ci":[0.6794,0.8072],"pass_all3_ci":[0.4969,0.641],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.6909,"low":0.68},"efforts_requested":{"high":59486,"low":48862,"unset":23,"none":3},"versions":["2.4.6"],"vs_leader":{"diff":-0.02423,"ci":[-0.06416,0.01499],"tied":true,"n_tasks":198}},"openclaw":{"cost_per_trial":0.1053,"cost_per_pass":0.3139,"tokens_per_trial":4796000.0,"calls_per_trial":66.97,"model_s_per_trial":1258.0,"agent_s_per_trial":1759.0,"rerun_flip":0.221,"pass_all3":0.2136,"pass_mean3":0.3556,"pass_any3":0.545,"n_trials":1209,"n_scored":1188,"n_pass":402,"n_infra":21,"cost_usd":125.3,"n_tasks3":396,"macro":0.3556,"ci":[0.3105,0.4008],"pass_all3_ci":[0.1741,0.253],"benchmarks":{"deep-swe":{"cost_per_trial":0.1546,"cost_per_pass":0.7422,"tokens_per_trial":7360000.0,"calls_per_trial":89.65,"model_s_per_trial":1808.0,"agent_s_per_trial":2473.0,"rerun_flip":0.2312,"pass_all3":0.07658,"pass_mean3":0.2132,"pass_any3":0.4234,"n_trials":680,"n_scored":666,"n_pass":142,"n_infra":14,"cost_usd":103.2,"n_tasks3":222,"rate":0.2132,"ci":[0.1599,0.2665],"pass_all3_ci":[0.04076,0.1124],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04232,"cost_per_pass":0.08528,"tokens_per_trial":1525000.0,"calls_per_trial":38.03,"model_s_per_trial":555.1,"agent_s_per_trial":848.6,"rerun_flip":0.2107,"pass_all3":0.3506,"pass_mean3":0.4981,"pass_any3":0.6667,"n_trials":529,"n_scored":522,"n_pass":260,"n_infra":7,"cost_usd":22.09,"n_tasks3":174,"rate":0.4981,"ci":[0.4255,0.5707],"pass_all3_ci":[0.2811,0.4201],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.3272,"low":0.3841},"efforts_requested":{"low":40868,"high":38276,"unset":413,"None":1},"versions":["2026.10.4"],"vs_leader":{"diff":-0.354,"ci":[-0.4026,-0.3094],"tied":false,"n_tasks":198}},"opencode":{"cost_per_trial":0.1071,"cost_per_pass":0.1533,"tokens_per_trial":8826000.0,"calls_per_trial":79.74,"model_s_per_trial":538.1,"agent_s_per_trial":798.9,"rerun_flip":0.2288,"pass_all3":0.5227,"pass_mean3":0.706,"pass_any3":0.866,"n_trials":1199,"n_scored":1188,"n_pass":832,"n_infra":11,"cost_usd":127.3,"n_tasks3":396,"macro":0.706,"ci":[0.6618,0.7502],"pass_all3_ci":[0.4752,0.5702],"benchmarks":{"deep-swe":{"cost_per_trial":0.1681,"cost_per_pass":0.2554,"tokens_per_trial":14850000.0,"calls_per_trial":122.1,"model_s_per_trial":792.6,"agent_s_per_trial":1075.0,"rerun_flip":0.2853,"pass_all3":0.4189,"pass_mean3":0.6592,"pass_any3":0.8468,"n_trials":676,"n_scored":666,"n_pass":439,"n_infra":10,"cost_usd":111.9,"n_tasks3":222,"rate":0.6592,"ci":[0.5978,0.7206],"pass_all3_ci":[0.3552,0.4826],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.02935,"cost_per_pass":0.03908,"tokens_per_trial":1134000.0,"calls_per_trial":25.66,"model_s_per_trial":213.4,"agent_s_per_trial":446.8,"rerun_flip":0.1724,"pass_all3":0.6264,"pass_mean3":0.7529,"pass_any3":0.8851,"n_trials":523,"n_scored":522,"n_pass":393,"n_infra":1,"cost_usd":15.32,"n_tasks3":174,"rate":0.7529,"ci":[0.6897,0.816],"pass_all3_ci":[0.556,0.6969],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.6941,"low":0.718},"efforts_requested":{"high":49241,"low":45491},"versions":["2.2.6"],"vs_leader":{"diff":-0.003676,"ci":[-0.0422,0.03477],"tied":true,"n_tasks":198}},"pi":{"cost_per_trial":0.09672,"cost_per_pass":0.1472,"tokens_per_trial":5638000.0,"calls_per_trial":86.39,"model_s_per_trial":813.2,"agent_s_per_trial":1326.0,"rerun_flip":0.2494,"pass_all3":0.4733,"pass_mean3":0.6661,"pass_any3":0.8473,"n_trials":1363,"n_scored":1188,"n_pass":780,"n_infra":175,"cost_usd":129.6,"n_tasks3":396,"macro":0.6661,"ci":[0.6209,0.7114],"pass_all3_ci":[0.4259,0.5207],"benchmarks":{"deep-swe":{"cost_per_trial":0.1507,"cost_per_pass":0.2563,"tokens_per_trial":9413000.0,"calls_per_trial":134.5,"model_s_per_trial":1163.0,"agent_s_per_trial":1865.0,"rerun_flip":0.3033,"pass_all3":0.3604,"pass_mean3":0.5871,"pass_any3":0.8153,"n_trials":841,"n_scored":666,"n_pass":391,"n_infra":175,"cost_usd":115.0,"n_tasks3":222,"rate":0.5871,"ci":[0.5234,0.6508],"pass_all3_ci":[0.2982,0.4225],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.02791,"cost_per_pass":0.03744,"tokens_per_trial":822500.0,"calls_per_trial":24.99,"model_s_per_trial":366.7,"agent_s_per_trial":637.0,"rerun_flip":0.1954,"pass_all3":0.5862,"pass_mean3":0.7452,"pass_any3":0.8793,"n_trials":522,"n_scored":522,"n_pass":389,"n_infra":0,"cost_usd":14.57,"n_tasks3":174,"rate":0.7452,"ci":[0.6814,0.809],"pass_all3_ci":[0.5146,0.6578],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.6709,"low":0.6614},"efforts_requested":{"high":54571,"low":48056},"versions":["1.2.0"],"vs_leader":{"diff":-0.04354,"ci":[-0.08379,-0.004447],"tied":false,"n_tasks":198}},"qwen-code":{"cost_per_trial":0.1131,"cost_per_pass":0.1713,"tokens_per_trial":5836000.0,"calls_per_trial":69.92,"model_s_per_trial":772.5,"agent_s_per_trial":1017.0,"rerun_flip":0.2372,"pass_all3":0.4743,"pass_mean3":0.668,"pass_any3":0.8301,"n_trials":1206,"n_scored":1188,"n_pass":785,"n_infra":18,"cost_usd":136.4,"n_tasks3":396,"macro":0.668,"ci":[0.6226,0.7135],"pass_all3_ci":[0.4266,0.522],"benchmarks":{"deep-swe":{"cost_per_trial":0.1675,"cost_per_pass":0.2753,"tokens_per_trial":9378000.0,"calls_per_trial":104.0,"model_s_per_trial":1058.0,"agent_s_per_trial":1307.0,"rerun_flip":0.2943,"pass_all3":0.3739,"pass_mean3":0.6081,"pass_any3":0.8153,"n_trials":683,"n_scored":666,"n_pass":405,"n_infra":17,"cost_usd":113.5,"n_tasks3":222,"rate":0.6081,"ci":[0.5449,0.6713],"pass_all3_ci":[0.3112,0.4365],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04386,"cost_per_pass":0.06029,"tokens_per_trial":1316000.0,"calls_per_trial":26.46,"model_s_per_trial":408.3,"agent_s_per_trial":647.3,"rerun_flip":0.1801,"pass_all3":0.5747,"pass_mean3":0.728,"pass_any3":0.8448,"n_trials":523,"n_scored":522,"n_pass":380,"n_infra":1,"cost_usd":22.89,"n_tasks3":174,"rate":0.728,"ci":[0.6629,0.793],"pass_all3_ci":[0.5028,0.6466],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_effort":{"high":0.6669,"low":0.6692},"efforts_requested":{"high":43232,"low":38084,"unset":1746},"versions":["0.25.2"],"vs_leader":{"diff":-0.04165,"ci":[-0.08,-0.005227],"tied":false,"n_tasks":198}}},"ranking":["hermes","opencode","deepseek-harness","claude-code","codex","mini-swe-agent","kimi-code","qwen-code","pi","openclaw"],"efforts":["high","low"],"noise":{"rerun_flip":0.2403,"harness_flip":0.3023,"effort_flip":0.2429,"by_benchmark":{"deep-swe":{"rerun_flip":0.2959,"harness_flip":0.3687,"effort_flip":0.2945},"terminal-bench-2.1":{"rerun_flip":0.1847,"harness_flip":0.2359,"effort_flip":0.1912}},"harness_pairs":[{"harnesses":["hermes","opencode"],"flip":0.2904},{"harnesses":["hermes","deepseek-harness"],"flip":0.2983},{"harnesses":["hermes","claude-code"],"flip":0.2676},{"harnesses":["hermes","codex"],"flip":0.2821},{"harnesses":["hermes","mini-swe-agent"],"flip":0.3019},{"harnesses":["hermes","kimi-code"],"flip":0.2924},{"harnesses":["hermes","qwen-code"],"flip":0.2893},{"harnesses":["hermes","pi"],"flip":0.2973},{"harnesses":["hermes","openclaw"],"flip":0.4765},{"harnesses":["opencode","deepseek-harness"],"flip":0.2496},{"harnesses":["opencode","claude-code"],"flip":0.2402},{"harnesses":["opencode","codex"],"flip":0.2411},{"harnesses":["opencode","mini-swe-agent"],"flip":0.2552},{"harnesses":["opencode","kimi-code"],"flip":0.2604},{"harnesses":["opencode","qwen-code"],"flip":0.2577},{"harnesses":["opencode","pi"],"flip":0.2435},{"harnesses":["opencode","openclaw"],"flip":0.4602},{"harnesses":["deepseek-harness","claude-code"],"flip":0.2636},{"harnesses":["deepseek-harness","codex"],"flip":0.2651},{"harnesses":["deepseek-harness","mini-swe-agent"],"flip":0.2582},{"harnesses":["deepseek-harness","kimi-code"],"flip":0.2752},{"harnesses":["deepseek-harness","qwen-code"],"flip":0.2637},{"harnesses":["deepseek-harness","pi"],"flip":0.263},{"harnesses":["deepseek-harness","openclaw"],"flip":0.4481},{"harnesses":["claude-code","codex"],"flip":0.2605},{"harnesses":["claude-code","mini-swe-agent"],"flip":0.2555},{"harnesses":["claude-code","kimi-code"],"flip":0.2572},{"harnesses":["claude-code","qwen-code"],"flip":0.2523},{"harnesses":["claude-code","pi"],"flip":0.2502},{"harnesses":["claude-code","openclaw"],"flip":0.4474},{"harnesses":["codex","mini-swe-agent"],"flip":0.2647},{"harnesses":["codex","kimi-code"],"flip":0.2697},{"harnesses":["codex","qwen-code"],"flip":0.2535},{"harnesses":["codex","pi"],"flip":0.2613},{"harnesses":["codex","openclaw"],"flip":0.444},{"harnesses":["mini-swe-agent","kimi-code"],"flip":0.2598},{"harnesses":["mini-swe-agent","qwen-code"],"flip":0.2638},{"harnesses":["mini-swe-agent","pi"],"flip":0.2595},{"harnesses":["mini-swe-agent","openclaw"],"flip":0.4513},{"harnesses":["kimi-code","qwen-code"],"flip":0.2639},{"harnesses":["kimi-code","pi"],"flip":0.2785},{"harnesses":["kimi-code","openclaw"],"flip":0.4517},{"harnesses":["qwen-code","pi"],"flip":0.2531},{"harnesses":["qwen-code","openclaw"],"flip":0.425},{"harnesses":["pi","openclaw"],"flip":0.4402}]},"leader":"hermes"}}}}},"_all":{"stages":{"full":{"efforts":{"_all":{"harnesses":{"claude-code":{"cost_per_trial":0.1451,"cost_per_pass":0.5173,"tokens_per_trial":6462000.0,"calls_per_trial":78.7,"model_s_per_trial":1869.0,"agent_s_per_trial":2314.0,"rerun_flip":0.2184,"pass_all3":0.3209,"pass_mean3":0.4915,"pass_any3":0.6486,"n_trials":3093,"n_scored":2970,"n_pass":1631,"n_infra":123,"cost_usd":421.1,"n_tasks3":990,"macro":0.4915,"ci":[0.4636,0.5194],"pass_all3_ci":[0.2952,0.3466],"benchmarks":{"deep-swe":{"cost_per_trial":0.2303,"cost_per_pass":2.086,"tokens_per_trial":10630000.0,"calls_per_trial":121.4,"model_s_per_trial":2852.0,"agent_s_per_trial":3371.0,"rerun_flip":0.2172,"pass_all3":0.2568,"pass_mean3":0.4299,"pass_any3":0.5826,"n_trials":1769,"n_scored":1665,"n_pass":841,"n_infra":104,"cost_usd":370.8,"n_tasks3":555,"rate":0.4299,"ci":[0.3969,0.4629],"pass_all3_ci":[0.2267,0.2868],"n_tasks":555,"n_tasks_expected":555},"terminal-bench-2.1":{"cost_per_trial":0.0363,"cost_per_pass":0.07251,"tokens_per_trial":1138000.0,"calls_per_trial":24.28,"model_s_per_trial":614.5,"agent_s_per_trial":965.5,"rerun_flip":0.2197,"pass_all3":0.3851,"pass_mean3":0.553,"pass_any3":0.7146,"n_trials":1324,"n_scored":1305,"n_pass":790,"n_infra":19,"cost_usd":50.32,"n_tasks3":435,"rate":0.553,"ci":[0.5084,0.5976],"pass_all3_ci":[0.3445,0.4257],"n_tasks":435,"n_tasks_expected":435}},"complete":true,"per_model":{"RecursiveSI/flint":0.1726,"deepseek-ai/DeepSeek-V4.1-Flash":0.6938,"zai-org/GLM-5.3-Flash":0.6079},"efforts_requested":{"high":82954,"low":72970,"medium":62284},"versions":["2.1.300"],"vs_leader":{"diff":-0.02816,"ci":[-0.05109,-0.006574],"tied":false,"n_tasks":198}},"codex":{"cost_per_trial":0.1685,"cost_per_pass":0.7061,"tokens_per_trial":8221000.0,"calls_per_trial":92.06,"model_s_per_trial":1516.0,"agent_s_per_trial":1877.0,"rerun_flip":0.2313,"pass_all3":0.2681,"pass_mean3":0.4448,"pass_any3":0.6154,"n_trials":3424,"n_scored":2962,"n_pass":1439,"n_infra":462,"cost_usd":469.4,"n_tasks3":983,"macro":0.4447,"ci":[0.4171,0.4722],"pass_all3_ci":[0.2435,0.2928],"benchmarks":{"deep-swe":{"cost_per_trial":0.2662,"cost_per_pass":3.924,"tokens_per_trial":13270000.0,"calls_per_trial":135.5,"model_s_per_trial":2323.0,"agent_s_per_trial":2750.0,"rerun_flip":0.2404,"pass_all3":0.1486,"pass_mean3":0.3251,"pass_any3":0.5096,"n_trials":1791,"n_scored":1662,"n_pass":636,"n_infra":129,"cost_usd":409.3,"n_tasks3":553,"rate":0.3248,"ci":[0.2932,0.3564],"pass_all3_ci":[0.1242,0.1731],"n_tasks":555,"n_tasks_expected":555},"terminal-bench-2.1":{"cost_per_trial":0.043,"cost_per_pass":0.0833,"tokens_per_trial":1743000.0,"calls_per_trial":36.32,"model_s_per_trial":477.3,"agent_s_per_trial":755.6,"rerun_flip":0.2222,"pass_all3":0.3876,"pass_mean3":0.5646,"pass_any3":0.7213,"n_trials":1633,"n_scored":1300,"n_pass":803,"n_infra":333,"cost_usd":60.03,"n_tasks3":430,"rate":0.5645,"ci":[0.5199,0.6091],"pass_all3_ci":[0.346,0.4293],"n_tasks":435,"n_tasks_expected":435}},"complete":true,"per_model":{"RecursiveSI/flint":0.1718,"deepseek-ai/DeepSeek-V4.1-Flash":0.6885,"zai-org/GLM-5.3-Flash":0.4736},"efforts_requested":{"high":97794,"low":90761,"medium":68826},"versions":["0.162.1"],"vs_leader":{"diff":-0.07496,"ci":[-0.09812,-0.04955],"tied":false,"n_tasks":198}},"deepseek-harness":{"cost_per_trial":0.1788,"cost_per_pass":0.5252,"tokens_per_trial":10640000.0,"calls_per_trial":113.0,"model_s_per_trial":1474.0,"agent_s_per_trial":1767.0,"rerun_flip":0.2131,"pass_all3":0.3457,"pass_mean3":0.5039,"pass_any3":0.6657,"n_trials":3384,"n_scored":2969,"n_pass":1659,"n_infra":415,"cost_usd":544.4,"n_tasks3":989,"macro":0.5042,"ci":[0.4757,0.5328],"pass_all3_ci":[0.3189,0.3725],"benchmarks":{"deep-swe":{"cost_per_trial":0.2832,"cost_per_pass":1.538,"tokens_per_trial":16950000.0,"calls_per_trial":159.1,"model_s_per_trial":2183.0,"agent_s_per_trial":2526.0,"rerun_flip":0.2232,"pass_all3":0.2814,"pass_mean3":0.4504,"pass_any3":0.6168,"n_trials":1951,"n_scored":1664,"n_pass":872,"n_infra":287,"cost_usd":484.2,"n_tasks3":554,"rate":0.451,"ci":[0.4168,0.4851],"pass_all3_ci":[0.251,0.3119],"n_tasks":555,"n_tasks_expected":555},"terminal-bench-2.1":{"cost_per_trial":0.04557,"cost_per_pass":0.09736,"tokens_per_trial":2581000.0,"calls_per_trial":54.19,"model_s_per_trial":569.2,"agent_s_per_trial":799.0,"rerun_flip":0.2031,"pass_all3":0.41,"pass_mean3":0.5575,"pass_any3":0.7146,"n_trials":1433,"n_scored":1305,"n_pass":787,"n_infra":128,"cost_usd":60.26,"n_tasks3":435,"rate":0.5575,"ci":[0.512,0.6029],"pass_all3_ci":[0.3669,0.453],"n_tasks":435,"n_tasks_expected":435}},"complete":true,"per_model":{"RecursiveSI/flint":0.2068,"deepseek-ai/DeepSeek-V4.1-Flash":0.7047,"zai-org/GLM-5.3-Flash":0.6011},"efforts_requested":{"medium":110267,"high":94843,"low":86603},"versions":["0.2.2-rc.1"],"vs_leader":{"diff":-0.01541,"ci":[-0.04186,0.009073],"tied":true,"n_tasks":198}},"hermes":{"cost_per_trial":0.1746,"cost_per_pass":0.6535,"tokens_per_trial":8485000.0,"calls_per_trial":81.91,"model_s_per_trial":1932.0,"agent_s_per_trial":2503.0,"rerun_flip":0.2181,"pass_all3":0.3312,"pass_mean3":0.5008,"pass_any3":0.6584,"n_trials":3175,"n_scored":2968,"n_pass":1660,"n_infra":207,"cost_usd":473.6,"n_tasks3":989,"macro":0.5006,"ci":[0.4724,0.5289],"pass_all3_ci":[0.3043,0.3581],"benchmarks":{"deep-swe":{"cost_per_trial":0.2815,"cost_per_pass":1.888,"tokens_per_trial":14080000.0,"calls_per_trial":124.6,"model_s_per_trial":2927.0,"agent_s_per_trial":3688.0,"rerun_flip":0.2486,"pass_all3":0.2658,"pass_mean3":0.4562,"pass_any3":0.6387,"n_trials":1861,"n_scored":1663,"n_pass":880,"n_infra":198,"cost_usd":422.1,"n_tasks3":554,"rate":0.456,"ci":[0.4219,0.49],"pass_all3_ci":[0.2348,0.2969],"n_tasks":555,"n_tasks_expected":555},"terminal-bench-2.1":{"cost_per_trial":0.03871,"cost_per_pass":0.08158,"tokens_per_trial":1367000.0,"calls_per_trial":27.69,"model_s_per_trial":670.2,"agent_s_per_trial":998.8,"rerun_flip":0.1877,"pass_all3":0.3966,"pass_mean3":0.5453,"pass_any3":0.6782,"n_trials":1314,"n_scored":1305,"n_pass":780,"n_infra":9,"cost_usd":51.44,"n_tasks3":435,"rate":0.5453,"ci":[0.5007,0.59],"pass_all3_ci":[0.3534,0.4397],"n_tasks":435,"n_tasks_expected":435}},"complete":true,"per_model":{"RecursiveSI/flint":0.1883,"deepseek-ai/DeepSeek-V4.1-Flash":0.7097,"zai-org/GLM-5.3-Flash":0.6039},"efforts_requested":{"medium":78600,"high":71570,"low":58726,"none":2143,"unset":1221},"versions":["v0.22.1"],"vs_leader":{"diff":-0.01898,"ci":[-0.04023,0.001874],"tied":true,"n_tasks":198}},"kimi-code":{"cost_per_trial":0.1881,"cost_per_pass":0.6262,"tokens_per_trial":9392000.0,"calls_per_trial":79.48,"model_s_per_trial":1783.0,"agent_s_per_trial":2090.0,"rerun_flip":0.2282,"pass_all3":0.3288,"pass_mean3":0.505,"pass_any3":0.6707,"n_trials":3146,"n_scored":2968,"n_pass":1650,"n_infra":178,"cost_usd":512.3,"n_tasks3":988,"macro":0.5048,"ci":[0.4759,0.5336],"pass_all3_ci":[0.3021,0.3555],"benchmarks":{"deep-swe":{"cost_per_trial":0.3033,"cost_per_pass":1.727,"tokens_per_trial":15460000.0,"calls_per_trial":120.1,"model_s_per_trial":2655.0,"agent_s_per_trial":3001.0,"rerun_flip":0.2648,"pass_all3":0.2438,"pass_mean3":0.4417,"pass_any3":0.6402,"n_trials":1825,"n_scored":1663,"n_pass":844,"n_infra":162,"cost_usd":457.4,"n_tasks3":553,"rate":0.4412,"ci":[0.4057,0.4767],"pass_all3_ci":[0.2138,0.2738],"n_tasks":555,"n_tasks_expected":555},"terminal-bench-2.1":{"cost_per_trial":0.04147,"cost_per_pass":0.0823,"tokens_per_trial":1662000.0,"calls_per_trial":27.72,"model_s_per_trial":672.8,"agent_s_per_trial":930.7,"rerun_flip":0.1916,"pass_all3":0.4138,"pass_mean3":0.5683,"pass_any3":0.7011,"n_trials":1321,"n_scored":1305,"n_pass":806,"n_infra":16,"cost_usd":54.93,"n_tasks3":435,"rate":0.5683,"ci":[0.5232,0.6135],"pass_all3_ci":[0.3707,0.4569],"n_tasks":435,"n_tasks_expected":435}},"complete":true,"per_model":{"RecursiveSI/flint":0.2165,"deepseek-ai/DeepSeek-V4.1-Flash":0.6824,"zai-org/GLM-5.3-Flash":0.6154},"efforts_requested":{"high":80558,"low":71105,"medium":65648},"versions":["2.2.0"],"vs_leader":{"diff":-0.01486,"ci":[-0.03577,0.004993],"tied":true,"n_tasks":198}},"mini-swe-agent":{"cost_per_trial":0.1796,"cost_per_pass":0.5356,"tokens_per_trial":8420000.0,"calls_per_trial":87.15,"model_s_per_trial":1691.0,"agent_s_per_trial":2104.0,"rerun_flip":0.2358,"pass_all3":0.3248,"pass_mean3":0.5071,"pass_any3":0.6789,"n_trials":3388,"n_scored":2969,"n_pass":1659,"n_infra":419,"cost_usd":547.9,"n_tasks3":989,"macro":0.5069,"ci":[0.4782,0.5357],"pass_all3_ci":[0.2983,0.3514],"benchmarks":{"deep-swe":{"cost_per_trial":0.2867,"cost_per_pass":1.457,"tokens_per_trial":13760000.0,"calls_per_trial":127.4,"model_s_per_trial":2478.0,"agent_s_per_trial":2989.0,"rerun_flip":0.2533,"pass_all3":0.2493,"pass_mean3":0.4408,"pass_any3":0.6298,"n_trials":2059,"n_scored":1664,"n_pass":845,"n_infra":395,"cost_usd":487.6,"n_tasks3":554,"rate":0.4404,"ci":[0.4051,0.4758],"pass_all3_ci":[0.2191,0.2795],"n_tasks":555,"n_tasks_expected":555},"terminal-bench-2.1":{"cost_per_trial":0.0432,"cost_per_pass":0.07932,"tokens_per_trial":1621000.0,"calls_per_trial":35.86,"model_s_per_trial":688.5,"agent_s_per_trial":977.4,"rerun_flip":0.2184,"pass_all3":0.4004,"pass_mean3":0.5734,"pass_any3":0.728,"n_trials":1329,"n_scored":1305,"n_pass":814,"n_infra":24,"cost_usd":60.33,"n_tasks3":435,"rate":0.5734,"ci":[0.5285,0.6184],"pass_all3_ci":[0.3577,0.443],"n_tasks":435,"n_tasks_expected":435}},"complete":true,"per_model":{"RecursiveSI/flint":0.2135,"deepseek-ai/DeepSeek-V4.1-Flash":0.6855,"zai-org/GLM-5.3-Flash":0.6219},"efforts_requested":{"high":108328,"low":89315,"medium":56365,"unset":23,"none":3},"versions":["2.4.6"],"vs_leader":{"diff":-0.01269,"ci":[-0.03413,0.006529],"tied":true,"n_tasks":198}},"openclaw":{"cost_per_trial":0.1958,"cost_per_pass":1.432,"tokens_per_trial":8708000.0,"calls_per_trial":118.6,"model_s_per_trial":1525.0,"agent_s_per_trial":2032.0,"rerun_flip":0.1762,"pass_all3":0.15,"pass_mean3":0.2706,"pass_any3":0.4117,"n_trials":3094,"n_scored":2966,"n_pass":894,"n_infra":128,"cost_usd":617.9,"n_tasks3":986,"macro":0.2704,"ci":[0.2455,0.2954],"pass_all3_ci":[0.1287,0.1714],"benchmarks":{"deep-swe":{"cost_per_trial":0.2929,"cost_per_pass":1.191,"tokens_per_trial":12360000.0,"calls_per_trial":154.1,"model_s_per_trial":2172.0,"agent_s_per_trial":2863.0,"rerun_flip":0.1877,"pass_all3":0.05736,"pass_mean3":0.1797,"pass_any3":0.3383,"n_trials":1765,"n_scored":1663,"n_pass":358,"n_infra":102,"cost_usd":532.0,"n_tasks3":553,"rate":0.1794,"ci":[0.1524,0.2065],"pass_all3_ci":[0.0394,0.07532],"n_tasks":555,"n_tasks_expected":555},"terminal-bench-2.1":{"cost_per_trial":0.07171,"cost_per_pass":0.3917,"tokens_per_trial":4052000.0,"calls_per_trial":73.3,"model_s_per_trial":697.3,"agent_s_per_trial":971.9,"rerun_flip":0.1648,"pass_all3":0.2427,"pass_mean3":0.3614,"pass_any3":0.4852,"n_trials":1329,"n_scored":1303,"n_pass":536,"n_infra":26,"cost_usd":85.85,"n_tasks3":433,"rate":0.3614,"ci":[0.3206,0.4022],"pass_all3_ci":[0.2055,0.2799],"n_tasks":435,"n_tasks_expected":435}},"complete":true,"per_model":{"RecursiveSI/flint":0.05556,"deepseek-ai/DeepSeek-V4.1-Flash":0.3556,"zai-org/GLM-5.3-Flash":0.4001},"efforts_requested":{"low":120193,"high":113103,"medium":94120,"unset":452,"None":1},"versions":["2026.10.4"],"vs_leader":{"diff":-0.2492,"ci":[-0.2824,-0.2179],"tied":false,"n_tasks":198}},"opencode":{"cost_per_trial":0.1494,"cost_per_pass":0.4518,"tokens_per_trial":8671000.0,"calls_per_trial":88.78,"model_s_per_trial":1316.0,"agent_s_per_trial":1779.0,"rerun_flip":0.2162,"pass_all3":0.3555,"pass_mean3":0.5197,"pass_any3":0.6797,"n_trials":3034,"n_scored":2968,"n_pass":1691,"n_infra":66,"cost_usd":412.2,"n_tasks3":989,"macro":0.5196,"ci":[0.4911,0.5482],"pass_all3_ci":[0.3279,0.3831],"benchmarks":{"deep-swe":{"cost_per_trial":0.2448,"cost_per_pass":1.776,"tokens_per_trial":14570000.0,"calls_per_trial":135.6,"model_s_per_trial":2028.0,"agent_s_per_trial":2607.0,"rerun_flip":0.2446,"pass_all3":0.2493,"pass_mean3":0.4367,"pass_any3":0.6161,"n_trials":1710,"n_scored":1663,"n_pass":847,"n_infra":47,"cost_usd":374.2,"n_tasks3":554,"rate":0.4364,"ci":[0.4025,0.4704],"pass_all3_ci":[0.2195,0.2791],"n_tasks":555,"n_tasks_expected":555},"terminal-bench-2.1":{"cost_per_trial":0.02813,"cost_per_pass":0.0491,"tokens_per_trial":1164000.0,"calls_per_trial":29.24,"model_s_per_trial":411.6,"agent_s_per_trial":728.0,"rerun_flip":0.1877,"pass_all3":0.4617,"pass_mean3":0.6028,"pass_any3":0.7433,"n_trials":1324,"n_scored":1305,"n_pass":844,"n_infra":19,"cost_usd":37.93,"n_tasks3":435,"rate":0.6028,"ci":[0.5573,0.6483],"pass_all3_ci":[0.4161,0.5073],"n_tasks":435,"n_tasks_expected":435}},"complete":true,"per_model":{"RecursiveSI/flint":0.2291,"deepseek-ai/DeepSeek-V4.1-Flash":0.706,"zai-org/GLM-5.3-Flash":0.6237},"efforts_requested":{"high":86989,"medium":76599,"low":75717},"versions":["2.2.6"]},"pi":{"cost_per_trial":0.09714,"cost_per_pass":0.2943,"tokens_per_trial":4391000.0,"calls_per_trial":76.24,"model_s_per_trial":1396.0,"agent_s_per_trial":1882.0,"rerun_flip":0.2207,"pass_all3":0.3208,"pass_mean3":0.4873,"pass_any3":0.6518,"n_trials":3246,"n_scored":2970,"n_pass":1604,"n_infra":276,"cost_usd":311.0,"n_tasks3":990,"macro":0.4873,"ci":[0.4592,0.5154],"pass_all3_ci":[0.2945,0.3471],"benchmarks":{"deep-swe":{"cost_per_trial":0.1538,"cost_per_pass":0.9232,"tokens_per_trial":7223000.0,"calls_per_trial":116.5,"model_s_per_trial":2067.0,"agent_s_per_trial":2710.0,"rerun_flip":0.2382,"pass_all3":0.2297,"pass_mean3":0.4089,"pass_any3":0.5871,"n_trials":1917,"n_scored":1665,"n_pass":794,"n_infra":252,"cost_usd":277.2,"n_tasks3":555,"rate":0.4089,"ci":[0.3748,0.443],"pass_all3_ci":[0.2001,0.2594],"n_tasks":555,"n_tasks_expected":555},"terminal-bench-2.1":{"cost_per_trial":0.02481,"cost_per_pass":0.04909,"tokens_per_trial":778500.0,"calls_per_trial":24.82,"model_s_per_trial":539.7,"agent_s_per_trial":825.3,"rerun_flip":0.2031,"pass_all3":0.4119,"pass_mean3":0.5658,"pass_any3":0.7165,"n_trials":1329,"n_scored":1305,"n_pass":810,"n_infra":24,"cost_usd":33.85,"n_tasks3":435,"rate":0.5658,"ci":[0.5215,0.61],"pass_all3_ci":[0.3694,0.4544],"n_tasks":435,"n_tasks_expected":435}},"complete":true,"per_model":{"RecursiveSI/flint":0.1801,"deepseek-ai/DeepSeek-V4.1-Flash":0.6661,"zai-org/GLM-5.3-Flash":0.6157},"efforts_requested":{"high":87614,"low":75725,"medium":54191},"versions":["1.2.0"],"vs_leader":{"diff":-0.03228,"ci":[-0.05298,-0.01245],"tied":false,"n_tasks":198}},"qwen-code":{"cost_per_trial":0.117,"cost_per_pass":0.3419,"tokens_per_trial":3846000.0,"calls_per_trial":53.85,"model_s_per_trial":1203.0,"agent_s_per_trial":1425.0,"rerun_flip":0.2126,"pass_all3":0.2863,"pass_mean3":0.4472,"pass_any3":0.6052,"n_trials":3041,"n_scored":2970,"n_pass":1495,"n_infra":71,"cost_usd":392.0,"n_tasks3":990,"macro":0.4472,"ci":[0.4201,0.4743],"pass_all3_ci":[0.2614,0.3113],"benchmarks":{"deep-swe":{"cost_per_trial":0.1767,"cost_per_pass":5.134,"tokens_per_trial":6019000.0,"calls_per_trial":76.63,"model_s_per_trial":1651.0,"agent_s_per_trial":1883.0,"rerun_flip":0.2182,"pass_all3":0.2087,"pass_mean3":0.3734,"pass_any3":0.536,"n_trials":1729,"n_scored":1665,"n_pass":744,"n_infra":64,"cost_usd":333.6,"n_tasks3":555,"rate":0.3734,"ci":[0.3429,0.4038],"pass_all3_ci":[0.1806,0.2368],"n_tasks":555,"n_tasks_expected":555},"terminal-bench-2.1":{"cost_per_trial":0.04086,"cost_per_pass":0.08286,"tokens_per_trial":1074000.0,"calls_per_trial":24.78,"model_s_per_trial":632.2,"agent_s_per_trial":840.2,"rerun_flip":0.2069,"pass_all3":0.364,"pass_mean3":0.5211,"pass_any3":0.6743,"n_trials":1312,"n_scored":1305,"n_pass":751,"n_infra":7,"cost_usd":58.35,"n_tasks3":435,"rate":0.5211,"ci":[0.4771,0.565],"pass_all3_ci":[0.3239,0.4041],"n_tasks":435,"n_tasks_expected":435}},"complete":true,"per_model":{"RecursiveSI/flint":0.1275,"deepseek-ai/DeepSeek-V4.1-Flash":0.668,"zai-org/GLM-5.3-Flash":0.5461},"efforts_requested":{"high":71015,"low":64577,"medium":27175,"unset":1882},"versions":["0.25.2"],"vs_leader":{"diff":-0.0724,"ci":[-0.098,-0.04839],"tied":false,"n_tasks":198}}},"ranking":["opencode","mini-swe-agent","kimi-code","deepseek-harness","hermes","claude-code","pi","qwen-code","codex","openclaw"],"models":["RecursiveSI/flint","deepseek-ai/DeepSeek-V4.1-Flash","zai-org/GLM-5.3-Flash"],"noise":{"rerun_flip":0.2339,"harness_flip":0.2821,"model_flip":0.3282,"effort_flip":0.2679,"by_benchmark":{"deep-swe":{"rerun_flip":0.2634,"harness_flip":0.3143,"model_flip":0.3862,"effort_flip":0.3173},"terminal-bench-2.1":{"rerun_flip":0.2044,"harness_flip":0.2499,"model_flip":0.2701,"effort_flip":0.2185}},"harness_pairs":[{"harnesses":["opencode","mini-swe-agent"],"flip":0.2571},{"harnesses":["opencode","kimi-code"],"flip":0.2525},{"harnesses":["opencode","deepseek-harness"],"flip":0.2548},{"harnesses":["opencode","hermes"],"flip":0.264},{"harnesses":["opencode","claude-code"],"flip":0.2568},{"harnesses":["opencode","pi"],"flip":0.2473},{"harnesses":["opencode","qwen-code"],"flip":0.2686},{"harnesses":["opencode","codex"],"flip":0.2801},{"harnesses":["opencode","openclaw"],"flip":0.3833},{"harnesses":["mini-swe-agent","kimi-code"],"flip":0.2527},{"harnesses":["mini-swe-agent","deepseek-harness"],"flip":0.2519},{"harnesses":["mini-swe-agent","hermes"],"flip":0.2681},{"harnesses":["mini-swe-agent","claude-code"],"flip":0.2516},{"harnesses":["mini-swe-agent","pi"],"flip":0.2531},{"harnesses":["mini-swe-agent","qwen-code"],"flip":0.2644},{"harnesses":["mini-swe-agent","codex"],"flip":0.2843},{"harnesses":["mini-swe-agent","openclaw"],"flip":0.3656},{"harnesses":["kimi-code","deepseek-harness"],"flip":0.259},{"harnesses":["kimi-code","hermes"],"flip":0.2584},{"harnesses":["kimi-code","claude-code"],"flip":0.2503},{"harnesses":["kimi-code","pi"],"flip":0.2545},{"harnesses":["kimi-code","qwen-code"],"flip":0.2568},{"harnesses":["kimi-code","codex"],"flip":0.281},{"harnesses":["kimi-code","openclaw"],"flip":0.3662},{"harnesses":["deepseek-harness","hermes"],"flip":0.2678},{"harnesses":["deepseek-harness","claude-code"],"flip":0.2609},{"harnesses":["deepseek-harness","pi"],"flip":0.2621},{"harnesses":["deepseek-harness","qwen-code"],"flip":0.252},{"harnesses":["deepseek-harness","codex"],"flip":0.2858},{"harnesses":["deepseek-harness","openclaw"],"flip":0.3557},{"harnesses":["hermes","claude-code"],"flip":0.2494},{"harnesses":["hermes","pi"],"flip":0.2635},{"harnesses":["hermes","qwen-code"],"flip":0.2639},{"harnesses":["hermes","codex"],"flip":0.2911},{"harnesses":["hermes","openclaw"],"flip":0.3723},{"harnesses":["claude-code","pi"],"flip":0.2479},{"harnesses":["claude-code","qwen-code"],"flip":0.2511},{"harnesses":["claude-code","codex"],"flip":0.2913},{"harnesses":["claude-code","openclaw"],"flip":0.3596},{"harnesses":["pi","qwen-code"],"flip":0.2522},{"harnesses":["pi","codex"],"flip":0.2797},{"harnesses":["pi","openclaw"],"flip":0.3566},{"harnesses":["qwen-code","codex"],"flip":0.2723},{"harnesses":["qwen-code","openclaw"],"flip":0.3294},{"harnesses":["codex","openclaw"],"flip":0.3455}]},"leader":"opencode","efforts":["default","high","low"]},"high":{"harnesses":{"claude-code":{"cost_per_trial":0.1465,"cost_per_pass":0.2271,"tokens_per_trial":5669000.0,"calls_per_trial":69.83,"model_s_per_trial":1060.0,"agent_s_per_trial":1410.0,"rerun_flip":0.2513,"pass_all3":0.4574,"pass_mean3":0.6559,"pass_any3":0.8343,"n_trials":1242,"n_scored":1188,"n_pass":776,"n_infra":54,"cost_usd":180.9,"n_tasks3":396,"macro":0.6559,"ci":[0.6098,0.702],"pass_all3_ci":[0.4096,0.5053],"benchmarks":{"deep-swe":{"cost_per_trial":0.2271,"cost_per_pass":0.3612,"tokens_per_trial":9315000.0,"calls_per_trial":106.5,"model_s_per_trial":1484.0,"agent_s_per_trial":1866.0,"rerun_flip":0.3033,"pass_all3":0.3919,"pass_mean3":0.6336,"pass_any3":0.8468,"n_trials":711,"n_scored":666,"n_pass":422,"n_infra":45,"cost_usd":157.4,"n_tasks3":222,"rate":0.6336,"ci":[0.5713,0.696],"pass_all3_ci":[0.3288,0.455],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04367,"cost_per_pass":0.06486,"tokens_per_trial":1017000.0,"calls_per_trial":23.07,"model_s_per_trial":517.8,"agent_s_per_trial":829.0,"rerun_flip":0.1992,"pass_all3":0.523,"pass_mean3":0.6782,"pass_any3":0.8218,"n_trials":531,"n_scored":522,"n_pass":354,"n_infra":9,"cost_usd":23.49,"n_tasks3":174,"rate":0.6782,"ci":[0.6104,0.7459],"pass_all3_ci":[0.4511,0.5948],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.6932,"zai-org/GLM-5.3-Flash":0.6186},"efforts_requested":{"high":82954},"versions":["2.1.300"],"vs_leader":{"diff":-0.01944,"ci":[-0.05374,0.01209],"tied":true,"n_tasks":198}},"codex":{"cost_per_trial":0.1355,"cost_per_pass":0.2454,"tokens_per_trial":6569000.0,"calls_per_trial":82.32,"model_s_per_trial":633.9,"agent_s_per_trial":1034.0,"rerun_flip":0.2869,"pass_all3":0.3802,"pass_mean3":0.6031,"pass_any3":0.8107,"n_trials":1355,"n_scored":1188,"n_pass":701,"n_infra":167,"cost_usd":165.0,"n_tasks3":396,"macro":0.6031,"ci":[0.5577,0.6485],"pass_all3_ci":[0.3356,0.4249],"benchmarks":{"deep-swe":{"cost_per_trial":0.2012,"cost_per_pass":0.4716,"tokens_per_trial":10420000.0,"calls_per_trial":119.2,"model_s_per_trial":857.0,"agent_s_per_trial":1346.0,"rerun_flip":0.3363,"pass_all3":0.2432,"pass_mean3":0.4955,"pass_any3":0.7477,"n_trials":704,"n_scored":666,"n_pass":330,"n_infra":38,"cost_usd":137.9,"n_tasks3":222,"rate":0.4955,"ci":[0.4335,0.5575],"pass_all3_ci":[0.1905,0.296],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.05166,"cost_per_pass":0.07429,"tokens_per_trial":1661000.0,"calls_per_trial":35.25,"model_s_per_trial":349.2,"agent_s_per_trial":634.7,"rerun_flip":0.2375,"pass_all3":0.5172,"pass_mean3":0.7107,"pass_any3":0.8736,"n_trials":651,"n_scored":522,"n_pass":371,"n_infra":129,"cost_usd":27.12,"n_tasks3":174,"rate":0.7107,"ci":[0.6447,0.7768],"pass_all3_ci":[0.4455,0.589],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.6934,"zai-org/GLM-5.3-Flash":0.5128},"efforts_requested":{"high":97794},"versions":["0.162.1"],"vs_leader":{"diff":-0.07223,"ci":[-0.1008,-0.04379],"tied":false,"n_tasks":198}},"deepseek-harness":{"cost_per_trial":0.2035,"cost_per_pass":0.3095,"tokens_per_trial":8308000.0,"calls_per_trial":80.85,"model_s_per_trial":1218.0,"agent_s_per_trial":1557.0,"rerun_flip":0.2219,"pass_all3":0.5014,"pass_mean3":0.6743,"pass_any3":0.8351,"n_trials":1345,"n_scored":1187,"n_pass":799,"n_infra":158,"cost_usd":252.7,"n_tasks3":395,"macro":0.6752,"ci":[0.6296,0.7207],"pass_all3_ci":[0.4533,0.5496],"benchmarks":{"deep-swe":{"cost_per_trial":0.3287,"cost_per_pass":0.5072,"tokens_per_trial":13950000.0,"calls_per_trial":124.4,"model_s_per_trial":1818.0,"agent_s_per_trial":2237.0,"rerun_flip":0.2523,"pass_all3":0.4569,"pass_mean3":0.659,"pass_any3":0.8369,"n_trials":773,"n_scored":665,"n_pass":439,"n_infra":108,"cost_usd":227.9,"n_tasks3":221,"rate":0.6607,"ci":[0.5994,0.7219],"pass_all3_ci":[0.3924,0.5214],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04404,"cost_per_pass":0.06465,"tokens_per_trial":1113000.0,"calls_per_trial":25.39,"model_s_per_trial":454.0,"agent_s_per_trial":690.5,"rerun_flip":0.1916,"pass_all3":0.546,"pass_mean3":0.6897,"pass_any3":0.8333,"n_trials":572,"n_scored":522,"n_pass":360,"n_infra":50,"cost_usd":24.79,"n_tasks3":174,"rate":0.6897,"ci":[0.6226,0.7567],"pass_all3_ci":[0.4744,0.6176],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.717,"zai-org/GLM-5.3-Flash":0.6333},"efforts_requested":{"high":94843,"low":1133},"versions":["0.2.2-rc.1"],"vs_leader":{"diff":-0.0001812,"ci":[-0.03448,0.03187],"tied":true,"n_tasks":198}},"hermes":{"cost_per_trial":0.1464,"cost_per_pass":0.2233,"tokens_per_trial":6908000.0,"calls_per_trial":61.42,"model_s_per_trial":918.6,"agent_s_per_trial":1571.0,"rerun_flip":0.2578,"pass_all3":0.464,"pass_mean3":0.6717,"pass_any3":0.8507,"n_trials":1259,"n_scored":1188,"n_pass":797,"n_infra":71,"cost_usd":183.2,"n_tasks3":396,"macro":0.6717,"ci":[0.6262,0.7172],"pass_all3_ci":[0.4158,0.5121],"benchmarks":{"deep-swe":{"cost_per_trial":0.2279,"cost_per_pass":0.3572,"tokens_per_trial":11380000.0,"calls_per_trial":90.19,"model_s_per_trial":1226.0,"agent_s_per_trial":2116.0,"rerun_flip":0.2973,"pass_all3":0.4279,"pass_mean3":0.6652,"pass_any3":0.8739,"n_trials":733,"n_scored":666,"n_pass":443,"n_infra":67,"cost_usd":161.0,"n_tasks3":222,"rate":0.6652,"ci":[0.605,0.7253],"pass_all3_ci":[0.3645,0.4914],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04236,"cost_per_pass":0.0624,"tokens_per_trial":1199000.0,"calls_per_trial":24.71,"model_s_per_trial":526.6,"agent_s_per_trial":875.6,"rerun_flip":0.2184,"pass_all3":0.5,"pass_mean3":0.6782,"pass_any3":0.8276,"n_trials":526,"n_scored":522,"n_pass":354,"n_infra":4,"cost_usd":22.24,"n_tasks3":174,"rate":0.6782,"ci":[0.6102,0.7462],"pass_all3_ci":[0.4275,0.5725],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.7298,"zai-org/GLM-5.3-Flash":0.6135},"efforts_requested":{"high":71570,"none":1016,"unset":381},"versions":["v0.22.1"],"vs_leader":{"diff":-0.003676,"ci":[-0.03632,0.02942],"tied":true,"n_tasks":198}},"kimi-code":{"cost_per_trial":0.1571,"cost_per_pass":0.2406,"tokens_per_trial":7363000.0,"calls_per_trial":67.88,"model_s_per_trial":1042.0,"agent_s_per_trial":1343.0,"rerun_flip":0.2623,"pass_all3":0.4509,"pass_mean3":0.6584,"pass_any3":0.842,"n_trials":1256,"n_scored":1187,"n_pass":776,"n_infra":69,"cost_usd":197.2,"n_tasks3":395,"macro":0.6581,"ci":[0.6121,0.7042],"pass_all3_ci":[0.4028,0.4989],"benchmarks":{"deep-swe":{"cost_per_trial":0.2433,"cost_per_pass":0.3908,"tokens_per_trial":11970000.0,"calls_per_trial":101.0,"model_s_per_trial":1407.0,"agent_s_per_trial":1763.0,"rerun_flip":0.3138,"pass_all3":0.3845,"pass_mean3":0.6214,"pass_any3":0.8507,"n_trials":726,"n_scored":665,"n_pass":413,"n_infra":61,"cost_usd":172.1,"n_tasks3":221,"rate":0.6209,"ci":[0.5581,0.6837],"pass_all3_ci":[0.3215,0.4475],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04725,"cost_per_pass":0.06796,"tokens_per_trial":1497000.0,"calls_per_trial":25.75,"model_s_per_trial":577.6,"agent_s_per_trial":807.1,"rerun_flip":0.2107,"pass_all3":0.5172,"pass_mean3":0.6954,"pass_any3":0.8333,"n_trials":530,"n_scored":522,"n_pass":363,"n_infra":8,"cost_usd":25.09,"n_tasks3":174,"rate":0.6954,"ci":[0.6283,0.7625],"pass_all3_ci":[0.4446,0.5899],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.6634,"zai-org/GLM-5.3-Flash":0.6529},"efforts_requested":{"high":80558},"versions":["2.2.0"],"vs_leader":{"diff":-0.0172,"ci":[-0.04501,0.009243],"tied":true,"n_tasks":198}},"mini-swe-agent":{"cost_per_trial":0.1905,"cost_per_pass":0.2892,"tokens_per_trial":9216000.0,"calls_per_trial":91.21,"model_s_per_trial":1034.0,"agent_s_per_trial":1517.0,"rerun_flip":0.2465,"pass_all3":0.4664,"pass_mean3":0.6677,"pass_any3":0.8362,"n_trials":1397,"n_scored":1188,"n_pass":789,"n_infra":209,"cost_usd":250.6,"n_tasks3":396,"macro":0.6677,"ci":[0.6219,0.7135],"pass_all3_ci":[0.4185,0.5144],"benchmarks":{"deep-swe":{"cost_per_trial":0.2965,"cost_per_pass":0.4682,"tokens_per_trial":15080000.0,"calls_per_trial":134.0,"model_s_per_trial":1376.0,"agent_s_per_trial":2003.0,"rerun_flip":0.2823,"pass_all3":0.4099,"pass_mean3":0.6381,"pass_any3":0.8333,"n_trials":865,"n_scored":666,"n_pass":425,"n_infra":199,"cost_usd":221.3,"n_tasks3":222,"rate":0.6381,"ci":[0.576,0.7003],"pass_all3_ci":[0.3469,0.473],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.05527,"cost_per_pass":0.07961,"tokens_per_trial":1733000.0,"calls_per_trial":36.64,"model_s_per_trial":598.7,"agent_s_per_trial":898.5,"rerun_flip":0.2107,"pass_all3":0.523,"pass_mean3":0.6973,"pass_any3":0.8391,"n_trials":532,"n_scored":522,"n_pass":364,"n_infra":10,"cost_usd":29.26,"n_tasks3":174,"rate":0.6973,"ci":[0.6304,0.7643],"pass_all3_ci":[0.4506,0.5954],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.6909,"zai-org/GLM-5.3-Flash":0.6446},"efforts_requested":{"high":108328,"unset":23,"none":3,"low":2},"versions":["2.4.6"],"vs_leader":{"diff":-0.007611,"ci":[-0.03946,0.02312],"tied":true,"n_tasks":198}},"openclaw":{"cost_per_trial":0.2263,"cost_per_pass":0.5873,"tokens_per_trial":7014000.0,"calls_per_trial":96.12,"model_s_per_trial":1833.0,"agent_s_per_trial":2548.0,"rerun_flip":0.24,"pass_all3":0.2176,"pass_mean3":0.3768,"pass_any3":0.5698,"n_trials":1251,"n_scored":1184,"n_pass":430,"n_infra":67,"cost_usd":271.0,"n_tasks3":392,"macro":0.3763,"ci":[0.3303,0.4224],"pass_all3_ci":[0.1776,0.2577],"benchmarks":{"deep-swe":{"cost_per_trial":0.3664,"cost_per_pass":1.244,"tokens_per_trial":11510000.0,"calls_per_trial":144.5,"model_s_per_trial":2826.0,"agent_s_per_trial":3842.0,"rerun_flip":0.2808,"pass_all3":0.08649,"pass_mean3":0.269,"pass_any3":0.5058,"n_trials":721,"n_scored":664,"n_pass":178,"n_infra":57,"cost_usd":246.2,"n_tasks3":220,"rate":0.268,"ci":[0.2115,0.3245],"pass_all3_ci":[0.04869,0.1243],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04727,"cost_per_pass":0.09735,"tokens_per_trial":1271000.0,"calls_per_trial":34.32,"model_s_per_trial":564.3,"agent_s_per_trial":895.5,"rerun_flip":0.1992,"pass_all3":0.3487,"pass_mean3":0.4847,"pass_any3":0.6337,"n_trials":530,"n_scored":520,"n_pass":252,"n_infra":10,"cost_usd":24.8,"n_tasks3":172,"rate":0.4847,"ci":[0.412,0.5573],"pass_all3_ci":[0.2789,0.4186],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.3272,"zai-org/GLM-5.3-Flash":0.4255},"efforts_requested":{"high":113103,"low":454,"unset":126},"versions":["2026.10.4"],"vs_leader":{"diff":-0.299,"ci":[-0.3461,-0.2543],"tied":false,"n_tasks":198}},"opencode":{"cost_per_trial":0.1442,"cost_per_pass":0.2168,"tokens_per_trial":7278000.0,"calls_per_trial":73.22,"model_s_per_trial":670.5,"agent_s_per_trial":1129.0,"rerun_flip":0.2678,"pass_all3":0.4682,"pass_mean3":0.6753,"pass_any3":0.8698,"n_trials":1198,"n_scored":1188,"n_pass":795,"n_infra":10,"cost_usd":171.3,"n_tasks3":396,"macro":0.6753,"ci":[0.63,0.7207],"pass_all3_ci":[0.4208,0.5155],"benchmarks":{"deep-swe":{"cost_per_trial":0.2308,"cost_per_pass":0.3729,"tokens_per_trial":12230000.0,"calls_per_trial":112.3,"model_s_per_trial":962.6,"agent_s_per_trial":1543.0,"rerun_flip":0.3363,"pass_all3":0.3559,"pass_mean3":0.6246,"pass_any3":0.8604,"n_trials":671,"n_scored":666,"n_pass":416,"n_infra":5,"cost_usd":153.7,"n_tasks3":222,"rate":0.6246,"ci":[0.562,0.6872],"pass_all3_ci":[0.294,0.4177],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.03372,"cost_per_pass":0.04648,"tokens_per_trial":964300.0,"calls_per_trial":23.35,"model_s_per_trial":297.8,"agent_s_per_trial":600.0,"rerun_flip":0.1992,"pass_all3":0.5805,"pass_mean3":0.7261,"pass_any3":0.8793,"n_trials":527,"n_scored":522,"n_pass":379,"n_infra":5,"cost_usd":17.6,"n_tasks3":174,"rate":0.7261,"ci":[0.6609,0.7912],"pass_all3_ci":[0.5089,0.652],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.6941,"zai-org/GLM-5.3-Flash":0.6566},"efforts_requested":{"high":86989},"versions":["2.2.6"]},"pi":{"cost_per_trial":0.114,"cost_per_pass":0.1739,"tokens_per_trial":4478000.0,"calls_per_trial":73.75,"model_s_per_trial":907.2,"agent_s_per_trial":1464.0,"rerun_flip":0.2387,"pass_all3":0.4772,"pass_mean3":0.6629,"pass_any3":0.8352,"n_trials":1322,"n_scored":1188,"n_pass":779,"n_infra":134,"cost_usd":147.8,"n_tasks3":396,"macro":0.6629,"ci":[0.6173,0.7085],"pass_all3_ci":[0.4296,0.5247],"benchmarks":{"deep-swe":{"cost_per_trial":0.1802,"cost_per_pass":0.2983,"tokens_per_trial":7497000.0,"calls_per_trial":114.5,"model_s_per_trial":1281.0,"agent_s_per_trial":2066.0,"rerun_flip":0.2973,"pass_all3":0.3739,"pass_mean3":0.6036,"pass_any3":0.8198,"n_trials":792,"n_scored":666,"n_pass":402,"n_infra":126,"cost_usd":132.2,"n_tasks3":222,"rate":0.6036,"ci":[0.5403,0.6669],"pass_all3_ci":[0.3113,0.4365],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.02947,"cost_per_pass":0.04081,"tokens_per_trial":626300.0,"calls_per_trial":21.8,"model_s_per_trial":430.7,"agent_s_per_trial":695.9,"rerun_flip":0.1801,"pass_all3":0.5805,"pass_mean3":0.7222,"pass_any3":0.8506,"n_trials":530,"n_scored":522,"n_pass":377,"n_infra":8,"cost_usd":15.62,"n_tasks3":174,"rate":0.7222,"ci":[0.6568,0.7876],"pass_all3_ci":[0.5088,0.6521],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.6709,"zai-org/GLM-5.3-Flash":0.655},"efforts_requested":{"high":87614},"versions":["1.2.0"],"vs_leader":{"diff":-0.01243,"ci":[-0.04145,0.01373],"tied":true,"n_tasks":198}},"qwen-code":{"cost_per_trial":0.1664,"cost_per_pass":0.2773,"tokens_per_trial":4802000.0,"calls_per_trial":60.57,"model_s_per_trial":964.3,"agent_s_per_trial":1252.0,"rerun_flip":0.2463,"pass_all3":0.4314,"pass_mean3":0.6195,"pass_any3":0.8009,"n_trials":1221,"n_scored":1188,"n_pass":730,"n_infra":33,"cost_usd":202.9,"n_tasks3":396,"macro":0.6195,"ci":[0.5727,0.6663],"pass_all3_ci":[0.3839,0.4789],"benchmarks":{"deep-swe":{"cost_per_trial":0.2548,"cost_per_pass":0.4488,"tokens_per_trial":7746000.0,"calls_per_trial":89.3,"model_s_per_trial":1327.0,"agent_s_per_trial":1648.0,"rerun_flip":0.2973,"pass_all3":0.3514,"pass_mean3":0.5781,"pass_any3":0.7973,"n_trials":695,"n_scored":666,"n_pass":385,"n_infra":29,"cost_usd":174.5,"n_tasks3":222,"rate":0.5781,"ci":[0.5143,0.6418],"pass_all3_ci":[0.2896,0.4131],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.05354,"cost_per_pass":0.08236,"tokens_per_trial":1047000.0,"calls_per_trial":23.91,"model_s_per_trial":501.6,"agent_s_per_trial":747.6,"rerun_flip":0.1954,"pass_all3":0.5115,"pass_mean3":0.6609,"pass_any3":0.8046,"n_trials":526,"n_scored":522,"n_pass":345,"n_infra":4,"cost_usd":28.41,"n_tasks3":174,"rate":0.6609,"ci":[0.5925,0.7293],"pass_all3_ci":[0.4394,0.5836],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.6669,"zai-org/GLM-5.3-Flash":0.5721},"efforts_requested":{"high":71015,"unset":940},"versions":["0.25.2"],"vs_leader":{"diff":-0.05584,"ci":[-0.0892,-0.02335],"tied":false,"n_tasks":198}}},"ranking":["opencode","deepseek-harness","hermes","mini-swe-agent","pi","kimi-code","claude-code","qwen-code","codex","openclaw"],"models":["deepseek-ai/DeepSeek-V4.1-Flash","zai-org/GLM-5.3-Flash"],"noise":{"rerun_flip":0.252,"harness_flip":0.3082,"model_flip":0.3122,"by_benchmark":{"deep-swe":{"rerun_flip":0.2997,"harness_flip":0.3629,"model_flip":0.3707},"terminal-bench-2.1":{"rerun_flip":0.2042,"harness_flip":0.2536,"model_flip":0.2536}},"harness_pairs":[{"harnesses":["opencode","deepseek-harness"],"flip":0.2771},{"harnesses":["opencode","hermes"],"flip":0.2876},{"harnesses":["opencode","mini-swe-agent"],"flip":0.2759},{"harnesses":["opencode","pi"],"flip":0.25},{"harnesses":["opencode","kimi-code"],"flip":0.2688},{"harnesses":["opencode","claude-code"],"flip":0.2795},{"harnesses":["opencode","qwen-code"],"flip":0.2807},{"harnesses":["opencode","codex"],"flip":0.2945},{"harnesses":["opencode","openclaw"],"flip":0.4381},{"harnesses":["deepseek-harness","hermes"],"flip":0.2907},{"harnesses":["deepseek-harness","mini-swe-agent"],"flip":0.2619},{"harnesses":["deepseek-harness","pi"],"flip":0.265},{"harnesses":["deepseek-harness","kimi-code"],"flip":0.2812},{"harnesses":["deepseek-harness","claude-code"],"flip":0.2812},{"harnesses":["deepseek-harness","qwen-code"],"flip":0.2625},{"harnesses":["deepseek-harness","codex"],"flip":0.3178},{"harnesses":["deepseek-harness","openclaw"],"flip":0.4088},{"harnesses":["hermes","mini-swe-agent"],"flip":0.296},{"harnesses":["hermes","pi"],"flip":0.2737},{"harnesses":["hermes","kimi-code"],"flip":0.2857},{"harnesses":["hermes","claude-code"],"flip":0.2722},{"harnesses":["hermes","qwen-code"],"flip":0.2859},{"harnesses":["hermes","codex"],"flip":0.3238},{"harnesses":["hermes","openclaw"],"flip":0.4329},{"harnesses":["mini-swe-agent","pi"],"flip":0.2658},{"harnesses":["mini-swe-agent","kimi-code"],"flip":0.2711},{"harnesses":["mini-swe-agent","claude-code"],"flip":0.2663},{"harnesses":["mini-swe-agent","qwen-code"],"flip":0.2858},{"harnesses":["mini-swe-agent","codex"],"flip":0.3063},{"harnesses":["mini-swe-agent","openclaw"],"flip":0.4183},{"harnesses":["pi","kimi-code"],"flip":0.2599},{"harnesses":["pi","claude-code"],"flip":0.2678},{"harnesses":["pi","qwen-code"],"flip":0.2622},{"harnesses":["pi","codex"],"flip":0.3012},{"harnesses":["pi","openclaw"],"flip":0.4111},{"harnesses":["kimi-code","claude-code"],"flip":0.2752},{"harnesses":["kimi-code","qwen-code"],"flip":0.2733},{"harnesses":["kimi-code","codex"],"flip":0.3068},{"harnesses":["kimi-code","openclaw"],"flip":0.4034},{"harnesses":["claude-code","qwen-code"],"flip":0.2654},{"harnesses":["claude-code","codex"],"flip":0.3281},{"harnesses":["claude-code","openclaw"],"flip":0.4088},{"harnesses":["qwen-code","codex"],"flip":0.3028},{"harnesses":["qwen-code","openclaw"],"flip":0.3912},{"harnesses":["codex","openclaw"],"flip":0.4089}]},"leader":"opencode"},"low":{"harnesses":{"claude-code":{"cost_per_trial":0.1054,"cost_per_pass":0.1666,"tokens_per_trial":4418000.0,"calls_per_trial":61.42,"model_s_per_trial":763.8,"agent_s_per_trial":1106.0,"rerun_flip":0.2514,"pass_all3":0.4433,"pass_mean3":0.6459,"pass_any3":0.8204,"n_trials":1245,"n_scored":1188,"n_pass":761,"n_infra":57,"cost_usd":130.5,"n_tasks3":396,"macro":0.6459,"ci":[0.5998,0.692],"pass_all3_ci":[0.3959,0.4907],"benchmarks":{"deep-swe":{"cost_per_trial":0.1596,"cost_per_pass":0.2682,"tokens_per_trial":7165000.0,"calls_per_trial":92.26,"model_s_per_trial":1048.0,"agent_s_per_trial":1429.0,"rerun_flip":0.2883,"pass_all3":0.3694,"pass_mean3":0.6021,"pass_any3":0.8018,"n_trials":715,"n_scored":666,"n_pass":401,"n_infra":49,"cost_usd":111.2,"n_tasks3":222,"rate":0.6021,"ci":[0.539,0.6652],"pass_all3_ci":[0.3071,0.4316],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.0363,"cost_per_pass":0.0533,"tokens_per_trial":912500.0,"calls_per_trial":22.07,"model_s_per_trial":401.8,"agent_s_per_trial":693.3,"rerun_flip":0.2146,"pass_all3":0.5172,"pass_mean3":0.6897,"pass_any3":0.8391,"n_trials":530,"n_scored":522,"n_pass":360,"n_infra":8,"cost_usd":19.28,"n_tasks3":174,"rate":0.6897,"ci":[0.6226,0.7567],"pass_all3_ci":[0.4458,0.5887],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.6945,"zai-org/GLM-5.3-Flash":0.5973},"efforts_requested":{"low":72970},"versions":["2.1.300"],"vs_leader":{"diff":-0.008543,"ci":[-0.04427,0.02685],"tied":true,"n_tasks":198}},"codex":{"cost_per_trial":0.1166,"cost_per_pass":0.2414,"tokens_per_trial":5510000.0,"calls_per_trial":76.4,"model_s_per_trial":550.9,"agent_s_per_trial":939.5,"rerun_flip":0.2714,"pass_all3":0.3571,"pass_mean3":0.5591,"pass_any3":0.7641,"n_trials":1365,"n_scored":1188,"n_pass":647,"n_infra":177,"cost_usd":143.0,"n_tasks3":396,"macro":0.5591,"ci":[0.5138,0.6043],"pass_all3_ci":[0.3136,0.4006],"benchmarks":{"deep-swe":{"cost_per_trial":0.1723,"cost_per_pass":0.5045,"tokens_per_trial":8670000.0,"calls_per_trial":109.9,"model_s_per_trial":755.2,"agent_s_per_trial":1233.0,"rerun_flip":0.3243,"pass_all3":0.2027,"pass_mean3":0.4399,"pass_any3":0.6892,"n_trials":701,"n_scored":666,"n_pass":293,"n_infra":35,"cost_usd":118.7,"n_tasks3":222,"rate":0.4399,"ci":[0.3796,0.5003],"pass_all3_ci":[0.1526,0.2528],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04557,"cost_per_pass":0.07136,"tokens_per_trial":1478000.0,"calls_per_trial":33.71,"model_s_per_trial":290.2,"agent_s_per_trial":564.5,"rerun_flip":0.2184,"pass_all3":0.5115,"pass_mean3":0.6782,"pass_any3":0.8391,"n_trials":664,"n_scored":522,"n_pass":354,"n_infra":142,"cost_usd":24.31,"n_tasks3":174,"rate":0.6782,"ci":[0.6109,0.7454],"pass_all3_ci":[0.4409,0.5821],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.6837,"zai-org/GLM-5.3-Flash":0.4344},"efforts_requested":{"low":90761},"versions":["0.162.1"],"vs_leader":{"diff":-0.09537,"ci":[-0.1313,-0.05777],"tied":false,"n_tasks":198}},"deepseek-harness":{"cost_per_trial":0.1305,"cost_per_pass":0.2124,"tokens_per_trial":5966000.0,"calls_per_trial":71.47,"model_s_per_trial":887.0,"agent_s_per_trial":1203.0,"rerun_flip":0.2472,"pass_all3":0.4392,"pass_mean3":0.6307,"pass_any3":0.81,"n_trials":1316,"n_scored":1188,"n_pass":746,"n_infra":128,"cost_usd":163.6,"n_tasks3":396,"macro":0.6307,"ci":[0.5841,0.6773],"pass_all3_ci":[0.3919,0.4865],"benchmarks":{"deep-swe":{"cost_per_trial":0.2045,"cost_per_pass":0.3429,"tokens_per_trial":9886000.0,"calls_per_trial":108.2,"model_s_per_trial":1289.0,"agent_s_per_trial":1678.0,"rerun_flip":0.2913,"pass_all3":0.3784,"pass_mean3":0.6081,"pass_any3":0.8153,"n_trials":762,"n_scored":666,"n_pass":405,"n_infra":96,"cost_usd":144.0,"n_tasks3":222,"rate":0.6081,"ci":[0.5453,0.6709],"pass_all3_ci":[0.3159,0.4409],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.03621,"cost_per_pass":0.05606,"tokens_per_trial":963500.0,"calls_per_trial":24.55,"model_s_per_trial":374.0,"agent_s_per_trial":598.0,"rerun_flip":0.2031,"pass_all3":0.5,"pass_mean3":0.6533,"pass_any3":0.8046,"n_trials":554,"n_scored":522,"n_pass":341,"n_infra":32,"cost_usd":19.57,"n_tasks3":174,"rate":0.6533,"ci":[0.5846,0.7219],"pass_all3_ci":[0.429,0.571],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.6924,"zai-org/GLM-5.3-Flash":0.569},"efforts_requested":{"low":84903},"versions":["0.2.2-rc.1"],"vs_leader":{"diff":-0.02374,"ci":[-0.06239,0.01133],"tied":true,"n_tasks":198}},"hermes":{"cost_per_trial":0.09672,"cost_per_pass":0.1544,"tokens_per_trial":4294000.0,"calls_per_trial":50.65,"model_s_per_trial":660.8,"agent_s_per_trial":1214.0,"rerun_flip":0.2723,"pass_all3":0.4298,"pass_mean3":0.642,"pass_any3":0.8382,"n_trials":1280,"n_scored":1188,"n_pass":758,"n_infra":92,"cost_usd":121.4,"n_tasks3":396,"macro":0.642,"ci":[0.5958,0.6881],"pass_all3_ci":[0.3824,0.4772],"benchmarks":{"deep-swe":{"cost_per_trial":0.1426,"cost_per_pass":0.2468,"tokens_per_trial":6840000.0,"calls_per_trial":71.18,"model_s_per_trial":828.1,"agent_s_per_trial":1573.0,"rerun_flip":0.3453,"pass_all3":0.3423,"pass_mean3":0.6096,"pass_any3":0.8604,"n_trials":757,"n_scored":666,"n_pass":406,"n_infra":91,"cost_usd":101.5,"n_tasks3":222,"rate":0.6096,"ci":[0.5477,0.6715],"pass_all3_ci":[0.2814,0.4033],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.03812,"cost_per_pass":0.05653,"tokens_per_trial":1046000.0,"calls_per_trial":24.45,"model_s_per_trial":447.4,"agent_s_per_trial":754.8,"rerun_flip":0.1992,"pass_all3":0.5172,"pass_mean3":0.6743,"pass_any3":0.8161,"n_trials":523,"n_scored":522,"n_pass":352,"n_infra":1,"cost_usd":19.9,"n_tasks3":174,"rate":0.6743,"ci":[0.606,0.7427],"pass_all3_ci":[0.4446,0.5899],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.6896,"zai-org/GLM-5.3-Flash":0.5943},"efforts_requested":{"low":58726,"none":1031,"unset":411},"versions":["v0.22.1"],"vs_leader":{"diff":-0.01245,"ci":[-0.04928,0.02258],"tied":true,"n_tasks":198}},"kimi-code":{"cost_per_trial":0.1084,"cost_per_pass":0.1738,"tokens_per_trial":5352000.0,"calls_per_trial":59.85,"model_s_per_trial":738.6,"agent_s_per_trial":1030.0,"rerun_flip":0.2552,"pass_all3":0.4403,"pass_mean3":0.6397,"pass_any3":0.8231,"n_trials":1231,"n_scored":1188,"n_pass":753,"n_infra":43,"cost_usd":134.0,"n_tasks3":396,"macro":0.6397,"ci":[0.5936,0.6858],"pass_all3_ci":[0.3934,0.4872],"benchmarks":{"deep-swe":{"cost_per_trial":0.1645,"cost_per_pass":0.287,"tokens_per_trial":8582000.0,"calls_per_trial":87.87,"model_s_per_trial":999.6,"agent_s_per_trial":1327.0,"rerun_flip":0.3303,"pass_all3":0.3288,"pass_mean3":0.5916,"pass_any3":0.8243,"n_trials":707,"n_scored":666,"n_pass":394,"n_infra":41,"cost_usd":114.8,"n_tasks3":222,"rate":0.5916,"ci":[0.529,0.6542],"pass_all3_ci":[0.2691,0.3885],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.03683,"cost_per_pass":0.05366,"tokens_per_trial":1230000.0,"calls_per_trial":24.11,"model_s_per_trial":405.6,"agent_s_per_trial":651.1,"rerun_flip":0.1801,"pass_all3":0.5517,"pass_mean3":0.6877,"pass_any3":0.8218,"n_trials":524,"n_scored":522,"n_pass":359,"n_infra":2,"cost_usd":19.23,"n_tasks3":174,"rate":0.6877,"ci":[0.6203,0.7552],"pass_all3_ci":[0.4794,0.624],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.7014,"zai-org/GLM-5.3-Flash":0.5779},"efforts_requested":{"low":71105},"versions":["2.2.0"],"vs_leader":{"diff":-0.01476,"ci":[-0.05287,0.01965],"tied":true,"n_tasks":198}},"mini-swe-agent":{"cost_per_trial":0.1238,"cost_per_pass":0.1985,"tokens_per_trial":5868000.0,"calls_per_trial":75.18,"model_s_per_trial":755.5,"agent_s_per_trial":1145.0,"rerun_flip":0.2816,"pass_all3":0.4185,"pass_mean3":0.6396,"pass_any3":0.8409,"n_trials":1344,"n_scored":1188,"n_pass":751,"n_infra":156,"cost_usd":161.4,"n_tasks3":396,"macro":0.6396,"ci":[0.5935,0.6857],"pass_all3_ci":[0.3714,0.4656],"benchmarks":{"deep-swe":{"cost_per_trial":0.1867,"cost_per_pass":0.3253,"tokens_per_trial":9320000.0,"calls_per_trial":107.1,"model_s_per_trial":984.9,"agent_s_per_trial":1469.0,"rerun_flip":0.3333,"pass_all3":0.3198,"pass_mean3":0.5781,"pass_any3":0.8198,"n_trials":814,"n_scored":666,"n_pass":385,"n_infra":148,"cost_usd":138.3,"n_tasks3":222,"rate":0.5781,"ci":[0.5142,0.6419],"pass_all3_ci":[0.2594,0.3802],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04366,"cost_per_pass":0.06308,"tokens_per_trial":1464000.0,"calls_per_trial":34.48,"model_s_per_trial":462.9,"agent_s_per_trial":731.5,"rerun_flip":0.2299,"pass_all3":0.5172,"pass_mean3":0.7011,"pass_any3":0.8621,"n_trials":530,"n_scored":522,"n_pass":366,"n_infra":8,"cost_usd":23.06,"n_tasks3":174,"rate":0.7011,"ci":[0.6349,0.7674],"pass_all3_ci":[0.445,0.5894],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.68,"zai-org/GLM-5.3-Flash":0.5992},"efforts_requested":{"low":89313},"versions":["2.4.6"],"vs_leader":{"diff":-0.01481,"ci":[-0.05118,0.01916],"tied":true,"n_tasks":198}},"openclaw":{"cost_per_trial":0.2077,"cost_per_pass":0.5678,"tokens_per_trial":6967000.0,"calls_per_trial":100.9,"model_s_per_trial":1223.0,"agent_s_per_trial":1894.0,"rerun_flip":0.258,"pass_all3":0.198,"pass_mean3":0.3794,"pass_any3":0.585,"n_trials":1237,"n_scored":1188,"n_pass":435,"n_infra":49,"cost_usd":252.6,"n_tasks3":396,"macro":0.3794,"ci":[0.333,0.4257],"pass_all3_ci":[0.159,0.2369],"benchmarks":{"deep-swe":{"cost_per_trial":0.3212,"cost_per_pass":1.137,"tokens_per_trial":10930000.0,"calls_per_trial":145.8,"model_s_per_trial":1827.0,"agent_s_per_trial":2769.0,"rerun_flip":0.2823,"pass_all3":0.08559,"pass_mean3":0.2703,"pass_any3":0.509,"n_trials":705,"n_scored":666,"n_pass":180,"n_infra":39,"cost_usd":219.5,"n_tasks3":222,"rate":0.2703,"ci":[0.2127,0.3279],"pass_all3_ci":[0.04811,0.1231],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.06283,"cost_per_pass":0.1326,"tokens_per_trial":1915000.0,"calls_per_trial":43.52,"model_s_per_trial":451.4,"agent_s_per_trial":777.0,"rerun_flip":0.2337,"pass_all3":0.3103,"pass_mean3":0.4885,"pass_any3":0.6609,"n_trials":532,"n_scored":522,"n_pass":255,"n_infra":10,"cost_usd":33.14,"n_tasks3":174,"rate":0.4885,"ci":[0.416,0.561],"pass_all3_ci":[0.243,0.3777],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.3841,"zai-org/GLM-5.3-Flash":0.3747},"efforts_requested":{"low":119523,"unset":323,"None":1},"versions":["2026.10.4"],"vs_leader":{"diff":-0.275,"ci":[-0.3214,-0.2282],"tied":false,"n_tasks":198}},"opencode":{"cost_per_trial":0.09473,"cost_per_pass":0.1471,"tokens_per_trial":5147000.0,"calls_per_trial":63.73,"model_s_per_trial":478.9,"agent_s_per_trial":883.1,"rerun_flip":0.2572,"pass_all3":0.4558,"pass_mean3":0.6544,"pass_any3":0.8416,"n_trials":1208,"n_scored":1188,"n_pass":771,"n_infra":20,"cost_usd":112.5,"n_tasks3":396,"macro":0.6544,"ci":[0.6087,0.7001],"pass_all3_ci":[0.4085,0.5031],"benchmarks":{"deep-swe":{"cost_per_trial":0.1477,"cost_per_pass":0.2442,"tokens_per_trial":8557000.0,"calls_per_trial":96.3,"model_s_per_trial":681.3,"agent_s_per_trial":1179.0,"rerun_flip":0.2883,"pass_all3":0.3829,"pass_mean3":0.6096,"pass_any3":0.8153,"n_trials":676,"n_scored":666,"n_pass":406,"n_infra":10,"cost_usd":98.4,"n_tasks3":222,"rate":0.6096,"ci":[0.5469,0.6723],"pass_all3_ci":[0.3205,0.4452],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.02709,"cost_per_pass":0.03928,"tokens_per_trial":796400.0,"calls_per_trial":22.18,"model_s_per_trial":220.6,"agent_s_per_trial":505.7,"rerun_flip":0.2261,"pass_all3":0.5287,"pass_mean3":0.6992,"pass_any3":0.8678,"n_trials":532,"n_scored":522,"n_pass":365,"n_infra":10,"cost_usd":14.14,"n_tasks3":174,"rate":0.6992,"ci":[0.633,0.7655],"pass_all3_ci":[0.4577,0.5998],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.718,"zai-org/GLM-5.3-Flash":0.5909},"efforts_requested":{"low":75717},"versions":["2.2.6"]},"pi":{"cost_per_trial":0.08156,"cost_per_pass":0.1335,"tokens_per_trial":3431000.0,"calls_per_trial":63.74,"model_s_per_trial":684.2,"agent_s_per_trial":1187.0,"rerun_flip":0.2893,"pass_all3":0.3958,"pass_mean3":0.619,"pass_any3":0.8297,"n_trials":1305,"n_scored":1188,"n_pass":726,"n_infra":117,"cost_usd":105.0,"n_tasks3":396,"macro":0.619,"ci":[0.5724,0.6656],"pass_all3_ci":[0.3493,0.4423],"benchmarks":{"deep-swe":{"cost_per_trial":0.1264,"cost_per_pass":0.228,"tokens_per_trial":5688000.0,"calls_per_trial":97.33,"model_s_per_trial":954.8,"agent_s_per_trial":1614.0,"rerun_flip":0.3333,"pass_all3":0.2973,"pass_mean3":0.5541,"pass_any3":0.7973,"n_trials":774,"n_scored":666,"n_pass":369,"n_infra":108,"cost_usd":92.18,"n_tasks3":222,"rate":0.5541,"ci":[0.4898,0.6183],"pass_all3_ci":[0.2387,0.3559],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.02435,"cost_per_pass":0.03572,"tokens_per_trial":552100.0,"calls_per_trial":20.88,"model_s_per_trial":338.9,"agent_s_per_trial":642.2,"rerun_flip":0.2452,"pass_all3":0.4943,"pass_mean3":0.6839,"pass_any3":0.8621,"n_trials":531,"n_scored":522,"n_pass":357,"n_infra":9,"cost_usd":12.84,"n_tasks3":174,"rate":0.6839,"ci":[0.6166,0.7512],"pass_all3_ci":[0.4222,0.5663],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.6614,"zai-org/GLM-5.3-Flash":0.5765},"efforts_requested":{"low":75725},"versions":["1.2.0"],"vs_leader":{"diff":-0.03544,"ci":[-0.06721,-0.004167],"tied":false,"n_tasks":198}},"qwen-code":{"cost_per_trial":0.1252,"cost_per_pass":0.2218,"tokens_per_trial":3871000.0,"calls_per_trial":55.07,"model_s_per_trial":670.0,"agent_s_per_trial":941.1,"rerun_flip":0.278,"pass_all3":0.3759,"pass_mean3":0.5946,"pass_any3":0.7929,"n_trials":1214,"n_scored":1188,"n_pass":698,"n_infra":26,"cost_usd":153.3,"n_tasks3":396,"macro":0.5946,"ci":[0.5478,0.6415],"pass_all3_ci":[0.3303,0.4215],"benchmarks":{"deep-swe":{"cost_per_trial":0.1876,"cost_per_pass":0.3654,"tokens_per_trial":6158000.0,"calls_per_trial":79.41,"model_s_per_trial":906.4,"agent_s_per_trial":1209.0,"rerun_flip":0.3453,"pass_all3":0.2748,"pass_mean3":0.536,"pass_any3":0.7928,"n_trials":690,"n_scored":666,"n_pass":357,"n_infra":24,"cost_usd":129.5,"n_tasks3":222,"rate":0.536,"ci":[0.4722,0.5999],"pass_all3_ci":[0.2181,0.3315],"n_tasks":222,"n_tasks_expected":222},"terminal-bench-2.1":{"cost_per_trial":0.04562,"cost_per_pass":0.07222,"tokens_per_trial":954000.0,"calls_per_trial":24.02,"model_s_per_trial":368.4,"agent_s_per_trial":599.1,"rerun_flip":0.2107,"pass_all3":0.477,"pass_mean3":0.6533,"pass_any3":0.7931,"n_trials":524,"n_scored":522,"n_pass":341,"n_infra":2,"cost_usd":23.83,"n_tasks3":174,"rate":0.6533,"ci":[0.5848,0.7217],"pass_all3_ci":[0.4057,0.5483],"n_tasks":174,"n_tasks_expected":174}},"complete":true,"per_model":{"deepseek-ai/DeepSeek-V4.1-Flash":0.6692,"zai-org/GLM-5.3-Flash":0.5201},"efforts_requested":{"low":64577,"unset":845},"versions":["0.25.2"],"vs_leader":{"diff":-0.05978,"ci":[-0.1001,-0.02063],"tied":false,"n_tasks":198}}},"ranking":["opencode","claude-code","hermes","kimi-code","mini-swe-agent","deepseek-harness","pi","qwen-code","codex","openclaw"],"models":["deepseek-ai/DeepSeek-V4.1-Flash","zai-org/GLM-5.3-Flash"],"noise":{"rerun_flip":0.2662,"harness_flip":0.3177,"model_flip":0.3442,"by_benchmark":{"deep-swe":{"rerun_flip":0.3162,"harness_flip":0.3744,"model_flip":0.4018},"terminal-bench-2.1":{"rerun_flip":0.2161,"harness_flip":0.261,"model_flip":0.2866}},"harness_pairs":[{"harnesses":["opencode","claude-code"],"flip":0.279},{"harnesses":["opencode","hermes"],"flip":0.2984},{"harnesses":["opencode","kimi-code"],"flip":0.2777},{"harnesses":["opencode","mini-swe-agent"],"flip":0.2858},{"harnesses":["opencode","deepseek-harness"],"flip":0.281},{"harnesses":["opencode","pi"],"flip":0.2846},{"harnesses":["opencode","qwen-code"],"flip":0.3128},{"harnesses":["opencode","codex"],"flip":0.3138},{"harnesses":["opencode","openclaw"],"flip":0.4307},{"harnesses":["claude-code","hermes"],"flip":0.2797},{"harnesses":["claude-code","kimi-code"],"flip":0.2714},{"harnesses":["claude-code","mini-swe-agent"],"flip":0.2815},{"harnesses":["claude-code","deepseek-harness"],"flip":0.281},{"harnesses":["claude-code","pi"],"flip":0.2766},{"harnesses":["claude-code","qwen-code"],"flip":0.293},{"harnesses":["claude-code","codex"],"flip":0.3145},{"harnesses":["claude-code","openclaw"],"flip":0.4196},{"harnesses":["hermes","kimi-code"],"flip":0.286},{"harnesses":["hermes","mini-swe-agent"],"flip":0.2988},{"harnesses":["hermes","deepseek-harness"],"flip":0.2974},{"harnesses":["hermes","pi"],"flip":0.311},{"harnesses":["hermes","qwen-code"],"flip":0.3094},{"harnesses":["hermes","codex"],"flip":0.3153},{"harnesses":["hermes","openclaw"],"flip":0.4238},{"harnesses":["kimi-code","mini-swe-agent"],"flip":0.2817},{"harnesses":["kimi-code","deepseek-harness"],"flip":0.2745},{"harnesses":["kimi-code","pi"],"flip":0.2984},{"harnesses":["kimi-code","qwen-code"],"flip":0.2911},{"harnesses":["kimi-code","codex"],"flip":0.3021},{"harnesses":["kimi-code","openclaw"],"flip":0.4194},{"harnesses":["mini-swe-agent","deepseek-harness"],"flip":0.2798},{"harnesses":["mini-swe-agent","pi"],"flip":0.2913},{"harnesses":["mini-swe-agent","qwen-code"],"flip":0.2985},{"harnesses":["mini-swe-agent","codex"],"flip":0.3188},{"harnesses":["mini-swe-agent","openclaw"],"flip":0.411},{"harnesses":["deepseek-harness","pi"],"flip":0.3019},{"harnesses":["deepseek-harness","qwen-code"],"flip":0.2894},{"harnesses":["deepseek-harness","codex"],"flip":0.31},{"harnesses":["deepseek-harness","openclaw"],"flip":0.3928},{"harnesses":["pi","qwen-code"],"flip":0.3009},{"harnesses":["pi","codex"],"flip":0.3214},{"harnesses":["pi","openclaw"],"flip":0.4144},{"harnesses":["qwen-code","codex"],"flip":0.3061},{"harnesses":["qwen-code","openclaw"],"flip":0.3828},{"harnesses":["codex","openclaw"],"flip":0.3878}]},"leader":"opencode"}}}}}},"runs":[{"run_id":"20261111-004515-flint-full-default-399e","created_at":"2026-11-11T00:45:15.108873+00:00","model":"RecursiveSI/flint","provider":"hf-endpoint","stage":"full","reasoning_effort":"default","harnesses":["mini-swe-agent"],"harness_versions":{"mini-swe-agent":"2.4.6"},"benchmarks":["deep-swe"],"n_trials":333,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":2,"low":0.78,"high":1.61,"per_trial":{"deep-swe":{"mean":0.3883,"reserve":0.805,"n_observed":6626,"source":"observed"}}},"plan":{"deep-swe":{"n_tasks":111,"n_reused":331,"n_new":2,"n_units":2}},"code_revision":"27cac20fab79c58d238b77c2516a242306b4a1d5","harness_adapters":{"mini-swe-agent":null},"harness_locks":{"mini-swe-agent":null},"invalid":null,"state":"completed","benches":{"deep-swe":{"units":{"pending":0,"running":0,"done":2,"failed":0},"n_slots":2,"n_new":4,"n_scored":1,"n_reused":331,"cost_new_usd":0.12,"by_harness":{"mini-swe-agent":{"infra":3,"fail":1}}}},"n_new":4,"n_scored":1,"n_slots":2,"n_reused":331,"cost_new_usd":0.12,"updated_at":"2026-11-11T01:35:21.838638+00:00"},{"run_id":"20261110-160505-flint-full-default-7122","created_at":"2026-11-10T16:05:05.554415+00:00","model":"RecursiveSI/flint","provider":"hf-endpoint","stage":"full","reasoning_effort":"default","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":3,"low":0.81,"high":1.69,"per_trial":{"terminal-bench-2.1":{"mean":0.0381,"reserve":0.0847,"n_observed":4953,"source":"observed"},"deep-swe":{"mean":0.3883,"reserve":0.8043,"n_observed":6623,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":260,"n_new":1,"n_units":1},"deep-swe":{"n_tasks":111,"n_reused":331,"n_new":2,"n_units":1}},"code_revision":"385e5fdcf0c3f6fcbf4986b2f28fa4de6bea338f","harness_adapters":{"deepseek-harness":null},"harness_locks":{"deepseek-harness":null},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":1,"n_new":2,"n_scored":1,"n_reused":260,"cost_new_usd":0.06,"by_harness":{"deepseek-harness":{"infra":1,"fail":1}}},"deep-swe":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":2,"n_new":3,"n_scored":2,"n_reused":331,"cost_new_usd":0.9005,"by_harness":{"deepseek-harness":{"infra":1,"fail":2}}}},"n_new":5,"n_scored":3,"n_slots":3,"n_reused":591,"cost_new_usd":0.9605,"updated_at":"2026-11-10T17:25:57.547374+00:00"},{"run_id":"20261110-155020-glm-5-3-flash-full-high-cbd8","created_at":"2026-11-10T15:50:20.164798+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":2,"low":0.73,"high":1.45,"per_trial":{"terminal-bench-2.1":{"mean":0.0441,"reserve":0.1064,"n_observed":5448,"source":"observed"},"deep-swe":{"mean":0.3674,"reserve":0.7228,"n_observed":6558,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":261,"n_new":0,"n_units":0},"deep-swe":{"n_tasks":111,"n_reused":331,"n_new":2,"n_units":1}},"code_revision":"385e5fdcf0c3f6fcbf4986b2f28fa4de6bea338f","harness_adapters":{"deepseek-harness":null},"harness_locks":{"deepseek-harness":null},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":261,"cost_new_usd":0,"by_harness":{}},"deep-swe":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":2,"n_new":5,"n_scored":1,"n_reused":331,"cost_new_usd":0.3628,"by_harness":{"deepseek-harness":{"infra":4,"pass":1}}}},"n_new":5,"n_scored":1,"n_slots":2,"n_reused":592,"cost_new_usd":0.3628,"updated_at":"2026-11-10T18:19:07.558225+00:00"},{"run_id":"20261110-100133-deepseek-v4-1-flash-full-high-7357","created_at":"2026-11-10T10:01:33.720997+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1"],"n_trials":261,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":1,"low":0.04,"high":0.09,"per_trial":{"terminal-bench-2.1":{"mean":0.0389,"reserve":0.0933,"n_observed":3601,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":260,"n_new":1,"n_units":1}},"code_revision":"385e5fdcf0c3f6fcbf4986b2f28fa4de6bea338f","harness_adapters":{"deepseek-harness":null},"harness_locks":{"deepseek-harness":null},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":1,"n_new":1,"n_scored":1,"n_reused":260,"cost_new_usd":0.0152,"by_harness":{"deepseek-harness":{"fail":1}}}},"n_new":1,"n_scored":1,"n_slots":1,"n_reused":260,"cost_new_usd":0.0152,"updated_at":"2026-11-10T10:08:16.027647+00:00"},{"run_id":"20261110-082921-glm-5-3-flash-full-high-8e46","created_at":"2026-11-10T08:29:21.132618+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1"],"n_trials":261,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":1,"low":0.04,"high":0.11,"per_trial":{"terminal-bench-2.1":{"mean":0.0441,"reserve":0.1064,"n_observed":5446,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":260,"n_new":1,"n_units":1}},"code_revision":"385e5fdcf0c3f6fcbf4986b2f28fa4de6bea338f","harness_adapters":{"deepseek-harness":null},"harness_locks":{"deepseek-harness":null},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":1,"n_new":2,"n_scored":1,"n_reused":260,"cost_new_usd":0.0088,"by_harness":{"deepseek-harness":{"infra":1,"fail":1}}}},"n_new":2,"n_scored":1,"n_slots":1,"n_reused":260,"cost_new_usd":0.0088,"updated_at":"2026-11-10T09:33:48.467244+00:00"},{"run_id":"20261110-015037-deepseek-v4-1-flash-full-low-b146","created_at":"2026-11-10T01:50:37.996915+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["mini-swe-agent"],"harness_versions":{"mini-swe-agent":"2.4.6"},"benchmarks":["deep-swe"],"n_trials":333,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":5,"low":0.84,"high":1.55,"per_trial":{"deep-swe":{"mean":0.1682,"reserve":0.3107,"n_observed":4147,"source":"observed"}}},"plan":{"deep-swe":{"n_tasks":111,"n_reused":328,"n_new":5,"n_units":4}},"code_revision":"c694e3286e24bb710316fd9e6509f0df96561a1b","harness_adapters":{"mini-swe-agent":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"completed","benches":{"deep-swe":{"units":{"pending":0,"running":0,"done":4,"failed":0},"n_slots":5,"n_new":5,"n_scored":5,"n_reused":328,"cost_new_usd":0.7546,"by_harness":{"mini-swe-agent":{"pass":3,"fail":2}}}},"n_new":5,"n_scored":5,"n_slots":5,"n_reused":328,"cost_new_usd":0.7546,"updated_at":"2026-11-10T02:24:05.021913+00:00"},{"run_id":"20261110-005013-glm-5-3-flash-full-high-e836","created_at":"2026-11-10T00:50:13.592925+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1"],"n_trials":261,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":7,"low":0.31,"high":0.74,"per_trial":{"terminal-bench-2.1":{"mean":0.0438,"reserve":0.106,"n_observed":5438,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":254,"n_new":7,"n_units":4}},"code_revision":"c694e3286e24bb710316fd9e6509f0df96561a1b","harness_adapters":{"deepseek-harness":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":4,"running":0,"done":0,"failed":0},"n_slots":7,"n_new":0,"n_scored":0,"n_reused":254,"cost_new_usd":0,"by_harness":{}}},"n_new":0,"n_scored":0,"n_slots":7,"n_reused":254,"cost_new_usd":0,"updated_at":"2026-11-10T00:51:05.004275+00:00"},{"run_id":"20261110-004920-glm-5-3-flash-full-high-4268","created_at":"2026-11-10T00:49:20.047953+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1"],"n_trials":261,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":7,"low":0.31,"high":0.74,"per_trial":{"terminal-bench-2.1":{"mean":0.0438,"reserve":0.106,"n_observed":5438,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":254,"n_new":7,"n_units":4}},"code_revision":"c694e3286e24bb710316fd9e6509f0df96561a1b","harness_adapters":{"deepseek-harness":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":4,"failed":0},"n_slots":7,"n_new":7,"n_scored":7,"n_reused":254,"cost_new_usd":1.945,"by_harness":{"deepseek-harness":{"pass":3,"fail":4}}}},"n_new":7,"n_scored":7,"n_slots":7,"n_reused":254,"cost_new_usd":1.945,"updated_at":"2026-11-10T02:39:47.236611+00:00"},{"run_id":"20261109-223043-deepseek-v4-1-flash-full-high-695c","created_at":"2026-11-09T22:30:43.812461+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":20.0,"estimate_usd":{"new_trials":23,"low":3.05,"high":5.98,"per_trial":{"terminal-bench-2.1":{"mean":0.037,"reserve":0.0907,"n_observed":3067,"source":"observed"},"deep-swe":{"mean":0.2061,"reserve":0.3906,"n_observed":4058,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":251,"n_new":10,"n_units":5},"deep-swe":{"n_tasks":111,"n_reused":320,"n_new":13,"n_units":8}},"code_revision":"ab75e592d94214a8898290b594db869ce46acf98","harness_adapters":{"deepseek-harness":null},"harness_locks":{},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":5,"failed":0},"n_slots":10,"n_new":10,"n_scored":10,"n_reused":251,"cost_new_usd":1.032,"by_harness":{"deepseek-harness":{"pass":5,"fail":5}}},"deep-swe":{"units":{"pending":0,"running":0,"done":8,"failed":0},"n_slots":13,"n_new":13,"n_scored":13,"n_reused":320,"cost_new_usd":2.402,"by_harness":{"deepseek-harness":{"pass":9,"fail":4}}}},"n_new":23,"n_scored":23,"n_slots":23,"n_reused":571,"cost_new_usd":3.434,"updated_at":"2026-11-10T00:26:45.635075+00:00"},{"run_id":"20261109-220831-glm-5-3-flash-full-high-5a07","created_at":"2026-11-09T22:08:31.989154+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["mini-swe-agent"],"harness_versions":{"mini-swe-agent":"2.4.6"},"benchmarks":["terminal-bench-2.1"],"n_trials":261,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":1,"low":0.04,"high":0.1,"per_trial":{"terminal-bench-2.1":{"mean":0.0431,"reserve":0.1044,"n_observed":5416,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":260,"n_new":1,"n_units":1}},"code_revision":"ab75e592d94214a8898290b594db869ce46acf98","harness_adapters":{"mini-swe-agent":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":1,"n_new":1,"n_scored":1,"n_reused":260,"cost_new_usd":0.5401,"by_harness":{"mini-swe-agent":{"fail":1}}}},"n_new":1,"n_scored":1,"n_slots":1,"n_reused":260,"cost_new_usd":0.5401,"updated_at":"2026-11-09T22:46:00.828410+00:00"},{"run_id":"20261109-220819-glm-5-3-flash-full-low-a654","created_at":"2026-11-09T22:08:19.820953+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"low","harnesses":["claude-code","mini-swe-agent"],"harness_versions":{"claude-code":"2.1.300","mini-swe-agent":"2.4.6"},"benchmarks":["terminal-bench-2.1"],"n_trials":522,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":4,"low":0.16,"high":0.37,"per_trial":{"terminal-bench-2.1":{"mean":0.0407,"reserve":0.0925,"n_observed":5393,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":518,"n_new":4,"n_units":3}},"code_revision":"ab75e592d94214a8898290b594db869ce46acf98","harness_adapters":{"claude-code":null,"mini-swe-agent":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":3,"failed":0},"n_slots":4,"n_new":4,"n_scored":4,"n_reused":518,"cost_new_usd":0.7621,"by_harness":{"claude-code":{"pass":1,"fail":1},"mini-swe-agent":{"fail":2}}}},"n_new":4,"n_scored":4,"n_slots":4,"n_reused":518,"cost_new_usd":0.7621,"updated_at":"2026-11-09T22:44:58.601506+00:00"},{"run_id":"20261109-220732-deepseek-v4-1-flash-full-low-d1a0","created_at":"2026-11-09T22:07:32.858715+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":20.0,"estimate_usd":{"new_trials":32,"low":3.91,"high":7.44,"per_trial":{"terminal-bench-2.1":{"mean":0.0308,"reserve":0.0755,"n_observed":3030,"source":"observed"},"deep-swe":{"mean":0.1699,"reserve":0.3147,"n_observed":3942,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":250,"n_new":11,"n_units":8},"deep-swe":{"n_tasks":111,"n_reused":312,"n_new":21,"n_units":11}},"code_revision":"ab75e592d94214a8898290b594db869ce46acf98","harness_adapters":{"deepseek-harness":null},"harness_locks":{},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":8,"failed":0},"n_slots":11,"n_new":11,"n_scored":11,"n_reused":250,"cost_new_usd":0.3474,"by_harness":{"deepseek-harness":{"pass":8,"fail":3}}},"deep-swe":{"units":{"pending":0,"running":0,"done":11,"failed":0},"n_slots":21,"n_new":23,"n_scored":21,"n_reused":312,"cost_new_usd":3.071,"by_harness":{"deepseek-harness":{"fail":11,"pass":10,"infra":2}}}},"n_new":34,"n_scored":32,"n_slots":32,"n_reused":562,"cost_new_usd":3.419,"updated_at":"2026-11-10T00:13:34.131278+00:00"},{"run_id":"20261109-213916-deepseek-v4-1-flash-full-high-80fb","created_at":"2026-11-09T21:39:16.975312+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":1,"low":0.21,"high":0.39,"per_trial":{"terminal-bench-2.1":{"mean":0.037,"reserve":0.0907,"n_observed":3058,"source":"observed"},"deep-swe":{"mean":0.2062,"reserve":0.3918,"n_observed":4028,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":261,"n_new":0,"n_units":0},"deep-swe":{"n_tasks":111,"n_reused":332,"n_new":1,"n_units":1}},"code_revision":"ab75e592d94214a8898290b594db869ce46acf98","harness_adapters":{"deepseek-harness":null},"harness_locks":{},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":261,"cost_new_usd":0,"by_harness":{}},"deep-swe":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":1,"n_new":1,"n_scored":1,"n_reused":332,"cost_new_usd":0.2139,"by_harness":{"deepseek-harness":{"pass":1}}}},"n_new":1,"n_scored":1,"n_slots":1,"n_reused":593,"cost_new_usd":0.2139,"updated_at":"2026-11-09T22:24:14.123616+00:00"},{"run_id":"20261109-193018-flint-full-default-6c10","created_at":"2026-11-09T19:30:18.357394+00:00","model":"RecursiveSI/flint","provider":"hf-endpoint","stage":"full","reasoning_effort":"default","harnesses":["pi","claude-code","kimi-code"],"harness_versions":{"pi":"1.2.0","claude-code":"2.1.300","kimi-code":"2.2.0"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":2376,"cost_cap_usd":20.0,"estimate_usd":{"new_trials":10,"low":2.42,"high":5.3,"per_trial":{"terminal-bench-2.1":{"mean":0.0382,"reserve":0.0843,"n_observed":4679,"source":"observed"},"deep-swe":{"mean":0.3787,"reserve":0.8277,"n_observed":6026,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":1040,"n_new":4,"n_units":2},"deep-swe":{"n_tasks":111,"n_reused":1326,"n_new":6,"n_units":4}},"code_revision":"91872c3e3899591730bad2c7d6e0db8561bd4c87","harness_adapters":{"pi":null,"claude-code":null,"kimi-code":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":2,"failed":0},"n_slots":4,"n_new":4,"n_scored":4,"n_reused":1040,"cost_new_usd":0.1176,"by_harness":{"claude-code":{"pass":1},"pi":{"fail":3}}},"deep-swe":{"units":{"pending":0,"running":0,"done":4,"failed":0},"n_slots":6,"n_new":9,"n_scored":5,"n_reused":1326,"cost_new_usd":1.421,"by_harness":{"kimi-code":{"fail":2,"infra":4},"pi":{"fail":2}}}},"n_new":13,"n_scored":9,"n_slots":10,"n_reused":2366,"cost_new_usd":1.538,"updated_at":"2026-11-09T23:11:38.009026+00:00"},{"run_id":"20261109-192407-glm-5-3-flash-full-high-2936","created_at":"2026-11-09T19:24:07.710070+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":2,"low":0.67,"high":1.26,"per_trial":{"terminal-bench-2.1":{"mean":0.0406,"reserve":0.0995,"n_observed":5108,"source":"observed"},"deep-swe":{"mean":0.3327,"reserve":0.6325,"n_observed":5891,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":261,"n_new":0,"n_units":0},"deep-swe":{"n_tasks":111,"n_reused":331,"n_new":2,"n_units":1}},"code_revision":"91872c3e3899591730bad2c7d6e0db8561bd4c87","harness_adapters":{"deepseek-harness":null},"harness_locks":{},"invalid":null,"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":261,"cost_new_usd":0,"by_harness":{}},"deep-swe":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":2,"n_new":6,"n_scored":0,"n_reused":331,"cost_new_usd":0.3163,"by_harness":{"deepseek-harness":{"infra":5,"cancelled":1}}}},"n_new":6,"n_scored":0,"n_slots":2,"n_reused":592,"cost_new_usd":0.3163,"updated_at":"2026-11-10T00:47:58.497655+00:00"},{"run_id":"20261109-164117-deepseek-v4-1-flash-full-low-6672","created_at":"2026-11-09T16:41:17.040399+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["codex"],"harness_versions":{"codex":"0.162.1"},"benchmarks":["terminal-bench-2.1"],"n_trials":261,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":2,"low":0.06,"high":0.16,"per_trial":{"terminal-bench-2.1":{"mean":0.0321,"reserve":0.078,"n_observed":3729,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":259,"n_new":2,"n_units":2}},"code_revision":"91872c3e3899591730bad2c7d6e0db8561bd4c87","harness_adapters":{"codex":"responses-bridge@7011f496679d"},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":2,"failed":0},"n_slots":2,"n_new":2,"n_scored":2,"n_reused":259,"cost_new_usd":0.1001,"by_harness":{"codex":{"pass":1,"fail":1}}}},"n_new":2,"n_scored":2,"n_slots":2,"n_reused":259,"cost_new_usd":0.1001,"updated_at":"2026-11-09T19:59:39.664620+00:00"},{"run_id":"20261109-164104-deepseek-v4-1-flash-full-high-1a2f","created_at":"2026-11-09T16:41:04.291665+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["mini-swe-agent"],"harness_versions":{"mini-swe-agent":"2.4.6"},"benchmarks":["terminal-bench-2.1"],"n_trials":261,"cost_cap_usd":5.0,"estimate_usd":{"new_trials":1,"low":0.04,"high":0.09,"per_trial":{"terminal-bench-2.1":{"mean":0.039,"reserve":0.0929,"n_observed":3775,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":260,"n_new":1,"n_units":1}},"code_revision":"91872c3e3899591730bad2c7d6e0db8561bd4c87","harness_adapters":{"mini-swe-agent":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":1,"n_new":1,"n_scored":1,"n_reused":260,"cost_new_usd":0.1082,"by_harness":{"mini-swe-agent":{"pass":1}}}},"n_new":1,"n_scored":1,"n_slots":1,"n_reused":260,"cost_new_usd":0.1082,"updated_at":"2026-11-09T20:58:55.244024+00:00"},{"run_id":"20261109-164059-deepseek-v4-1-flash-full-low-9bdf","created_at":"2026-11-09T16:40:59.209442+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","mini-swe-agent","pi"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","mini-swe-agent":"2.4.6","pi":"1.2.0"},"benchmarks":["deep-swe"],"n_trials":1332,"cost_cap_usd":100.0,"estimate_usd":{"new_trials":141,"low":22.76,"high":41.17,"per_trial":{"deep-swe":{"mean":0.1614,"reserve":0.292,"n_observed":4868,"source":"observed"}}},"plan":{"deep-swe":{"n_tasks":111,"n_reused":1191,"n_new":141,"n_units":62}},"code_revision":"91872c3e3899591730bad2c7d6e0db8561bd4c87","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","mini-swe-agent":null,"pi":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"completed","benches":{"deep-swe":{"units":{"pending":0,"running":0,"done":62,"failed":0},"n_slots":141,"n_new":143,"n_scored":141,"n_reused":1191,"cost_new_usd":22.62,"by_harness":{"claude-code":{"pass":10,"fail":2},"mini-swe-agent":{"pass":57,"fail":18},"pi":{"pass":34,"fail":18,"infra":2},"codex":{"fail":2}}}},"n_new":143,"n_scored":141,"n_slots":141,"n_reused":1191,"cost_new_usd":22.62,"updated_at":"2026-11-10T01:30:15.617371+00:00"},{"run_id":"20261109-164053-deepseek-v4-1-flash-full-high-0e68","created_at":"2026-11-09T16:40:53.517496+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["claude-code","codex","mini-swe-agent","pi"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","mini-swe-agent":"2.4.6","pi":"1.2.0"},"benchmarks":["deep-swe"],"n_trials":1332,"cost_cap_usd":150.0,"estimate_usd":{"new_trials":184,"low":36.78,"high":68.23,"per_trial":{"deep-swe":{"mean":0.1999,"reserve":0.3708,"n_observed":4902,"source":"observed"}}},"plan":{"deep-swe":{"n_tasks":111,"n_reused":1148,"n_new":184,"n_units":73}},"code_revision":"91872c3e3899591730bad2c7d6e0db8561bd4c87","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","mini-swe-agent":null,"pi":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"completed","benches":{"deep-swe":{"units":{"pending":0,"running":0,"done":73,"failed":0},"n_slots":184,"n_new":186,"n_scored":184,"n_reused":1148,"cost_new_usd":36.72,"by_harness":{"mini-swe-agent":{"pass":83,"fail":24},"pi":{"pass":42,"fail":19,"infra":1},"claude-code":{"pass":7,"fail":5},"codex":{"fail":3,"pass":1,"infra":1}}}},"n_new":186,"n_scored":184,"n_slots":184,"n_reused":1148,"cost_new_usd":36.72,"updated_at":"2026-11-10T00:34:21.348301+00:00"},{"run_id":"20261109-145011-deepseek-v4-1-flash-full-low-8515","created_at":"2026-11-09T14:50:11.954388+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["codex","hermes","kimi-code","pi"],"harness_versions":{"codex":"0.162.1","hermes":"v0.22.1","kimi-code":"2.2.0","pi":"1.2.0"},"benchmarks":["terminal-bench-2.1"],"n_trials":1044,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":6,"low":0.2,"high":0.48,"per_trial":{"terminal-bench-2.1":{"mean":0.0333,"reserve":0.0801,"n_observed":3921,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":1038,"n_new":6,"n_units":4}},"code_revision":"8e0b575ac9376a670a3fdd1978a33f7e273affb5","harness_adapters":{"codex":"responses-bridge@7011f496679d","hermes":null,"kimi-code":null,"pi":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":4,"running":0,"done":0,"failed":0},"n_slots":6,"n_new":0,"n_scored":0,"n_reused":1038,"cost_new_usd":0,"by_harness":{}}},"n_new":0,"n_scored":0,"n_slots":6,"n_reused":1038,"cost_new_usd":0,"updated_at":"2026-11-09T16:32:55.119529+00:00"},{"run_id":"20261109-113521-deepseek-v4-1-flash-full-low-39eb","created_at":"2026-11-09T11:35:21.506012+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","hermes","kimi-code","pi"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","hermes":"v0.22.1","kimi-code":"2.2.0","pi":"1.2.0"},"benchmarks":["terminal-bench-2.1"],"n_trials":1305,"cost_cap_usd":5.0,"estimate_usd":{"new_trials":6,"low":0.2,"high":0.48,"per_trial":{"terminal-bench-2.1":{"mean":0.0332,"reserve":0.0795,"n_observed":3811,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":1299,"n_new":6,"n_units":4}},"code_revision":"647d98743a5af9f3bd0cad2c326b5be98b4d33ff","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","hermes":null,"kimi-code":null,"pi":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":4,"running":0,"done":0,"failed":0},"n_slots":6,"n_new":0,"n_scored":0,"n_reused":1299,"cost_new_usd":0,"by_harness":{}}},"n_new":0,"n_scored":0,"n_slots":6,"n_reused":1299,"cost_new_usd":0,"updated_at":"2026-11-09T14:48:05.224824+00:00"},{"run_id":"20261109-113417-deepseek-v4-1-flash-full-high-dd52","created_at":"2026-11-09T11:34:17.556881+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["hermes","kimi-code","mini-swe-agent"],"harness_versions":{"hermes":"v0.22.1","kimi-code":"2.2.0","mini-swe-agent":"2.4.6"},"benchmarks":["terminal-bench-2.1"],"n_trials":783,"cost_cap_usd":5.0,"estimate_usd":{"new_trials":5,"low":0.2,"high":0.48,"per_trial":{"terminal-bench-2.1":{"mean":0.0404,"reserve":0.0965,"n_observed":3848,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":778,"n_new":5,"n_units":5}},"code_revision":"647d98743a5af9f3bd0cad2c326b5be98b4d33ff","harness_adapters":{"hermes":null,"kimi-code":null,"mini-swe-agent":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":4,"running":0,"done":1,"failed":0},"n_slots":5,"n_new":1,"n_scored":1,"n_reused":778,"cost_new_usd":0.0562,"by_harness":{"hermes":{"pass":1}}}},"n_new":1,"n_scored":1,"n_slots":5,"n_reused":778,"cost_new_usd":0.0562,"updated_at":"2026-11-09T16:32:55.117885+00:00"},{"run_id":"20261109-113343-glm-5-3-flash-full-low-d4d8","created_at":"2026-11-09T11:33:43.583320+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","mini-swe-agent","pi","qwen-code"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","mini-swe-agent":"2.4.6","pi":"1.2.0","qwen-code":"0.25.2"},"benchmarks":["terminal-bench-2.1"],"n_trials":1305,"cost_cap_usd":5.0,"estimate_usd":{"new_trials":10,"low":0.38,"high":0.86,"per_trial":{"terminal-bench-2.1":{"mean":0.0375,"reserve":0.086,"n_observed":4989,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":1295,"n_new":10,"n_units":5}},"code_revision":"647d98743a5af9f3bd0cad2c326b5be98b4d33ff","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","mini-swe-agent":null,"pi":null,"qwen-code":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["adaptive-rejection-sampler__KeXYg5K","make-doom-for-mips__7tLaAsr","make-doom-for-mips__A5C72FT","make-mips-interpreter__LQHyRvX"],"outcome":"infra","reason":"Ended on a provider error inside the stream: the last model call's stream, answered with status 200, ended with the router's error event (finish_reason 'error', 'JSON error injected into SSE stream'; docs/incidents/provider-stream-errors.json); a provider failure like a stream that broke off, run again, pass or fail alike (meter.stream_error, 2026-11-09).","from_release":"release-time-reruns"}],"reason":""},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":5,"failed":0},"n_slots":10,"n_new":10,"n_scored":10,"n_reused":1295,"cost_new_usd":1.525,"by_harness":{"claude-code":{"fail":2},"mini-swe-agent":{"fail":4},"pi":{"fail":2},"codex":{"fail":1},"qwen-code":{"pass":1}}}},"n_new":10,"n_scored":10,"n_slots":10,"n_reused":1295,"cost_new_usd":1.525,"updated_at":"2026-11-09T14:43:01.667751+00:00"},{"run_id":"20261109-113239-glm-5-3-flash-full-high-e1b4","created_at":"2026-11-09T11:32:39.737398+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["claude-code","hermes","kimi-code","mini-swe-agent","pi","qwen-code"],"harness_versions":{"claude-code":"2.1.300","hermes":"v0.22.1","kimi-code":"2.2.0","mini-swe-agent":"2.4.6","pi":"1.2.0","qwen-code":"0.25.2"},"benchmarks":["terminal-bench-2.1"],"n_trials":1566,"cost_cap_usd":5.0,"estimate_usd":{"new_trials":21,"low":0.84,"high":2.02,"per_trial":{"terminal-bench-2.1":{"mean":0.04,"reserve":0.096,"n_observed":4973,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":1545,"n_new":21,"n_units":10}},"code_revision":"647d98743a5af9f3bd0cad2c326b5be98b4d33ff","harness_adapters":{"claude-code":null,"hermes":null,"kimi-code":null,"mini-swe-agent":null,"pi":null,"qwen-code":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["make-mips-interpreter__gmzCenj"],"outcome":"infra","reason":"Ended on a provider error inside the stream: the last model call's stream, answered with status 200, ended with the router's error event (finish_reason 'error', 'JSON error injected into SSE stream'; docs/incidents/provider-stream-errors.json); a provider failure like a stream that broke off, run again, pass or fail alike (meter.stream_error, 2026-11-09).","from_release":"release-time-reruns"}],"reason":""},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":10,"failed":0},"n_slots":21,"n_new":21,"n_scored":21,"n_reused":1545,"cost_new_usd":2.18,"by_harness":{"mini-swe-agent":{"fail":3,"pass":2},"qwen-code":{"fail":2},"claude-code":{"fail":4},"hermes":{"fail":2},"kimi-code":{"pass":2,"fail":2},"pi":{"fail":3,"pass":1}}}},"n_new":21,"n_scored":21,"n_slots":21,"n_reused":1545,"cost_new_usd":2.18,"updated_at":"2026-11-09T15:00:57.479487+00:00"},{"run_id":"20261109-112518-deepseek-v4-1-flash-full-low-058b","created_at":"2026-11-09T11:25:18.503637+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","hermes","kimi-code","mini-swe-agent","pi","qwen-code"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","hermes":"v0.22.1","kimi-code":"2.2.0","mini-swe-agent":"2.4.6","pi":"1.2.0","qwen-code":"0.25.2"},"benchmarks":["deep-swe"],"n_trials":2331,"cost_cap_usd":100.0,"estimate_usd":{"new_trials":270,"low":45.55,"high":86.16,"per_trial":{"deep-swe":{"mean":0.1687,"reserve":0.3191,"n_observed":5041,"source":"observed"}}},"plan":{"deep-swe":{"n_tasks":111,"n_reused":2061,"n_new":270,"n_units":93}},"code_revision":"647d98743a5af9f3bd0cad2c326b5be98b4d33ff","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","hermes":null,"kimi-code":null,"mini-swe-agent":null,"pi":null,"qwen-code":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":{"rules":[{"benchmarks":["deep-swe"],"trials":["adaptix-name-mapping-aliases__qxdaWRQ","aiomonitor-task-snapshots-diff__fciqqj4","aiomonitor-task-snapshots-diff__whMEzij","anko-default-function-arguments__HuhHtNU","arktype-json-schema-refs-depende__Y6dDp3m"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08).","from_release":"release-time-reruns"}],"reason":""},"state":"cancelled","benches":{"deep-swe":{"units":{"pending":81,"running":0,"done":12,"failed":0},"n_slots":270,"n_new":19,"n_scored":19,"n_reused":2061,"cost_new_usd":1.345,"by_harness":{"hermes":{"pass":4,"fail":6},"kimi-code":{"fail":1},"mini-swe-agent":{"fail":6},"qwen-code":{"pass":1},"pi":{"pass":1}}}},"n_new":19,"n_scored":19,"n_slots":270,"n_reused":2061,"cost_new_usd":1.345,"updated_at":"2026-11-09T16:32:55.117738+00:00"},{"run_id":"20261109-112451-deepseek-v4-1-flash-full-high-6bd0","created_at":"2026-11-09T11:24:51.720098+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["claude-code","codex","hermes","kimi-code","mini-swe-agent","pi","qwen-code"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","hermes":"v0.22.1","kimi-code":"2.2.0","mini-swe-agent":"2.4.6","pi":"1.2.0","qwen-code":"0.25.2"},"benchmarks":["deep-swe"],"n_trials":2331,"cost_cap_usd":150.0,"estimate_usd":{"new_trials":335,"low":70.52,"high":136.9,"per_trial":{"deep-swe":{"mean":0.2105,"reserve":0.4087,"n_observed":5095,"source":"observed"}}},"plan":{"deep-swe":{"n_tasks":111,"n_reused":1996,"n_new":335,"n_units":93}},"code_revision":"647d98743a5af9f3bd0cad2c326b5be98b4d33ff","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","hermes":null,"kimi-code":null,"mini-swe-agent":null,"pi":null,"qwen-code":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"cancelled","benches":{"deep-swe":{"units":{"pending":88,"running":0,"done":5,"failed":0},"n_slots":335,"n_new":7,"n_scored":7,"n_reused":1996,"cost_new_usd":1.264,"by_harness":{"hermes":{"pass":3,"fail":1},"mini-swe-agent":{"pass":2},"kimi-code":{"pass":1}}}},"n_new":7,"n_scored":7,"n_slots":335,"n_reused":1996,"cost_new_usd":1.264,"updated_at":"2026-11-09T16:32:55.117369+00:00"},{"run_id":"20261109-111048-glm-5-3-flash-full-low-e58b","created_at":"2026-11-09T11:10:48.357226+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","hermes","kimi-code","mini-swe-agent","pi"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","hermes":"v0.22.1","kimi-code":"2.2.0","mini-swe-agent":"2.4.6","pi":"1.2.0"},"benchmarks":["deep-swe"],"n_trials":1998,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":11,"low":2.79,"high":5.16,"per_trial":{"deep-swe":{"mean":0.2535,"reserve":0.4692,"n_observed":6029,"source":"observed"}}},"plan":{"deep-swe":{"n_tasks":111,"n_reused":1987,"n_new":11,"n_units":7}},"code_revision":"647d98743a5af9f3bd0cad2c326b5be98b4d33ff","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","hermes":null,"kimi-code":null,"mini-swe-agent":null,"pi":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"completed","benches":{"deep-swe":{"units":{"pending":0,"running":0,"done":7,"failed":0},"n_slots":11,"n_new":11,"n_scored":11,"n_reused":1987,"cost_new_usd":2.062,"by_harness":{"hermes":{"fail":1},"kimi-code":{"fail":2,"pass":1},"pi":{"pass":1},"mini-swe-agent":{"pass":2},"claude-code":{"pass":2},"codex":{"fail":2}}}},"n_new":11,"n_scored":11,"n_slots":11,"n_reused":1987,"cost_new_usd":2.062,"updated_at":"2026-11-09T14:56:57.046954+00:00"},{"run_id":"20261109-110747-glm-5-3-flash-full-high-3cc9","created_at":"2026-11-09T11:07:47.703669+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["claude-code","codex","hermes","kimi-code","mini-swe-agent","pi"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","hermes":"v0.22.1","kimi-code":"2.2.0","mini-swe-agent":"2.4.6","pi":"1.2.0"},"benchmarks":["deep-swe"],"n_trials":1998,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":16,"low":5.49,"high":10.7,"per_trial":{"deep-swe":{"mean":0.3433,"reserve":0.6689,"n_observed":5971,"source":"observed"}}},"plan":{"deep-swe":{"n_tasks":111,"n_reused":1982,"n_new":16,"n_units":11}},"code_revision":"647d98743a5af9f3bd0cad2c326b5be98b4d33ff","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","hermes":null,"kimi-code":null,"mini-swe-agent":null,"pi":null},"harness_locks":{},"from_release":"release-time-reruns","invalid":null,"state":"completed","benches":{"deep-swe":{"units":{"pending":0,"running":0,"done":11,"failed":0},"n_slots":16,"n_new":16,"n_scored":16,"n_reused":1982,"cost_new_usd":6.101,"by_harness":{"mini-swe-agent":{"fail":2,"pass":1},"hermes":{"pass":1,"fail":2},"kimi-code":{"fail":4,"pass":1},"claude-code":{"pass":2},"pi":{"pass":2},"codex":{"fail":1}}}},"n_new":16,"n_scored":16,"n_slots":16,"n_reused":1982,"cost_new_usd":6.101,"updated_at":"2026-11-09T13:28:55.173935+00:00"},{"run_id":"20261109-093753-flint-full-default-e893","created_at":"2026-11-09T09:37:53.911828+00:00","model":"RecursiveSI/flint","provider":"hf-endpoint","stage":"full","reasoning_effort":"default","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":500.0,"estimate_usd":{"new_trials":594,"low":141.8,"high":301.2,"per_trial":{"terminal-bench-2.1":{"mean":0.0375,"reserve":0.0835,"n_observed":4345,"source":"observed"},"deep-swe":{"mean":0.3963,"reserve":0.839,"n_observed":5683,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":261,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":333,"n_units":111}},"code_revision":"39d3f3d127111b798c0278c10df2897ad737ac62","harness_adapters":{"deepseek-harness":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__ApFTTHZ","fix-ocaml-gc__Xyc8hv2","fix-ocaml-gc__jpBEzEK"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":261,"n_new":305,"n_scored":260,"n_reused":0,"cost_new_usd":15.85,"by_harness":{"deepseek-harness":{"fail":171,"infra":45,"pass":89}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":333,"n_new":413,"n_scored":331,"n_reused":0,"cost_new_usd":111.3,"by_harness":{"deepseek-harness":{"fail":303,"infra":82,"pass":28}}}},"n_new":718,"n_scored":591,"n_slots":594,"n_reused":0,"cost_new_usd":127.1,"updated_at":"2026-11-09T16:40:48.270747+00:00"},{"run_id":"20261109-093728-deepseek-v4-1-flash-full-high-d9d2","created_at":"2026-11-09T09:37:28.613842+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":150.0,"estimate_usd":{"new_trials":594,"low":80.81,"high":162.4,"per_trial":{"terminal-bench-2.1":{"mean":0.0403,"reserve":0.0966,"n_observed":3782,"source":"observed"},"deep-swe":{"mean":0.2111,"reserve":0.4119,"n_observed":5035,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":261,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":333,"n_units":111}},"code_revision":"39d3f3d127111b798c0278c10df2897ad737ac62","harness_adapters":{"deepseek-harness":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["compile-compcert__5JMLUjv","count-dataset-tokens__EkDUbKQ","fix-ocaml-gc__gHt5Swo","fix-ocaml-gc__k93dvMX","fix-ocaml-gc__qMR6fju","raman-fitting__V8WEbYq"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["deep-swe"],"trials":["go-git-worktree-merge-conflicts__QSryQUw","goreleaser-retry-publish-auditin__2CafNtF","happy-dom-abort-pending-body-rea__gBvEwmH","happy-dom-abort-pending-body-rea__inCzqsk","igel-persist-feature-schema__HP2jSgq","igel-persist-feature-schema__adhJUJZ","ipython-session-bundle-replay__YZq4Go2","ipython-session-bundle-replay__an8VwCi","ipython-session-bundle-replay__knffoiT","kcp-go-multiplexed-kcp-streams__X9ViGdF","kcp-go-multiplexed-kcp-streams__n7dNiMq","kombu-single-active-consumer-pri__tB4qC2R","koota-deferred-mutation-buffer__EFes39w"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08)."},{"benchmarks":["terminal-bench-2.1"],"trials":["largest-eigenval__dV3gCcY","llm-inference-batching-scheduler__hAPptKQ","make-mips-interpreter__Z8zxoTJ","make-mips-interpreter__dFWsnEq","make-mips-interpreter__yCgiDhp","model-extraction-relu-logits__FxMUHPm","model-extraction-relu-logits__Xx2SqvA","model-extraction-relu-logits__gT2Xka5","mteb-retrieve__a29fhHp","mteb-retrieve__jzta4uT","mteb-retrieve__pTK4acc"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08)."}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30).; Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":261,"n_new":266,"n_scored":261,"n_reused":0,"cost_new_usd":9.855,"by_harness":{"deepseek-harness":{"pass":195,"fail":66,"infra":5}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":333,"n_new":349,"n_scored":332,"n_reused":0,"cost_new_usd":69.07,"by_harness":{"deepseek-harness":{"pass":220,"infra":17,"fail":112}}}},"n_new":615,"n_scored":593,"n_slots":594,"n_reused":0,"cost_new_usd":78.92,"updated_at":"2026-11-09T21:25:19.411464+00:00"},{"run_id":"20261109-093723-deepseek-v4-1-flash-full-low-4a8e","created_at":"2026-11-09T09:37:23.192595+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":150.0,"estimate_usd":{"new_trials":594,"low":64.98,"high":128.0,"per_trial":{"terminal-bench-2.1":{"mean":0.0332,"reserve":0.0806,"n_observed":3738,"source":"observed"},"deep-swe":{"mean":0.1691,"reserve":0.3211,"n_observed":4972,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":261,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":333,"n_units":111}},"code_revision":"39d3f3d127111b798c0278c10df2897ad737ac62","harness_adapters":{"deepseek-harness":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["dna-assembly__fXEHYfc","fix-ocaml-gc__8zbshj2","fix-ocaml-gc__aPV6pR9","fix-ocaml-gc__dH8LELS","path-tracing-reverse__3uknabm","torch-pipeline-parallelism__SRyqnyq"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["deep-swe"],"trials":["effect-sse-httpapi-streaming__kwDiZb8","go-git-worktree-merge-conflicts__Nqz7LgN","goreleaser-retry-publish-auditin__aosJKDM","helm-array-merge-strategies__BukfRx7","helm-array-merge-strategies__pxZgFn9","httpx-multipart-response-parsing__3VTBGwL","httpx-multipart-response-parsing__EcaAP53","igel-persist-feature-schema__RGaNsrs","igel-persist-feature-schema__xy6x32j","ipython-session-bundle-replay__CQsdrjM","ipython-session-bundle-replay__CcezR5N","ipython-session-bundle-replay__ckRu9ug","kcp-go-multiplexed-kcp-streams__5UVdy9x","kcp-go-multiplexed-kcp-streams__hvLd52R","kcp-go-multiplexed-kcp-streams__neBoFsm","kombu-virtual-queue-dead-letteri__hJwYgvm","kombu-virtual-queue-dead-letteri__uDT5mDa","koota-entity-snapshot-rollback__23qULKn","koota-entity-snapshot-rollback__nbsXZdJ","koota-entity-snapshot-rollback__qSwf2N7","koota-query-predicates__dnVF8DY"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08)."},{"benchmarks":["terminal-bench-2.1"],"trials":["chess-best-move__KVeSShA","llm-inference-batching-scheduler__iBWJFUK","make-doom-for-mips__dywD2hw","make-doom-for-mips__kPxSSv5","multi-source-data-merger__uXXuHgW","nginx-request-logging__WX38K3C","nginx-request-logging__XFwLeUQ","openssl-selfsigned-cert__26SnZF7","overfull-hbox__F23hmgR","overfull-hbox__pB5miKd","password-recovery__c8Zecoa"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08)."}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30).; Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":261,"n_new":264,"n_scored":261,"n_reused":0,"cost_new_usd":9.316,"by_harness":{"deepseek-harness":{"pass":187,"fail":74,"infra":3}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":333,"n_new":342,"n_scored":333,"n_reused":0,"cost_new_usd":56.9,"by_harness":{"deepseek-harness":{"pass":211,"infra":9,"fail":122}}}},"n_new":606,"n_scored":594,"n_slots":594,"n_reused":0,"cost_new_usd":66.22,"updated_at":"2026-11-09T20:33:02.638555+00:00"},{"run_id":"20261109-093716-glm-5-3-flash-full-high-eb54","created_at":"2026-11-09T09:37:16.245736+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":250.0,"estimate_usd":{"new_trials":594,"low":125.9,"high":251.9,"per_trial":{"terminal-bench-2.1":{"mean":0.0403,"reserve":0.0967,"n_observed":4720,"source":"observed"},"deep-swe":{"mean":0.3466,"reserve":0.6806,"n_observed":5858,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":261,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":333,"n_units":111}},"code_revision":"39d3f3d127111b798c0278c10df2897ad737ac62","harness_adapters":{"deepseek-harness":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__5gmYqvY","fix-ocaml-gc__MDqyhN9","fix-ocaml-gc__aGU9AJn"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["terminal-bench-2.1"],"trials":["adaptive-rejection-sampler__8vaHmjQ","filter-js-from-html__9aCzpkM","make-mips-interpreter__6hDhYpV","make-mips-interpreter__WZe6AKK","make-mips-interpreter__nz2EgpY","path-tracing-reverse__Nm9JZ8i","regex-chess__CNpL6jC","regex-chess__SytjJeB"],"outcome":"infra","reason":"Ended on a provider error inside the stream: the last model call's stream, answered with status 200, ended with the router's error event (finish_reason 'error', 'JSON error injected into SSE stream'; docs/incidents/provider-stream-errors.json); a provider failure like a stream that broke off, run again, pass or fail alike (meter.stream_error, 2026-11-09)."}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30).; Ended on a provider error inside the stream: the last model call's stream, answered with status 200, ended with the router's error event (finish_reason 'error', 'JSON error injected into SSE stream'; docs/incidents/provider-stream-errors.json); a provider failure like a stream that broke off, run again, pass or fail alike (meter.stream_error, 2026-11-09)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":261,"n_new":286,"n_scored":261,"n_reused":0,"cost_new_usd":11.93,"by_harness":{"deepseek-harness":{"fail":95,"infra":25,"pass":166}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":333,"n_new":400,"n_scored":331,"n_reused":0,"cost_new_usd":155.9,"by_harness":{"deepseek-harness":{"pass":208,"infra":69,"fail":123}}}},"n_new":686,"n_scored":592,"n_slots":594,"n_reused":0,"cost_new_usd":167.8,"updated_at":"2026-11-09T15:37:52.119642+00:00"},{"run_id":"20261109-093711-glm-5-3-flash-full-low-bfde","created_at":"2026-11-09T09:37:11.202759+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"low","harnesses":["deepseek-harness"],"harness_versions":{"deepseek-harness":"0.2.2-rc.1"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":150.0,"estimate_usd":{"new_trials":588,"low":94.87,"high":182.2,"per_trial":{"terminal-bench-2.1":{"mean":0.0381,"reserve":0.0868,"n_observed":4737,"source":"observed"},"deep-swe":{"mean":0.2577,"reserve":0.4841,"n_observed":5847,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":3,"n_new":258,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":3,"n_new":330,"n_units":111}},"code_revision":"39d3f3d127111b798c0278c10df2897ad737ac62","harness_adapters":{"deepseek-harness":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__EWYpv6P","fix-ocaml-gc__aTD7eAg"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":258,"n_new":276,"n_scored":258,"n_reused":3,"cost_new_usd":9.883,"by_harness":{"deepseek-harness":{"fail":106,"infra":18,"pass":152}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":330,"n_new":394,"n_scored":330,"n_reused":3,"cost_new_usd":83.51,"by_harness":{"deepseek-harness":{"pass":185,"infra":64,"fail":145}}}},"n_new":670,"n_scored":588,"n_slots":588,"n_reused":6,"cost_new_usd":93.39,"updated_at":"2026-11-09T16:02:00.829246+00:00"},{"run_id":"20261109-010940-deepseek-v4-1-flash-full-low-2547","created_at":"2026-11-09T01:09:40.780443+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":1,"low":0.17,"high":0.32,"per_trial":{"terminal-bench-2.1":{"mean":0.0336,"reserve":0.0805,"n_observed":3504,"source":"observed"},"deep-swe":{"mean":0.1695,"reserve":0.3225,"n_observed":4627,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":261,"n_new":0,"n_units":0},"deep-swe":{"n_tasks":111,"n_reused":332,"n_new":1,"n_units":1}},"code_revision":"fb3c9714154a9cdc36f0e8cce0549be4b21a5eab","harness_adapters":{"opencode":null},"harness_locks":{},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":261,"cost_new_usd":0,"by_harness":{}},"deep-swe":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":1,"n_new":3,"n_scored":1,"n_reused":332,"cost_new_usd":0.144,"by_harness":{"opencode":{"infra":2,"fail":1}}}},"n_new":3,"n_scored":1,"n_slots":1,"n_reused":593,"cost_new_usd":0.144,"updated_at":"2026-11-09T02:02:37.459510+00:00"},{"run_id":"20261108-222008-flint-full-default-8c99","created_at":"2026-11-08T22:20:08.816723+00:00","model":"RecursiveSI/flint","provider":"hf-endpoint","stage":"full","reasoning_effort":"default","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":1188,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":4,"low":1.6,"high":3.41,"per_trial":{"terminal-bench-2.1":{"mean":0.0376,"reserve":0.082,"n_observed":4277,"source":"observed"},"deep-swe":{"mean":0.4,"reserve":0.8516,"n_observed":5235,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":522,"n_new":0,"n_units":0},"deep-swe":{"n_tasks":111,"n_reused":662,"n_new":4,"n_units":1}},"code_revision":"fb3c9714154a9cdc36f0e8cce0549be4b21a5eab","harness_adapters":{"opencode":null},"harness_locks":{},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":522,"cost_new_usd":0,"by_harness":{}},"deep-swe":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":4,"n_new":10,"n_scored":2,"n_reused":662,"cost_new_usd":0.3376,"by_harness":{"opencode":{"infra":6}}}},"n_new":10,"n_scored":2,"n_slots":4,"n_reused":1184,"cost_new_usd":0.3376,"updated_at":"2026-11-09T00:32:13.414822+00:00"},{"run_id":"20261108-081736-deepseek-v4-1-flash-full-high-d38d","created_at":"2026-11-08T08:17:36.622009+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":350.0,"estimate_usd":{"new_trials":440,"low":60.5,"high":121.2,"per_trial":{"terminal-bench-2.1":{"mean":0.0396,"reserve":0.0944,"n_observed":3177,"source":"observed"},"deep-swe":{"mean":0.2072,"reserve":0.4045,"n_observed":4176,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":78,"n_new":183,"n_units":61},"deep-swe":{"n_tasks":111,"n_reused":76,"n_new":257,"n_units":87}},"code_revision":"3fd2cb6d68d38c0988c9721850822b0dcf5dc4f7","harness_adapters":{"opencode":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__V8gR2LX","fix-ocaml-gc__bBLwTME","fix-ocaml-gc__zhE2Jy3","raman-fitting__Ndb8yC9","regex-chess__eRH4MLy","regex-chess__fyzUcUt"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":61,"failed":0},"n_slots":183,"n_new":183,"n_scored":183,"n_reused":78,"cost_new_usd":6.565,"by_harness":{"opencode":{"fail":40,"pass":143}}},"deep-swe":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":257,"n_new":261,"n_scored":257,"n_reused":76,"cost_new_usd":48.65,"by_harness":{"opencode":{"pass":162,"fail":95,"infra":4}}}},"n_new":444,"n_scored":440,"n_slots":440,"n_reused":154,"cost_new_usd":55.22,"updated_at":"2026-11-09T01:07:43.593913+00:00"},{"run_id":"20261108-081551-deepseek-v4-1-flash-full-low-6c62","created_at":"2026-11-08T08:15:51.727368+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":300.0,"estimate_usd":{"new_trials":438,"low":48.77,"high":95.21,"per_trial":{"terminal-bench-2.1":{"mean":0.0328,"reserve":0.0784,"n_observed":3139,"source":"observed"},"deep-swe":{"mean":0.1677,"reserve":0.3171,"n_observed":4125,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":78,"n_new":183,"n_units":61},"deep-swe":{"n_tasks":111,"n_reused":78,"n_new":255,"n_units":85}},"code_revision":"3fd2cb6d68d38c0988c9721850822b0dcf5dc4f7","harness_adapters":{"opencode":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__8G4HD7r","fix-ocaml-gc__KTUAeht","fix-ocaml-gc__imvb7wM"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":61,"failed":0},"n_slots":183,"n_new":183,"n_scored":183,"n_reused":78,"cost_new_usd":5.064,"by_harness":{"opencode":{"pass":139,"fail":44}}},"deep-swe":{"units":{"pending":0,"running":0,"done":85,"failed":0},"n_slots":255,"n_new":258,"n_scored":254,"n_reused":78,"cost_new_usd":39.11,"by_harness":{"opencode":{"fail":86,"pass":168,"infra":4}}}},"n_new":441,"n_scored":437,"n_slots":438,"n_reused":156,"cost_new_usd":44.18,"updated_at":"2026-11-09T01:05:18.510636+00:00"},{"run_id":"20261108-063020-flint-full-default-9473","created_at":"2026-11-08T06:30:20.379247+00:00","model":"RecursiveSI/flint","provider":"hf-endpoint","stage":"full","reasoning_effort":"default","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":522.2,"estimate_usd":{"new_trials":594,"low":146.2,"high":316.6,"per_trial":{"terminal-bench-2.1":{"mean":0.0405,"reserve":0.0886,"n_observed":3545,"source":"observed"},"deep-swe":{"mean":0.4073,"reserve":0.8813,"n_observed":4562,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":261,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":333,"n_units":111}},"code_revision":"4cfc4782dd19017e98bdac595743a3eac1797dd1","harness_adapters":{"opencode":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__6kpRVxV","fix-ocaml-gc__LV7WGy4"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":261,"n_new":265,"n_scored":261,"n_reused":0,"cost_new_usd":6.181,"by_harness":{"opencode":{"fail":159,"infra":4,"pass":102}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":333,"n_new":357,"n_scored":331,"n_reused":0,"cost_new_usd":122.1,"by_harness":{"opencode":{"pass":25,"fail":306,"infra":26}}}},"n_new":622,"n_scored":592,"n_slots":594,"n_reused":0,"cost_new_usd":128.3,"updated_at":"2026-11-08T13:46:25.041923+00:00"},{"run_id":"20261107-210227-deepseek-v4-1-flash-full-high-341e","created_at":"2026-11-07T21:02:27.527165+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":329.7,"estimate_usd":{"new_trials":594,"low":80.05,"high":160.8,"per_trial":{"terminal-bench-2.1":{"mean":0.0403,"reserve":0.0967,"n_observed":2997,"source":"observed"},"deep-swe":{"mean":0.2088,"reserve":0.4071,"n_observed":3994,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":261,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":333,"n_units":111}},"code_revision":"d9c0044ff910b66e5b3f3199223966f86290f510","harness_adapters":{"opencode":null},"harness_locks":{},"invalid":null,"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":61,"running":0,"done":26,"failed":0},"n_slots":261,"n_new":79,"n_scored":78,"n_reused":0,"cost_new_usd":2.016,"by_harness":{"opencode":{"pass":56,"fail":22,"infra":1}}},"deep-swe":{"units":{"pending":85,"running":0,"done":26,"failed":0},"n_slots":333,"n_new":78,"n_scored":76,"n_reused":0,"cost_new_usd":13.16,"by_harness":{"opencode":{"pass":54,"fail":22,"cancelled":2}}}},"n_new":157,"n_scored":154,"n_slots":594,"n_reused":0,"cost_new_usd":15.18,"updated_at":"2026-11-08T08:10:55.123909+00:00"},{"run_id":"20261107-210054-deepseek-v4-1-flash-full-low-a24b","created_at":"2026-11-07T21:00:54.772688+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":299.6,"estimate_usd":{"new_trials":594,"low":64.94,"high":127.0,"per_trial":{"terminal-bench-2.1":{"mean":0.0333,"reserve":0.0801,"n_observed":2969,"source":"observed"},"deep-swe":{"mean":0.1689,"reserve":0.3186,"n_observed":3953,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":261,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":333,"n_units":111}},"code_revision":"d9c0044ff910b66e5b3f3199223966f86290f510","harness_adapters":{"opencode":null},"harness_locks":{},"invalid":null,"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":61,"running":0,"done":26,"failed":0},"n_slots":261,"n_new":78,"n_scored":78,"n_reused":0,"cost_new_usd":1.679,"by_harness":{"opencode":{"pass":64,"fail":14}}},"deep-swe":{"units":{"pending":84,"running":0,"done":27,"failed":0},"n_slots":333,"n_new":79,"n_scored":78,"n_reused":0,"cost_new_usd":11.15,"by_harness":{"opencode":{"pass":55,"fail":23,"cancelled":1}}}},"n_new":157,"n_scored":156,"n_slots":594,"n_reused":0,"cost_new_usd":12.83,"updated_at":"2026-11-08T08:02:57.278460+00:00"},{"run_id":"20261107-205922-glm-5-3-flash-full-high-f31a","created_at":"2026-11-07T20:59:22.047517+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":341.2,"estimate_usd":{"new_trials":594,"low":134.3,"high":268.7,"per_trial":{"terminal-bench-2.1":{"mean":0.0416,"reserve":0.0979,"n_observed":3786,"source":"observed"},"deep-swe":{"mean":0.3708,"reserve":0.7303,"n_observed":4534,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":261,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":333,"n_units":111}},"code_revision":"d9c0044ff910b66e5b3f3199223966f86290f510","harness_adapters":{"opencode":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__9WL9Tto","fix-ocaml-gc__XkeqJLJ","fix-ocaml-gc__jiqob73","regex-chess__EoBUK9d"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":261,"n_new":265,"n_scored":261,"n_reused":0,"cost_new_usd":9.021,"by_harness":{"opencode":{"pass":190,"fail":71,"infra":4}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":333,"n_new":334,"n_scored":333,"n_reused":0,"cost_new_usd":92.18,"by_harness":{"opencode":{"pass":200,"fail":133,"infra":1}}}},"n_new":599,"n_scored":594,"n_slots":594,"n_reused":0,"cost_new_usd":101.2,"updated_at":"2026-11-08T02:20:48.416807+00:00"},{"run_id":"20261107-205835-glm-5-3-flash-full-low-de4d","created_at":"2026-11-07T20:58:35.042784+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"low","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":292.9,"estimate_usd":{"new_trials":588,"low":105.7,"high":202.3,"per_trial":{"terminal-bench-2.1":{"mean":0.0402,"reserve":0.0868,"n_observed":3771,"source":"observed"},"deep-swe":{"mean":0.2888,"reserve":0.5452,"n_observed":4516,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":3,"n_new":258,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":3,"n_new":330,"n_units":111}},"code_revision":"d9c0044ff910b66e5b3f3199223966f86290f510","harness_adapters":{"opencode":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__ED8xU8S","fix-ocaml-gc__TAUKgrK","fix-ocaml-gc__kaQn6Jo"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":258,"n_new":268,"n_scored":258,"n_reused":3,"cost_new_usd":7.379,"by_harness":{"opencode":{"pass":165,"fail":93,"infra":10}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":330,"n_new":334,"n_scored":330,"n_reused":3,"cost_new_usd":47.58,"by_harness":{"opencode":{"pass":182,"fail":148,"infra":4}}}},"n_new":602,"n_scored":588,"n_slots":588,"n_reused":6,"cost_new_usd":54.96,"updated_at":"2026-11-08T02:05:49.413671+00:00"},{"run_id":"20261107-194031-deepseek-v4-1-flash-full-high-63b7","created_at":"2026-11-07T19:40:31.680784+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":219.8,"estimate_usd":{"new_trials":593,"low":79.91,"high":160.5,"per_trial":{"terminal-bench-2.1":{"mean":0.0403,"reserve":0.0967,"n_observed":2994,"source":"observed"},"deep-swe":{"mean":0.2085,"reserve":0.4066,"n_observed":3984,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":1,"n_new":260,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":333,"n_units":111}},"code_revision":"4b6296af6a9b515a80dac0ad622598dbdb8eabb5","harness_adapters":{"opencode":"responses-bridge@7011f496679d"},"harness_locks":{},"invalid":null,"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":87,"running":0,"done":0,"failed":0},"n_slots":260,"n_new":0,"n_scored":0,"n_reused":1,"cost_new_usd":0,"by_harness":{}},"deep-swe":{"units":{"pending":111,"running":0,"done":0,"failed":0},"n_slots":333,"n_new":0,"n_scored":0,"n_reused":0,"cost_new_usd":0,"by_harness":{}}},"n_new":0,"n_scored":0,"n_slots":593,"n_reused":1,"cost_new_usd":0,"updated_at":"2026-11-07T20:00:57.135248+00:00"},{"run_id":"20261107-193930-deepseek-v4-1-flash-full-low-41c0","created_at":"2026-11-07T19:39:30.653197+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":199.7,"estimate_usd":{"new_trials":592,"low":64.84,"high":126.8,"per_trial":{"terminal-bench-2.1":{"mean":0.0333,"reserve":0.0801,"n_observed":2967,"source":"observed"},"deep-swe":{"mean":0.1688,"reserve":0.3184,"n_observed":3949,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2,"n_new":259,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":333,"n_units":111}},"code_revision":"4b6296af6a9b515a80dac0ad622598dbdb8eabb5","harness_adapters":{"opencode":"responses-bridge@7011f496679d"},"harness_locks":{},"invalid":null,"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":87,"running":0,"done":0,"failed":0},"n_slots":259,"n_new":0,"n_scored":0,"n_reused":2,"cost_new_usd":0,"by_harness":{}},"deep-swe":{"units":{"pending":111,"running":0,"done":0,"failed":0},"n_slots":333,"n_new":0,"n_scored":0,"n_reused":0,"cost_new_usd":0,"by_harness":{}}},"n_new":0,"n_scored":0,"n_slots":592,"n_reused":2,"cost_new_usd":0,"updated_at":"2026-11-07T19:53:53.376519+00:00"},{"run_id":"20261107-190835-flint-full-default-3496","created_at":"2026-11-07T19:08:35.169571+00:00","model":"RecursiveSI/flint","provider":"hf-endpoint","stage":"full","reasoning_effort":"default","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":1188,"cost_cap_usd":700.0,"estimate_usd":{"new_trials":1188,"low":292.8,"high":641.8,"per_trial":{"terminal-bench-2.1":{"mean":0.0422,"reserve":0.0943,"n_observed":3119,"source":"observed"},"deep-swe":{"mean":0.4065,"reserve":0.8897,"n_observed":3990,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":522,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":666,"n_units":111}},"code_revision":"bc61d980c3f7e07a433a3c43b85eca04098c6742","harness_adapters":{"opencode":"responses-bridge@7011f496679d"},"harness_locks":{},"invalid":{"rules":[{"harnesses":["opencode"],"trials":["count-dataset-tokens__PDf7UYh","extract-elf__yqKeoRs"],"outcome":"infra","reason":"OpenCode 2.2.6's websearch tool asked for a search provider, which `opencode run` cannot answer, and the run ended (\"Cancelled a request for input this run cannot answer: Web Search\"). Our setup's failure: infrastructure, run again with websearch off (docs/decisions.md, OpenCode 2's web search is off, 2026-11-07)."}],"reason":"OpenCode 2.2.6's websearch tool asked for a search provider, which `opencode run` cannot answer, and the run ended (\"Cancelled a request for input this run cannot answer: Web Search\"). Our setup's failure: infrastructure, run again with websearch off (docs/decisions.md, OpenCode 2's web search is off, 2026-11-07)."},"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":63,"running":0,"done":24,"failed":0},"n_slots":522,"n_new":80,"n_scored":37,"n_reused":0,"cost_new_usd":1.05,"by_harness":{"opencode":{"cancelled":24,"pass":11,"infra":1,"fail":8}}},"deep-swe":{"units":{"pending":87,"running":0,"done":24,"failed":0},"n_slots":666,"n_new":48,"n_scored":0,"n_reused":0,"cost_new_usd":2.69,"by_harness":{"opencode":{"cancelled":24}}}},"n_new":128,"n_scored":37,"n_slots":1188,"n_reused":0,"cost_new_usd":3.74,"updated_at":"2026-11-07T20:01:50.621850+00:00"},{"run_id":"20261107-183615-deepseek-v4-1-flash-full-high-4765","created_at":"2026-11-07T18:36:15.005322+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":220.0,"estimate_usd":{"new_trials":594,"low":79.86,"high":160.4,"per_trial":{"terminal-bench-2.1":{"mean":0.0402,"reserve":0.0967,"n_observed":2989,"source":"observed"},"deep-swe":{"mean":0.2083,"reserve":0.4058,"n_observed":3975,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":261,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":333,"n_units":111}},"code_revision":"6338e01bba0eb41064efe8885ea2b148e53dfc6a","harness_adapters":{"opencode":"responses-bridge@7011f496679d"},"harness_locks":{},"invalid":{"rules":[{"harnesses":["opencode"],"trials":["abs-module-cache-flags__Spt2t8M"],"outcome":"infra","reason":"OpenCode 2.2.6's websearch tool asked for a search provider, which `opencode run` cannot answer, and the run ended (\"Cancelled a request for input this run cannot answer: Web Search\"). Our setup's failure: infrastructure, run again with websearch off (docs/decisions.md, OpenCode 2's web search is off, 2026-11-07)."}],"reason":"OpenCode 2.2.6's websearch tool asked for a search provider, which `opencode run` cannot answer, and the run ended (\"Cancelled a request for input this run cannot answer: Web Search\"). Our setup's failure: infrastructure, run again with websearch off (docs/decisions.md, OpenCode 2's web search is off, 2026-11-07)."},"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":86,"running":0,"done":1,"failed":0},"n_slots":261,"n_new":2,"n_scored":1,"n_reused":0,"cost_new_usd":0.0662,"by_harness":{"opencode":{"pass":1,"cancelled":1}}},"deep-swe":{"units":{"pending":110,"running":0,"done":1,"failed":0},"n_slots":333,"n_new":2,"n_scored":1,"n_reused":0,"cost_new_usd":0.1265,"by_harness":{"opencode":{"pass":1,"cancelled":1}}}},"n_new":4,"n_scored":2,"n_slots":594,"n_reused":0,"cost_new_usd":0.1927,"updated_at":"2026-11-07T19:35:54.492757+00:00"},{"run_id":"20261107-183515-deepseek-v4-1-flash-full-low-573e","created_at":"2026-11-07T18:35:15.301635+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":200.0,"estimate_usd":{"new_trials":594,"low":64.81,"high":126.8,"per_trial":{"terminal-bench-2.1":{"mean":0.0332,"reserve":0.0801,"n_observed":2959,"source":"observed"},"deep-swe":{"mean":0.1686,"reserve":0.3179,"n_observed":3937,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":261,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":333,"n_units":111}},"code_revision":"6338e01bba0eb41064efe8885ea2b148e53dfc6a","harness_adapters":{"opencode":"responses-bridge@7011f496679d"},"harness_locks":{},"invalid":{"rules":[{"harnesses":["opencode"],"trials":["abs-module-cache-flags__JeyA6DS","abs-module-cache-flags__ihJ6HU5","abs-module-cache-flags__oJ2xAbd"],"outcome":"infra","reason":"OpenCode 2.2.6's websearch tool asked for a search provider, which `opencode run` cannot answer, and the run ended (\"Cancelled a request for input this run cannot answer: Web Search\"). Our setup's failure: infrastructure, run again with websearch off (docs/decisions.md, OpenCode 2's web search is off, 2026-11-07)."}],"reason":"OpenCode 2.2.6's websearch tool asked for a search provider, which `opencode run` cannot answer, and the run ended (\"Cancelled a request for input this run cannot answer: Web Search\"). Our setup's failure: infrastructure, run again with websearch off (docs/decisions.md, OpenCode 2's web search is off, 2026-11-07)."},"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":86,"running":0,"done":1,"failed":0},"n_slots":261,"n_new":3,"n_scored":2,"n_reused":0,"cost_new_usd":0.0658,"by_harness":{"opencode":{"pass":2,"cancelled":1}}},"deep-swe":{"units":{"pending":110,"running":0,"done":1,"failed":0},"n_slots":333,"n_new":3,"n_scored":3,"n_reused":0,"cost_new_usd":0.2173,"by_harness":{"opencode":{"pass":3}}}},"n_new":6,"n_scored":5,"n_slots":594,"n_reused":0,"cost_new_usd":0.2831,"updated_at":"2026-11-07T19:34:59.144002+00:00"},{"run_id":"20261107-183414-glm-5-3-flash-full-high-38f4","created_at":"2026-11-07T18:34:14.961027+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":350.0,"estimate_usd":{"new_trials":594,"low":135.5,"high":268.2,"per_trial":{"terminal-bench-2.1":{"mean":0.0434,"reserve":0.1009,"n_observed":3451,"source":"observed"},"deep-swe":{"mean":0.373,"reserve":0.7264,"n_observed":4285,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":261,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":333,"n_units":111}},"code_revision":"6338e01bba0eb41064efe8885ea2b148e53dfc6a","harness_adapters":{"opencode":"responses-bridge@7011f496679d"},"harness_locks":{},"invalid":{"rules":[{"harnesses":["opencode"],"trials":["abs-stepped-slices__EjuBXmA","abs-stepped-slices__XU7TZXj","abs-stepped-slices__ZogLyCm","actionlint-action-pinning-lint__hikQKai","bandit-structured-nosec-directiv__Z2Tqjij","build-pmars__WCJBRLf","build-pov-ray__uorF9YM","build-pov-ray__zhz7Unv"],"outcome":"infra","reason":"OpenCode 2.2.6's websearch tool asked for a search provider, which `opencode run` cannot answer, and the run ended (\"Cancelled a request for input this run cannot answer: Web Search\"). Our setup's failure: infrastructure, run again with websearch off (docs/decisions.md, OpenCode 2's web search is off, 2026-11-07)."}],"reason":"OpenCode 2.2.6's websearch tool asked for a search provider, which `opencode run` cannot answer, and the run ended (\"Cancelled a request for input this run cannot answer: Web Search\"). Our setup's failure: infrastructure, run again with websearch off (docs/decisions.md, OpenCode 2's web search is off, 2026-11-07)."},"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":48,"running":0,"done":39,"failed":0},"n_slots":261,"n_new":94,"n_scored":68,"n_reused":0,"cost_new_usd":1.473,"by_harness":{"opencode":{"fail":10,"cancelled":26,"pass":58}}},"deep-swe":{"units":{"pending":72,"running":0,"done":39,"failed":0},"n_slots":333,"n_new":62,"n_scored":24,"n_reused":0,"cost_new_usd":7.285,"by_harness":{"opencode":{"pass":14,"cancelled":38,"fail":10}}}},"n_new":156,"n_scored":92,"n_slots":594,"n_reused":0,"cost_new_usd":8.758,"updated_at":"2026-11-07T19:40:54.704327+00:00"},{"run_id":"20261107-183314-glm-5-3-flash-full-low-b96c","created_at":"2026-11-07T18:33:14.643736+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"low","harnesses":["opencode"],"harness_versions":{"opencode":"2.2.6"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":594,"cost_cap_usd":300.0,"estimate_usd":{"new_trials":588,"low":107.2,"high":205.3,"per_trial":{"terminal-bench-2.1":{"mean":0.0425,"reserve":0.0936,"n_observed":3433,"source":"observed"},"deep-swe":{"mean":0.2917,"reserve":0.5491,"n_observed":4223,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":3,"n_new":258,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":3,"n_new":330,"n_units":111}},"code_revision":"6338e01bba0eb41064efe8885ea2b148e53dfc6a","harness_adapters":{"opencode":"responses-bridge@7011f496679d"},"harness_locks":{},"invalid":{"rules":[{"harnesses":["opencode"],"trials":["abs-stepped-slices__uTRoYUH","actionlint-action-pinning-lint__cQPMMWU","build-pov-ray__AoWTrTu","build-pov-ray__G4xaNLT","build-pov-ray__UtsRhjb"],"outcome":"infra","reason":"OpenCode 2.2.6's websearch tool asked for a search provider, which `opencode run` cannot answer, and the run ended (\"Cancelled a request for input this run cannot answer: Web Search\"). Our setup's failure: infrastructure, run again with websearch off (docs/decisions.md, OpenCode 2's web search is off, 2026-11-07)."}],"reason":"OpenCode 2.2.6's websearch tool asked for a search provider, which `opencode run` cannot answer, and the run ended (\"Cancelled a request for input this run cannot answer: Web Search\"). Our setup's failure: infrastructure, run again with websearch off (docs/decisions.md, OpenCode 2's web search is off, 2026-11-07)."},"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":48,"running":0,"done":39,"failed":0},"n_slots":258,"n_new":95,"n_scored":75,"n_reused":3,"cost_new_usd":1.052,"by_harness":{"opencode":{"pass":56,"fail":19,"cancelled":20}}},"deep-swe":{"units":{"pending":72,"running":0,"done":39,"failed":0},"n_slots":330,"n_new":77,"n_scored":42,"n_reused":3,"cost_new_usd":5.559,"by_harness":{"opencode":{"pass":26,"fail":16,"cancelled":34,"infra":1}}}},"n_new":172,"n_scored":117,"n_slots":588,"n_reused":6,"cost_new_usd":6.611,"updated_at":"2026-11-07T19:47:26.245593+00:00"},{"run_id":"20261030-130315-glm-5-3-flash-full-low-99c8","created_at":"2026-10-30T13:03:15.075961+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":5346,"cost_cap_usd":3.0,"estimate_usd":{"new_trials":1,"low":0.28,"high":0.51,"per_trial":{"terminal-bench-2.1":{"mean":0.0417,"reserve":0.0882,"n_observed":3171,"source":"observed"},"deep-swe":{"mean":0.2794,"reserve":0.5104,"n_observed":3935,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2349,"n_new":0,"n_units":0},"deep-swe":{"n_tasks":111,"n_reused":2996,"n_new":1,"n_units":1}},"code_revision":"8753fcffa2cb8b59934c7f5cbb7e22add01311d9","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","pi":null,"opencode":"responses-bridge@7011f496679d","hermes":null,"mini-swe-agent":null,"kimi-code":null,"qwen-code":null,"openclaw":null},"harness_locks":{},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":2349,"cost_new_usd":0,"by_harness":{}},"deep-swe":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":1,"n_new":1,"n_scored":1,"n_reused":2996,"cost_new_usd":0.6402,"by_harness":{"hermes":{"pass":1}}}},"n_new":1,"n_scored":1,"n_slots":1,"n_reused":5345,"cost_new_usd":0.6402,"updated_at":"2026-10-30T14:55:37.028619+00:00"},{"run_id":"20261030-130311-deepseek-v4-1-flash-full-high-f8e6","created_at":"2026-10-30T13:03:11.154934+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":5346,"cost_cap_usd":222.0,"estimate_usd":{"new_trials":862,"low":126.5,"high":247.5,"per_trial":{"terminal-bench-2.1":{"mean":0.0404,"reserve":0.0965,"n_observed":2649,"source":"observed"},"deep-swe":{"mean":0.1968,"reserve":0.3769,"n_observed":3345,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2073,"n_new":276,"n_units":46},"deep-swe":{"n_tasks":111,"n_reused":2411,"n_new":586,"n_units":96}},"code_revision":"8753fcffa2cb8b59934c7f5cbb7e22add01311d9","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","pi":null,"opencode":"responses-bridge@7011f496679d","hermes":null,"mini-swe-agent":null,"kimi-code":null,"qwen-code":null,"openclaw":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["torch-pipeline-parallelism__DFWZiSV"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["deep-swe"],"trials":["meriyah-explicit-resource-declar__Xye7GzN"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["deep-swe"],"trials":["abs-module-cache-flags__BZrozC4","abs-stepped-slices__LtB8ZTB","abs-stepped-slices__fBTbjbv","actionlint-action-pinning-lint__EmGccdL","actionlint-action-pinning-lint__FMEKjrk","adaptix-name-mapping-aliases__rcf9LZs","anko-typed-variable-bindings__GBBpFSL","anko-typed-variable-bindings__YMd3zSz","awilix-async-container-initializ__Sm3FsXh","bandit-structured-nosec-directiv__jcvvnDm","claude-code-by-agents-recursive__MjuiYTx","dasel-html-document-format__2W2DtFM","dynamodb-toolbox-lazy-recursive__jsEZTSf","dynamodb-toolbox-lazy-recursive__xwYqT5h","etree-xml-diff-patch__Q6tAe7P","etree-xml-diff-patch__vVP9TKE","gql-incremental-graphql-delivery__mLNM3tH","happy-dom-deterministic-intersec__kMZy9ny"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08).","from_release":"release-time-reruns"}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":46,"failed":0},"n_slots":276,"n_new":277,"n_scored":276,"n_reused":2073,"cost_new_usd":10.7,"by_harness":{"codex":{"pass":102,"fail":36},"opencode":{"fail":42,"pass":96,"infra":1}}},"deep-swe":{"units":{"pending":0,"running":0,"done":96,"failed":0},"n_slots":586,"n_new":601,"n_scored":583,"n_reused":2411,"cost_new_usd":157.0,"by_harness":{"mini-swe-agent":{"fail":36,"pass":23},"kimi-code":{"fail":14,"pass":12,"infra":3},"codex":{"pass":121,"fail":88,"infra":7},"opencode":{"pass":111,"fail":102,"infra":8},"hermes":{"fail":17,"pass":25},"pi":{"fail":15,"pass":10},"qwen-code":{"fail":5,"pass":3},"claude-code":{"fail":1}}}},"n_new":878,"n_scored":859,"n_slots":862,"n_reused":4484,"cost_new_usd":167.7,"updated_at":"2026-11-02T04:37:51.865041+00:00"},{"run_id":"20261030-130307-deepseek-v4-1-flash-full-low-45b5","created_at":"2026-10-30T13:03:07.187170+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":5346,"cost_cap_usd":187.0,"estimate_usd":{"new_trials":860,"low":101.1,"high":190.3,"per_trial":{"terminal-bench-2.1":{"mean":0.033,"reserve":0.078,"n_observed":2617,"source":"observed"},"deep-swe":{"mean":0.1575,"reserve":0.289,"n_observed":3298,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2073,"n_new":276,"n_units":46},"deep-swe":{"n_tasks":111,"n_reused":2413,"n_new":584,"n_units":99}},"code_revision":"8753fcffa2cb8b59934c7f5cbb7e22add01311d9","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","pi":null,"opencode":"responses-bridge@7011f496679d","hermes":null,"mini-swe-agent":null,"kimi-code":null,"qwen-code":null,"openclaw":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["path-tracing-reverse__M4n2DXs","regex-chess__twMH4uP"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["deep-swe"],"trials":["effect-sse-httpapi-streaming__SmeBux5","effect-sse-httpapi-streaming__wtguQA2"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":46,"failed":0},"n_slots":276,"n_new":277,"n_scored":276,"n_reused":2073,"cost_new_usd":9.374,"by_harness":{"codex":{"pass":107,"fail":31},"opencode":{"fail":43,"pass":95,"infra":1}}},"deep-swe":{"units":{"pending":0,"running":0,"done":99,"failed":0},"n_slots":584,"n_new":596,"n_scored":582,"n_reused":2413,"cost_new_usd":130.8,"by_harness":{"mini-swe-agent":{"pass":21,"fail":13},"codex":{"pass":126,"fail":83,"infra":6},"hermes":{"pass":45,"fail":32},"opencode":{"pass":111,"fail":103,"infra":8},"pi":{"pass":9,"fail":15},"qwen-code":{"pass":3,"fail":4},"kimi-code":{"pass":12,"fail":5}}}},"n_new":873,"n_scored":858,"n_slots":860,"n_reused":4486,"cost_new_usd":140.1,"updated_at":"2026-11-02T04:39:54.893558+00:00"},{"run_id":"20261030-071636-deepseek-v4-1-flash-full-high-3a38","created_at":"2026-10-30T07:16:36.331401+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":5346,"cost_cap_usd":194.0,"estimate_usd":{"new_trials":755,"low":101.8,"high":200.4,"per_trial":{"terminal-bench-2.1":{"mean":0.0405,"reserve":0.0965,"n_observed":2625,"source":"observed"},"deep-swe":{"mean":0.197,"reserve":0.3769,"n_observed":3309,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2049,"n_new":300,"n_units":50},"deep-swe":{"n_tasks":111,"n_reused":2542,"n_new":455,"n_units":77}},"code_revision":"8753fcffa2cb8b59934c7f5cbb7e22add01311d9","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","pi":null,"opencode":"responses-bridge@7011f496679d","hermes":null,"mini-swe-agent":null,"kimi-code":null,"qwen-code":null,"openclaw":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["deep-swe"],"trials":["gql-incremental-graphql-delivery__VMexnd8","gql-incremental-graphql-delivery__Zf7c6Ly","gql-incremental-graphql-delivery__d9Gf6T2","happy-dom-abort-pending-body-rea__HUxmert","happy-dom-abort-pending-body-rea__MXQCEAb","happy-dom-abort-pending-body-rea__u8tndYH","happy-dom-deterministic-intersec__DoHrLgd","happy-dom-deterministic-intersec__UfYsLoX","helm-array-merge-strategies__FtkZ3X4","helm-array-merge-strategies__Hf4Nvaw","helm-array-merge-strategies__YfNcovG","helm-array-merge-strategies__ksX6SBu","helm-array-merge-strategies__qUNSNHb","helm-array-merge-strategies__qakR4Ne"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["terminal-bench-2.1"],"trials":["install-windows-3.11__SZmyCk4","install-windows-3.11__puQ6FTd"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08).","from_release":"release-time-reruns"}],"reason":""},"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":46,"running":0,"done":4,"failed":0},"n_slots":300,"n_new":24,"n_scored":24,"n_reused":2049,"cost_new_usd":0.7664,"by_harness":{"codex":{"pass":10,"fail":2},"opencode":{"fail":10,"pass":2}}},"deep-swe":{"units":{"pending":72,"running":0,"done":5,"failed":0},"n_slots":455,"n_new":36,"n_scored":31,"n_reused":2542,"cost_new_usd":6.342,"by_harness":{"opencode":{"infra":5,"fail":13,"pass":3},"codex":{"fail":9,"pass":6}}}},"n_new":60,"n_scored":55,"n_slots":755,"n_reused":4591,"cost_new_usd":7.109,"updated_at":"2026-10-30T13:01:34.734686+00:00"},{"run_id":"20261030-071632-deepseek-v4-1-flash-full-low-8c2f","created_at":"2026-10-30T07:16:32.326721+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":5346,"cost_cap_usd":159.0,"estimate_usd":{"new_trials":756,"low":81.63,"high":153.5,"per_trial":{"terminal-bench-2.1":{"mean":0.033,"reserve":0.078,"n_observed":2593,"source":"observed"},"deep-swe":{"mean":0.1573,"reserve":0.2854,"n_observed":3261,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2049,"n_new":300,"n_units":50},"deep-swe":{"n_tasks":111,"n_reused":2541,"n_new":456,"n_units":78}},"code_revision":"8753fcffa2cb8b59934c7f5cbb7e22add01311d9","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","pi":null,"opencode":"responses-bridge@7011f496679d","hermes":null,"mini-swe-agent":null,"kimi-code":null,"qwen-code":null,"openclaw":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["deep-swe"],"trials":["goreleaser-retry-publish-auditin__hDSvkyz","gql-incremental-graphql-delivery__LLVqcuL","happy-dom-abort-pending-body-rea__3wTZ6pw","happy-dom-deterministic-intersec__TAXPSxU","happy-dom-deterministic-intersec__uvouLBg","helm-array-merge-strategies__pfbKxPb"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08).","from_release":"release-time-reruns"}],"reason":""},"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":46,"running":0,"done":4,"failed":0},"n_slots":300,"n_new":24,"n_scored":24,"n_reused":2049,"cost_new_usd":0.6882,"by_harness":{"codex":{"pass":12},"opencode":{"pass":7,"fail":5}}},"deep-swe":{"units":{"pending":73,"running":0,"done":5,"failed":0},"n_slots":456,"n_new":37,"n_scored":29,"n_reused":2541,"cost_new_usd":6.578,"by_harness":{"opencode":{"fail":9,"pass":7,"infra":8},"codex":{"fail":6,"pass":7}}}},"n_new":61,"n_scored":53,"n_slots":756,"n_reused":4590,"cost_new_usd":7.266,"updated_at":"2026-10-30T13:01:34.502859+00:00"},{"run_id":"20261030-015116-deepseek-v4-1-flash-full-high-6f85","created_at":"2026-10-30T01:51:16.014738+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":5346,"cost_cap_usd":217.0,"estimate_usd":{"new_trials":916,"low":119.9,"high":236.6,"per_trial":{"terminal-bench-2.1":{"mean":0.041,"reserve":0.0969,"n_observed":2540,"source":"observed"},"deep-swe":{"mean":0.1957,"reserve":0.3747,"n_observed":3229,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":1965,"n_new":384,"n_units":64},"deep-swe":{"n_tasks":111,"n_reused":2465,"n_new":532,"n_units":91}},"code_revision":"7f0d18028727885f702f55aec1bf84e6001d797c","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","pi":null,"opencode":"responses-bridge@7011f496679d","hermes":null,"mini-swe-agent":null,"kimi-code":null,"qwen-code":null,"openclaw":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__EK7WhHx","fix-ocaml-gc__G9bfGVH","fix-ocaml-gc__YZjzXY3","fix-ocaml-gc__dUD7qbJ","fix-ocaml-gc__rHqSyBC"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["deep-swe"],"trials":["dynamodb-toolbox-lazy-recursive__WUvcBpZ","effect-sse-httpapi-streaming__ieMAnmT","fastapi-implicit-head-options__3g8CYvR","fastapi-implicit-head-options__FSLyiJF","fastapi-implicit-head-options__GDbzDoe","fd-deterministic-multi-key-sorti__DH9XLXH","fd-deterministic-multi-key-sorti__KgbsYbb","fd-deterministic-multi-key-sorti__jXidJWq","go-critic-doc-link-checker__2X6qQKz","go-critic-doc-link-checker__VPmmHmR","go-critic-doc-link-checker__ZNiy2jC","go-git-worktree-merge-conflicts__68iWQ8E"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08).","from_release":"release-time-reruns"}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":50,"running":0,"done":14,"failed":0},"n_slots":384,"n_new":87,"n_scored":84,"n_reused":1965,"cost_new_usd":2.092,"by_harness":{"codex":{"fail":12,"pass":30},"opencode":{"fail":13,"pass":29,"infra":3}}},"deep-swe":{"units":{"pending":77,"running":0,"done":14,"failed":0},"n_slots":532,"n_new":80,"n_scored":77,"n_reused":2465,"cost_new_usd":20.08,"by_harness":{"opencode":{"pass":15,"infra":3,"fail":22},"codex":{"pass":33,"fail":7}}}},"n_new":167,"n_scored":161,"n_slots":916,"n_reused":4430,"cost_new_usd":22.17,"updated_at":"2026-10-30T07:12:06.828490+00:00"},{"run_id":"20261030-015111-deepseek-v4-1-flash-full-low-1200","created_at":"2026-10-30T01:51:11.808951+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":5346,"cost_cap_usd":175.0,"estimate_usd":{"new_trials":911,"low":96.01,"high":180.7,"per_trial":{"terminal-bench-2.1":{"mean":0.0335,"reserve":0.0788,"n_observed":2513,"source":"observed"},"deep-swe":{"mean":0.1566,"reserve":0.2835,"n_observed":3185,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":1970,"n_new":379,"n_units":64},"deep-swe":{"n_tasks":111,"n_reused":2465,"n_new":532,"n_units":90}},"code_revision":"7f0d18028727885f702f55aec1bf84e6001d797c","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","pi":null,"opencode":"responses-bridge@7011f496679d","hermes":null,"mini-swe-agent":null,"kimi-code":null,"qwen-code":null,"openclaw":null},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__6m5JZLW","fix-ocaml-gc__DMgwpkS","fix-ocaml-gc__Nxm6QRQ","fix-ocaml-gc__Z2vucFg","fix-ocaml-gc__cfwAtN2","fix-ocaml-gc__zYjbuux"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["deep-swe"],"trials":["effect-sse-httpapi-streaming__cXeuLkc"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["deep-swe"],"trials":["dynamodb-toolbox-conditional-att__MzSa5jq","effect-sse-httpapi-streaming__HhYWSZh","expr-try-catch-errors__4eMDrWy","expr-try-catch-errors__NqqvbUV","expr-try-catch-errors__TQ3AjEd","fastapi-implicit-head-options__ZArQQAG","geo-shapeindex-serialization__CzdbkJS","geo-shapeindex-serialization__WWPXRgi","geo-shapeindex-serialization__tuXR3PB","go-critic-doc-link-checker__TfA8Su6","go-critic-doc-link-checker__cdrc25S","go-critic-doc-link-checker__zKCFNJe"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["terminal-bench-2.1"],"trials":["financial-document-processor__WhEAdWC","gpt2-codegolf__mzZCX43"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":50,"running":0,"done":14,"failed":0},"n_slots":379,"n_new":80,"n_scored":79,"n_reused":1970,"cost_new_usd":1.48,"by_harness":{"codex":{"fail":12,"pass":27,"infra":1},"opencode":{"fail":13,"pass":27}}},"deep-swe":{"units":{"pending":78,"running":0,"done":12,"failed":0},"n_slots":532,"n_new":77,"n_scored":76,"n_reused":2465,"cost_new_usd":14.15,"by_harness":{"opencode":{"fail":20,"pass":15},"codex":{"pass":25,"fail":16,"infra":1}}}},"n_new":157,"n_scored":155,"n_slots":911,"n_reused":4435,"cost_new_usd":15.63,"updated_at":"2026-10-30T07:12:06.583609+00:00"},{"run_id":"20261029-230128-glm-5-3-flash-full-high-30ad","created_at":"2026-10-29T23:01:28.420597+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":5346,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":33,"low":2.4,"high":5.15,"per_trial":{"terminal-bench-2.1":{"mean":0.0436,"reserve":0.1013,"n_observed":3162,"source":"observed"},"deep-swe":{"mean":0.3625,"reserve":0.7026,"n_observed":4030,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2319,"n_new":30,"n_units":21},"deep-swe":{"n_tasks":111,"n_reused":2994,"n_new":3,"n_units":3}},"code_revision":"43e0550f8fa99d3b5d29754e9b445a39e947108c","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","pi":null,"opencode":"responses-bridge@7011f496679d","hermes":null,"mini-swe-agent":null,"kimi-code":null,"qwen-code":null,"openclaw":null},"harness_locks":{},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":21,"failed":0},"n_slots":30,"n_new":30,"n_scored":30,"n_reused":2319,"cost_new_usd":0.5938,"by_harness":{"opencode":{"fail":10,"pass":10},"codex":{"pass":7,"fail":3}}},"deep-swe":{"units":{"pending":0,"running":0,"done":3,"failed":0},"n_slots":3,"n_new":5,"n_scored":3,"n_reused":2994,"cost_new_usd":1.815,"by_harness":{"opencode":{"infra":2,"pass":1,"fail":2}}}},"n_new":35,"n_scored":33,"n_slots":33,"n_reused":5313,"cost_new_usd":2.409,"updated_at":"2026-10-30T02:33:23.392319+00:00"},{"run_id":"20261029-230124-glm-5-3-flash-full-low-45d6","created_at":"2026-10-29T23:01:24.690453+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":5346,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":37,"low":2.02,"high":4.09,"per_trial":{"terminal-bench-2.1":{"mean":0.0418,"reserve":0.0878,"n_observed":3136,"source":"observed"},"deep-swe":{"mean":0.2794,"reserve":0.5104,"n_observed":3934,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2314,"n_new":35,"n_units":25},"deep-swe":{"n_tasks":111,"n_reused":2995,"n_new":2,"n_units":1}},"code_revision":"43e0550f8fa99d3b5d29754e9b445a39e947108c","harness_adapters":{"claude-code":null,"codex":"responses-bridge@7011f496679d","pi":null,"opencode":"responses-bridge@7011f496679d","hermes":null,"mini-swe-agent":null,"kimi-code":null,"qwen-code":null,"openclaw":null},"harness_locks":{},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":25,"failed":0},"n_slots":35,"n_new":35,"n_scored":35,"n_reused":2314,"cost_new_usd":1.393,"by_harness":{"codex":{"pass":11,"fail":7},"opencode":{"fail":7,"pass":10}}},"deep-swe":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":2,"n_new":2,"n_scored":2,"n_reused":2995,"cost_new_usd":0.3238,"by_harness":{"opencode":{"pass":1,"fail":1}}}},"n_new":37,"n_scored":37,"n_slots":37,"n_reused":5309,"cost_new_usd":1.717,"updated_at":"2026-10-30T02:33:56.543426+00:00"},{"run_id":"20261029-102538-deepseek-v4-1-flash-full-high-4caf","created_at":"2026-10-29T10:25:38.987344+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["codex","opencode"],"harness_versions":{"codex":"0.162.1","opencode":"1.19.0"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":1188,"cost_cap_usd":250.0,"estimate_usd":{"new_trials":1188,"low":152.3,"high":298.2,"per_trial":{"terminal-bench-2.1":{"mean":0.0418,"reserve":0.0975,"n_observed":2389,"source":"observed"},"deep-swe":{"mean":0.1959,"reserve":0.3714,"n_observed":3080,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":522,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":666,"n_units":111}},"code_revision":"5e012a859c672d33e925d5251deaffd88733e45e","harness_adapters":{"codex":"responses-bridge@7011f496679d","opencode":"responses-bridge@7011f496679d"},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["deep-swe"],"trials":["adaptix-name-mapping-aliases__CArpTfi","adaptix-name-mapping-aliases__uB6rReo","adaptix-name-mapping-aliases__yxBAunG","anko-default-function-arguments__UEPv2gr","anko-default-function-arguments__Xs9b5qn","anko-default-function-arguments__tayNjtF","arcane-drift-detection-baselines__Y9dbzYm","csstree-shorthand-expansion-comp__oBhX3J4","drizzle-orm-window-function-buil__L4UPz69","drizzle-orm-window-function-buil__SL2j7YU","dynamodb-toolbox-conditional-att__Nn4Tjvo","dynamodb-toolbox-conditional-att__PFruYy8"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08).","from_release":"release-time-reruns"}],"reason":""},"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":64,"running":0,"done":23,"failed":0},"n_slots":522,"n_new":155,"n_scored":138,"n_reused":0,"cost_new_usd":4.337,"by_harness":{"codex":{"pass":55,"infra":9,"fail":14},"opencode":{"pass":47,"infra":8,"fail":22}}},"deep-swe":{"units":{"pending":91,"running":0,"done":20,"failed":0},"n_slots":666,"n_new":149,"n_scored":134,"n_reused":0,"cost_new_usd":28.47,"by_harness":{"codex":{"pass":52,"fail":16,"infra":4},"opencode":{"infra":11,"pass":33,"fail":33}}}},"n_new":304,"n_scored":272,"n_slots":1188,"n_reused":0,"cost_new_usd":32.8,"updated_at":"2026-10-30T01:46:19.252773+00:00"},{"run_id":"20261029-102535-deepseek-v4-1-flash-full-low-2550","created_at":"2026-10-29T10:25:35.089636+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["codex","opencode"],"harness_versions":{"codex":"0.162.1","opencode":"1.19.0"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":1188,"cost_cap_usd":210.0,"estimate_usd":{"new_trials":1188,"low":120.2,"high":225.9,"per_trial":{"terminal-bench-2.1":{"mean":0.0341,"reserve":0.0797,"n_observed":2362,"source":"observed"},"deep-swe":{"mean":0.1538,"reserve":0.2767,"n_observed":3038,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":522,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":666,"n_units":111}},"code_revision":"5e012a859c672d33e925d5251deaffd88733e45e","harness_adapters":{"codex":"responses-bridge@7011f496679d","opencode":"responses-bridge@7011f496679d"},"harness_locks":{},"invalid":null,"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":64,"running":0,"done":23,"failed":0},"n_slots":522,"n_new":154,"n_scored":143,"n_reused":0,"cost_new_usd":3.578,"by_harness":{"codex":{"fail":19,"pass":53,"infra":4},"opencode":{"pass":47,"fail":24,"infra":7}}},"deep-swe":{"units":{"pending":90,"running":0,"done":21,"failed":0},"n_slots":666,"n_new":148,"n_scored":134,"n_reused":0,"cost_new_usd":31.38,"by_harness":{"codex":{"pass":46,"fail":23,"infra":4},"opencode":{"pass":36,"fail":29,"infra":10}}}},"n_new":302,"n_scored":277,"n_slots":1188,"n_reused":0,"cost_new_usd":34.96,"updated_at":"2026-10-30T01:46:19.028076+00:00"},{"run_id":"20261029-101931-flint-full-default-0bb1","created_at":"2026-10-29T10:19:31.752821+00:00","model":"RecursiveSI/flint","provider":"hf-endpoint","stage":"full","reasoning_effort":"default","harnesses":["codex","opencode"],"harness_versions":{"codex":"0.162.1","opencode":"1.19.0"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":1188,"cost_cap_usd":700.0,"estimate_usd":{"new_trials":1188,"low":256.6,"high":558.2,"per_trial":{"terminal-bench-2.1":{"mean":0.0433,"reserve":0.0938,"n_observed":2347,"source":"observed"},"deep-swe":{"mean":0.3513,"reserve":0.7646,"n_observed":3103,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":522,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":666,"n_units":111}},"code_revision":"1bbf0aab64892b65f525b7858f8409918f7b9b80","harness_adapters":{"codex":"responses-bridge@7011f496679d","opencode":"responses-bridge@7011f496679d"},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__5uZq7Y5","fix-ocaml-gc__WuihKUA"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["terminal-bench-2.1"],"trials":["password-recovery__Dr8gFxJ"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":522,"n_new":633,"n_scored":507,"n_reused":0,"cost_new_usd":19.96,"by_harness":{"codex":{"fail":176,"infra":62,"pass":80},"opencode":{"fail":216,"infra":64,"pass":35}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":666,"n_new":796,"n_scored":663,"n_reused":0,"cost_new_usd":478.5,"by_harness":{"codex":{"pass":13,"fail":317,"infra":56},"opencode":{"fail":333,"infra":77}}}},"n_new":1429,"n_scored":1170,"n_slots":1188,"n_reused":0,"cost_new_usd":498.5,"updated_at":"2026-10-30T15:47:02.178756+00:00"},{"run_id":"20261029-101318-glm-5-3-flash-full-high-93f7","created_at":"2026-10-29T10:13:18.698343+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["codex","opencode"],"harness_versions":{"codex":"0.162.1","opencode":"1.19.0"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":1188,"cost_cap_usd":520.0,"estimate_usd":{"new_trials":1188,"low":254.9,"high":497.2,"per_trial":{"terminal-bench-2.1":{"mean":0.046,"reserve":0.1072,"n_observed":2403,"source":"observed"},"deep-swe":{"mean":0.3467,"reserve":0.6625,"n_observed":3252,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":522,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":666,"n_units":111}},"code_revision":"ddadff4feab27fcf6bb3e52909edac2c3bb86d9e","harness_adapters":{"codex":"responses-bridge@7011f496679d","opencode":"responses-bridge@7011f496679d"},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__7TVtosv","fix-ocaml-gc__HUdxYL4","fix-ocaml-gc__bC9jPnm","fix-ocaml-gc__mwd5kqU"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["deep-swe"],"trials":["ipython-session-bundle-replay__tHYnyM3","textual-kitty-key-phases__LQsvMeV"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":522,"n_new":768,"n_scored":492,"n_reused":0,"cost_new_usd":27.43,"by_harness":{"codex":{"pass":171,"infra":120,"fail":80},"opencode":{"fail":110,"infra":156,"pass":131}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":666,"n_new":792,"n_scored":663,"n_reused":0,"cost_new_usd":333.1,"by_harness":{"codex":{"pass":117,"fail":216,"infra":22},"opencode":{"fail":220,"pass":110,"infra":107}}}},"n_new":1560,"n_scored":1155,"n_slots":1188,"n_reused":0,"cost_new_usd":360.5,"updated_at":"2026-10-29T22:51:06.965662+00:00"},{"run_id":"20261029-101315-glm-5-3-flash-full-low-7601","created_at":"2026-10-29T10:13:15.181260+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"low","harnesses":["codex","opencode"],"harness_versions":{"codex":"0.162.1","opencode":"1.19.0"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":1188,"cost_cap_usd":420.0,"estimate_usd":{"new_trials":1188,"low":196.0,"high":352.4,"per_trial":{"terminal-bench-2.1":{"mean":0.0445,"reserve":0.0946,"n_observed":2374,"source":"observed"},"deep-swe":{"mean":0.2594,"reserve":0.455,"n_observed":3188,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":0,"n_new":522,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":0,"n_new":666,"n_units":111}},"code_revision":"ddadff4feab27fcf6bb3e52909edac2c3bb86d9e","harness_adapters":{"codex":"responses-bridge@7011f496679d","opencode":"responses-bridge@7011f496679d"},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__8y5Y6zw","fix-ocaml-gc__Bb9Uvh7","fix-ocaml-gc__TfbxSFp","fix-ocaml-gc__pNRPty3"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["deep-swe"],"trials":["prometheus-typed-label-sorting__hCowg8z","scc-bounded-memory-spilling__LMtsfD4"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["terminal-bench-2.1"],"trials":["path-tracing-reverse__XBpfrpC"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"}],"reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":522,"n_new":772,"n_scored":487,"n_reused":0,"cost_new_usd":25.22,"by_harness":{"codex":{"pass":148,"infra":134,"fail":95},"opencode":{"fail":120,"infra":151,"pass":124}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":666,"n_new":767,"n_scored":664,"n_reused":0,"cost_new_usd":272.4,"by_harness":{"codex":{"fail":244,"pass":89,"infra":21},"opencode":{"fail":231,"pass":100,"infra":82}}}},"n_new":1539,"n_scored":1151,"n_slots":1188,"n_reused":0,"cost_new_usd":297.6,"updated_at":"2026-10-29T22:48:51.108016+00:00"},{"run_id":"20261025-161003-flint-full-default-5596","created_at":"2026-10-25T16:10:03.598258+00:00","model":"RecursiveSI/flint","provider":"hf-endpoint","stage":"full","reasoning_effort":"default","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":5346,"cost_cap_usd":2300.0,"estimate_usd":{"new_trials":5124,"low":946.7,"high":2562.0,"per_trial":{"terminal-bench-2.1":{"mean":0.0436,"reserve":0.058,"n_observed":165,"source":"observed"},"deep-swe":{"mean":0.2914,"reserve":0.834,"n_observed":86,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":144,"n_new":2205,"n_units":85},"deep-swe":{"n_tasks":111,"n_reused":78,"n_new":2919,"n_units":110}},"code_revision":"cef141cc822d789a8f1a4687be5fac740d62a79a","harness_adapters":{},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["build-pov-ray__DBAnJKE","build-pov-ray__HvfYtfq","build-pov-ray__KiL9DGN","build-pov-ray__To7c9en","build-pov-ray__XKQT8fy","build-pov-ray__ZTuog7H","code-from-image__gtSKTEy","compile-compcert__UQwuEHH","crack-7z-hash__E4KEEU8","db-wal-recovery__DAWLrdp","db-wal-recovery__ayGvLjV","dna-assembly__NZrQMYP","dna-insert__kSG4Xxn","extract-elf__fau9uT4","extract-elf__gt88fJB","headless-terminal__8f34mdM","mteb-leaderboard__EZFmJr5","mteb-leaderboard__EjEX3M7","mteb-leaderboard__J9fdP5X","mteb-leaderboard__eWvPPM3","mteb-leaderboard__iLcbwwJ","mteb-leaderboard__t4yjQm2","mteb-retrieve__c3UeNmV","polyglot-rust-c__xkVXH9X","protein-assembly__4StR8rv","protein-assembly__bCq6ajk","regex-chess__xR7ZRth","torch-pipeline-parallelism__siPueiG","torch-tensor-parallelism__vRYS6sx","torch-tensor-parallelism__wpjEwht"],"outcome":"fail","reason":"Terminal-Bench answer key retrieved over the network, confirmed by reading the trajectory (docs/incidents/terminal-bench-answer-key-confirmed.json); counts as a fail, pass or fail (Joel, 2026-10-29)."},{"benchmarks":["deep-swe"],"trials":["actionlint-action-pinning-lint__zfwmQAD","adaptix-name-mapping-aliases__gdTFjLV","anko-default-function-arguments__9HbSP6w","anko-default-function-arguments__ZWSnRqE","arktype-json-schema-refs-depende__phu2t7U","arktype-json-schema-refs-depende__q8BSLB6","bandit-interprocedural-taint-che__8wxzBzY","boa-hierarchical-evaluation-canc__9Ktruah","drizzle-orm-window-function-buil__sc6zcYQ","dynamodb-toolbox-conditional-att__JcivXmP","dynamodb-toolbox-conditional-att__PBxo7zE","dynamodb-toolbox-conditional-att__dmyEtfh","dynamodb-toolbox-lazy-recursive__44LA9e4","dynamodb-toolbox-lazy-recursive__SMB5NAj","effect-sse-httpapi-streaming__2jimMvZ","expr-try-catch-errors__D6EpnTy","expr-try-catch-errors__xzrmAMP","expr-try-catch-errors__yz6kszN","fastapi-implicit-head-options__4nHzXco","fastapi-implicit-head-options__AHd3gbD","fastapi-implicit-head-options__noqZ2QL","katex-multicolumn-array-spans__Yw343ya","kea-atomic-signal-selectors__KRCW376","kgateway-consistent-hash-policy__N33ManH","koota-composite-trait-aspects__kVzStja","koota-composite-trait-aspects__o3YrABK","koota-composite-trait-aspects__scoep2o","koota-pair-relation-tracking__Pas9Zzc","koota-pair-relation-tracking__RbSkx5C","koota-pair-relation-tracking__uGEJnnb","koota-query-predicates__Y5ifjw7","koota-query-predicates__tK3ofJY","kysely-window-grouping-helpers__8beA3G6","kysely-window-grouping-helpers__JHL3oPn","narwhals-rolling-window-suite__o6GkUem","numba-stencil-boundary-modes__iNWxsTj","optique-conditional-option-depen__52xXdEy","optique-conditional-option-depen__ZXhrzKS","oxvg-structural-selector-preserv__UxStwFg","pebble-durability-wait-apis__c8UMqP5","python-statemachine-state-data-s__2xpeNxP","python-statemachine-state-data-s__SzFuzGM","scriggo-method-declarations__PwaZJ8k","scriggo-method-declarations__oapF7ZK","scriggo-method-declarations__yDqD9RK","sql-formatter-bigquery-pipe-form__h8MWMHp","tengo-destructuring-bindings__bA6Bwan","textual-kitty-key-phases__2iDnqWA","valibot-recursive-schema-composi__28WkLcq","valibot-recursive-schema-composi__H2hH9xp","wasmi-trap-coredumps__LFU9C52","wasmi-trap-coredumps__UxAJKQu","yaegi-go-embed-directives__4H6g7gY","yaegi-go-embed-directives__m2MGWnr"],"outcome":"infra","reason":"Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."},{"benchmarks":["terminal-bench-2.1"],"trials":["chess-best-move__CWcNXFb","chess-best-move__bvcas9s","chess-best-move__cqpZeRv","chess-best-move__jTUpPjV","chess-best-move__uVQYgoo","chess-best-move__zZgQco7","code-from-image__2CzmGQ4","code-from-image__DQG6XJ5","code-from-image__HMTHoUY","code-from-image__SXs9B7J","extract-moves-from-video__9K7eZ6A","extract-moves-from-video__NzN8Xh3","extract-moves-from-video__SAjdm63","extract-moves-from-video__dmPnJDW","extract-moves-from-video__hbPuNnE","financial-document-processor__L7MjWzd","financial-document-processor__qExcH9R","gcode-to-text__TUJP2QP","install-windows-3.11__JhKMHmq","path-tracing__KTprqMS","path-tracing__SUcMLRk","pytorch-model-cli__mugqgzb","pytorch-model-cli__xBn7pLk","raman-fitting__CSkoyhk","raman-fitting__PHadAGJ","raman-fitting__aaVwMjW","train-fasttext__BSd9FD3","video-processing__G7MdqDs","video-processing__RYXoHL5","video-processing__bKVAac7","video-processing__bMRaMyA","video-processing__iQZTHJ8"],"outcome":"infra","reason":"Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."},{"benchmarks":["terminal-bench-2.1"],"trials":["build-pov-ray__bxm6iH6","code-from-image__tgmkYzK","count-dataset-tokens__kPUL6AV","crack-7z-hash__donFLAh","fix-ocaml-gc__45gGvyh","fix-ocaml-gc__4cDX8oS","fix-ocaml-gc__76yEgxn","fix-ocaml-gc__7HeipLJ","fix-ocaml-gc__HHNUZUv","fix-ocaml-gc__UzPmbqT","fix-ocaml-gc__dAXFN5A","fix-ocaml-gc__gHMgjbw","fix-ocaml-gc__meKnhK5","fix-ocaml-gc__pmrWgJe","fix-ocaml-gc__sgEvVek","fix-ocaml-gc__spCtyyN"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["deep-swe"],"trials":["effect-sse-httpapi-streaming__YaaMSJr","effect-sse-httpapi-streaming__ZBkUGMA","effect-sse-httpapi-streaming__bxeCBsG"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["terminal-bench-2.1"],"trials":["password-recovery__7KmagQD","password-recovery__A74xsF9","password-recovery__bNe78t4","winning-avg-corewars__jiLRWoW"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"}],"reason":"Terminal-Bench answer key retrieved over the network, confirmed by reading the trajectory (docs/incidents/terminal-bench-answer-key-confirmed.json); counts as a fail, pass or fail (Joel, 2026-10-29).; Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29).; Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"failed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":85,"failed":0},"n_slots":2205,"n_new":2257,"n_scored":2205,"n_reused":144,"cost_new_usd":94.48,"by_harness":{"claude-code":{"pass":70,"fail":171,"infra":1},"codex":{"fail":144,"pass":100,"infra":18},"hermes":{"fail":156,"pass":90,"infra":4},"kimi-code":{"pass":84,"fail":162,"infra":6},"mini-swe-agent":{"pass":83,"infra":6,"fail":163},"openclaw":{"fail":218,"pass":28,"infra":6},"opencode":{"fail":198,"pass":48,"infra":6},"pi":{"fail":171,"pass":73,"infra":4},"qwen-code":{"pass":60,"fail":186,"infra":1}}},"deep-swe":{"units":{"pending":0,"running":0,"done":109,"failed":1},"n_slots":2919,"n_new":3144,"n_scored":2911,"n_reused":78,"cost_new_usd":1065.0,"by_harness":{"claude-code":{"pass":15,"fail":307,"infra":10},"codex":{"fail":304,"pass":17,"infra":21},"hermes":{"fail":297,"pass":26,"infra":40},"kimi-code":{"fail":291,"pass":33,"infra":55},"mini-swe-agent":{"fail":290,"pass":34,"infra":45},"openclaw":{"fail":327,"infra":6},"opencode":{"fail":319,"infra":29,"pass":2},"pi":{"fail":302,"pass":20,"infra":16},"qwen-code":{"fail":325,"pass":2,"infra":11}}}},"n_new":5401,"n_scored":5116,"n_slots":5124,"n_reused":222,"cost_new_usd":1160.0,"updated_at":"2026-10-28T19:09:53.937747+00:00"},{"run_id":"20261025-050351-flint-full-default-cf79","created_at":"2026-10-25T05:03:51.185474+00:00","model":"RecursiveSI/flint","provider":"hf-endpoint","stage":"full","reasoning_effort":"default","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe"],"n_trials":5346,"cost_cap_usd":2300.0,"estimate_usd":{"new_trials":5286,"low":866.7,"high":1904.0,"per_trial":{"terminal-bench-2.1":{"mean":0.0707,"reserve":0.1893,"n_observed":33,"source":"observed"},"deep-swe":{"mean":0.2367,"reserve":0.4934,"n_observed":27,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":33,"n_new":2316,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":27,"n_new":2970,"n_units":111}},"code_revision":"3b0fa395c048f199af620977f09adb0caceb49cd","harness_adapters":{},"harness_locks":{},"invalid":null,"state":"cancelled","benches":{"terminal-bench-2.1":{"units":{"pending":81,"running":0,"done":6,"failed":0},"n_slots":2316,"n_new":136,"n_scored":109,"n_reused":33,"cost_new_usd":4.866,"by_harness":{"claude-code":{"fail":10,"pass":2,"infra":5},"codex":{"pass":5,"fail":6,"infra":4,"cancelled":1},"hermes":{"infra":6,"fail":10,"pass":2},"kimi-code":{"infra":2,"fail":9,"pass":3},"mini-swe-agent":{"fail":9,"pass":3},"openclaw":{"fail":12},"opencode":{"fail":10,"infra":3,"pass":2,"cancelled":2},"pi":{"fail":10,"infra":1,"cancelled":1,"pass":4},"qwen-code":{"fail":10,"infra":2,"pass":2}}},"deep-swe":{"units":{"pending":104,"running":0,"done":7,"failed":0},"n_slots":2970,"n_new":59,"n_scored":51,"n_reused":27,"cost_new_usd":18.67,"by_harness":{"claude-code":{"pass":2,"fail":6,"cancelled":2},"codex":{"pass":3,"fail":5},"hermes":{"pass":3,"cancelled":3,"fail":2},"kimi-code":{"pass":2,"fail":2},"mini-swe-agent":{"pass":1,"fail":3,"cancelled":1},"openclaw":{"fail":3,"cancelled":1},"opencode":{"fail":8},"pi":{"fail":6,"pass":2},"qwen-code":{"fail":3,"cancelled":1}}}},"n_new":195,"n_scored":160,"n_slots":5286,"n_reused":60,"cost_new_usd":23.54,"updated_at":"2026-10-25T15:42:40.763839+00:00"},{"run_id":"20261023-183218-deepseek-v4-1-flash-full-low-3849","created_at":"2026-10-23T18:32:18.708099+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe","featurebench"],"n_trials":9531,"cost_cap_usd":10.0,"estimate_usd":{"new_trials":1,"low":0.15,"high":0.28,"per_trial":{"terminal-bench-2.1":{"mean":0.0341,"reserve":0.0797,"n_observed":2362,"source":"observed"},"deep-swe":{"mean":0.1537,"reserve":0.2767,"n_observed":3045,"source":"observed"},"featurebench":{"mean":0.0251,"reserve":0.0467,"n_observed":4191,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2349,"n_new":0,"n_units":0},"deep-swe":{"n_tasks":111,"n_reused":2996,"n_new":1,"n_units":1},"featurebench":{"n_tasks":155,"n_reused":4185,"n_new":0,"n_units":0}},"code_revision":"a5f02e629ce1cb646d3f58d1df1c5c1db188fefb","harness_adapters":{},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["build-pov-ray__fsAkDev","count-dataset-tokens__6vHjZYE","count-dataset-tokens__QPD5DPY","db-wal-recovery__gLCUHTu","extract-elf__7QAbkta","extract-elf__8bptGaK","extract-elf__GBTWNni","extract-elf__PYVk3HL","extract-elf__VLC6EcU","extract-elf__ZCdH5Nc","extract-elf__ZEEq6aG","extract-elf__amFmKjJ","extract-elf__as4eoNm","extract-elf__ggCJ24f","extract-elf__kYYPR4J","extract-elf__p57cvtx","feal-differential-cryptanalysis__83tUdK7","fix-code-vulnerability__FymrykQ","gpt2-codegolf__gevpDEf","gpt2-codegolf__xJVfedR","headless-terminal__CqaZr3g","make-doom-for-mips__UURTzpr","make-doom-for-mips__nX8yXNC","mteb-leaderboard__9fYFYbx","mteb-leaderboard__D7H3cQj","mteb-leaderboard__JKgoSMb","mteb-leaderboard__QvqTzba","mteb-leaderboard__Tr8hNTm","mteb-leaderboard__qMaGjWz","mteb-retrieve__BPcpae2","mteb-retrieve__CcqNLgb","mteb-retrieve__QcJwKDU","mteb-retrieve__y4XRzCU","raman-fitting__7aJLSst","raman-fitting__MRJUwVM","raman-fitting__bgtNQ4Y","raman-fitting__iLwwaWz","raman-fitting__rMzfhE2","raman-fitting__sSryaKL","regex-chess__SpG8nCB","torch-pipeline-parallelism__7Wy56Sk","torch-pipeline-parallelism__7evUVkz","torch-pipeline-parallelism__HM9opK2","torch-pipeline-parallelism__ZJYCH4n","torch-tensor-parallelism__qG8JYzz"],"outcome":"fail","reason":"Terminal-Bench answer key retrieved over the network, confirmed by reading the trajectory (docs/incidents/terminal-bench-answer-key-confirmed.json); counts as a fail, pass or fail (Joel, 2026-10-29)."},{"benchmarks":["deep-swe"],"trials":["katex-multicolumn-array-spans__M9A5g5f","scriggo-method-declarations__ADoFPrj","scriggo-method-declarations__zzTATW9","wasmi-trap-coredumps__RoE2MFM"],"outcome":"infra","reason":"Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."},{"benchmarks":["terminal-bench-2.1"],"trials":["build-pov-ray__JzvVAs6","build-pov-ray__nZXLEzX","build-pov-ray__qKL2Eyz","chess-best-move__85Diahd","chess-best-move__EukojDm","chess-best-move__F85f9wB","chess-best-move__Ftz3qx7","chess-best-move__cmBbQjC","chess-best-move__vXvmun2","code-from-image__GeFCHG9","code-from-image__UdzTKwT","code-from-image__XY2nguu","code-from-image__qrjWrKp","code-from-image__sXxxUHN","code-from-image__swoNWpT","extract-moves-from-video__GnNScHU","extract-moves-from-video__ZC3Yjjv","extract-moves-from-video__oqb8wox","extract-moves-from-video__pgZNYkP","financial-document-processor__4b8roDb","financial-document-processor__DAMbDMA","financial-document-processor__MNdf5sb","financial-document-processor__WAa2ymp","financial-document-processor__kDH9wfJ","gcode-to-text__3w9E5rN","gcode-to-text__AkpxJW5","gcode-to-text__PTGgF77","gcode-to-text__QZYczLp","gcode-to-text__kuyTkz2","gcode-to-text__xsx7Y9a","install-windows-3.11__CHCkXcV","install-windows-3.11__Hg8sgZg","install-windows-3.11__WGAbuff","install-windows-3.11__qUHZ73u","install-windows-3.11__wtbKppJ","make-mips-interpreter__PbSq7DA","make-mips-interpreter__gTG9Uba","make-mips-interpreter__zGajeJ2","path-tracing-reverse__KQXsq7v","path-tracing-reverse__M9dMmpJ","path-tracing__Qvkjf9v","path-tracing__bDAoosi","path-tracing__bzQmvRs","path-tracing__tvdSUEG","pytorch-model-cli__6qdyrBz","pytorch-model-cli__7XvyGhn","pytorch-model-cli__JufjzDQ","pytorch-model-cli__maLTuAm","raman-fitting__5Uoc8L3","raman-fitting__ErfdpW2","raman-fitting__GkQ92Dp","raman-fitting__Z7xufNn","raman-fitting__q85rVQA","raman-fitting__vCNUbKs","sam-cell-seg__7Xt2ALp","sam-cell-seg__Mra7Qjz","sam-cell-seg__Q3SekNW","sam-cell-seg__ZdR4EQu","sam-cell-seg__teDcKg4","sam-cell-seg__zjGJNXw","video-processing__A9QaxtX","video-processing__GkHoj9W","video-processing__KwAwMJN","video-processing__QGfwXSZ","video-processing__pTzLCpw","video-processing__vx3XJVZ"],"outcome":"infra","reason":"Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."}],"reason":"Terminal-Bench answer key retrieved over the network, confirmed by reading the trajectory (docs/incidents/terminal-bench-answer-key-confirmed.json); counts as a fail, pass or fail (Joel, 2026-10-29).; Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":2349,"cost_new_usd":0,"by_harness":{}},"deep-swe":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":1,"n_new":1,"n_scored":1,"n_reused":2996,"cost_new_usd":0.0889,"by_harness":{"claude-code":{"pass":1}}},"featurebench":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":4185,"cost_new_usd":0,"by_harness":{}}},"n_new":1,"n_scored":1,"n_slots":1,"n_reused":9530,"cost_new_usd":0.0889,"updated_at":"2026-10-23T19:47:53.356951+00:00"},{"run_id":"20261023-183213-glm-5-3-flash-full-high-cc70","created_at":"2026-10-23T18:32:13.578572+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe","featurebench"],"n_trials":9531,"cost_cap_usd":15.0,"estimate_usd":{"new_trials":8,"low":0.82,"high":1.53,"per_trial":{"terminal-bench-2.1":{"mean":0.0456,"reserve":0.1057,"n_observed":2465,"source":"observed"},"deep-swe":{"mean":0.3453,"reserve":0.6567,"n_observed":3269,"source":"observed"},"featurebench":{"mean":0.0671,"reserve":0.1246,"n_observed":4363,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2349,"n_new":0,"n_units":0},"deep-swe":{"n_tasks":111,"n_reused":2996,"n_new":1,"n_units":1},"featurebench":{"n_tasks":155,"n_reused":4178,"n_new":7,"n_units":2}},"code_revision":"a5f02e629ce1cb646d3f58d1df1c5c1db188fefb","harness_adapters":{},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["raman-fitting__x4kYS8Z"],"outcome":"fail","reason":"Terminal-Bench answer key retrieved over the network, confirmed by reading the trajectory (docs/incidents/terminal-bench-answer-key-confirmed.json); counts as a fail, pass or fail (Joel, 2026-10-29)."},{"benchmarks":["deep-swe"],"trials":["quill-shared-toolbar-focus__5kaoTcx","quill-shared-toolbar-focus__8uknjar","scriggo-method-declarations__28Cicxw","vitest-duration-sharding__NzY45DD","wasmi-trap-coredumps__ChtsUvy"],"outcome":"infra","reason":"Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."},{"benchmarks":["terminal-bench-2.1"],"trials":["build-pov-ray__irAWy3h","chess-best-move__4bcVeHP","chess-best-move__JWP6JQv","chess-best-move__NnvjaHZ","chess-best-move__WpoB9no","chess-best-move__rjA6u6W","chess-best-move__tqYZM5Z","code-from-image__6TZNMju","code-from-image__EPqqb9H","code-from-image__JKsrZ6K","code-from-image__YdZrhQg","code-from-image__d3oUL5x","code-from-image__yfjJGTC","extract-moves-from-video__YCwkYEE","extract-moves-from-video__t4QvQND","extract-moves-from-video__xtmexqu","financial-document-processor__6aJekMt","financial-document-processor__Pd3c3bM","financial-document-processor__XrWnj4q","financial-document-processor__dgrZnfZ","financial-document-processor__erZM6e7","gcode-to-text__BeyY6P4","gcode-to-text__L5Gj3LT","gcode-to-text__VVcwBVJ","gcode-to-text__nSisLo8","install-windows-3.11__6A6Zwgr","install-windows-3.11__aiBHMcs","install-windows-3.11__hJaThmA","install-windows-3.11__tbWv8Aa","make-mips-interpreter__7L7NWKU","path-tracing-reverse__NLXGxLT","path-tracing__4iQkFtT","path-tracing__YRVM4oH","path-tracing__davKPtj","path-tracing__pczwcGQ","path-tracing__tmXRWWH","pytorch-model-cli__6ns2iDk","pytorch-model-cli__gtDEvWb","pytorch-model-cli__i62namF","pytorch-model-cli__zYKe5gm","raman-fitting__eZfgFYV","raman-fitting__j4kmNNo","raman-fitting__qX4PxRk","sam-cell-seg__AebX32s","sam-cell-seg__WjMZhXq","sam-cell-seg__v9R6kYt","video-processing__3XZhNSb","video-processing__A6Zej4N","video-processing__UmT5UyQ","video-processing__ndbgrRs","video-processing__o2hHshn","video-processing__wETnZEc"],"outcome":"infra","reason":"Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."}],"reason":"Terminal-Bench answer key retrieved over the network, confirmed by reading the trajectory (docs/incidents/terminal-bench-answer-key-confirmed.json); counts as a fail, pass or fail (Joel, 2026-10-29).; Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":2349,"cost_new_usd":0,"by_harness":{}},"deep-swe":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":1,"n_new":2,"n_scored":1,"n_reused":2996,"cost_new_usd":0.4804,"by_harness":{"openclaw":{"infra":1,"pass":1}}},"featurebench":{"units":{"pending":0,"running":0,"done":2,"failed":0},"n_slots":7,"n_new":0,"n_scored":0,"n_reused":4178,"cost_new_usd":0,"by_harness":{}}},"n_new":2,"n_scored":1,"n_slots":8,"n_reused":9523,"cost_new_usd":0.4804,"updated_at":"2026-10-23T22:55:20.184113+00:00"},{"run_id":"20261023-183209-glm-5-3-flash-full-low-fe4c","created_at":"2026-10-23T18:32:09.652711+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe","featurebench"],"n_trials":9531,"cost_cap_usd":15.0,"estimate_usd":{"new_trials":7,"low":0.79,"high":1.48,"per_trial":{"terminal-bench-2.1":{"mean":0.0441,"reserve":0.094,"n_observed":2431,"source":"observed"},"deep-swe":{"mean":0.2594,"reserve":0.455,"n_observed":3188,"source":"observed"},"featurebench":{"mean":0.0604,"reserve":0.1281,"n_observed":4329,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2347,"n_new":2,"n_units":2},"deep-swe":{"n_tasks":111,"n_reused":2995,"n_new":2,"n_units":1},"featurebench":{"n_tasks":155,"n_reused":4182,"n_new":3,"n_units":2}},"code_revision":"a5f02e629ce1cb646d3f58d1df1c5c1db188fefb","harness_adapters":{},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["crack-7z-hash__6B9wmNV"],"outcome":"fail","reason":"Terminal-Bench answer key retrieved over the network, confirmed by reading the trajectory (docs/incidents/terminal-bench-answer-key-confirmed.json); counts as a fail, pass or fail (Joel, 2026-10-29)."},{"benchmarks":["deep-swe"],"trials":["arktype-json-schema-refs-depende__99FjQ6y","kgateway-consistent-hash-policy__xY4B397","oxvg-structural-selector-preserv__Nu4Sh4S","quill-shared-toolbar-focus__3LgHrue","quill-shared-toolbar-focus__JXKxvBk"],"outcome":"infra","reason":"Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."},{"benchmarks":["terminal-bench-2.1"],"trials":["chess-best-move__69kyYAE","chess-best-move__RBL27ku","chess-best-move__WJEjYw4","chess-best-move__Wg43MnV","chess-best-move__mekneVm","chess-best-move__uAKpKW3","code-from-image__BbYFD5U","code-from-image__LDsZxjk","code-from-image__ciUxmXU","code-from-image__f8a57hK","code-from-image__j8J7Ppn","code-from-image__vvb8Dgn","extract-moves-from-video__8W2XcnA","extract-moves-from-video__B9YUM3e","extract-moves-from-video__HNJXzhT","extract-moves-from-video__Tid2S54","extract-moves-from-video__rsbXtDv","financial-document-processor__JWSWaor","financial-document-processor__LDiB94H","financial-document-processor__bRL7Lu6","financial-document-processor__ksfh95L","gcode-to-text__32SDm5X","gcode-to-text__NUuv26b","gcode-to-text__iWQfNxR","gcode-to-text__oMEv7Pk","gcode-to-text__oVRUMbQ","gcode-to-text__sK57W6u","install-windows-3.11__6eR7v6K","install-windows-3.11__JZ7qe63","install-windows-3.11__fQWq85c","install-windows-3.11__i4pRHEw","path-tracing-reverse__mk8wLqB","path-tracing-reverse__qcmBpFT","path-tracing__67DANUV","path-tracing__LwgtseT","path-tracing__PeX93R3","path-tracing__Zu6pMmv","pytorch-model-cli__7jFFwzB","pytorch-model-cli__KeQHc4i","raman-fitting__73SB642","raman-fitting__nQNcfps","raman-fitting__nVTCjxZ","regex-chess__p4rSkKh","sam-cell-seg__DEBEgzp","sam-cell-seg__FExKSXj","sam-cell-seg__ib5mhsV","sam-cell-seg__qyRmatQ","video-processing__7NmtwUq","video-processing__8tSptZe","video-processing__KAM9Ctv","video-processing__YGD4hT5","video-processing__h6xUjWa"],"outcome":"infra","reason":"Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."}],"reason":"Terminal-Bench answer key retrieved over the network, confirmed by reading the trajectory (docs/incidents/terminal-bench-answer-key-confirmed.json); counts as a fail, pass or fail (Joel, 2026-10-29).; Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":2,"failed":0},"n_slots":2,"n_new":2,"n_scored":2,"n_reused":2347,"cost_new_usd":0.1725,"by_harness":{"pi":{"fail":1,"pass":1}}},"deep-swe":{"units":{"pending":0,"running":0,"done":1,"failed":0},"n_slots":2,"n_new":2,"n_scored":2,"n_reused":2995,"cost_new_usd":0.1104,"by_harness":{"pi":{"fail":1,"pass":1}}},"featurebench":{"units":{"pending":0,"running":0,"done":2,"failed":0},"n_slots":3,"n_new":0,"n_scored":0,"n_reused":4182,"cost_new_usd":0,"by_harness":{}}},"n_new":4,"n_scored":4,"n_slots":7,"n_reused":9524,"cost_new_usd":0.2829,"updated_at":"2026-10-23T19:52:38.936001+00:00"},{"run_id":"20261023-105153-deepseek-v4-1-flash-full-high-7530","created_at":"2026-10-23T10:51:53.006110+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe","featurebench"],"n_trials":9531,"cost_cap_usd":2.0,"estimate_usd":{"new_trials":2,"low":0.08,"high":0.19,"per_trial":{"terminal-bench-2.1":{"mean":0.0417,"reserve":0.0974,"n_observed":2389,"source":"observed"},"deep-swe":{"mean":0.1958,"reserve":0.3714,"n_observed":3089,"source":"observed"},"featurebench":{"mean":0.0379,"reserve":0.0807,"n_observed":4208,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2347,"n_new":2,"n_units":2},"deep-swe":{"n_tasks":111,"n_reused":2997,"n_new":0,"n_units":0},"featurebench":{"n_tasks":155,"n_reused":4185,"n_new":0,"n_units":0}},"code_revision":"d31ad630b3e6ad060f74d61b78ab9c9ddee86ffd","harness_adapters":{},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["terminal-bench-2.1"],"trials":["bn-fit-modify__EjWPD6Q","build-pov-ray__HuyDRkU","configure-git-webserver__5wGpVZt","count-dataset-tokens__BhQqmrx","count-dataset-tokens__cfxDwSU","count-dataset-tokens__qWUbHqU","db-wal-recovery__8aL4vg8","db-wal-recovery__L8rNKZh","db-wal-recovery__kVEEmDT","db-wal-recovery__kcacnod","dna-insert__DC92Pst","dna-insert__Gab2Rex","extract-elf__7T6kuGv","extract-elf__9wLW3vP","extract-elf__CycTtna","extract-elf__JyDazz9","extract-elf__KNV5dcK","extract-elf__Syx56JM","extract-elf__WFuB737","extract-elf__cKZDXBm","extract-elf__fBwFGaF","extract-elf__fZzRCUC","extract-elf__qqacUPc","extract-elf__sUiAVnt","filter-js-from-html__iEVTh68","filter-js-from-html__yH8tPMg","gpt2-codegolf__H22hyM8","gpt2-codegolf__QkriHER","gpt2-codegolf__Z9LJ8r6","headless-terminal__YMurycd","headless-terminal__dm7gveY","mteb-leaderboard__FF5GvoG","mteb-leaderboard__FFqJhKh","mteb-leaderboard__U9udi9g","mteb-leaderboard__sKS53b5","mteb-retrieve__3zXLaFh","mteb-retrieve__58uhfYn","mteb-retrieve__FDPyFdv","mteb-retrieve__VHyrVX7","mteb-retrieve__g33SzTn","mteb-retrieve__hyxzv8V","mteb-retrieve__ixLcxyz","mteb-retrieve__nCiEHKd","mteb-retrieve__sf4BNcS","mteb-retrieve__vjXwPP6","mteb-retrieve__xR6VELS","protein-assembly__ELA5nY8","raman-fitting__UynfqyR","raman-fitting__cZGi9GJ","regex-chess__Hcgcq9H","sanitize-git-repo__ayMZRgh","torch-pipeline-parallelism__9RB4HZ9","torch-pipeline-parallelism__DLTDCkS","torch-pipeline-parallelism__M9KoCZ2","torch-pipeline-parallelism__c4PMsw7","torch-pipeline-parallelism__fcpCDHL","torch-tensor-parallelism__DXbvvPM"],"outcome":"fail","reason":"Terminal-Bench answer key retrieved over the network, confirmed by reading the trajectory (docs/incidents/terminal-bench-answer-key-confirmed.json); counts as a fail, pass or fail (Joel, 2026-10-29)."},{"benchmarks":["deep-swe"],"trials":["arktype-json-schema-refs-depende__43zF6R4","arktype-json-schema-refs-depende__n7v4rDQ","dynamodb-toolbox-lazy-recursive__y6apKWo","expr-try-catch-errors__5ASkZpK","koota-pair-relation-tracking__gmQq4kQ","koota-query-predicates__FonPYz7","meriyah-explicit-resource-declar__PHRzxNn","opa-template-string-reconstructi__uMX2iCd","oxvg-structural-selector-preserv__57ZdSRc","scriggo-method-declarations__8zFMquE","scriggo-method-declarations__hYuthM9","scriggo-method-declarations__rqAPyqX","wasmi-trap-coredumps__AiiT2rJ","wasmi-trap-coredumps__sijgito"],"outcome":"infra","reason":"Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."},{"benchmarks":["terminal-bench-2.1"],"trials":["build-pov-ray__Cunjpg4","build-pov-ray__U22aQHA","build-pov-ray__YbPyK38","build-pov-ray__ZMh73Z5","build-pov-ray__ffg335n","build-pov-ray__nGBFWjW","caffe-cifar-10__aQ24SCY","caffe-cifar-10__fdT3KHS","chess-best-move__E9r6VoW","chess-best-move__Ec9kZfd","chess-best-move__VaaLbUX","chess-best-move__nfLc832","chess-best-move__sfTgaPJ","chess-best-move__wLdAxDc","code-from-image__8V4SrSn","code-from-image__HBfqq4s","code-from-image__M79KP54","code-from-image__MdNKLP8","code-from-image__QkMK7cZ","code-from-image__qGSpn2j","extract-moves-from-video__3yso389","extract-moves-from-video__Zrna7qj","extract-moves-from-video__zK7DkBc","financial-document-processor__F2Rrstd","financial-document-processor__kb94HFc","financial-document-processor__mWGRQmW","financial-document-processor__ryokyzu","financial-document-processor__z8WxUaG","gcode-to-text__4QjLG7r","gcode-to-text__u3EYVE3","gcode-to-text__vVB8Xai","gcode-to-text__vWqip4x","install-windows-3.11__4euohxt","install-windows-3.11__HDGoKD5","install-windows-3.11__g5wsPYF","install-windows-3.11__kWNdYmx","install-windows-3.11__uuz9rwY","install-windows-3.11__vBAC4eA","make-mips-interpreter__7cdwrez","make-mips-interpreter__S2vJQct","make-mips-interpreter__cCpjDez","path-tracing-reverse__AZXHnnx","path-tracing-reverse__Aj9jqBq","path-tracing-reverse__tpjLamA","path-tracing__7XUdouS","path-tracing__EGEB7nR","path-tracing__YNPCRsu","path-tracing__iHKx3BE","path-tracing__rB3c8Bb","pytorch-model-cli__NKkmSkG","raman-fitting__Mk3BA9q","raman-fitting__Q4wVy4L","raman-fitting__f3GsnLz","raman-fitting__qRZdc7s","raman-fitting__wjzGHHh","raman-fitting__z3UZCAh","sam-cell-seg__3FRRXNU","sam-cell-seg__V362SXt","sam-cell-seg__sVVoZqS","sam-cell-seg__xTSGYfm","sam-cell-seg__xoku9ps","sam-cell-seg__yHaxTyA","video-processing__hmVJufn","video-processing__i5AFHsh","video-processing__qQnUb92","video-processing__rpFX7gA","video-processing__t9e4RuU","video-processing__vbu6mEk","winning-avg-corewars__GuuQDia"],"outcome":"infra","reason":"Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."}],"reason":"Terminal-Bench answer key retrieved over the network, confirmed by reading the trajectory (docs/incidents/terminal-bench-answer-key-confirmed.json); counts as a fail, pass or fail (Joel, 2026-10-29).; Ended on a request our Responses bridge rejected (docs/incidents/responses-bridge-rejections.json); infrastructure failure, run again, pass or fail alike (Joel, 2026-10-29)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":2,"failed":0},"n_slots":2,"n_new":2,"n_scored":2,"n_reused":2347,"cost_new_usd":0.1996,"by_harness":{"qwen-code":{"pass":1},"openclaw":{"fail":1}}},"deep-swe":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":2997,"cost_new_usd":0,"by_harness":{}},"featurebench":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":4185,"cost_new_usd":0,"by_harness":{}}},"n_new":2,"n_scored":2,"n_slots":2,"n_reused":9529,"cost_new_usd":0.1996,"updated_at":"2026-10-23T12:20:49.941030+00:00"},{"run_id":"20261023-105150-glm-5-3-flash-full-high-2b66","created_at":"2026-10-23T10:51:50.082231+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe","featurebench"],"n_trials":9531,"cost_cap_usd":20.0,"estimate_usd":{"new_trials":45,"low":8.8,"high":16.53,"per_trial":{"terminal-bench-2.1":{"mean":0.0456,"reserve":0.1057,"n_observed":2465,"source":"observed"},"deep-swe":{"mean":0.3438,"reserve":0.6481,"n_observed":3244,"source":"observed"},"featurebench":{"mean":0.0658,"reserve":0.1217,"n_observed":4323,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2349,"n_new":0,"n_units":0},"deep-swe":{"n_tasks":111,"n_reused":2976,"n_new":21,"n_units":15},"featurebench":{"n_tasks":155,"n_reused":4161,"n_new":24,"n_units":5}},"code_revision":"d31ad630b3e6ad060f74d61b78ab9c9ddee86ffd","harness_adapters":{},"harness_locks":{},"invalid":null,"state":"cost_stopped","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":2349,"cost_new_usd":0,"by_harness":{}},"deep-swe":{"units":{"pending":0,"running":0,"done":15,"failed":0},"n_slots":21,"n_new":25,"n_scored":20,"n_reused":2976,"cost_new_usd":13.64,"by_harness":{"openclaw":{"fail":10,"infra":4,"pass":8,"cancelled":1},"codex":{"fail":1},"pi":{"fail":1}}},"featurebench":{"units":{"pending":0,"running":0,"done":5,"failed":0},"n_slots":24,"n_new":0,"n_scored":0,"n_reused":4161,"cost_new_usd":0,"by_harness":{}}},"n_new":25,"n_scored":20,"n_slots":45,"n_reused":9486,"cost_new_usd":13.64,"updated_at":"2026-10-23T17:07:01.809316+00:00"},{"run_id":"20261023-105147-glm-5-3-flash-full-low-242a","created_at":"2026-10-23T10:51:47.087681+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe","featurebench"],"n_trials":9531,"cost_cap_usd":5.0,"estimate_usd":{"new_trials":17,"low":1.39,"high":2.74,"per_trial":{"terminal-bench-2.1":{"mean":0.0435,"reserve":0.0939,"n_observed":2429,"source":"observed"},"deep-swe":{"mean":0.2585,"reserve":0.4526,"n_observed":3186,"source":"observed"},"featurebench":{"mean":0.0602,"reserve":0.127,"n_observed":4306,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2347,"n_new":2,"n_units":2},"deep-swe":{"n_tasks":111,"n_reused":2995,"n_new":2,"n_units":2},"featurebench":{"n_tasks":155,"n_reused":4172,"n_new":13,"n_units":3}},"code_revision":"d31ad630b3e6ad060f74d61b78ab9c9ddee86ffd","harness_adapters":{},"harness_locks":{},"invalid":null,"state":"cost_stopped","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":2,"failed":0},"n_slots":2,"n_new":2,"n_scored":2,"n_reused":2347,"cost_new_usd":1.532,"by_harness":{"openclaw":{"fail":2}}},"deep-swe":{"units":{"pending":0,"running":0,"done":2,"failed":0},"n_slots":2,"n_new":2,"n_scored":2,"n_reused":2995,"cost_new_usd":3.343,"by_harness":{"openclaw":{"fail":2}}},"featurebench":{"units":{"pending":0,"running":0,"done":3,"failed":0},"n_slots":13,"n_new":0,"n_scored":0,"n_reused":4172,"cost_new_usd":0,"by_harness":{}}},"n_new":4,"n_scored":4,"n_slots":17,"n_reused":9514,"cost_new_usd":4.875,"updated_at":"2026-10-23T14:58:54.309586+00:00"},{"run_id":"20261023-065049-deepseek-v4-1-flash-full-high-3d35","created_at":"2026-10-23T06:50:49.069757+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe","featurebench"],"n_trials":9531,"cost_cap_usd":15.0,"estimate_usd":{"new_trials":13,"low":2.24,"high":4.3,"per_trial":{"terminal-bench-2.1":{"mean":0.0417,"reserve":0.0972,"n_observed":2387,"source":"observed"},"deep-swe":{"mean":0.1959,"reserve":0.3735,"n_observed":3078,"source":"observed"},"featurebench":{"mean":0.0379,"reserve":0.0807,"n_observed":4208,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2347,"n_new":2,"n_units":2},"deep-swe":{"n_tasks":111,"n_reused":2986,"n_new":11,"n_units":10},"featurebench":{"n_tasks":155,"n_reused":4185,"n_new":0,"n_units":0}},"code_revision":"79724c46d1e0fd13b6fe484a3ae9f533fc162fbb","harness_adapters":{},"harness_locks":{},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":2,"failed":0},"n_slots":2,"n_new":3,"n_scored":2,"n_reused":2347,"cost_new_usd":0.1899,"by_harness":{"claude-code":{"fail":2,"infra":1}}},"deep-swe":{"units":{"pending":0,"running":0,"done":10,"failed":0},"n_slots":11,"n_new":12,"n_scored":11,"n_reused":2986,"cost_new_usd":1.86,"by_harness":{"hermes":{"pass":4,"fail":5},"claude-code":{"fail":1},"codex":{"infra":1,"pass":1}}},"featurebench":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":4185,"cost_new_usd":0,"by_harness":{}}},"n_new":15,"n_scored":13,"n_slots":13,"n_reused":9518,"cost_new_usd":2.05,"updated_at":"2026-10-23T08:13:22.124868+00:00"},{"run_id":"20261023-065046-deepseek-v4-1-flash-full-low-92cb","created_at":"2026-10-23T06:50:46.144706+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe","featurebench"],"n_trials":9531,"cost_cap_usd":15.0,"estimate_usd":{"new_trials":10,"low":1.54,"high":2.77,"per_trial":{"terminal-bench-2.1":{"mean":0.0341,"reserve":0.0797,"n_observed":2362,"source":"observed"},"deep-swe":{"mean":0.1537,"reserve":0.2767,"n_observed":3036,"source":"observed"},"featurebench":{"mean":0.0251,"reserve":0.0467,"n_observed":4191,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":2349,"n_new":0,"n_units":0},"deep-swe":{"n_tasks":111,"n_reused":2987,"n_new":10,"n_units":10},"featurebench":{"n_tasks":155,"n_reused":4185,"n_new":0,"n_units":0}},"code_revision":"79724c46d1e0fd13b6fe484a3ae9f533fc162fbb","harness_adapters":{},"harness_locks":{},"invalid":null,"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":2349,"cost_new_usd":0,"by_harness":{}},"deep-swe":{"units":{"pending":0,"running":0,"done":10,"failed":0},"n_slots":10,"n_new":12,"n_scored":9,"n_reused":2987,"cost_new_usd":1.531,"by_harness":{"hermes":{"pass":5,"fail":3},"claude-code":{"infra":3,"pass":1}}},"featurebench":{"units":{"pending":0,"running":0,"done":0,"failed":0},"n_slots":0,"n_new":0,"n_scored":0,"n_reused":4185,"cost_new_usd":0,"by_harness":{}}},"n_new":12,"n_scored":9,"n_slots":10,"n_reused":9521,"cost_new_usd":1.531,"updated_at":"2026-10-23T08:10:47.455888+00:00"},{"run_id":"20261020-011839-deepseek-v4-1-flash-full-high-23dc","created_at":"2026-10-20T01:18:39.723265+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"high","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe","featurebench"],"n_trials":9531,"cost_cap_usd":1800.0,"estimate_usd":{"new_trials":8183,"low":749.5,"high":1437.0,"per_trial":{"terminal-bench-2.1":{"mean":0.0431,"reserve":0.1072,"n_observed":487,"source":"observed"},"deep-swe":{"mean":0.2077,"reserve":0.3886,"n_observed":520,"source":"observed"},"featurebench":{"mean":0.037,"reserve":0.065,"n_observed":465,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":450,"n_new":1899,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":448,"n_new":2549,"n_units":111},"featurebench":{"n_tasks":155,"n_reused":450,"n_new":3735,"n_units":155}},"code_revision":"3710a5c0c56dbe16a3e741738f57991c617b080a","harness_adapters":{},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["deep-swe"],"trials":["abs-stepped-slices__SqJEKw2","bandit-structured-nosec-directiv__2mtdBPp","clack-async-autocomplete-options__777Fmrd","go-genai-streamed-function-args__rmUkqjt","koota-composite-trait-aspects__3y2bA46","koota-deferred-mutation-buffer__U5j6Gz4","koota-deferred-mutation-buffer__d5FiAqb","oxvg-structural-selector-preserv__hNFaRtY","skrub-duration-encoding__68T968a"]},{"benchmarks":["terminal-bench-2.1"],"trials":["path-tracing-reverse__7vTjMRC","reshard-c4-data__h3n8F9d"],"reason":"Provider stream broke off or was empty (the meter logged the last call as 200 before it learned to log these as 599, provider failure); infra, re-run."},{"benchmarks":["deep-swe"],"trials":["abs-module-cache-flags__utMmnY6","abs-stepped-slices__BiLtBqZ","abs-stepped-slices__JaaWWmA","abs-stepped-slices__drJ4M78","actionlint-action-pinning-lint__C5G2GkG","actionlint-action-pinning-lint__RRNso6U","actionlint-action-pinning-lint__SUXPw6j","actionlint-action-pinning-lint__kWAoUq3","adaptix-name-mapping-aliases__M9P9r9n","adaptix-name-mapping-aliases__VgW6cH6","aiomonitor-task-snapshots-diff__ce5iLDW","aiomonitor-task-snapshots-diff__x9ZXDqm","anko-default-function-arguments__54nyzhr","anko-default-function-arguments__NhfTYzu","anko-typed-variable-bindings__2px8mqC","anko-typed-variable-bindings__2q64FVB","anko-typed-variable-bindings__SRZQmKe","anko-typed-variable-bindings__XGibxnP","anko-typed-variable-bindings__awkP8iK","anko-typed-variable-bindings__eUQ9FmH","anko-typed-variable-bindings__gzNdwfZ","arktype-json-schema-refs-depende__smCNWRc","awilix-async-container-initializ__TLih4ca","bandit-structured-nosec-directiv__QRwuvh5","bandit-structured-nosec-directiv__phQEspp","claude-code-by-agents-recursive__EuJSkU9","claude-code-by-agents-recursive__oS9rWKR","csstree-shorthand-expansion-comp__HBW9k7H","dasel-html-document-format__8JpQqbA","dasel-html-document-format__NEPPM6s","dasel-html-document-format__w3Kdg6j","dateutil-rfc5545-timezone-intero__3bp3z8X","dateutil-rfc5545-timezone-intero__KzQ6uSq","dateutil-rfc5545-timezone-intero__QUk5QZw","dateutil-rfc5545-timezone-intero__XToBSA7","dateutil-rfc5545-timezone-intero__bwXktJw","dateutil-rfc5545-timezone-intero__fAregea","dateutil-rfc5545-timezone-intero__rhiA5Xf","dateutil-rfc5545-timezone-intero__t9VzKqF","dateutil-rfc5545-timezone-intero__zEjNoYU","dynamodb-toolbox-conditional-att__7nvm4zj","dynamodb-toolbox-lazy-recursive__5Wxxrzi","dynamodb-toolbox-lazy-recursive__6zthGig","dynamodb-toolbox-lazy-recursive__wsyg4ok","effect-sse-httpapi-streaming__CQVsmh6","effect-sse-httpapi-streaming__KaZMHyJ","effect-sse-httpapi-streaming__hFFxg4m","etree-xml-diff-patch__CCLLYbg","etree-xml-diff-patch__Kg3pQSx","etree-xml-diff-patch__b88qrQU","fd-deterministic-multi-key-sorti__ErCYqbQ","geo-shapeindex-serialization__S5Ztb83","go-critic-doc-link-checker__PDoXhJa","go-git-worktree-merge-conflicts__2935ZY7","go-git-worktree-merge-conflicts__CyfgNMV","go-git-worktree-merge-conflicts__tsTkC8m","go-git-worktree-merge-conflicts__xb45fFH","goreleaser-retry-publish-auditin__UdLFdKd","goreleaser-retry-publish-auditin__wFBAzqr","gql-incremental-graphql-delivery__9K86tQn","gql-incremental-graphql-delivery__Vexnaia","gql-incremental-graphql-delivery__ZSEeJVG","happy-dom-deterministic-intersec__TBEH4Jm","happy-dom-deterministic-intersec__ntExmgT","httpx-deterministic-cookie-store__PqwoEvQ","httpx-deterministic-cookie-store__WXbfku9","httpx-multipart-response-parsing__6cxKmWd","httpx-multipart-response-parsing__m5aKCYx","httpx-multipart-response-parsing__pZUmA9L","httpx-streaming-json-iteration__dU9Sa4i","httpx-streaming-json-iteration__qJQ7P6X","httpx-streaming-json-iteration__xx3iC29","katex-multicolumn-array-spans__D7MsdNd","kgateway-consistent-hash-policy__tR9NG7X","kombu-single-active-consumer-pri__6scJs7t","kombu-single-active-consumer-pri__d2dqigN","kombu-virtual-queue-dead-letteri__9K3i6fk","kombu-virtual-queue-dead-letteri__BJ8VqQj","kombu-virtual-queue-dead-letteri__rDD3sDv","koota-composite-trait-aspects__RDsfUJY","koota-deferred-mutation-buffer__ZeeheEx","koota-deferred-mutation-buffer__oiSPjR5","koota-entity-snapshot-rollback__Di4tyyo","koota-entity-snapshot-rollback__ePRuxTf","kysely-window-grouping-helpers__sD9x83K","mashumaro-flattened-dataclass-fi__NqYgCs5","mashumaro-flattened-dataclass-fi__mtzRmYQ","mobly-grouped-test-barriers__SEnkpA5","mobly-grouped-test-barriers__W7URURm","narwhals-rolling-window-suite__uMwZPWY","numba-stencil-boundary-modes__7wZXNvu","obsidian-linter-auto-table-of-co__87xbQ8z","obsidian-linter-auto-table-of-co__NfTVsr6","obsidian-linter-auto-table-of-co__Y5FZkMy","obsidian-linter-auto-table-of-co__dL4qCNN","obsidian-linter-auto-table-of-co__n9t7Wyk","obsidian-linter-auto-table-of-co__tx7XCYA","obsidian-linter-link-format-conv__YC6kqnM","opa-template-string-reconstructi__3F9N6uo","oxvg-structural-selector-preserv__j3KNVqv","oxvg-structural-selector-preserv__vwseCzr","participle-grammar-conflict-anal__PTETmvS","pest-character-class-coalescing__HGKEKeD","pest-character-class-coalescing__QDjAzRE","pest-character-class-coalescing__QSWmya6","pest-character-class-coalescing__Shr8HxU","pest-character-class-coalescing__WDKxPGV","pest-character-class-coalescing__aNrmFcJ","pest-character-class-coalescing__dcuaDPx","pest-character-class-coalescing__nK7fB4e","prometheus-transactional-reload__B6JbJrf","prometheus-transactional-reload__PEscrYk","prometheus-typed-label-sorting__S72P4tA","psd-tools-blend-range-api__qiRdXSf","python-statemachine-state-data-s__pMseB4D","query-persist-restored-query-sta__2zAwpSu","query-persist-restored-query-sta__JVi335Z","quill-shared-toolbar-focus__9Z9N6BF","quill-shared-toolbar-focus__AUxu269","quill-shared-toolbar-focus__FEcrosh","quill-shared-toolbar-focus__YDFo5n8","quill-shared-toolbar-focus__tsNsLVU","quill-shared-toolbar-focus__yRZr3A4","returns-validated-error-accumula__HC6u5Bj","sql-formatter-bigquery-pipe-form__H9JSK6n","sql-formatter-bigquery-pipe-form__kEkbqTE","sql-formatter-bigquery-pipe-form__sZXGkod","sql-formatter-bigquery-pipe-form__v92AGZg","sqlfmt-create-table-ddl-formatti__6PzGv6M","sqlfmt-create-table-ddl-formatti__G5qPMHX","sqlfmt-create-table-ddl-formatti__giniTSn","superjson-error-stack-serializat__VTg2gSt","superjson-error-stack-serializat__ZnwSKUu","task-task-graph-export__mdJW336","tengo-callable-instance-isolatio__EUW78GU","tengo-destructuring-bindings__EocWKeu","tengo-destructuring-bindings__GaHo98M","tengo-destructuring-bindings__NbVeNM9","termenv-preserve-ansi-resets__2b8uK9d","termenv-preserve-ansi-resets__3tmiZbK","termenv-preserve-ansi-resets__46ntqCR","termenv-preserve-ansi-resets__HeVqX9X","termenv-preserve-ansi-resets__wNaSNND","testem-bail-on-test-failure__4xz7rPY","testem-bail-on-test-failure__7ttYz76","testem-bail-on-test-failure__LZ4ExyA","testem-bail-on-test-failure__LshnrwL","testem-bail-on-test-failure__WnEkjms","testem-bail-on-test-failure__dfF4v7q","testem-bail-on-test-failure__hf3KcS8","testem-per-launcher-reports__p9jfNeu","textual-kitty-key-phases__PAwkPSP","textual-richlog-follow-state__RyufEyK","textual-richlog-follow-state__mGRESWJ","textual-richlog-follow-state__rfP8CuN","updo-policy-alerting__6M3HS4w","valibot-recursive-schema-composi__WCXbxFf","vitest-duration-sharding__bvqpCDj","wasmi-trap-coredumps__4NxbydR","wasmi-trap-coredumps__Q7mvLPC","wasmi-trap-coredumps__YdKLabF","ytt-jsonpath-query-api__QXqLoFM"],"outcome":"infra","reason":"Looked up its answer online while this DeepSWE run mistakenly had network access, confirmed by reading the trajectory (docs/incidents/atif-scan-readings.json); run again offline, DeepSWE's own setup (Joel, 2026-10-30)."},{"benchmarks":["terminal-bench-2.1"],"trials":["count-dataset-tokens__9VFu7tn","count-dataset-tokens__aZTs3XR","count-dataset-tokens__kRECQ4k","count-dataset-tokens__ycFM7ko","count-dataset-tokens__zfhf5hU","extract-elf__XgsYL7r","financial-document-processor__auEvjVV","fix-ocaml-gc__3AE68Gd","fix-ocaml-gc__3bV8heb","fix-ocaml-gc__5WHDtZX","fix-ocaml-gc__GAk6G9H","fix-ocaml-gc__HKDs6UN","fix-ocaml-gc__QerNYnN","fix-ocaml-gc__Wy2nF5K","fix-ocaml-gc__ZUkffon","fix-ocaml-gc__g7WtiXD","fix-ocaml-gc__qFC9f2v","fix-ocaml-gc__qxnvzSv","fix-ocaml-gc__wzcgk6e","path-tracing-reverse__n2P42Yc","path-tracing-reverse__oneuehc","raman-fitting__A6XzFAy","regex-chess__Uxogh3i"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["deep-swe"],"trials":["abs-stepped-slices__egXs6Vo","actionlint-action-pinning-lint__2jMYhL8","actionlint-action-pinning-lint__88Vfnku","actionlint-action-pinning-lint__9583Dwn","actionlint-action-pinning-lint__b6n8qQj","adaptix-name-mapping-aliases__2UXgkAe","adaptix-name-mapping-aliases__4V7xHWT","aiomonitor-task-snapshots-diff__MJcMBc3","aiomonitor-task-snapshots-diff__Piynddq","aiomonitor-task-snapshots-diff__TtYxxSL","aiomonitor-task-snapshots-diff__kQkdHRR","anko-default-function-arguments__fv7RzJK","anko-typed-variable-bindings__9sHZAmu","anko-typed-variable-bindings__gnY6d4Z","awilix-async-container-initializ__N3ydP4s","bandit-interprocedural-taint-che__dTCHBqs","bandit-interprocedural-taint-che__pZHVegq","bandit-structured-nosec-directiv__N7y63wa","boa-hierarchical-evaluation-canc__MSWDywP","cattrs-partial-structuring-recov__xefwqMA","clack-async-autocomplete-options__5MuhMEc","clack-async-autocomplete-options__i7jrvLr","cliffy-config-file-parsing__UREdQET","csstree-shorthand-expansion-comp__Baor3DZ","csstree-shorthand-expansion-comp__aq42s78","csstree-shorthand-expansion-comp__gYvSwvo","dasel-html-document-format__vXTYYxh","dasel-html-document-format__ysmKeuT","dateutil-rfc5545-timezone-intero__9VP5qqw","dateutil-rfc5545-timezone-intero__Je4LuBn","drizzle-orm-window-function-buil__2RYaFtX","drizzle-orm-window-function-buil__HqSYXC5","drizzle-orm-window-function-buil__Ny9rwHU","drizzle-orm-window-function-buil__xR6AxJo","dynamodb-toolbox-conditional-att__N5j3HxZ","dynamodb-toolbox-lazy-recursive__8CUUHTo","dynamodb-toolbox-lazy-recursive__LdtHyn2","effect-sse-httpapi-streaming__meMHmpR","etree-xml-diff-patch__CZXhsFy","expr-try-catch-errors__J6rBKpw","fastapi-implicit-head-options__3hbX4uy","fastapi-implicit-head-options__TXRNaNs","fastapi-implicit-head-options__eKe2nNG","fd-deterministic-multi-key-sorti__Au2TexN","fd-deterministic-multi-key-sorti__CDkkFxy","geo-shapeindex-serialization__BgxEXcM","go-critic-doc-link-checker__A3v8yav","go-critic-doc-link-checker__wu6bbCq","go-critic-doc-link-checker__xRAsXC5","go-critic-doc-link-checker__zTnGCon","go-genai-streamed-function-args__2HqWXcT","go-genai-streamed-function-args__HRXCT5R","go-genai-streamed-function-args__PgnUa2Z","go-genai-streamed-function-args__VGaHwqx","go-genai-streamed-function-args__fpfKqFf","go-genai-streamed-function-args__oGaiq7e","go-git-worktree-merge-conflicts__DBUy62J","go-git-worktree-merge-conflicts__USyvEMo","go-git-worktree-merge-conflicts__cmsGjpK","go-git-worktree-merge-conflicts__dUP4eTh","goreleaser-retry-publish-auditin__AX3cms3","goreleaser-retry-publish-auditin__FtGZ9WT","goreleaser-retry-publish-auditin__JSsLF2f","goreleaser-retry-publish-auditin__zc8mn7Q","happy-dom-abort-pending-body-rea__My58nLL","helm-array-merge-strategies__XFjxYRt","helm-array-merge-strategies__rmcfkqY","httpx-deterministic-cookie-store__2WfEfx5","httpx-deterministic-cookie-store__82UH4X3","httpx-deterministic-cookie-store__GASUsCB","httpx-deterministic-cookie-store__qPeSSA5","httpx-multipart-response-parsing__t4tGJcw","httpx-streaming-json-iteration__7VygAoy","httpx-streaming-json-iteration__9Pj7roJ","igel-persist-feature-schema__4892x8y","igel-persist-feature-schema__GNfPWiU","igel-persist-feature-schema__GzZ5Ctk","igel-persist-feature-schema__qsWTTeM","katex-multicolumn-array-spans__KUApzn3","katex-multicolumn-array-spans__kGw8Zdx","kombu-single-active-consumer-pri__ZprTDxW","kombu-virtual-queue-dead-letteri__enWHaaR","kombu-virtual-queue-dead-letteri__es7mtqh","koota-deferred-mutation-buffer__BZN3tpx","koota-deferred-mutation-buffer__ZCCWnNv","koota-deferred-mutation-buffer__ZodSMuL","koota-entity-snapshot-rollback__PKPkriA","koota-entity-snapshot-rollback__UkhfqGa","kysely-window-grouping-helpers__5u46prA","kysely-window-grouping-helpers__b8SPkzq","kysely-window-grouping-helpers__iAFVAAL","kysely-window-grouping-helpers__jFtmCom","kysely-window-grouping-helpers__qtR4Pcm","langchain-request-coalescing__jvV2M5u","langchain-request-coalescing__ve4h3Za","mashumaro-flattened-dataclass-fi__4oCEqrZ","mashumaro-flattened-dataclass-fi__5GK45JJ","mashumaro-flattened-dataclass-fi__ohqEqEp","mashumaro-flattened-dataclass-fi__tM6TWQK","mashumaro-flattened-dataclass-fi__tVp7oMu","mobly-grouped-test-barriers__7Hj3m5R","narwhals-rolling-window-suite__3CC3ar7","narwhals-rolling-window-suite__EgYbGZR","numba-stencil-boundary-modes__GLmeVEd","obsidian-linter-auto-table-of-co__Rr8XVgX","obsidian-linter-auto-table-of-co__YXCzjSJ","obsidian-linter-auto-table-of-co__dKizcCj","obsidian-linter-auto-table-of-co__pncYZmS","obsidian-linter-auto-table-of-co__sQDxGMR","obsidian-linter-scoped-ignore-ma__UZRevVq","obsidian-linter-scoped-ignore-ma__x9YauRU","onedump-dump-encryption-pipeline__4SiHL4R","onedump-dump-encryption-pipeline__9T4hgPk","opa-rego-rule-profiling__CWYiGWs","opa-rego-rule-profiling__t8TNJzc","oxvg-structural-selector-preserv__8zZfRbc","oxvg-structural-selector-preserv__dqzw8XU","oxvg-structural-selector-preserv__fG6cTVf","participle-grammar-conflict-anal__p9HBo2S","participle-grammar-conflict-anal__qqGz6yx","pest-character-class-coalescing__Bqc7Kka","pest-character-class-coalescing__jCjCgNV","prometheus-typed-label-sorting__CfCHWtv","prometheus-typed-label-sorting__F3YYGuu","prometheus-typed-label-sorting__z3dHF5y","psd-tools-blend-range-api__KwTXzV4","psd-tools-blend-range-api__kWMA38M","query-persist-restored-query-sta__uy4PqNv","returns-validated-error-accumula__NZrdfqZ","returns-validated-error-accumula__i28KfLn","returns-validated-error-accumula__i63QhGE","scc-bounded-memory-spilling__vENjTVA","skrub-duration-encoding__HHAq94r","sql-formatter-bigquery-pipe-form__CVLQa52","sql-formatter-bigquery-pipe-form__kZHNDeD","sqlfmt-create-table-ddl-formatti__9j9SocK","task-task-graph-export__N3XeGXT","task-task-graph-export__jmtBDzS","tengo-callable-instance-isolatio__aDb3PNC","tengo-callable-instance-isolatio__ctHiv3P","tengo-callable-instance-isolatio__esfqdDN","tengo-callable-instance-isolatio__h8MKvpc","tengo-callable-instance-isolatio__nMp643P","testem-bail-on-test-failure__9qpgCtm","testem-bail-on-test-failure__XGcuxNL","testem-bail-on-test-failure__jma7f7p","textual-kitty-key-phases__uhppniP","textual-kitty-key-phases__z5P967S","tomlkit-toml-table-converters__KTKFkCh","tomlkit-toml-table-converters__agxdKNB","tomlkit-toml-table-converters__oP6DqiL","true-myth-iterable-collection-co__Ne92q7L","ts-pattern-match-each__CZPj2Cd","ts-pattern-match-each__sHTnt7H","ts-pattern-match-each__xQ8r8Sx","updo-policy-alerting__7jsfatE","updo-policy-alerting__CjbPKFx","updo-policy-alerting__HTBT6Y2","updo-policy-alerting__Zi7J2f2","valibot-recursive-schema-composi__HqYUZiV","valibot-recursive-schema-composi__reqWBGV","wasmi-trap-coredumps__96D2PEs"],"outcome":"infra","reason":"Looked up its answer online while this DeepSWE run mistakenly had network access, confirmed by reading the trajectory (docs/incidents/atif-scan-readings.json); run again offline, DeepSWE's own setup (Joel, 2026-10-30).","from_release":"release-time-reruns"},{"benchmarks":["deep-swe"],"trials":["effect-sse-httpapi-streaming__DSsHeNW","effect-sse-httpapi-streaming__oSZd5XG"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["terminal-bench-2.1"],"trials":["path-tracing-reverse__UFsRByi"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["terminal-bench-2.1"],"trials":["code-from-image__BZ6shpD","fix-ocaml-gc__A2Q7NKA","fix-ocaml-gc__VTL6obV","fix-ocaml-gc__jv457q6","fix-ocaml-gc__tJXxhfa"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30).","from_release":"release-time-reruns"}],"reason":"Answer key retrieved with network access the benchmark did not intend (docs/incidents/2026-10-22-deepswe-network.md): DeepSWE trials that reached the benchmark's own published solutions, tests or trial viewer. Re-run offline.; Provider stream broke off or was empty (the meter logged the last call as 200 before it learned to log these as 599, provider failure); infra, re-run.; Looked up its answer online while this DeepSWE run mistakenly had network access, confirmed by reading the trajectory (docs/incidents/atif-scan-readings.json); run again offline, DeepSWE's own setup (Joel, 2026-10-30).; Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":1899,"n_new":1936,"n_scored":1897,"n_reused":450,"cost_new_usd":78.52,"by_harness":{"claude-code":{"fail":48,"pass":161,"infra":26},"codex":{"fail":78,"pass":133,"infra":2},"hermes":{"fail":53,"pass":158},"kimi-code":{"fail":49,"pass":162,"infra":2},"mini-swe-agent":{"pass":172,"fail":39,"infra":1},"openclaw":{"fail":107,"pass":104,"infra":4},"opencode":{"fail":100,"pass":111,"infra":4},"pi":{"fail":39,"pass":172},"qwen-code":{"fail":48,"pass":163}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":2549,"n_new":2581,"n_scored":2547,"n_reused":448,"cost_new_usd":494.9,"by_harness":{"claude-code":{"pass":188,"fail":95,"infra":6},"codex":{"pass":195,"fail":88,"infra":5},"hermes":{"pass":227,"fail":56,"infra":1},"kimi-code":{"pass":186,"fail":97,"infra":1},"mini-swe-agent":{"pass":224,"fail":59,"infra":1},"openclaw":{"pass":52,"fail":231,"infra":7},"opencode":{"pass":113,"fail":170,"infra":6},"pi":{"pass":188,"fail":95,"infra":6},"qwen-code":{"pass":178,"fail":105,"infra":1}}},"featurebench":{"units":{"pending":0,"running":0,"done":155,"failed":0},"n_slots":3735,"n_new":0,"n_scored":0,"n_reused":450,"cost_new_usd":0,"by_harness":{}}},"n_new":4517,"n_scored":4444,"n_slots":8183,"n_reused":1348,"cost_new_usd":573.4,"updated_at":"2026-10-22T04:52:03.620626+00:00"},{"run_id":"20261020-011837-deepseek-v4-1-flash-full-low-7d5f","created_at":"2026-10-20T01:18:37.785897+00:00","model":"deepseek-ai/DeepSeek-V4.1-Flash","provider":"deepinfra","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe","featurebench"],"n_trials":9531,"cost_cap_usd":1500.0,"estimate_usd":{"new_trials":8182,"low":573.6,"high":1120.0,"per_trial":{"terminal-bench-2.1":{"mean":0.0339,"reserve":0.0793,"n_observed":456,"source":"observed"},"deep-swe":{"mean":0.1654,"reserve":0.3179,"n_observed":477,"source":"observed"},"featurebench":{"mean":0.0235,"reserve":0.0427,"n_observed":451,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":450,"n_new":1899,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":449,"n_new":2548,"n_units":111},"featurebench":{"n_tasks":155,"n_reused":450,"n_new":3735,"n_units":155}},"code_revision":"3710a5c0c56dbe16a3e741738f57991c617b080a","harness_adapters":{},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["deep-swe"],"trials":["drizzle-orm-window-function-buil__wHjNtRY","httpx-deterministic-cookie-store__oPJUovE","koota-deferred-mutation-buffer__fWAuLCn","narwhals-rolling-window-suite__v3Z9qBK","psd-tools-blend-range-api__6ykw262","python-statemachine-state-data-s__w826q2N","sqlfmt-create-table-ddl-formatti__rGv946K","valibot-recursive-schema-composi__QBhG5pY"]},{"benchmarks":["deep-swe"],"trials":["abs-stepped-slices__ZYLFL7F","actionlint-action-pinning-lint__AV38dsF","actionlint-action-pinning-lint__dTVBbxB","actionlint-action-pinning-lint__qCdCEpb","actionlint-action-pinning-lint__u5FJ5JQ","adaptix-name-mapping-aliases__bg8FVEd","adaptix-name-mapping-aliases__cCnAVUb","aiomonitor-task-snapshots-diff__DfcPJAp","aiomonitor-task-snapshots-diff__gCMEsEr","anko-default-function-arguments__Gumrbhf","anko-default-function-arguments__cV9Bm59","anko-default-function-arguments__cxBKeMZ","anko-typed-variable-bindings__MYpxUCJ","anko-typed-variable-bindings__gtfGVGV","arktype-json-schema-refs-depende__dW3jecW","arktype-json-schema-refs-depende__y4pQtx2","bandit-interprocedural-taint-che__xizyDHh","bandit-structured-nosec-directiv__27ZTECW","bandit-structured-nosec-directiv__i8b7CzD","bandit-structured-nosec-directiv__moBENUQ","boa-hierarchical-evaluation-canc__8chyct5","clack-async-autocomplete-options__6HRJAB8","clack-async-autocomplete-options__aGoTPN9","claude-code-by-agents-recursive__QwmVfTA","cliffy-config-file-parsing__WVUXaPE","cliffy-config-file-parsing__iDzN44u","csstree-shorthand-expansion-comp__772n57D","dasel-html-document-format__HPTHE98","dasel-html-document-format__cyyH8WE","dateutil-rfc5545-timezone-intero__84r9Fcx","dateutil-rfc5545-timezone-intero__QMUcCEm","dateutil-rfc5545-timezone-intero__ghPPeBu","dateutil-rfc5545-timezone-intero__ntxu4XR","dateutil-rfc5545-timezone-intero__ppPNvr5","dateutil-rfc5545-timezone-intero__sqYKYAQ","drizzle-orm-window-function-buil__Fhjkocx","dynamodb-toolbox-conditional-att__FYd8Wjg","dynamodb-toolbox-conditional-att__z5PtYxB","dynamodb-toolbox-lazy-recursive__DP8mYPX","dynamodb-toolbox-lazy-recursive__bHU7u6X","dynamodb-toolbox-lazy-recursive__nS5zYRb","dynamodb-toolbox-lazy-recursive__qs93VaV","effect-sse-httpapi-streaming__KAHbyrg","effect-sse-httpapi-streaming__NYapBcg","effect-sse-httpapi-streaming__UwLDZbp","effect-sse-httpapi-streaming__enNLGts","eicrud-keyset-pagination-cursor__vkmhVnU","go-critic-doc-link-checker__JWLXskp","go-critic-doc-link-checker__MKdhnfp","go-critic-doc-link-checker__QZxP4YF","go-critic-doc-link-checker__mXwT5hm","go-genai-streamed-function-args__5i7psrk","go-git-worktree-merge-conflicts__C4wGN2i","go-git-worktree-merge-conflicts__Wa64qUi","goreleaser-retry-publish-auditin__URsdR2f","goreleaser-retry-publish-auditin__ZPoTnXo","gql-incremental-graphql-delivery__JWfsDGo","gql-incremental-graphql-delivery__zt4zkjM","happy-dom-deterministic-intersec__qKwEmsk","helm-array-merge-strategies__BWDEGhN","httpx-deterministic-cookie-store__bH2AuFy","httpx-multipart-response-parsing__7m5wifK","httpx-multipart-response-parsing__YRUsTt3","httpx-multipart-response-parsing__YYPvFF5","httpx-multipart-response-parsing__ezG4M8D","httpx-multipart-response-parsing__mvBNtz6","httpx-streaming-json-iteration__b6MX5aA","httpx-streaming-json-iteration__iY8aZtY","katex-multicolumn-array-spans__sZ9nMkx","kcp-go-multiplexed-kcp-streams__Xe8Pzov","kombu-virtual-queue-dead-letteri__W7G8UVW","koota-deferred-mutation-buffer__LVeNmj5","koota-deferred-mutation-buffer__Tq9n7fA","koota-deferred-mutation-buffer__gCCvcoJ","koota-entity-snapshot-rollback__QPRRLTs","koota-entity-snapshot-rollback__e9KH3AQ","koota-entity-snapshot-rollback__gXpUoKT","koota-pair-relation-tracking__SGXkGK7","kysely-window-grouping-helpers__5oU5PoB","kysely-window-grouping-helpers__UMCoE3t","kysely-window-grouping-helpers__oUQv7jk","kysely-window-grouping-helpers__wWuPmPv","mashumaro-flattened-dataclass-fi__nL7e9d9","mobly-grouped-test-barriers__EEu3A7T","mobly-grouped-test-barriers__i5xHscv","narwhals-rolling-window-suite__bADuDQX","numba-stencil-boundary-modes__AQhr2St","obsidian-linter-auto-table-of-co__DC7Crau","obsidian-linter-auto-table-of-co__DoF3Fd4","obsidian-linter-auto-table-of-co__NMcw7KC","obsidian-linter-auto-table-of-co__TSPPFPc","obsidian-linter-auto-table-of-co__bJpBtKa","obsidian-linter-auto-table-of-co__m2nx6KL","ofetch-per-origin-circuit-breake__3onGHXq","ofetch-per-origin-circuit-breake__7ooNaEy","ofetch-per-origin-circuit-breake__EHmxpES","opa-rego-rule-profiling__WkPyEca","optique-conditional-option-depen__fwUMwTT","optique-conditional-option-depen__j7SjFqX","oxvg-structural-selector-preserv__KU6TYHx","participle-grammar-conflict-anal__mC9jLAq","participle-grammar-conflict-anal__mSrYMNF","pebble-durability-wait-apis__iZfk539","pest-character-class-coalescing__GNdUrw7","pest-character-class-coalescing__MukRzAD","pest-character-class-coalescing__W68naF5","pest-character-class-coalescing__WokpZnr","pest-character-class-coalescing__vcCBKnM","prometheus-transactional-reload__CC6Qh8N","prometheus-transactional-reload__eaCnrdg","prometheus-typed-label-sorting__B3Pnn3M","prometheus-typed-label-sorting__LZfnmG7","prometheus-typed-label-sorting__X66XsgA","prometheus-typed-label-sorting__dEzorud","pwntools-tube-multiplexing__ZXdfA5h","pwntools-tube-multiplexing__xLnoFw3","python-statemachine-state-data-s__nqGBEJa","python-statemachine-state-data-s__yCSBGUn","query-persist-restored-query-sta__AqeAEXn","query-persist-restored-query-sta__D8EmvGc","query-persist-restored-query-sta__vSuxpSe","quill-shared-toolbar-focus__CuQpsSr","quill-shared-toolbar-focus__aibB2pH","quill-shared-toolbar-focus__uDf7D3W","returns-validated-error-accumula__xj8Pa9E","skrub-duration-encoding__7zG4daG","skrub-duration-encoding__Kc24Vax","skrub-duration-encoding__nJss3bh","sql-formatter-bigquery-pipe-form__4muQRG5","sql-formatter-bigquery-pipe-form__Bt2uGFb","task-task-graph-export__QgdFeqL","task-task-graph-export__VYGdf6e","task-task-graph-export__wMPqicA","task-task-graph-export__yQtcbPe","tengo-callable-instance-isolatio__PYFHHcz","tengo-destructuring-bindings__AtxipyC","tengo-destructuring-bindings__rNvSnky","termenv-preserve-ansi-resets__NVPUn72","termenv-preserve-ansi-resets__TWSQTeo","termenv-preserve-ansi-resets__Y7TSz6d","termenv-preserve-ansi-resets__rEJurnf","testem-bail-on-test-failure__PGKfLKc","testem-bail-on-test-failure__cRDQYCY","testem-bail-on-test-failure__pGLsK7A","testem-per-launcher-reports__UvPF5WT","testem-per-launcher-reports__aNsdnFE","testem-per-launcher-reports__iG8w3zs","textual-kitty-key-phases__3r2AwDz","textual-kitty-key-phases__4E5W5yk","textual-kitty-key-phases__4SuCCxj","textual-kitty-key-phases__GDjC6eu","textual-richlog-follow-state__CUpJcjU","tomlkit-toml-table-converters__YeCTJao","valibot-recursive-schema-composi__JtN3wbb","wasmi-trap-coredumps__NMECoBQ","wasmi-trap-coredumps__U9ihrU5","yjs-map-conflict-detection__ps6mipE","ytt-jsonpath-query-api__3FweDxK","ytt-jsonpath-query-api__aMdhoH4"],"outcome":"infra","reason":"Looked up its answer online while this DeepSWE run mistakenly had network access, confirmed by reading the trajectory (docs/incidents/atif-scan-readings.json); run again offline, DeepSWE's own setup (Joel, 2026-10-30)."},{"benchmarks":["terminal-bench-2.1"],"trials":["count-dataset-tokens__VdVGREK","extract-elf__vSLZqMq","financial-document-processor__ZvAJmuR","fix-ocaml-gc__2cSU2w6","fix-ocaml-gc__6moavmj","fix-ocaml-gc__7H8jEKU","fix-ocaml-gc__9tM3gk3","fix-ocaml-gc__BKC6kJN","fix-ocaml-gc__FKvk6Np","fix-ocaml-gc__RLfAkqq","fix-ocaml-gc__RYG3esj","fix-ocaml-gc__ZrQfyqG","fix-ocaml-gc__aCPa4F9","fix-ocaml-gc__cBucq3M","fix-ocaml-gc__cunnNY2","fix-ocaml-gc__dwtpcPM","fix-ocaml-gc__oN2RSH7","fix-ocaml-gc__pSnMpfR","mteb-leaderboard__auzWhpT","path-tracing-reverse__c38gT3k","torch-tensor-parallelism__yxJXSv5"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["deep-swe"],"trials":["actionlint-action-pinning-lint__R2Pknau","adaptix-name-mapping-aliases__rrdGkQ7","aiomonitor-task-snapshots-diff__b2m94rZ","aiomonitor-task-snapshots-diff__cjE9LRz","anko-default-function-arguments__uSKo6Fs","anko-default-function-arguments__umLtY3n","anko-typed-variable-bindings__c8YVNZH","arktype-json-schema-refs-depende__oinxAUb","awilix-async-container-initializ__Mv64msi","awilix-async-container-initializ__UzBuWVQ","bandit-interprocedural-taint-che__mN3USWD","bandit-interprocedural-taint-che__pYJByrj","cattrs-partial-structuring-recov__Yso9A3Q","csstree-shorthand-expansion-comp__cWUYEDg","csstree-shorthand-expansion-comp__kRDNago","csstree-shorthand-expansion-comp__w4QW93j","dasel-html-document-format__JhdNSMJ","dateutil-rfc5545-timezone-intero__QuAk35r","dateutil-rfc5545-timezone-intero__frpgpt9","dateutil-rfc5545-timezone-intero__qkA3EwS","drizzle-orm-window-function-buil__4P2ca5o","drizzle-orm-window-function-buil__7tj4oPa","drizzle-orm-window-function-buil__9WAn6yp","drizzle-orm-window-function-buil__PXMpwof","drizzle-orm-window-function-buil__YJbnkue","drizzle-orm-window-function-buil__vQJ7PFY","drizzle-orm-window-function-buil__wp9p6TN","dynamodb-toolbox-lazy-recursive__kKgR4v6","effect-sse-httpapi-streaming__mTGYHrQ","effect-sse-httpapi-streaming__naYAWah","effect-sse-httpapi-streaming__zWYY88W","etree-xml-diff-patch__sW9GvzH","fastapi-implicit-head-options__DvAFyWL","fastapi-implicit-head-options__hjrpL7J","fastapi-implicit-head-options__tZn4XDB","fd-deterministic-multi-key-sorti__2CuBMtX","fd-deterministic-multi-key-sorti__RxzCFf3","geo-shapeindex-serialization__MDHV56k","geo-shapeindex-serialization__f8R3H8v","go-critic-doc-link-checker__23FH6r6","go-genai-streamed-function-args__BuBMJJh","go-genai-streamed-function-args__C8Nt9qz","go-genai-streamed-function-args__UYkCZSL","go-genai-streamed-function-args__mVxPcV8","go-genai-streamed-function-args__nbFHiTJ","go-genai-streamed-function-args__pZaEagF","go-git-worktree-merge-conflicts__9PjA2pG","go-git-worktree-merge-conflicts__9wAvmAj","go-git-worktree-merge-conflicts__Byu2NTj","go-git-worktree-merge-conflicts__Q9RF7HT","go-git-worktree-merge-conflicts__cYS9tMm","goreleaser-retry-publish-auditin__V6Jbm3h","goreleaser-retry-publish-auditin__kzWPqY7","happy-dom-abort-pending-body-rea__9MHv7tx","happy-dom-abort-pending-body-rea__F8imftw","happy-dom-abort-pending-body-rea__PkZzrYg","happy-dom-abort-pending-body-rea__hZ2dMke","happy-dom-abort-pending-body-rea__qoTYztv","helm-array-merge-strategies__xQ6XPmZ","httpx-deterministic-cookie-store__ScetD3f","httpx-deterministic-cookie-store__pGUmtAE","httpx-multipart-response-parsing__9jFaXHf","httpx-multipart-response-parsing__RJMtKvc","httpx-streaming-json-iteration__3s5eUpX","httpx-streaming-json-iteration__9qNFZHh","httpx-streaming-json-iteration__AQsqJM3","igel-persist-feature-schema__zSMAoJb","katex-multicolumn-array-spans__C6aBGhP","katex-multicolumn-array-spans__hD9oSgM","kombu-single-active-consumer-pri__4YdVwGw","kombu-single-active-consumer-pri__oAxaHGX","kombu-virtual-queue-dead-letteri__AmJbDTC","kombu-virtual-queue-dead-letteri__dv4aceg","koota-entity-snapshot-rollback__D5vcpbX","koota-entity-snapshot-rollback__uqWUVxL","langchain-request-coalescing__NkaqenE","mashumaro-flattened-dataclass-fi__28AMphA","mashumaro-flattened-dataclass-fi__Jj6fEmA","mashumaro-flattened-dataclass-fi__SAmzmKK","mashumaro-flattened-dataclass-fi__fonfSRv","mashumaro-flattened-dataclass-fi__p89naBG","mashumaro-flattened-dataclass-fi__yjiF5yn","mobly-grouped-test-barriers__qgWA4kT","narwhals-rolling-window-suite__5EMExXD","narwhals-rolling-window-suite__ivC9bCR","obsidian-linter-auto-table-of-co__8JPuxdU","obsidian-linter-auto-table-of-co__PCpMQvV","obsidian-linter-auto-table-of-co__i96pRzM","obsidian-linter-scoped-ignore-ma__4uPC2To","obsidian-linter-scoped-ignore-ma__7akWcM3","obsidian-linter-scoped-ignore-ma__nMWmHVr","onedump-dump-encryption-pipeline__YMGtsik","onedump-dump-encryption-pipeline__qK3L8ep","opa-rego-rule-profiling__SHXE2A4","opa-rego-rule-profiling__kvycvys","oxvg-structural-selector-preserv__MjRDzVV","prometheus-typed-label-sorting__2H9LXrx","prometheus-typed-label-sorting__oVB7fMt","prometheus-typed-label-sorting__yNL3FDX","psd-tools-blend-range-api__hT2htNd","pwntools-tube-multiplexing__UAJPhha","python-statemachine-state-data-s__FYYN5RS","query-persist-restored-query-sta__6Az6MfM","quill-shared-toolbar-focus__fMY6q5f","returns-validated-error-accumula__FZGX4nz","returns-validated-error-accumula__zLaBHGt","scriggo-method-declarations__dB6eE8z","skrub-duration-encoding__5r7B83H","skrub-duration-encoding__Rje748G","skrub-duration-encoding__f4Bpqpg","sql-formatter-bigquery-pipe-form__HzgaF6S","superjson-error-stack-serializat__bQuU2Nq","task-task-graph-export__iefKtVg","tengo-callable-instance-isolatio__2SC9Vz5","tengo-callable-instance-isolatio__CkMxPTM","tengo-callable-instance-isolatio__Y3LMSHH","tengo-callable-instance-isolatio__gVTfNmT","tengo-callable-instance-isolatio__rFMu44w","termenv-preserve-ansi-resets__2ynfvpC","termenv-preserve-ansi-resets__5aXkVti","termenv-preserve-ansi-resets__GCE7TrV","termenv-preserve-ansi-resets__gKeTFkC","testem-bail-on-test-failure__4MnD2sW","testem-bail-on-test-failure__TBaQnty","testem-bail-on-test-failure__Win9EKp","testem-bail-on-test-failure__tpocmoU","testem-per-launcher-reports__c3XzLfh","textual-kitty-key-phases__JFone4h","textual-kitty-key-phases__jde8PY9","textual-richlog-follow-state__2C7YHWw","textual-richlog-follow-state__qZYC62C","tomlkit-toml-table-converters__JqRvC3j","tomlkit-toml-table-converters__TR8NX6d","tomlkit-toml-table-converters__is9Jwag","true-myth-iterable-collection-co__NYSb5Mt","true-myth-iterable-collection-co__aAZvcJj","valibot-recursive-schema-composi__3sRax3E","valibot-recursive-schema-composi__ojvimkc","valibot-recursive-schema-composi__r5uJNTC","valibot-recursive-schema-composi__rgG8eTQ","vitest-duration-sharding__5euM9k5","wasmi-trap-coredumps__PjpND2p","yaegi-go-embed-directives__isRFgG9","ytt-jsonpath-query-api__nEFo5KT"],"outcome":"infra","reason":"Looked up its answer online while this DeepSWE run mistakenly had network access, confirmed by reading the trajectory (docs/incidents/atif-scan-readings.json); run again offline, DeepSWE's own setup (Joel, 2026-10-30).","from_release":"release-time-reruns"},{"benchmarks":["terminal-bench-2.1"],"trials":["path-tracing-reverse__9m9SApy"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__7uhes8N","fix-ocaml-gc__ajSHfU2","fix-ocaml-gc__i9ofsG2","fix-ocaml-gc__vRNBawh"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30).","from_release":"release-time-reruns"}],"reason":"Answer key retrieved with network access the benchmark did not intend (docs/incidents/2026-10-22-deepswe-network.md): DeepSWE trials that reached the benchmark's own published solutions, tests or trial viewer. Re-run offline.; Looked up its answer online while this DeepSWE run mistakenly had network access, confirmed by reading the trajectory (docs/incidents/atif-scan-readings.json); run again offline, DeepSWE's own setup (Joel, 2026-10-30).; Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":1899,"n_new":1925,"n_scored":1899,"n_reused":450,"cost_new_usd":65.17,"by_harness":{"claude-code":{"pass":160,"infra":10,"fail":51},"codex":{"pass":153,"fail":58,"infra":3},"hermes":{"pass":152,"fail":59,"infra":1},"kimi-code":{"pass":159,"fail":52},"mini-swe-agent":{"pass":174,"fail":37},"openclaw":{"fail":100,"pass":111,"infra":5},"opencode":{"pass":141,"fail":70,"infra":6},"pi":{"fail":45,"pass":166},"qwen-code":{"pass":162,"fail":49,"infra":1}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":2548,"n_new":2583,"n_scored":2546,"n_reused":449,"cost_new_usd":387.7,"by_harness":{"claude-code":{"pass":179,"fail":102,"infra":8},"codex":{"pass":190,"fail":93,"infra":6},"hermes":{"pass":229,"fail":54,"infra":3},"kimi-code":{"pass":189,"fail":94,"infra":1},"mini-swe-agent":{"pass":195,"fail":88,"infra":2},"openclaw":{"pass":70,"fail":213,"infra":7},"opencode":{"pass":157,"fail":126,"infra":6},"pi":{"pass":189,"fail":95,"infra":3},"qwen-code":{"pass":172,"fail":111,"infra":1}}},"featurebench":{"units":{"pending":0,"running":0,"done":155,"failed":0},"n_slots":3735,"n_new":0,"n_scored":0,"n_reused":450,"cost_new_usd":0,"by_harness":{}}},"n_new":4508,"n_scored":4445,"n_slots":8182,"n_reused":1349,"cost_new_usd":452.9,"updated_at":"2026-10-22T01:56:34.882181+00:00"},{"run_id":"20261018-191102-glm-5-3-flash-full-high-0df9","created_at":"2026-10-18T19:11:02.676045+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"high","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe","featurebench"],"n_trials":9558,"cost_cap_usd":2400.0,"estimate_usd":{"new_trials":8303,"low":1316.0,"high":2360.0,"per_trial":{"terminal-bench-2.1":{"mean":0.0428,"reserve":0.0942,"n_observed":472,"source":"observed"},"deep-swe":{"mean":0.4043,"reserve":0.7118,"n_observed":468,"source":"observed"},"featurebench":{"mean":0.0534,"reserve":0.0956,"n_observed":460,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":403,"n_new":1946,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":450,"n_new":2547,"n_units":111},"featurebench":{"n_tasks":156,"n_reused":402,"n_new":3810,"n_units":156}},"code_revision":"0583ba55ac8ddaea3464c85b54828baeaa8d834a","harness_adapters":{},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["deep-swe"],"trials":["helm-array-merge-strategies__TxJFrL3","helm-array-merge-strategies__aRMonGH","kcp-go-multiplexed-kcp-streams__2Ls9Qhs","koota-composite-trait-aspects__NDpckds","koota-composite-trait-aspects__mmgwWqa","koota-query-predicates__DnsYL2A","koota-query-predicates__TG23t53","koota-query-predicates__xdevLks","mashumaro-flattened-dataclass-fi__fKAcxbT","mashumaro-flattened-dataclass-fi__wyHMZjY","meriyah-explicit-resource-declar__iQBQy9N","meriyah-explicit-resource-declar__k4JWxpL","participle-grammar-conflict-anal__G9mYrgj","python-statemachine-state-data-s__QmWbuEr","task-task-graph-export__QvKaNDd","wasmi-trap-coredumps__HjyJi4c"],"reason":"Provider stream broke off or was empty (the meter logged the last call as 200 before it learned to log these as 599, provider failure); infra, re-run."},{"benchmarks":["featurebench"],"trials":["astropy__astropy.b0db0daa.test_t__unx5iKh","mlflow__mlflow.93dab383.test_fil__4vtdB3k","mlflow__mlflow.93dab383.test_fil__79DguiY","mlflow__mlflow.93dab383.test_fil__Ej9txEH","mlflow__mlflow.93dab383.test_fil__NVLBSnD","mlflow__mlflow.93dab383.test_fil__YgTiD8f","mlflow__mlflow.93dab383.test_fil__u334JRz","mlflow__mlflow.93dab383.test_num__Rw4Q6md","mlflow__mlflow.93dab383.test_num__gVU7YKb","mlflow__mlflow.93dab383.test_num__kZKqojA","mlflow__mlflow.93dab383.test_num__uQXKgmp","pydata__xarray.97f3a746.test_coo__QtP7eWp","pydata__xarray.97f3a746.test_coo__Y6spY6y","pydata__xarray.97f3a746.test_tre__3VRTQKP"],"reason":"Provider stream broke off or was empty (the meter logged the last call as 200 before it learned to log these as 599, provider failure); infra, re-run."},{"benchmarks":["terminal-bench-2.1"],"trials":["count-dataset-tokens__8LweyEQ","db-wal-recovery__vZzHG47","extract-elf__9EC5MnC","fix-ocaml-gc__9gfzVy5","fix-ocaml-gc__AFsoFJs","fix-ocaml-gc__DktZgLr","fix-ocaml-gc__FBDqzMC","fix-ocaml-gc__baDug2r","fix-ocaml-gc__bhuyXPt","fix-ocaml-gc__kd5mwx5","fix-ocaml-gc__qoeZZep","fix-ocaml-gc__vCC5JMA","fix-ocaml-gc__yVZtYGP","fix-ocaml-gc__zSs2Pez","mteb-leaderboard__qZb7qVS"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["deep-swe"],"trials":["bandit-structured-nosec-directiv__nSUwxTe","clack-async-autocomplete-options__3w8W7WX","dateutil-rfc5545-timezone-intero__aYXfRaw","go-genai-streamed-function-args__LGcYLUe","happy-dom-abort-pending-body-rea__3HJhmfW","happy-dom-abort-pending-body-rea__8xeMBJc","happy-dom-abort-pending-body-rea__9tLCmcw","happy-dom-abort-pending-body-rea__J4Q4tCZ","happy-dom-abort-pending-body-rea__KeemCMB","happy-dom-abort-pending-body-rea__kovxUhQ","prometheus-typed-label-sorting__HAk4saU","tengo-callable-instance-isolatio__K5mkY7J","testem-bail-on-test-failure__XFQMNXk"],"outcome":"infra","reason":"Looked up its answer online while this DeepSWE run mistakenly had network access, confirmed by reading the trajectory (docs/incidents/atif-scan-readings.json); run again offline, DeepSWE's own setup (Joel, 2026-10-30).","from_release":"release-time-reruns"},{"benchmarks":["deep-swe"],"trials":["effect-sse-httpapi-streaming__hwReYpv","koota-composite-trait-aspects__dnxDk62"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["deep-swe"],"trials":["scriggo-method-declarations__7biD2SE","valibot-recursive-schema-composi__8ChQeUM"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["terminal-bench-2.1"],"trials":["password-recovery__VoxDJ2d","pytorch-model-cli__DxWPr9Q"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["terminal-bench-2.1"],"trials":["adaptive-rejection-sampler__Aocr9Fe","adaptive-rejection-sampler__MV7m3av","circuit-fibsqrt__8D4DKcN","circuit-fibsqrt__CEuaqrn","gpt2-codegolf__StLvQ7q","largest-eigenval__nqYaQ7A","largest-eigenval__wUjy59w","make-mips-interpreter__2wNcKBC","make-mips-interpreter__8SXhgzm","make-mips-interpreter__9ifZj6H","make-mips-interpreter__A35nUGZ","make-mips-interpreter__BpBfVt2","make-mips-interpreter__E9uLLtr","make-mips-interpreter__MeGJfak","make-mips-interpreter__MujD52M","make-mips-interpreter__mYkttSA","make-mips-interpreter__ubT3NX8","make-mips-interpreter__vZZUjRr","path-tracing-reverse__9tjcHwt","regex-chess__AWfZW2u","torch-pipeline-parallelism__7KGtNPW"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__552TF6y","fix-ocaml-gc__YhBFiUp","fix-ocaml-gc__qMXdgUe","fix-ocaml-gc__w9iPuLj"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30).","from_release":"release-time-reruns"}],"reason":"Provider stream broke off or was empty (the meter logged the last call as 200 before it learned to log these as 599, provider failure); infra, re-run.; Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":1946,"n_new":2009,"n_scored":1946,"n_reused":403,"cost_new_usd":92.13,"by_harness":{"claude-code":{"fail":76,"infra":15,"pass":135},"codex":{"fail":75,"pass":136,"infra":8},"hermes":{"fail":66,"pass":145,"infra":5},"kimi-code":{"fail":69,"pass":142,"infra":5},"mini-swe-agent":{"fail":84,"pass":174,"infra":3},"openclaw":{"pass":106,"fail":105,"infra":7},"opencode":{"pass":103,"fail":108,"infra":13},"pi":{"pass":148,"fail":63,"infra":5},"qwen-code":{"fail":81,"pass":130,"infra":2}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":2547,"n_new":2815,"n_scored":2545,"n_reused":450,"cost_new_usd":926.1,"by_harness":{"claude-code":{"pass":168,"fail":115,"infra":24},"codex":{"pass":148,"fail":134,"infra":29},"hermes":{"fail":121,"pass":162,"infra":21},"kimi-code":{"pass":177,"fail":106,"infra":25},"mini-swe-agent":{"pass":173,"fail":110,"infra":27},"openclaw":{"fail":193,"pass":90,"infra":43},"opencode":{"fail":203,"pass":80,"infra":50},"pi":{"pass":171,"fail":111,"infra":31},"qwen-code":{"pass":158,"fail":125,"infra":20}}},"featurebench":{"units":{"pending":0,"running":0,"done":156,"failed":0},"n_slots":3810,"n_new":0,"n_scored":0,"n_reused":402,"cost_new_usd":0,"by_harness":{}}},"n_new":4824,"n_scored":4491,"n_slots":8303,"n_reused":1255,"cost_new_usd":1018.0,"updated_at":"2026-10-19T23:56:11.858530+00:00"},{"run_id":"20261018-191100-glm-5-3-flash-full-low-3c31","created_at":"2026-10-18T19:11:00.847235+00:00","model":"zai-org/GLM-5.3-Flash","provider":"zai-org","stage":"full","reasoning_effort":"low","harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw"],"harness_versions":{"claude-code":"2.1.300","codex":"0.162.1","pi":"1.2.0","opencode":"1.19.0","hermes":"v0.22.1","mini-swe-agent":"2.4.6","kimi-code":"2.2.0","qwen-code":"0.25.2","openclaw":"2026.10.4"},"benchmarks":["terminal-bench-2.1","deep-swe","featurebench"],"n_trials":9558,"cost_cap_usd":2000.0,"estimate_usd":{"new_trials":8305,"low":1194.0,"high":1951.0,"per_trial":{"terminal-bench-2.1":{"mean":0.0515,"reserve":0.115,"n_observed":464,"source":"observed"},"deep-swe":{"mean":0.3417,"reserve":0.5338,"n_observed":460,"source":"observed"},"featurebench":{"mean":0.0585,"reserve":0.0964,"n_observed":455,"source":"observed"}}},"plan":{"terminal-bench-2.1":{"n_tasks":87,"n_reused":402,"n_new":1947,"n_units":87},"deep-swe":{"n_tasks":111,"n_reused":450,"n_new":2547,"n_units":111},"featurebench":{"n_tasks":156,"n_reused":401,"n_new":3811,"n_units":156}},"code_revision":"0583ba55ac8ddaea3464c85b54828baeaa8d834a","harness_adapters":{},"harness_locks":{},"invalid":{"rules":[{"benchmarks":["deep-swe"],"trials":["ink-grid-box-layout__gvh8x2U","wasmi-trap-coredumps__zQyDjox"],"reason":"Provider stream broke off or was empty (the meter logged the last call as 200 before it learned to log these as 599, provider failure); infra, re-run."},{"benchmarks":["featurebench"],"trials":["astropy__astropy.b0db0daa.test_t__jf7Ggo3","mlflow__mlflow.93dab383.test_fil__RZv29zy","mwaskom__seaborn.7001ebe7.test_p__8sVFHHc","mwaskom__seaborn.7001ebe7.test_p__9bYCqGK"],"reason":"Provider stream broke off or was empty (the meter logged the last call as 200 before it learned to log these as 599, provider failure); infra, re-run."},{"benchmarks":["terminal-bench-2.1"],"trials":["make-mips-interpreter__vEeXyjp","path-tracing-reverse__6i373oM"],"reason":"Provider stream broke off or was empty (the meter logged the last call as 200 before it learned to log these as 599, provider failure); infra, re-run."},{"benchmarks":["deep-swe"],"trials":["dynamodb-toolbox-lazy-recursive__gNhoWWa"],"outcome":"infra","reason":"Looked up its answer online while this DeepSWE run mistakenly had network access, confirmed by reading the trajectory (docs/incidents/atif-scan-readings.json); run again offline, DeepSWE's own setup (Joel, 2026-10-30)."},{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__2NPBBiv","fix-ocaml-gc__KPekByF","fix-ocaml-gc__UA64bFU","fix-ocaml-gc__XvWFQMD","fix-ocaml-gc__nGVitH8","fix-ocaml-gc__q3WpZWq","fix-ocaml-gc__r3zKNoc","fix-ocaml-gc__xpxxbz9","regex-chess__8KuMH3G"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},{"benchmarks":["deep-swe"],"trials":["anko-typed-variable-bindings__D3trXG2","effect-sse-httpapi-streaming__sJqnmLK","go-genai-streamed-function-args__a7Fs9R8","happy-dom-abort-pending-body-rea__YV6ebbr","happy-dom-abort-pending-body-rea__ey4GPUH","happy-dom-abort-pending-body-rea__wBYVQ34","tengo-callable-instance-isolatio__Cap66TA"],"outcome":"infra","reason":"Looked up its answer online while this DeepSWE run mistakenly had network access, confirmed by reading the trajectory (docs/incidents/atif-scan-readings.json); run again offline, DeepSWE's own setup (Joel, 2026-10-30).","from_release":"release-time-reruns"},{"benchmarks":["deep-swe"],"trials":["effect-sse-httpapi-streaming__3r75Wjh","effect-sse-httpapi-streaming__kHKgVw8"],"outcome":"infra","reason":"Ended because HF Sandbox's command stream broke: the sandbox client got a cut event line while the harness was working (JSONDecodeError in _iter_events; docs/incidents/sandbox-stream-errors.json); infrastructure failure, run again, pass or fail alike (scoring.sandbox_stream_broke, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["terminal-bench-2.1"],"trials":["adaptive-rejection-sampler__osL4t4o","circuit-fibsqrt__ZSKm6Ux","make-doom-for-mips__6kV7pMM","make-doom-for-mips__RmLJTRa","make-doom-for-mips__kkZmMNo","make-mips-interpreter__7h6tato","make-mips-interpreter__MMoQumU","make-mips-interpreter__VvkFbmz","make-mips-interpreter__qNggXkB"],"outcome":"infra","reason":"Ended on a provider stall: the last model call's stream ended without its answer (status 200, no usage, no finish reason; docs/incidents/provider-stalls.json); a provider failure, run again, pass or fail alike (scoring.unfinished, 2026-11-08).","from_release":"release-time-reruns"},{"benchmarks":["terminal-bench-2.1"],"trials":["fix-ocaml-gc__KEwFTz6"],"outcome":"fail","reason":"Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30).","from_release":"release-time-reruns"}],"reason":"Provider stream broke off or was empty (the meter logged the last call as 200 before it learned to log these as 599, provider failure); infra, re-run.; Looked up its answer online while this DeepSWE run mistakenly had network access, confirmed by reading the trajectory (docs/incidents/atif-scan-readings.json); run again offline, DeepSWE's own setup (Joel, 2026-10-30).; Cheating confirmed by reading the trajectory an atif-scan flag pointed to (docs/incidents/atif-scan-readings.json); counts as a fail (Joel, 2026-10-29 and 2026-10-30)."},"state":"completed","benches":{"terminal-bench-2.1":{"units":{"pending":0,"running":0,"done":87,"failed":0},"n_slots":1947,"n_new":1978,"n_scored":1947,"n_reused":402,"cost_new_usd":81.85,"by_harness":{"claude-code":{"pass":128,"fail":83,"infra":4},"codex":{"fail":96,"pass":115,"infra":4},"hermes":{"fail":66,"pass":145,"infra":2},"kimi-code":{"pass":139,"fail":72,"infra":2},"mini-swe-agent":{"pass":167,"fail":92,"infra":2},"openclaw":{"pass":96,"fail":115,"infra":8},"opencode":{"pass":98,"fail":113,"infra":4},"pi":{"pass":128,"fail":83,"infra":5},"qwen-code":{"pass":122,"fail":89}}},"deep-swe":{"units":{"pending":0,"running":0,"done":111,"failed":0},"n_slots":2547,"n_new":2754,"n_scored":2547,"n_reused":450,"cost_new_usd":666.4,"by_harness":{"claude-code":{"fail":125,"pass":158,"infra":24},"codex":{"pass":121,"fail":162,"infra":22},"hermes":{"pass":146,"fail":137,"infra":9},"kimi-code":{"pass":143,"fail":140,"infra":20},"mini-swe-agent":{"pass":154,"fail":129,"infra":24},"openclaw":{"fail":200,"pass":83,"infra":32},"opencode":{"fail":216,"pass":67,"infra":37},"pi":{"fail":136,"pass":147,"infra":23},"qwen-code":{"fail":154,"infra":16,"pass":129}}},"featurebench":{"units":{"pending":0,"running":0,"done":156,"failed":0},"n_slots":3811,"n_new":0,"n_scored":0,"n_reused":401,"cost_new_usd":0,"by_harness":{}}},"n_new":4732,"n_scored":4494,"n_slots":8305,"n_reused":1253,"cost_new_usd":748.2,"updated_at":"2026-10-19T22:29:29.713679+00:00"}],"alerts":[],"config":{"owner":"joelniklaus","bucket":"joelniklaus/harness-leaderboard","ranked_harnesses":["claude-code","codex","pi","opencode","hermes","mini-swe-agent","kimi-code","qwen-code","openclaw","deepseek-harness"],"harnesses":{"claude-code":{"display":"Claude Code","vendor":"Anthropic","color":"#E07A4F","logo":"claude-code.svg","lens":"https://harness-hub-harness-lens.hf.space/#/h/claude-code","wire_api":"Speaks Anthropic Messages to our proxy through its base URL setting; the proxy translates them to Chat Completions.","deviations":"Requests are translated from Anthropic Messages to Chat Completions inside the sandbox (tools, streaming, thinking preserved) because the router's Messages route ignores reasoning effort; that way Claude Code runs at the same, enforced effort as every other harness.","version":"2.1.300"},"codex":{"display":"Codex","vendor":"OpenAI","color":"#3B7CF6","logo":"codex.svg","lens":"https://harness-hub-harness-lens.hf.space/#/h/codex","wire_api":"Speaks only OpenAI Responses, through a model provider in its config pointed at our build of responses.js, which translates them to Chat Completions for our proxy.","deviations":"Requests go through the local responses.js bridge (the router's own copy rejects bodies over 100 KB). web_search and multi_agent are off: the Responses route accepts only function and mcp tools. The bridge is configured as Codex's own model provider, so Codex compacts long histories locally with its own prompt (its built-in OpenAI provider would ask OpenAI's server to compact). Its developer messages (base instructions, permissions) reach the model as system messages, and sampling stays at the provider's defaults (the router's translation drops the first and sets temperature and top_p to 1). Images from view_image reach the model in a user message after the tool results (Chat Completions tool messages carry text only).","version":"0.162.1"},"pi":{"display":"Pi","vendor":"Earendil","color":"#A855F7","logo":"pi.svg","lens":"https://harness-hub-harness-lens.hf.space/#/h/pi","wire_api":"Speaks Chat Completions through a custom provider in its model settings, pointed at our proxy.","deviations":"compat.supportsDeveloperRole=false (as in pi-ai's own GLM model entries): with thinking on, Pi otherwise sends the system prompt as role 'developer', which zai rejects. Pi 1.2.0 takes its own libraries (pi-ai, pi-agent-core, pi-codemode, pi-tui, pi-mcp) at any 1.x from 1.2.0 on, so each install took the newest: the board's Pi trials ran them at 1.3.0 (99.5% of the GLM-5.3-Flash trials, 95.6% of the DeepSeek-V4.1-Flash ones) or 1.4.0 (the rest, and every Flint trial).","version":"1.2.0"},"opencode":{"display":"OpenCode","vendor":"Anomaly","color":"#3E8E5E","logo":"opencode.svg","lens":"https://harness-hub-harness-lens.hf.space/#/h/opencode","wire_api":"Speaks Chat Completions through OpenCode's own Hugging Face provider definition (OpenAI-compatible), with its base URL pointed at our proxy.","deviations":"Reaches the model through its own Hugging Face provider definition (OpenAI-compatible, Chat Completions) with our proxy as base URL and the key in a variable of our own. Its web search tool is off: its first search asks which search service to use, which `opencode run` cannot answer, so the run ended with an error. Its image-generation tool is denied (the route serves no image model).","version":"2.2.6"},"hermes":{"display":"Hermes Agent","vendor":"Nous Research","color":"#FF9D00","logo":"hermes.png","lens":"https://harness-hub-harness-lens.hf.space/#/h/hermes-agent","wire_api":"Speaks Chat Completions through its custom provider, pointed at our proxy.","deviations":"custom provider with Chat Completions instead of openai-api (Responses, blocked by the router's 100 KB limit).","version":"v0.22.1"},"mini-swe-agent":{"display":"mini-swe-agent","vendor":"Princeton/Stanford","color":"#8C6D46","logo":"mini-swe-agent.svg","lens":"https://harness-hub-harness-lens.hf.space/#/h/mini-swe-agent","wire_api":"Speaks Chat Completions through LiteLLM's OpenAI client, with its base URL pointed at our proxy.","deviations":"Effort set via mini-swe-agent's model_kwargs on Chat Completions; Harbor's effort option would switch it to the Responses API. Installed with project uv config ignored and PyPI as index (some task repos pin uv or point at an offline index).","version":"2.4.6"},"kimi-code":{"display":"Kimi Code","vendor":"Moonshot AI","color":"#0EA5E9","logo":"kimi-code.svg","lens":"https://harness-hub-harness-lens.hf.space/#/h/kimi-code","wire_api":"Speaks Chat Completions through its model settings (OpenAI type), pointed at our proxy.","deviations":"Thinking is switched on in Kimi Code's config.toml so its effort setting is sent (otherwise it sends none, then 'medium').","version":"2.2.0"},"qwen-code":{"display":"Qwen Code","vendor":"Alibaba","color":"#EC4899","logo":"qwen-code.svg","lens":"https://harness-hub-harness-lens.hf.space/#/h/qwen-code","wire_api":"Speaks Chat Completions through its OpenAI-compatible settings, with its base URL pointed at our proxy.","deviations":"Effort set via Qwen Code's settings.json (model.generationConfig.samplingParams.reasoning_effort); it sends none by default for non-Qwen models.","version":"0.25.2"},"openclaw":{"display":"OpenClaw","vendor":"OpenClaw","color":"#EF3E36","logo":"openclaw.svg","lens":"https://harness-hub-harness-lens.hf.space/#/h/openclaw","wire_api":"Speaks Chat Completions through a custom provider in its config, pointed at our proxy.","deviations":"Installed on Node 24 (this OpenClaw release requires it; Harbor's adapter installs Node 22). Chat Completions custom provider with role 'system'.","version":"2026.10.4"},"deepseek-harness":{"display":"DeepSeek Harness","vendor":"DeepSeek","color":"#4D6BFE","logo":"deepseek-harness.svg","lens":"https://harness-hub-harness-lens.hf.space/#/h/deepseek-harness","wire_api":"Speaks Chat Completions through a custom provider in its profile, pointed at our proxy.","deviations":"A developer preview: every release is a prerelease, and the pin is what npm's latest tag serves. Runs its headless profile with full access (no sandbox, no approvals), as the other harnesses run. The route model is a custom openai-completions provider with the model's context window and longest reply, its system prompt as role 'system' (with reasoning on it would send 'developer', which zai rejects), and its web search off: it calls DeepSeek's search API, which needs a DeepSeek platform key the sandbox does not hold. Its trajectory comes from its JSON event stream, which cuts strings at 8 KiB; the full session log is kept next to it.","version":"0.2.2-rc.1"}},"benchmarks":{"terminal-bench-2.1":{"display":"Terminal-Bench 2.1","short":"TB 2.1","kind":"Terminal tasks: build, debug, configure, research","url":"https://github.com/harbor-framework/terminal-bench-2-1","commit":"d49e28f1e4ddd13d289e85a5f312a66750951932","n_upstream":89,"strata":"difficulty","version":"2.1","version_note":null,"commit_url":"https://github.com/harbor-framework/terminal-bench-2-1/commit/d49e28f1e4ddd13d289e85a5f312a66750951932","exclusions":[{"tasks":["qemu-startup","qemu-alpine-ssh"],"reason":"Needs a patched QEMU image to run on HF Sandbox (the upstream image fails without KVM)."}],"subset_sizes":{"smoke":3,"pilot":50,"full":87}},"deep-swe":{"display":"DeepSWE v1.1","short":"DeepSWE v1.1","kind":"Long-horizon features and fixes in real repos (Go, Python, TS/JS, Rust)","url":"https://deepswe.datacurve.ai","commit":"0b9fabbb63b9104d678fe965e1632f2dd9eaa2ea","n_upstream":113,"strata":"language","version":"v1.1","version_note":"upstream main of 2026-08-26, whose dataset manifest is datacurve/deep-swe-1-1; upstream had tagged only v1.0.0 as of 2026-10-25","commit_url":"https://github.com/datacurve-ai/deep-swe/commit/0b9fabbb63b9104d678fe965e1632f2dd9eaa2ea","exclusions":[{"tasks":["helm-unified-manifest-stream"],"reason":"Its reference solution fails 4 of its 5 target tests on HF Sandbox, so no harness can pass it."},{"tasks":["vulture-persistent-analysis-cache"],"reason":"Its grader fails correct solutions that add their options at the top of the DEFAULTS dict (13 of 15 early trials that passed all target tests failed this way)."}],"subset_sizes":{"smoke":3,"pilot":50,"full":111}}},"unscored_benchmarks":{"featurebench":{"display":"FeatureBench","short":"FeatureBench","url":"https://github.com/LiberCoders/FeatureBench","reason":"Not scored since 2026-10-25. Its task images held the reference solution and hidden tests, and most earlier GLM-5.3-Flash and DeepSeek-V4.1-Flash trials accessed them. The repaired setup passed its checks; scores return once every model's trials are re-run on it. All trials stay public in the bucket."}},"models":{"zai-org/GLM-5.3-Flash":{"display":"GLM-5.3-Flash","released":"0.1.0","org":"Z.ai","default_provider":"zai-org","pricing":{"zai-org":{"input":0.15,"cached_input":0.03,"output":0.5,"source":"https://x.com/Zai_org/status/2092616209426493766","as_of":"2026-08-26"}},"limits":{"zai-org":{"context_window":1048576,"max_output_tokens":131072,"strict_window":false,"input_modalities":["text","image"],"input_modalities_source":"Text and image (checked 2026-11-09): the HF router lists zai-org/GLM-5.3-Flash with architecture.input_modalities [\"text\", \"image\"] (GET router.huggingface.co/v1/models); its Hub card's pipeline is image-text-to-text (Glm5NextForConditionalGeneration); Z.ai files it under its VLM guide (docs.z.ai/guides/vlm/glm-5.3-flash). On the route every image an OpenCode 2 trial sent was answered 200 and grew the call's input (50 Terminal-Bench trials of the GLM full runs, .cache/release-v0.1.2/image-check).","source":"Z.ai: docs.z.ai/guides/vlm/glm-5.3-flash (context 1M, maximum output 128K) and the Chat Completion API reference (max_tokens up to 131072 for GLM-5.3-Flash); the model's config.json (max_position_embeddings 1048576). Checked on the route on 2026-11-06: a 1,035,024-token prompt was accepted, so the window is 1,048,576, not 1,000,000; max_completion_tokens above 131072 is accepted too (the provider caps it). Not strict: a 960,019-token prompt with max_tokens 128,000 (1,088,019 in total) was served on 2026-11-07, so Z.ai cuts the reply to the room left instead of refusing the request.","as_of":"2026-11-06"}},"reasoning_efforts":["low","high","max"],"reasoning_efforts_note":"zai-org route, 2026-10-16: low/high/max accepted; none/minimal/medium/xhigh rejected (error 1210). Unset = provider default (about 2x high).","capacity":{"zai-org":{"measured_at":"2026-10-17T07:53:26.051004+00:00","max_concurrent_requests":384,"top_level_tested":512,"duty_cycle":0.78,"max_concurrent_trials":393,"spent_usd":7.48,"note":"Load test 16-512 concurrent agent-like requests (~12k input tokens, streaming): no 429s, 1 5xx in ~14k; p95 latency 7.4 s at 384, 17.1 s at 512."}},"wire":"zai-org/GLM-5.3-Flash:zai-org","served_by":"HF Inference Providers","price_note":"zai-org list prices"},"deepseek-ai/DeepSeek-V4.1-Flash":{"display":"DeepSeek-V4.1-Flash","short":"DS-V4.1-Flash","released":"0.1.0","org":"DeepSeek","default_provider":"deepinfra","pricing":{"deepinfra":{"input":0.2,"cached_input":0.006,"output":0.6,"source":"https://deepinfra.com/deepseek-ai/DeepSeek-V4.1-Flash (list price; cached input = 3% of input per api.deepinfra.com/models; the 30% launch promotion is not applied)","as_of":"2026-10-18"}},"limits":{"deepinfra":{"context_window":1048576,"max_output_tokens":393216,"strict_window":true,"input_modalities":["text","image"],"input_modalities_source":"Text and image (checked 2026-11-09): the HF router lists deepseek-ai/DeepSeek-V4.1-Flash with architecture.input_modalities [\"text\", \"image\"] (GET router.huggingface.co/v1/models); its Hub card's pipeline is image-text-to-text; api.deepinfra.com/models/deepseek-ai/DeepSeek-V4.1-Flash tags it multimodal. Every image an OpenCode 2 trial sent was answered 200 and grew the call's input (69 Terminal-Bench trials of the DeepSeek full runs on DeepInfra and Novita, .cache/release-v0.1.2/image-check).","source":"api.deepinfra.com/models/deepseek-ai/DeepSeek-V4.1-Flash (max_tokens 1048576, max_output_tokens 393216); the HF router lists context_length 1048576 for deepinfra. max_completion_tokens 393216 and above is accepted on the route (2026-11-06). Strict: a 960,037-token prompt with max_tokens 128,000 was refused on 2026-11-07 ('This model configuration accepts at most 1048576 combined input and output tokens').","as_of":"2026-11-06"}},"reasoning_efforts":["low","high","max"],"reasoning_efforts_note":"deepinfra route: every reasoning_effort from minimal to xhigh is accepted, and none disables reasoning (2026-10-18). The model's chat template turns the effort into a budget hint in the prompt, so it is an instruction to the model, not a limit the server enforces: on 2026-11-07 a hard prompt reasoned to max_tokens 16,000 at every effort, unset included, and on a short coding prompt the effort moves the median by tens to a few hundred tokens; the meter sets the effort on every call. DeepInfra sets no fixed reasoning budget: 6 to 10% of the board's agent calls reasoned past 4,800 tokens, up to the reply limit (131,072 tokens until 2026-10-31, 393,216 since); it stops reasoning at 60% of a small reply limit (4,916 tokens on OpenClaw 2026.10.4's 8,192-token calls, 2,458 on 4,096), which an earlier note took for a budget of about 4,800. Novita and Together reason to max_tokens like DeepInfra; Baseten stops at about 12,000 reasoning tokens and forces an answer; Fireworks reports about 30% fewer reasoning tokens than it generates (Drive measurements/deepseek-providers-2026-11-07.md; harness-difference investigation D4).","capacity":{"deepinfra":{"measured_at":"2026-10-30T07:10:00+00:00","max_concurrent_requests":24,"top_level_tested":64,"duty_cycle":0.915,"max_concurrent_trials":20,"spent_usd":1.27,"note":"24 requests (20 trials): the highest level whose first-token latency matches what the board's other harnesses saw on 2026-10-23 (meter logs of runs 3849 and 7530: p50/p95 0.9/2.2 s) through a whole day. At 24 the Codex and OpenCode re-runs stayed at 0.8 to 1.1 s p50 and 1.4 to 2.0 s p95 in every hour of 2026-10-29. At 40 (2026-10-30 01:50 to 07:10 UTC, after a night load test on top of the running re-runs gave 1.0/1.7 s with 24 more requests) the hours from 02:00 to 04:00 UTC matched, but from 05:00 UTC p95 rose to 5.2 s, so the capacity went back to 24. Load tests 2026-10-29 10:10 and 10:20 UTC: 1.2/2.3 s at 24, 2.7/10.7 s at 32, 3.4/14.2 s at 48. On 2026-10-19 deepinfra served 128 at full speed. 2026-11-09, live step to 28 trials (16:01-17:06 UTC, US morning, on top of the running runs): decode per output token p50 rose from 6.1-6.5 ms (09:30-16:00 UTC at 20) to 7.0-8.5 ms per 10-minute bin and p95 from 12.2-13.5 to 14.4-17.8 ms, first-token p95 reached 4.6 s in one bin (3.0-3.6 s at 20), and Terminal-Bench agent timeouts were 5 of 46 trials (11%; 7% at 20): back to 20 at 17:06 UTC (capacity.json). At 20 DeepInfra also answered 406 calls with 429 and cut 131 streams without usage from 13:50 to 14:50 UTC the same day."}},"wire":"deepseek-ai/DeepSeek-V4.1-Flash:deepinfra","served_by":"HF Inference Providers","price_note":"deepinfra list prices"},"RecursiveSI/flint":{"display":"Flint","released":"0.1.0","org":"Recursive","default_provider":"hf-endpoint","routes":{"hf-endpoint":{"kind":"endpoint-pool","served_model":"RecursiveSI/flint","serving":{"repository":"RecursiveSI/flint","revision":"2a247be0c0aec895ccb0ac7ca92229ec6df37288","vendor":"gcp","region":"us-south1","instance_type":"nvidia-h200","instance_size":"x1","image":"lmsysorg/sglang:v0.5.16-cu130","image_digest":"sha256:7b6a35df9839fd593a94a1eaee82d7777f472225d9f3ad1f8a2e0cb2bd1785d0","port":30000,"kit":{"kind":"install-sh","serve_script":"serve_h200.sh"},"idle_minutes":30,"usd_per_replica_hour":5.0,"note":"No HF Inference Provider serves Flint (2026-10-23). Every pool endpoint is one BF16 replica on one H200 with Recursive's SGLang kit (sglang/ in the model repo: SGLang v0.5.16 + flint_sglang 0.1.4, FA3 attention, Triton MoE, 262,144-token context). The image digest is the one the kit's Dockerfile pins; the endpoints reference its tag, which resolved to this digest on Docker Hub on 2026-10-25. Add endpoints with scripts/endpoint.py clone (an exact copy of the first). The pool spans two namespaces, each with its H200 quota in gpu_quota (read live at every launch, launcher.size_pool): joelniklaus's 4 (flint-hlb, -2, -3, -8) first, then the huggingface org (flint-hlb-huggingface-1 to -8 created paused on 2026-11-07 when the org had 14 H200 free, -9 to -26 on 2026-11-08 for OpenCode 2 on Flint, -27 to -50 on 2026-11-09 with Ripple's pool cut to 4 endpoints (Joel, 2026-11-09 16:55 CET: 'put all the servers we have for flint use now'); since 2026-11-10 only -1 to -4 are listed, the org's H200 going to Ripple (Joel, 2026-11-10 00:45 CET: Ripple's release runs and four new harnesses' Ripple cells first; -5 to -50 exist paused outside the pool, re-add them from git history when Flint needs them); billed to the org; Joel, 2026-11-07: 'Max out my own quota and then use the hf quota in addition'), of which a launch takes only the org's free H200 minus 10 on weekdays and 6 on weekends. The other joelniklaus clones (flint-hlb-4 to -59 without -8, -13, -15 and -31) exist paused, held (owner-quota) and outside the pool since 2026-10-29. Since 2026-11-10 08:30 CET joelniklaus's 4 H200 serve Ripple's pool instead (Flint had no work queued and the 4 sat idle; Ripple's queue waited for org GPUs), so flint-hlb, -2, -3 and -8 exist paused outside Flint's listing too: when Flint needs them, re-add them from git history and take Ripple's two joelniklaus endpoints out (test_gpu_quota: the pools together list at most each namespace's quota)."},"gpu_quota":{"joelniklaus":4,"huggingface":64},"endpoints":[{"namespace":"huggingface","name":"flint-hlb-huggingface-1","url":"https://6aef6c4b62710132ca851d80.endpoints.huggingface.cloud"},{"namespace":"huggingface","name":"flint-hlb-huggingface-2","url":"https://6aef6c4cfef6e9d4f61271d6.endpoints.huggingface.cloud"},{"namespace":"huggingface","name":"flint-hlb-huggingface-3","url":"https://6aef6c4dfef6e9d4f61271d8.endpoints.huggingface.cloud"},{"namespace":"huggingface","name":"flint-hlb-huggingface-4","url":"https://6aef6c4d62710132ca851d84.endpoints.huggingface.cloud"}]}},"reasoning_efforts":["default"],"reasoning_efforts_note":"Flint decides per turn whether to reason; the model card and kit offer no effort switch or budget. Runs use the effort \"default\": the meter sends no reasoning_effort and the server's sampling defaults apply (temperature 1.0, top-p 0.95, top-k 20).","capacity":{"hf-endpoint":{"measured_at":"2026-10-23T21:49:55.045415+00:00","max_concurrent_requests":12,"top_level_tested":16,"duty_cycle":0.78,"max_concurrent_trials":12,"spent_usd":1.3,"replicas":1}},"pricing":{"hf-endpoint":{"input":0.03905,"cached_input":0.007809,"output":0.1301,"source":"derived (scripts/load_test.py): $5.0/h per replica / tokens one replica served per hour at 12 concurrent requests (1 replica(s), 144741 input incl. 138135 cached and 402 output tokens/s), in zai-org/GLM-5.3-Flash's input/cached/output price ratios","as_of":"2026-10-23"}},"limits":{"hf-endpoint":{"context_window":262144,"max_output_tokens":262144,"strict_window":true,"input_modalities":["text"],"input_modalities_source":"Text only (checked 2026-11-09): the model card (huggingface.co/RecursiveSI/flint: \"Flint supports text input and output\"; specifications \"Input / output: Text / text\"; FlintForCausalLM, no vision config). The pool's SGLang 0.5.16 answers a call with an image part 200 and drops the image: 19 OpenCode 2 trials that read an image file got input +21 to +226 tokens for 5-394 KB images, and all 19 failed (.cache/release-v0.1.2/image-check).","source":"The pool's SGLang kit serves a 262,144-token context (model card: 'including prompt and generated output'). SGLang has no separate output limit and refuses a request whose input plus max_tokens exceeds the context, so the longest reply is the whole window; OpenClaw lowers each request's max_completion_tokens to the window minus its input estimate.","as_of":"2026-11-06"}},"wire":"RecursiveSI/flint","served_by":"dedicated HF Inference Endpoint (1x nvidia-h200 per replica)","price_note":"priced from GPU-hours"}},"attempts":{"smoke":1,"pilot":1,"full":3},"default_caps_usd":{"smoke":15.0,"pilot":300.0}},"version":"0.1.5","debug":false}