[{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"cobench-prior-unspecified","comparability_caveats":["Prior environment details not re-established."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"co-prior","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Percentage of problems solved"},"attempts_per_task":{"status":"known","value":1},"benchmark_version_id":"cobench-2-1","comparability_caveats":["Weights unchanged from earlier card; environment changed."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"known","value":"API safety filter off"},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"co21-earlier","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":500},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"known","value":"Same 500 problems and evaluation code in Figure 2.3.4.1.A"},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Percentage of problems solved"},"attempts_per_task":{"status":"known","value":1},"benchmark_version_id":"cobench-2-1","comparability_caveats":["Opus 5.5 run 13 days later. One environment change estimated at 0–1 point; two unmeasured."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"known","value":"API safety filter off"},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"co21-later","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":500},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"known","value":"Same 500 problems and evaluation code in Figure 2.3.4.1.A"},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"openai-research-debugging-2026","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"openai-research-debugging-protocol","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§10.1.3.1","source_id":"src-astra"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":41},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"kernelgen-1p-2026","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"kernelgen-1p-protocol","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§10.1.3.2","source_id":"src-astra"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"nanogpt-2026","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One H100 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"nanogpt-protocol","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§10.1.3.3","source_id":"src-astra"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"clamp((trial − base)/(instruct − base), 0, 1); cheating trials assigned zero"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"posttrainbench-lite-2026","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One H100 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"posttrainbench-lite-protocol","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"known","value":true},"metric_id":"score","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§10.1.3.4","source_id":"src-astra"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":12},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"5 hours"}},{"aggregate_method":{"status":"known","value":"Average pass@1"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"grb-2026-08","comparability_caveats":["Bugged tasks retained."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"grb-current","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"pass1","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Scoped internal agent scaffold"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Printed p.37; PDF index 36; GRB Internal Benchmark","source_id":"src-grb"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":74},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"2 hours per task"}},{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"grb-2026-08","comparability_caveats":["No detailed transcript analysis of older runs."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"grb-earlier","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"pass1","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Printed p.37; PDF index 36; Results and footnote 10","source_id":"src-grb"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Mean replication score; standard error across runs"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"paperbench-v1","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"pb-basic","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"known","value":true},"metric_id":"replication","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":3},"scaffold":{"status":"known","value":"BasicAgent"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§5.2–5.3, Tables 4–5","source_id":"src-paperbench"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":20},"task_exclusions":{"status":"known","value":"Author code repositories blacklisted"},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"12 hours"}},{"aggregate_method":{"status":"known","value":"Mean replication score; standard error across runs"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"paperbench-v1","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"pb-iterative","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"known","value":true},"metric_id":"replication","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":3},"scaffold":{"status":"known","value":"IterativeAgent"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§5.2–5.3, Tables 4–5","source_id":"src-paperbench"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":20},"task_exclusions":{"status":"known","value":"Author code repositories blacklisted"},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"12 hours"}},{"aggregate_method":{"status":"known","value":"Mean replication score; standard error across runs"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"paperbench-v1","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"pb-iterative36","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"known","value":true},"metric_id":"replication","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":3},"scaffold":{"status":"known","value":"IterativeAgent"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§5.2–5.3, Tables 4–5","source_id":"src-paperbench"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":20},"task_exclusions":{"status":"known","value":"Author code repositories blacklisted"},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"36 hours"}},{"aggregate_method":{"status":"known","value":"Mean across repeated attempts; one SEM"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"mle-2024","comparability_caveats":["Number of seeds differs; comparison explicitly reported by authors."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"36 vCPUs, 440 GB RAM, one Nvidia A10 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"mle-o1-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"medal","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":16},"scaffold":{"status":"known","value":"AIDE"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":75},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"24 hours per run"}},{"aggregate_method":{"status":"known","value":"Mean across repeated attempts; one SEM"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"mle-2024","comparability_caveats":["Number of seeds differs; comparison explicitly reported by authors."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"36 vCPUs, 440 GB RAM, one Nvidia A10 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"mle-4o-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"medal","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":36},"scaffold":{"status":"known","value":"AIDE"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":75},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"24 hours per run"}},{"aggregate_method":{"status":"known","value":"Mean across repeated attempts; one SEM"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"mle-2024","comparability_caveats":["Number of seeds differs; comparison explicitly reported by authors."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"36 vCPUs, 440 GB RAM, one Nvidia A10 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"mle-llama-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"medal","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":3},"scaffold":{"status":"known","value":"AIDE"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":75},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"24 hours per run"}},{"aggregate_method":{"status":"known","value":"Mean across repeated attempts; one SEM"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"mle-2024","comparability_caveats":["Number of seeds differs; comparison explicitly reported by authors."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"36 vCPUs, 440 GB RAM, one Nvidia A10 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"mle-claude-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"medal","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":3},"scaffold":{"status":"known","value":"AIDE"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":75},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"24 hours per run"}},{"aggregate_method":{"status":"known","value":"Percentile rank against test-time-compute reference distribution"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"mle-revised-2026","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"mle-revised-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"percentile","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"known","value":"Up to three leaderboard submissions"},"source_refs":[{"locator":"§10.1.3.5","source_id":"src-astra"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":72},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Bootstrap max of 16; mean for hidden-score Scaling Law task"},"attempts_per_task":{"status":"known","value":16},"benchmark_version_id":"rebench-gemini3","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"rebench-gemini-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"known","value":false},"metric_id":"assessment","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"known","value":24},"scaffold":{"status":"known","value":"METR Modular with minimal changes"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Printed pp.13–15; PDF indices 13–15; Machine Learning R&D","source_id":"src-gemini3"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":5},"task_exclusions":{"status":"known","value":"Finetune GPT-2 for QA; Scaffolding for Rust Codecontest"},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"32 cumulative hours: 16 × 2-hour attempts"}},{"aggregate_method":{"status":"known","value":"Fitted 50% success horizon"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"th-1-0","comparability_caveats":["Version-specific task distribution; no evaluation dates supplied."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"th-1-0-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"p50","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Vivaria"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":170},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Fitted 50% success horizon"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"th-1-1","comparability_caveats":["Version-specific task distribution; no evaluation dates supplied."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"th-1-1-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"p50","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Inspect"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":228},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Mean over tasks and tested effort levels"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"ai4ai-v1","comparability_caveats":["Six effort levels for GPT, five for Claude, highest only for Kimi; costs differ."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One B300 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"a4-opus-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Claude Code"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.4, §3.1–3.2, Figure 2","source_id":"src-ai4ai"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":10},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"4 hours exploration; up to 12 hours independent retraining"}},{"aggregate_method":{"status":"known","value":"Mean over tasks and tested effort levels"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"ai4ai-v1","comparability_caveats":["Six effort levels for GPT, five for Claude, highest only for Kimi; costs differ."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One B300 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"a4-sol-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Codex"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.4, §3.1–3.2, Figure 2","source_id":"src-ai4ai"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":10},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"4 hours exploration; up to 12 hours independent retraining"}},{"aggregate_method":{"status":"known","value":"Mean over tasks and tested effort levels"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"ai4ai-v1","comparability_caveats":["Six effort levels for GPT, five for Claude, highest only for Kimi; costs differ."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One B300 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"a4-kimi-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Claude Code"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.4, §3.1–3.2, Figure 2","source_id":"src-ai4ai"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":10},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"4 hours exploration; up to 12 hours independent retraining"}},{"aggregate_method":{"status":"known","value":"Mean over tasks and tested effort levels"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"ai4ai-v1","comparability_caveats":["Six effort levels for GPT, five for Claude, highest only for Kimi; costs differ."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One B300 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"a4-sonnet-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Claude Code"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.4, §3.1–3.2, Figure 2","source_id":"src-ai4ai"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":10},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"4 hours exploration; up to 12 hours independent retraining"}},{"aggregate_method":{"status":"known","value":"Mean over tasks and tested effort levels"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"ai4ai-v1","comparability_caveats":["Six effort levels for GPT, five for Claude, highest only for Kimi; costs differ."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One B300 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"a4-terra-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Codex"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.4, §3.1–3.2, Figure 2","source_id":"src-ai4ai"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":10},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"4 hours exploration; up to 12 hours independent retraining"}},{"aggregate_method":{"status":"known","value":"Mean over tasks and tested effort levels"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"ai4ai-v1","comparability_caveats":["Six effort levels for GPT, five for Claude, highest only for Kimi; costs differ."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"known","value":"One B300 GPU"},"human_intervention":{"status":"not_reported","value":null},"id":"a4-luna-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"normalized","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"known","value":"Codex"},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.4, §3.1–3.2, Figure 2","source_id":"src-ai4ai"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":10},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"known","value":"4 hours exploration; up to 12 hours independent retraining"}},{"aggregate_method":{"status":"known","value":"Person-time-weighted automation categories"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"automation-aug26","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"known","value":"Human-directed or supervised work represented by automation levels"},"id":"automation-ai_leads","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"ai_leads","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§1; Appendix: Measuring AI-led R&D","source_id":"src-automation"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":378},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Person-time-weighted automation categories"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"automation-aug26","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"known","value":"Human-directed or supervised work represented by automation levels"},"id":"automation-autonomous","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"autonomous","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"§1; Appendix: Measuring AI-led R&D","source_id":"src-automation"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":378},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"known","value":"Estimated change in completion time"},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"productivity-early25","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"productivity-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"time_change","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"known","value":"Random assignment of issues to AI allowed or disallowed"},"source_refs":[{"locator":"Methodology; Core Result","source_id":"src-productivity"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"known","value":246},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"known","value":"Chosen tools, primarily Cursor with Claude 3.5/3.7 Sonnet"},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"productivity-late25","comparability_caveats":["Selection and concurrent-agent time measurement biases."],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"productivity-later-p","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"assessment","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"known","value":"Volunteer participation and self-selected submitted tasks"},"source_refs":[{"locator":"Introduction; Wider adoption section","source_id":"src-productivity-update"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"alphaevolve-2025","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"alpha-training_reduction","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"training_reduction","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Enhancing AI training and inference","source_id":"src-alphaevolve"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}},{"aggregate_method":{"status":"not_reported","value":null},"attempts_per_task":{"status":"not_reported","value":null},"benchmark_version_id":"alphaevolve-2025","comparability_caveats":[],"contamination_concerns":{"status":"not_reported","value":null},"evaluator_version":{"status":"not_reported","value":null},"filtering":{"status":"not_reported","value":null},"hardware":{"status":"not_reported","value":null},"human_intervention":{"status":"not_reported","value":null},"id":"alpha-kernel_speedup","inference_budget":{"status":"not_reported","value":null},"internet_access":{"status":"not_reported","value":null},"metric_id":"kernel_speedup","monetary_cost":{"status":"not_reported","value":null},"run_count":{"status":"not_reported","value":null},"scaffold":{"status":"not_reported","value":null},"selection_rule":{"status":"not_reported","value":null},"source_refs":[{"locator":"Enhancing AI training and inference","source_id":"src-alphaevolve"}],"split":{"status":"not_reported","value":null},"task_count":{"status":"not_reported","value":null},"task_exclusions":{"status":"not_reported","value":null},"task_snapshot":{"status":"not_reported","value":null},"token_budget":{"status":"not_reported","value":null},"tool_access":{"status":"not_reported","value":null},"training_budget":{"status":"not_reported","value":null},"wall_clock_budget":{"status":"not_reported","value":null}}]