[{"base_model_ids":["Claude Opus 5"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Claude Opus 5","id":"opus5","organization_id":"anthropic","release_date":null,"scaffold":{"status":"not_reported","value":null},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.3.4.1, Figure 2.3.4.1.A; printed pp.36–37, PDF indices 35–36","source_id":"src-opus55"}],"tool_configuration":{"status":"known","value":"API; API safety filter off; CoBench evaluation environment"}},{"base_model_ids":["Claude Mythos 5.1"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Claude Mythos 5.1","id":"mythos51","organization_id":"anthropic","release_date":null,"scaffold":{"status":"not_reported","value":null},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.3.4.1, Figure 2.3.4.1.A; printed pp.36–37, PDF indices 35–36","source_id":"src-opus55"}],"tool_configuration":{"status":"known","value":"API; API safety filter off; CoBench evaluation environment"}},{"base_model_ids":["Claude Opus 5.5"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Claude Opus 5.5","id":"opus55","organization_id":"anthropic","release_date":null,"scaffold":{"status":"not_reported","value":null},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§2.3.4.1, Figure 2.3.4.1.A; printed pp.36–37, PDF indices 35–36","source_id":"src-opus55"}],"tool_configuration":{"status":"known","value":"API; API safety filter off; CoBench evaluation environment"}},{"base_model_ids":["GPT-6 Astra"],"checkpoint":{"status":"not_reported","value":null},"display_name":"GPT-6 Astra","id":"astra","organization_id":"openai","release_date":null,"scaffold":{"status":"not_reported","value":null},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§10.1.3 and §§10.1.3.1–10.1.3.5; Table 18","source_id":"src-astra"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Gemini 3.7 Flash / GRB scaffold"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Gemini 3.7 Flash / GRB scaffold","id":"gemini37","organization_id":"deepmind","release_date":null,"scaffold":{"status":"known","value":"Scoped internal Google coding scaffold"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"GRB Internal Benchmark, printed p.37, PDF index 36","source_id":"src-grb"}],"tool_configuration":{"status":"known","value":"Custom tools; sub-agents and custom skills unavailable"}},{"base_model_ids":["Gemini 3.1 Pro / preliminary GRB runs"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Gemini 3.1 Pro / preliminary GRB runs","id":"gemini31","organization_id":"deepmind","release_date":null,"scaffold":{"status":"not_reported","value":null},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"GRB Internal Benchmark, printed p.37, PDF index 36","source_id":"src-grb"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Gemini 3 Pro / METR Modular adaptation"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Gemini 3 Pro / METR Modular adaptation","id":"gemini3","organization_id":"deepmind","release_date":null,"scaffold":{"status":"known","value":"METR Modular with minimal modifications"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"Machine Learning R&D, printed pp.13–15, PDF indices 13–15","source_id":"src-gemini3"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["o3-mini-high / BasicAgent"],"checkpoint":{"status":"known","value":"o3-mini-2025-01-31"},"display_name":"o3-mini-high / BasicAgent","id":"pb-o3","organization_id":"openai","release_date":null,"scaffold":{"status":"known","value":"BasicAgent"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3 and §§5.2–5.3; Tables 4–5","source_id":"src-paperbench"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["GPT-4o / BasicAgent"],"checkpoint":{"status":"known","value":"gpt-4o-2024-08-06"},"display_name":"GPT-4o / BasicAgent","id":"pb-4o","organization_id":"openai","release_date":null,"scaffold":{"status":"known","value":"BasicAgent"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3 and §§5.2–5.3; Tables 4–5","source_id":"src-paperbench"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Gemini 2.0 Flash / BasicAgent"],"checkpoint":{"status":"known","value":"gemini-2.0-flash"},"display_name":"Gemini 2.0 Flash / BasicAgent","id":"pb-gemini","organization_id":"deepmind","release_date":null,"scaffold":{"status":"known","value":"BasicAgent"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3 and §§5.2–5.3; Tables 4–5","source_id":"src-paperbench"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["o1-high / BasicAgent"],"checkpoint":{"status":"known","value":"o1-2024-12-17"},"display_name":"o1-high / BasicAgent","id":"pb-o1","organization_id":"openai","release_date":null,"scaffold":{"status":"known","value":"BasicAgent"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3 and §§5.2–5.3; Tables 4–5","source_id":"src-paperbench"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Claude 3.5 Sonnet (New) / BasicAgent"],"checkpoint":{"status":"known","value":"claude-3-5-sonnet-20241022"},"display_name":"Claude 3.5 Sonnet (New) / BasicAgent","id":"pb-claude","organization_id":"anthropic","release_date":null,"scaffold":{"status":"known","value":"BasicAgent"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3 and §§5.2–5.3; Tables 4–5","source_id":"src-paperbench"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["o3-mini-high / IterativeAgent"],"checkpoint":{"status":"not_reported","value":null},"display_name":"o3-mini-high / IterativeAgent","id":"pb-o3-iter","organization_id":"openai","release_date":null,"scaffold":{"status":"known","value":"IterativeAgent"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3 and §§5.2–5.3; Tables 4–5","source_id":"src-paperbench"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Claude 3.5 Sonnet (New) / IterativeAgent"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Claude 3.5 Sonnet (New) / IterativeAgent","id":"pb-claude-iter","organization_id":"anthropic","release_date":null,"scaffold":{"status":"known","value":"IterativeAgent"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3 and §§5.2–5.3; Tables 4–5","source_id":"src-paperbench"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["o1-high / IterativeAgent"],"checkpoint":{"status":"not_reported","value":null},"display_name":"o1-high / IterativeAgent","id":"pb-o1-iter","organization_id":"openai","release_date":null,"scaffold":{"status":"known","value":"IterativeAgent"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3 and §§5.2–5.3; Tables 4–5","source_id":"src-paperbench"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["o1-high / IterativeAgent, 36 h"],"checkpoint":{"status":"not_reported","value":null},"display_name":"o1-high / IterativeAgent, 36 h","id":"pb-o1-iter36","organization_id":"openai","release_date":null,"scaffold":{"status":"known","value":"IterativeAgent"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3 and §§5.2–5.3; Tables 4–5","source_id":"src-paperbench"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["o1-preview / AIDE"],"checkpoint":{"status":"not_reported","value":null},"display_name":"o1-preview / AIDE","id":"mle-o1","organization_id":"openai","release_date":null,"scaffold":{"status":"known","value":"AIDE"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3; Table 2","source_id":"src-mle"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["GPT-4o / AIDE"],"checkpoint":{"status":"not_reported","value":null},"display_name":"GPT-4o / AIDE","id":"mle-4o","organization_id":"openai","release_date":null,"scaffold":{"status":"known","value":"AIDE"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3; Table 2","source_id":"src-mle"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Llama 3.1 405B / AIDE"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Llama 3.1 405B / AIDE","id":"mle-llama","organization_id":"meta","release_date":null,"scaffold":{"status":"known","value":"AIDE"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3; Table 2","source_id":"src-mle"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Claude 3.5 Sonnet / AIDE"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Claude 3.5 Sonnet / AIDE","id":"mle-claude","organization_id":"anthropic","release_date":null,"scaffold":{"status":"known","value":"AIDE"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3; Table 2","source_id":"src-mle"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Claude Opus 4.5 / METR horizon evaluation"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Claude Opus 4.5 / METR horizon evaluation","id":"th-opus45","organization_id":"anthropic","release_date":null,"scaffold":{"status":"not_reported","value":null},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"Task-suite changes; Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["GPT-5 / METR horizon evaluation"],"checkpoint":{"status":"not_reported","value":null},"display_name":"GPT-5 / METR horizon evaluation","id":"th-gpt5","organization_id":"openai","release_date":null,"scaffold":{"status":"not_reported","value":null},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"Task-suite changes; Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Claude Opus 4 / METR horizon evaluation"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Claude Opus 4 / METR horizon evaluation","id":"th-opus4","organization_id":"anthropic","release_date":null,"scaffold":{"status":"not_reported","value":null},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"Task-suite changes; Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["o3 / METR horizon evaluation"],"checkpoint":{"status":"not_reported","value":null},"display_name":"o3 / METR horizon evaluation","id":"th-o3","organization_id":"openai","release_date":null,"scaffold":{"status":"not_reported","value":null},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"Task-suite changes; Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Claude 3.7 Sonnet / METR horizon evaluation"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Claude 3.7 Sonnet / METR horizon evaluation","id":"th-sonnet37","organization_id":"anthropic","release_date":null,"scaffold":{"status":"not_reported","value":null},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"Task-suite changes; Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Claude Opus 5 / Claude Code effort aggregate"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Claude Opus 5 / Claude Code effort aggregate","id":"a4-opus","organization_id":"anthropic","release_date":null,"scaffold":{"status":"known","value":"Claude Code"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3; Figure 2 and Table 2","source_id":"src-ai4ai"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["GPT-5.6 Sol / Codex effort aggregate"],"checkpoint":{"status":"not_reported","value":null},"display_name":"GPT-5.6 Sol / Codex effort aggregate","id":"a4-sol","organization_id":"openai","release_date":null,"scaffold":{"status":"known","value":"Codex"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3; Figure 2 and Table 2","source_id":"src-ai4ai"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Kimi K3 / Claude Code effort aggregate"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Kimi K3 / Claude Code effort aggregate","id":"a4-kimi","organization_id":"moonshot","release_date":null,"scaffold":{"status":"known","value":"Claude Code"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3; Figure 2 and Table 2","source_id":"src-ai4ai"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Claude Sonnet 5 / Claude Code effort aggregate"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Claude Sonnet 5 / Claude Code effort aggregate","id":"a4-sonnet","organization_id":"anthropic","release_date":null,"scaffold":{"status":"known","value":"Claude Code"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3; Figure 2 and Table 2","source_id":"src-ai4ai"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["GPT-5.6 Terra / Codex effort aggregate"],"checkpoint":{"status":"not_reported","value":null},"display_name":"GPT-5.6 Terra / Codex effort aggregate","id":"a4-terra","organization_id":"openai","release_date":null,"scaffold":{"status":"known","value":"Codex"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3; Figure 2 and Table 2","source_id":"src-ai4ai"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["GPT-5.6 Luna / Codex effort aggregate"],"checkpoint":{"status":"not_reported","value":null},"display_name":"GPT-5.6 Luna / Codex effort aggregate","id":"a4-luna","organization_id":"openai","release_date":null,"scaffold":{"status":"known","value":"Codex"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"§§2–3; Figure 2 and Table 2","source_id":"src-ai4ai"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Gemini Flash","Gemini Pro"],"checkpoint":{"status":"not_reported","value":null},"display_name":"AlphaEvolve / Gemini ensemble","id":"alphaevolve","organization_id":"deepmind","release_date":null,"scaffold":{"status":"known","value":"Evolutionary program search"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"Designing better algorithms; Enhancing AI training and inference","source_id":"src-alphaevolve"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Claude 3.5 Sonnet (New)","o3-mini"],"checkpoint":{"status":"not_reported","value":null},"display_name":"Darwin Gödel Machine / v1","id":"dgm","organization_id":"dgm-authors","release_date":null,"scaffold":{"status":"known","value":"Self-modifying coding agent archive; Claude 3.5 Sonnet (New) for self-modification and SWE-bench evaluation, o3-mini for Polyglot evaluation"},"scaffold_version":{"status":"not_reported","value":null},"source_refs":[{"locator":"v1 §§4.1–4.4; Figure 2; §4.1 specifies model roles","source_id":"src-dgm"}],"tool_configuration":{"status":"not_reported","value":null}},{"base_model_ids":["Claude Opus 4.7 (outer loop)","Gemini 3 Flash (selection inner loop)"],"checkpoint":{"status":"not_reported","value":null},"display_name":"AIDE² (September 2026 study)","id":"aide2","organization_id":"weco","release_date":null,"scaffold":{"status":"known","value":"AIDE² nested harness search"},"scaffold_version":{"status":"known","value":"arXiv:2609.26457v1"},"source_refs":[{"locator":"v1 §§2–3.6, especially ignition test §3.6","source_id":"src-aide2"}],"tool_configuration":{"status":"known","value":"Code editing and task evaluators; fixed selection rule"}}]