[{"benchmark_version_id":"cobench-2-1","comparison_group_id":"co21-reported","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"co21-opus5","limitations":["Source reports no statistically distinguishable difference (paired p≈0.2). Environment changes prevent attributing the point difference solely to the model.","Error bars shown without numeric endpoints; no interval was digitized."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"co21-earlier","publication_date":{"precision":"day","value":"2026-09-22"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"53.2%","source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"subject_id":"opus5","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":53.2},{"benchmark_version_id":"cobench-2-1","comparison_group_id":"co21-reported","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"co21-mythos51","limitations":["Source reports no statistically distinguishable difference (paired p≈0.2). Environment changes prevent attributing the point difference solely to the model.","Error bars shown without numeric endpoints; no interval was digitized."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"co21-earlier","publication_date":{"precision":"day","value":"2026-09-22"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"53.4%","source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"subject_id":"mythos51","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":53.4},{"benchmark_version_id":"cobench-2-1","comparison_group_id":"co21-reported","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"co21-opus55","limitations":["Source reports no statistically distinguishable difference (paired p≈0.2). Environment changes prevent attributing the point difference solely to the model.","Error bars shown without numeric endpoints; no interval was digitized."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"co21-later","publication_date":{"precision":"day","value":"2026-09-22"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"55.8%","source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"subject_id":"opus55","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":55.8},{"benchmark_version_id":"cobench-prior-unspecified","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"co-prior-opus5","limitations":["Historical score restated September 22; original publication/evaluation date not established. No comparison group assigned."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"co-prior","publication_date":{"precision":"day","value":"2026-09-22"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"59.6%","source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"subject_id":"opus5","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":59.6},{"benchmark_version_id":"cobench-prior-unspecified","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"co-prior-mythos51","limitations":["Historical score restated September 22; original publication/evaluation date not established. No comparison group assigned."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"co-prior","publication_date":{"precision":"day","value":"2026-09-22"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"57.6%","source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}],"subject_id":"mythos51","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":57.6},{"benchmark_version_id":"openai-research-debugging-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"openai-research-debugging-astra","limitations":["41 debugging bugs; six additional alignment-auditing tasks are described separately."],"metric_id":"score","operational_context":null,"outcome":"numeric","protocol_id":"openai-research-debugging-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"78.05%","source_refs":[{"locator":"§10.1.3.1","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":78.05},{"benchmark_version_id":"kernelgen-1p-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"kernelgen-1p-astra","limitations":["No exact chart value transcribed; missing is not zero."],"metric_id":"score","operational_context":null,"outcome":"not_reported","protocol_id":"kernelgen-1p-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":null,"source_refs":[{"locator":"§10.1.3.2","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"not_reported","value":null},{"benchmark_version_id":"nanogpt-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"nanogpt-astra","limitations":["No exact chart value transcribed; missing is not zero."],"metric_id":"score","operational_context":null,"outcome":"not_reported","protocol_id":"nanogpt-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":null,"source_refs":[{"locator":"§10.1.3.3","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":null},{"benchmark_version_id":"posttrainbench-lite-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"posttrainbench-lite-astra","limitations":["No exact chart value transcribed; missing is not zero."],"metric_id":"score","operational_context":null,"outcome":"not_reported","protocol_id":"posttrainbench-lite-protocol","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":null,"source_refs":[{"locator":"§10.1.3.4","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":null},{"benchmark_version_id":"grb-2026-08","comparison_group_id":"grb-limited","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"grb-flash37","limitations":["Bugged tasks retained; adjusted hypothetical 47% is not an observed score."],"metric_id":"pass1","operational_context":null,"outcome":"numeric","protocol_id":"grb-current","publication_date":{"precision":"month","value":"2026-08"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"27%","source_refs":[{"locator":"Printed p.37; PDF index 36; Results","source_id":"src-grb"}],"subject_id":"gemini37","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":27},{"benchmark_version_id":"grb-2026-08","comparison_group_id":"grb-limited","empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"grb-pro31","limitations":["Approximate preliminary estimate, not a confident baseline."],"metric_id":"pass1","operational_context":null,"outcome":"numeric","protocol_id":"grb-earlier","publication_date":{"precision":"month","value":"2026-08"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"around 16%","source_refs":[{"locator":"Printed p.37; PDF index 36; Results, footnote 10","source_id":"src-grb"}],"subject_id":"gemini31","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":16},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-basic-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-o3-result","limitations":[],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-basic","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"2.6 ± 0.2%","source_refs":[{"locator":"§5.2, Table 4","source_id":"src-paperbench"}],"subject_id":"pb-o3","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"± 0.2 percent","lower":null,"reported_text":"± 0.2 percent","source_ref":{"locator":"§5.2, Table 4","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":2.6},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-basic-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-4o-result","limitations":[],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-basic","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"4.1 ± 0.1%","source_refs":[{"locator":"§5.2, Table 4","source_id":"src-paperbench"}],"subject_id":"pb-4o","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"± 0.1 percent","lower":null,"reported_text":"± 0.1 percent","source_ref":{"locator":"§5.2, Table 4","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":4.1},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-basic-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-gemini-result","limitations":[],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-basic","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"3.2 ± 0.2%","source_refs":[{"locator":"§5.2, Table 4","source_id":"src-paperbench"}],"subject_id":"pb-gemini","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"± 0.2 percent","lower":null,"reported_text":"± 0.2 percent","source_ref":{"locator":"§5.2, Table 4","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":3.2},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-basic-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-o1-result","limitations":[],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-basic","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"13.2 ± 0.3%","source_refs":[{"locator":"§5.2, Table 4","source_id":"src-paperbench"}],"subject_id":"pb-o1","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"± 0.3 percent","lower":null,"reported_text":"± 0.3 percent","source_ref":{"locator":"§5.2, Table 4","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":13.2},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-basic-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-claude-result","limitations":[],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-basic","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"21.0 ± 0.8%","source_refs":[{"locator":"§5.2, Table 4","source_id":"src-paperbench"}],"subject_id":"pb-claude","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"± 0.8 percent","lower":null,"reported_text":"± 0.8 percent","source_ref":{"locator":"§5.2, Table 4","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":21},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-iterative-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-o3-iter-result","limitations":["Different scaffold or time budget from BasicAgent; no cross-protocol delta."],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-iterative","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"8.5 ± 0.8%","source_refs":[{"locator":"§5.3, Table 5","source_id":"src-paperbench"}],"subject_id":"pb-o3-iter","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"± 0.8 percent","lower":null,"reported_text":"± 0.8 percent","source_ref":{"locator":"§5.3, Table 5","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":8.5},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-iterative-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-claude-iter-result","limitations":["Different scaffold or time budget from BasicAgent; no cross-protocol delta."],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-iterative","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"16.1 ± 0.1%","source_refs":[{"locator":"§5.3, Table 5","source_id":"src-paperbench"}],"subject_id":"pb-claude-iter","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"± 0.1 percent","lower":null,"reported_text":"± 0.1 percent","source_ref":{"locator":"§5.3, Table 5","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":16.1},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-iterative-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-o1-iter-result","limitations":["Different scaffold or time budget from BasicAgent; no cross-protocol delta."],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-iterative","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"24.4 ± 0.7%","source_refs":[{"locator":"§5.3, Table 5","source_id":"src-paperbench"}],"subject_id":"pb-o1-iter","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"± 0.7 percent","lower":null,"reported_text":"± 0.7 percent","source_ref":{"locator":"§5.3, Table 5","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":24.4},{"benchmark_version_id":"paperbench-v1","comparison_group_id":"pb-iterative36-group","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"pb-o1-iter36-result","limitations":["Different scaffold or time budget from BasicAgent; no cross-protocol delta."],"metric_id":"replication","operational_context":null,"outcome":"numeric","protocol_id":"pb-iterative36","publication_date":{"precision":"day","value":"2025-04-02"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"26.0 ± 0.3%","source_refs":[{"locator":"§5.3, Table 5","source_id":"src-paperbench"}],"subject_id":"pb-o1-iter36","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"± 0.3 percent","lower":null,"reported_text":"± 0.3 percent","source_ref":{"locator":"§5.3, Table 5","source_id":"src-paperbench"},"type":"standard_error","upper":null},"unit":"percent","value":26},{"benchmark_version_id":"mle-2024","comparison_group_id":"mle-aide","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-o1-result","limitations":[],"metric_id":"medal","operational_context":null,"outcome":"numeric","protocol_id":"mle-o1-p","publication_date":{"precision":"day","value":"2024-10-09"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"16.9 ± 1.1%","source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"subject_id":"mle-o1","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"± 1.1 percent","lower":null,"reported_text":"± 1.1 percent","source_ref":{"locator":"§3.1, Table 2","source_id":"src-mle"},"type":"standard_error","upper":null},"unit":"percent","value":16.9},{"benchmark_version_id":"mle-2024","comparison_group_id":"mle-aide","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-4o-result","limitations":[],"metric_id":"medal","operational_context":null,"outcome":"numeric","protocol_id":"mle-4o-p","publication_date":{"precision":"day","value":"2024-10-09"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"8.7 ± 0.5%","source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"subject_id":"mle-4o","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"± 0.5 percent","lower":null,"reported_text":"± 0.5 percent","source_ref":{"locator":"§3.1, Table 2","source_id":"src-mle"},"type":"standard_error","upper":null},"unit":"percent","value":8.7},{"benchmark_version_id":"mle-2024","comparison_group_id":"mle-aide","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-llama-result","limitations":[],"metric_id":"medal","operational_context":null,"outcome":"numeric","protocol_id":"mle-llama-p","publication_date":{"precision":"day","value":"2024-10-09"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"3.0 ± 1.0%","source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"subject_id":"mle-llama","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"± 1.0 percent","lower":null,"reported_text":"± 1.0 percent","source_ref":{"locator":"§3.1, Table 2","source_id":"src-mle"},"type":"standard_error","upper":null},"unit":"percent","value":3},{"benchmark_version_id":"mle-2024","comparison_group_id":"mle-aide","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-claude-result","limitations":[],"metric_id":"medal","operational_context":null,"outcome":"numeric","protocol_id":"mle-claude-p","publication_date":{"precision":"day","value":"2024-10-09"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"7.6 ± 1.8%","source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}],"subject_id":"mle-claude","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"± 1.8 percent","lower":null,"reported_text":"± 1.8 percent","source_ref":{"locator":"§3.1, Table 2","source_id":"src-mle"},"type":"standard_error","upper":null},"unit":"percent","value":7.6},{"benchmark_version_id":"mle-revised-2026","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"mle-revised-astra","limitations":["Numeric chart not transcribed; revised metric cannot be joined to medal rates."],"metric_id":"percentile","operational_context":null,"outcome":"not_reported","protocol_id":"mle-revised-p","publication_date":{"precision":"day","value":"2026-09-03"},"publication_status":"published","qualitative_statement":null,"reported_value_text":null,"source_refs":[{"locator":"§10.1.3.5","source_id":"src-astra"}],"subject_id":"astra","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":null},{"benchmark_version_id":"rebench-gemini3","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"rebench-gemini3-assessment","limitations":["Qualitative risk assessment; exact graph scores withheld. Not comparable to full seven-task suite."],"metric_id":"assessment","operational_context":null,"outcome":"qualitative","protocol_id":"rebench-gemini-p","publication_date":{"precision":"month","value":"2025-11"},"publication_status":"published","qualitative_statement":"Google reports Gemini 3 Pro remains below its ML R&D alert threshold.","reported_value_text":null,"source_refs":[{"locator":"Machine Learning R&D, printed pp.13–15; PDF indices 13–15","source_id":"src-gemini3"}],"subject_id":"gemini3","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"qualitative","value":null},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-opus45","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"289","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-opus45","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":110,"reported_text":"[110, 1268]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":1268},"unit":"minutes","value":289},{"benchmark_version_id":"th-1-1","comparison_group_id":"th-1-1-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-1-th-opus45","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-1-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"320","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-opus45","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":170,"reported_text":"[170, 729]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":729},"unit":"minutes","value":320},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-gpt5","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"138","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-gpt5","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":68,"reported_text":"[68, 281]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":281},"unit":"minutes","value":138},{"benchmark_version_id":"th-1-1","comparison_group_id":"th-1-1-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-1-th-gpt5","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-1-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"214","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-gpt5","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":117,"reported_text":"[117, 480]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":480},"unit":"minutes","value":214},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-opus4","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"86","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-opus4","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":44,"reported_text":"[44, 144]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":144},"unit":"minutes","value":86},{"benchmark_version_id":"th-1-1","comparison_group_id":"th-1-1-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-1-th-opus4","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-1-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"101","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-opus4","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":58,"reported_text":"[58, 170]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":170},"unit":"minutes","value":101},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-o3","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"94","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-o3","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":48,"reported_text":"[48, 165]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":165},"unit":"minutes","value":94},{"benchmark_version_id":"th-1-1","comparison_group_id":"th-1-1-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-1-th-o3","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-1-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"121","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-o3","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":74,"reported_text":"[74, 201]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":201},"unit":"minutes","value":121},{"benchmark_version_id":"th-1-0","comparison_group_id":"th-1-0-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-0-th-sonnet37","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-0-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"56","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-sonnet37","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":28,"reported_text":"[28, 94]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":94},"unit":"minutes","value":56},{"benchmark_version_id":"th-1-1","comparison_group_id":"th-1-1-group","empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"th-1-1-th-sonnet37","limitations":["Historical snapshot, not the latest live dashboard estimate. Human-task duration, not AI runtime."],"metric_id":"p50","operational_context":null,"outcome":"numeric","protocol_id":"th-1-1-p","publication_date":{"precision":"day","value":"2026-01-29"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"60","source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}],"subject_id":"th-sonnet37","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":{"level":"Source-reported bootstrap interval; level not specified in this table","lower":32,"reported_text":"[32, 106]","source_ref":{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"},"type":"confidence_interval","upper":106},"unit":"minutes","value":60},{"benchmark_version_id":"ai4ai-v1","comparison_group_id":"ai4ai-means","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"a4-opus-mean","limitations":["Effort-grid aggregate; not equal-cost comparison."],"metric_id":"normalized","operational_context":null,"outcome":"numeric","protocol_id":"a4-opus-p","publication_date":{"precision":"day","value":"2026-08-20"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.250","source_refs":[{"locator":"§3.2, Figure 2 prose","source_id":"src-ai4ai"}],"subject_id":"a4-opus","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":0.25},{"benchmark_version_id":"ai4ai-v1","comparison_group_id":"ai4ai-means","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"a4-sol-mean","limitations":["Effort-grid aggregate; not equal-cost comparison."],"metric_id":"normalized","operational_context":null,"outcome":"numeric","protocol_id":"a4-sol-p","publication_date":{"precision":"day","value":"2026-08-20"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.191","source_refs":[{"locator":"§3.2, Figure 2 prose","source_id":"src-ai4ai"}],"subject_id":"a4-sol","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":0.191},{"benchmark_version_id":"ai4ai-v1","comparison_group_id":"ai4ai-means","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"a4-kimi-mean","limitations":["Effort-grid aggregate; not equal-cost comparison."],"metric_id":"normalized","operational_context":null,"outcome":"numeric","protocol_id":"a4-kimi-p","publication_date":{"precision":"day","value":"2026-08-20"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.174","source_refs":[{"locator":"§3.2, Figure 2 prose","source_id":"src-ai4ai"}],"subject_id":"a4-kimi","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":0.174},{"benchmark_version_id":"ai4ai-v1","comparison_group_id":"ai4ai-means","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"a4-sonnet-mean","limitations":["Effort-grid aggregate; not equal-cost comparison."],"metric_id":"normalized","operational_context":null,"outcome":"numeric","protocol_id":"a4-sonnet-p","publication_date":{"precision":"day","value":"2026-08-20"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.145","source_refs":[{"locator":"§3.2, Figure 2 prose","source_id":"src-ai4ai"}],"subject_id":"a4-sonnet","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":0.145},{"benchmark_version_id":"ai4ai-v1","comparison_group_id":"ai4ai-means","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"a4-terra-mean","limitations":["Effort-grid aggregate; not equal-cost comparison."],"metric_id":"normalized","operational_context":null,"outcome":"numeric","protocol_id":"a4-terra-p","publication_date":{"precision":"day","value":"2026-08-20"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.135","source_refs":[{"locator":"§3.2, Figure 2 prose","source_id":"src-ai4ai"}],"subject_id":"a4-terra","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":0.135},{"benchmark_version_id":"ai4ai-v1","comparison_group_id":"ai4ai-means","empirical_verification":"benchmark_author_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"a4-luna-mean","limitations":["Effort-grid aggregate; not equal-cost comparison."],"metric_id":"normalized","operational_context":null,"outcome":"numeric","protocol_id":"a4-luna-p","publication_date":{"precision":"day","value":"2026-08-20"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"0.117","source_refs":[{"locator":"§3.2, Figure 2 prose","source_id":"src-ai4ai"}],"subject_id":"a4-luna","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"normalized_score","value":0.117},{"benchmark_version_id":"automation-aug26","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":{"precision":"month","value":"2026-08"},"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"automation-ai_leads-aug26","limitations":["Not directly comparable across organizations without common methodology."],"metric_id":"ai_leads","operational_context":{"denominator":"Person-time-weighted categories in a fixed work basket","method":"estimate","population":"Anthropic model R&D work","sampling_method":"20% staff sampled weekly by department; Claude researches and rates tasks","timeframe":"August 2026; basket and weights constructed from July 2026"},"outcome":"numeric","protocol_id":"automation-ai_leads","publication_date":{"precision":"day","value":"2026-09-17"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"26%","source_refs":[{"locator":"§1; Appendix: Measuring AI-led R&D","source_id":"src-automation"}],"subject_id":"anthropic","subject_type":"organization","supersedes_id":null,"uncertainty":null,"unit":"percent","value":26},{"benchmark_version_id":"automation-aug26","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":{"precision":"month","value":"2026-08"},"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"automation-autonomous-aug26","limitations":["Self-report using Claude judges. Zero describes fully autonomous measured work, not absence of AI assistance."],"metric_id":"autonomous","operational_context":{"denominator":"Person-time-weighted categories in a fixed work basket","method":"estimate","population":"Anthropic model R&D work","sampling_method":"20% staff sampled weekly by department; Claude researches and rates tasks","timeframe":"August 2026; basket and weights constructed from July 2026"},"outcome":"numeric","protocol_id":"automation-autonomous","publication_date":{"precision":"day","value":"2026-09-17"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"No measured subset fully autonomous","source_refs":[{"locator":"§1; Appendix: Measuring AI-led R&D","source_id":"src-automation"}],"subject_id":"anthropic","subject_type":"organization","supersedes_id":null,"uncertainty":null,"unit":"percent","value":0},{"benchmark_version_id":"productivity-early25","comparison_group_id":null,"empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"productivity-slowdown","limitations":["Positive means slower. Not an estimate for all developers or current tools."],"metric_id":"time_change","operational_context":{"denominator":"246 randomized real repository issues","method":"experimental","population":"16 experienced open-source developers","sampling_method":"Recruited developers; within-study issue randomization","timeframe":"Early 2025"},"outcome":"numeric","protocol_id":"productivity-p","publication_date":{"precision":"day","value":"2025-07-10"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"19% longer","source_refs":[{"locator":"Core Result","source_id":"src-productivity"}],"subject_id":"metr","subject_type":"organization","supersedes_id":null,"uncertainty":null,"unit":"percent","value":19},{"benchmark_version_id":"productivity-late25","comparison_group_id":null,"empirical_verification":"independent_evaluation","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"productivity-selection","limitations":["Reported raw effects are not used as a comparable trend."],"metric_id":"assessment","operational_context":{"denominator":"Submitted tasks; substantial task and participant selection","method":"experimental","population":"10 returning and 47 newly recruited developers","sampling_method":"Voluntary recruitment; randomized submitted tasks","timeframe":"Experiment begun August 2025; reported February 2026"},"outcome":"qualitative","protocol_id":"productivity-later-p","publication_date":{"precision":"day","value":"2026-02-24"},"publication_status":"published","qualitative_statement":"METR reports that selection effects make the later experiment unreliable for estimating current productivity gains.","reported_value_text":null,"source_refs":[{"locator":"Introduction and selection-effects discussion","source_id":"src-productivity-update"}],"subject_id":"metr","subject_type":"organization","supersedes_id":null,"uncertainty":null,"unit":"qualitative","value":null},{"benchmark_version_id":"alphaevolve-2025","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"alpha-training_reduction-result","limitations":["Does not measure improvement in the optimizer across generations."],"metric_id":"training_reduction","operational_context":{"denominator":"Training time","method":"telemetry","population":"Gemini training / matrix multiplication kernel","sampling_method":"Developer-reported production application","timeframe":"Deployment reported May 2025"},"outcome":"numeric","protocol_id":"alpha-training_reduction","publication_date":{"precision":"day","value":"2025-05-14"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"1%","source_refs":[{"locator":"Enhancing AI training and inference","source_id":"src-alphaevolve"}],"subject_id":"alphaevolve","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":1},{"benchmark_version_id":"alphaevolve-2025","comparison_group_id":null,"empirical_verification":"lab_reported","evaluation_date":null,"extraction_review":{"notes":"Read the cited original source. No human extraction review or experimental replication claimed.","reviewed_at":"2026-09-26T04:47:27Z","reviewer":"Codex","status":"agent_checked"},"id":"alpha-kernel_speedup-result","limitations":["Does not measure improvement in the optimizer across generations."],"metric_id":"kernel_speedup","operational_context":{"denominator":"Kernel performance baseline","method":"telemetry","population":"Gemini training / matrix multiplication kernel","sampling_method":"Developer-reported production application","timeframe":"Deployment reported May 2025"},"outcome":"numeric","protocol_id":"alpha-kernel_speedup","publication_date":{"precision":"day","value":"2025-05-14"},"publication_status":"published","qualitative_statement":null,"reported_value_text":"23%","source_refs":[{"locator":"Enhancing AI training and inference","source_id":"src-alphaevolve"}],"subject_id":"alphaevolve","subject_type":"evaluated_system","supersedes_id":null,"uncertainty":null,"unit":"percent","value":23}]