[{"allowed_varying_dimensions":["evaluated_system","evaluation_environment"],"benchmark_version_id":"cobench-2-1","comparability":"limited","comparability_basis":"Source explicitly reports these systems under the same evaluation design.","id":"co21-reported","limitations":["Source reports no statistically distinguishable difference (paired p≈0.2). Environment changes prevent attributing the point difference solely to the model."],"metric_id":"score","protocol_ids":["co21-earlier","co21-later"],"source_refs":[{"locator":"§2.3.4.1, printed pp.36–37; PDF indices 35–36; Figure 2.3.4.1.A","source_id":"src-opus55"}]},{"allowed_varying_dimensions":["evaluated_system"],"benchmark_version_id":"grb-2026-08","comparability":"limited","comparability_basis":"Source explicitly reports these systems under the same evaluation design.","id":"grb-limited","limitations":["Rough older estimate; no automatic delta."],"metric_id":"pass1","protocol_ids":["grb-current","grb-earlier"],"source_refs":[{"locator":"Printed p.37; PDF index 36","source_id":"src-grb"}]},{"allowed_varying_dimensions":["evaluated_system"],"benchmark_version_id":"paperbench-v1","comparability":"direct","comparability_basis":"Source explicitly reports these systems under the same evaluation design.","id":"pb-basic-group","limitations":["Point differences do not establish significance."],"metric_id":"replication","protocol_ids":["pb-basic"],"source_refs":[{"locator":"§5.2–5.3, Tables 4–5","source_id":"src-paperbench"}]},{"allowed_varying_dimensions":["evaluated_system"],"benchmark_version_id":"paperbench-v1","comparability":"direct","comparability_basis":"Source explicitly reports these systems under the same evaluation design.","id":"pb-iterative-group","limitations":["Point differences do not establish significance."],"metric_id":"replication","protocol_ids":["pb-iterative"],"source_refs":[{"locator":"§5.2–5.3, Tables 4–5","source_id":"src-paperbench"}]},{"allowed_varying_dimensions":["evaluated_system"],"benchmark_version_id":"paperbench-v1","comparability":"direct","comparability_basis":"Source explicitly reports these systems under the same evaluation design.","id":"pb-iterative36-group","limitations":["Point differences do not establish significance."],"metric_id":"replication","protocol_ids":["pb-iterative36"],"source_refs":[{"locator":"§5.2–5.3, Tables 4–5","source_id":"src-paperbench"}]},{"allowed_varying_dimensions":["evaluated_system","run_count"],"benchmark_version_id":"mle-2024","comparability":"direct","comparability_basis":"Source explicitly reports these systems under the same evaluation design.","id":"mle-aide","limitations":["Different seed counts affect precision; no significance inference from rounded means."],"metric_id":"medal","protocol_ids":["mle-o1-p","mle-4o-p","mle-llama-p","mle-claude-p"],"source_refs":[{"locator":"§3.1, Table 2","source_id":"src-mle"}]},{"allowed_varying_dimensions":["evaluated_system"],"benchmark_version_id":"th-1-0","comparability":"limited","comparability_basis":"Source explicitly reports these systems under the same evaluation design.","id":"th-1-0-group","limitations":["Configurations and budgets not fully captured in this extraction. No automatic deltas or joined calendar trend."],"metric_id":"p50","protocol_ids":["th-1-0-p"],"source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}]},{"allowed_varying_dimensions":["evaluated_system"],"benchmark_version_id":"th-1-1","comparability":"limited","comparability_basis":"Source explicitly reports these systems under the same evaluation design.","id":"th-1-1-group","limitations":["Configurations and budgets not fully captured in this extraction. No automatic deltas or joined calendar trend."],"metric_id":"p50","protocol_ids":["th-1-1-p"],"source_refs":[{"locator":"Appendix: Changes to Model Horizon Estimates","source_id":"src-th11"}]},{"allowed_varying_dimensions":["evaluated_system","reasoning_effort","harness"],"benchmark_version_id":"ai4ai-v1","comparability":"limited","comparability_basis":"Source explicitly reports these systems under the same evaluation design.","id":"ai4ai-means","limitations":["Different effort grids and costs; no automatic delta."],"metric_id":"normalized","protocol_ids":["a4-opus-p","a4-sol-p","a4-kimi-p","a4-sonnet-p","a4-terra-p","a4-luna-p"],"source_refs":[{"locator":"§3.1–3.2","source_id":"src-ai4ai"}]}]