{"harness_tax":{"families":[{"family_id":"aime2025","label":"Aime2025","capability":"math","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"aa-aime2025/current","name":"aa-aime2025","harness":"artificial-analysis","n":1},{"key":"aime2025/current","name":"aime2025","harness":"published","n":16},{"key":"aime2025arcee/current","name":"aime2025arcee","harness":"published","n":6}]},{"family_id":"arxivmathaugust2026","label":"Arxivmathaugust2026","capability":"math","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"arxivmathaugust2026/current","name":"arxivmathaugust2026","harness":"published","n":2},{"key":"arxivmathaugust2026withtools/current","name":"arxivmathaugust2026withtools","harness":"published","n":2}]},{"family_id":"arxivmathjune2026","label":"Arxivmathjune2026","capability":"math","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"arxivmathjune2026/current","name":"arxivmathjune2026","harness":"published","n":1},{"key":"arxivmathjune2026withtools/current","name":"arxivmathjune2026withtools","harness":"published","n":1}]},{"family_id":"automationbench","label":"Automationbench","capability":"agentic-tool","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"aa-automation-bench/current","name":"aa-automation-bench","harness":"artificial-analysis","n":25},{"key":"automationbench/current","name":"automationbench","harness":"published","n":22}]},{"family_id":"benchcadvision2code","label":"Benchcadvision2Code","capability":"vision","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"benchcadvision2code/current","name":"benchcadvision2code","harness":"published","n":3},{"key":"benchcadvision2codewithtools/current","name":"benchcadvision2codewithtools","harness":"published","n":4}]},{"family_id":"chartography","label":"Chartography","capability":"vision","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"chartography/current","name":"chartography","harness":"published","n":5},{"key":"chartographywithtools/current","name":"chartographywithtools","harness":"published","n":5}]},{"family_id":"charxiv","label":"Charxiv","capability":"vision","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"charxiv/current","name":"charxiv","harness":"published","n":39},{"key":"charxivnotools/current","name":"charxivnotools","harness":"published","n":14}]},{"family_id":"gdppdf","label":"Gdppdf","capability":"agentic-tool","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"aa-gdp-pdf/current","name":"aa-gdp-pdf","harness":"artificial-analysis","n":24},{"key":"gdppdf/current","name":"gdppdf","harness":"published","n":2},{"key":"gdppdfwithtools/current","name":"gdppdfwithtools","harness":"published","n":1}]},{"family_id":"gpqa-diamond","label":"GPQA Diamond","capability":"knowledge","tier":"knowledge_tool_free","same_item_set":"verified","status":"active","pair_type":null,"audit_note":null,"origin":"seed","versions":[{"key":"gpqa-diamond/current","name":"gpqa-diamond","harness":"published","n":68},{"key":"vals-gpqa-diamond/current","name":"vals-gpqa-diamond","harness":"vals-ai","n":56},{"key":"aa-gpqa-diamond/current","name":"aa-gpqa-diamond","harness":"artificial-analysis","n":181}]},{"family_id":"hle","label":"Humanity's Last Exam","capability":"knowledge","tier":"protocol_layer","same_item_set":"needs_audit","status":"active","pair_type":null,"audit_note":"AA HLE protocol may include tools; this pair is the pivot case for the paper. Audit before claiming pure harness tax.","origin":"seed","versions":[{"key":"hle/current","name":"hle","harness":"published","n":59},{"key":"aa-hle/current","name":"aa-hle","harness":"artificial-analysis","n":195}]},{"family_id":"hle-protocol-variants","label":"HLE tools on/off (same harness)","capability":"knowledge","tier":"protocol_layer","same_item_set":"verified","status":"active","pair_type":"protocol_variant","audit_note":null,"origin":"seed","versions":[{"key":"hlenotools/current","name":"hlenotools","harness":"published","n":42},{"key":"hlewithtools/current","name":"hlewithtools","harness":"published","n":22}]},{"family_id":"ifbench","label":"IFBench","capability":"instruction-following","tier":"knowledge_tool_free","same_item_set":"verified","status":"active","pair_type":null,"audit_note":null,"origin":"seed","versions":[{"key":"ifbench/current","name":"ifbench","harness":"published","n":43},{"key":"aa-if-bench/current","name":"aa-if-bench","harness":"artificial-analysis","n":130}]},{"family_id":"livecodebench","label":"LiveCodeBench","capability":"coding","tier":"agentic","same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Versions differ across harnesses (v6 etc.) so cross-harness pairs conflate version and harness. Keep pairs but flag weak_item_set_alignment; do not use in headline aggregates until aligned versions exist.","origin":"seed","versions":[{"key":"livecodebench/current","name":"livecodebench","harness":"published","n":7},{"key":"livecodebench-v6/current","name":"livecodebench-v6","harness":"published","n":32},{"key":"livecodebench-pro/current","name":"livecodebench-pro","harness":"published","n":8},{"key":"valslivecodebench/current","name":"valslivecodebench","harness":"vals-ai","n":58},{"key":"aa-live-code-bench/current","name":"aa-live-code-bench","harness":"artificial-analysis","n":2}]},{"family_id":"math500","label":"Math500","capability":"math","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"aa-math500/current","name":"aa-math500","harness":"artificial-analysis","n":3},{"key":"math-500/current","name":"math-500","harness":"published","n":7}]},{"family_id":"mmlu-pro","label":"MMLU-Pro","capability":"knowledge","tier":"knowledge_tool_free","same_item_set":"needs_audit","status":"active","pair_type":null,"audit_note":"mmlu-pro-arcee is a re-run protocol variant; confirm item set and treat separately from plain published run if it differs.","origin":"seed","versions":[{"key":"mmlu-pro/current","name":"mmlu-pro","harness":"published","n":46},{"key":"vals-mmlu-pro/current","name":"vals-mmlu-pro","harness":"vals-ai","n":56},{"key":"mmlu-pro-arcee/current","name":"mmlu-pro-arcee","harness":"published","n":6}]},{"family_id":"mmmu-pro","label":"MMMU-Pro","capability":"vision","tier":"knowledge_tool_free","same_item_set":"verified","status":"active","pair_type":null,"audit_note":null,"origin":"seed","versions":[{"key":"mmmu-pro/current","name":"mmmu-pro","harness":"published","n":42},{"key":"aa-mmmu-pro/current","name":"aa-mmmu-pro","harness":"artificial-analysis","n":104}]},{"family_id":"riemannbench","label":"Riemannbench","capability":"math","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"riemannbench/current","name":"riemannbench","harness":"published","n":1},{"key":"riemannbenchwithtools/current","name":"riemannbenchwithtools","harness":"published","n":1}]},{"family_id":"scicode","label":"SciCode","capability":"coding","tier":"agentic","same_item_set":"needs_audit","status":"active","pair_type":null,"audit_note":"Confirm the AA 'aa-sci-code' run uses the same item set as the published SciCode runs.","origin":"seed","versions":[{"key":"scicode/current","name":"scicode","harness":"published","n":28},{"key":"aa-sci-code/current","name":"aa-sci-code","harness":"artificial-analysis","n":105}]},{"family_id":"swe-bench-verified","label":"SWE-bench Verified","capability":"coding","tier":"agentic","same_item_set":"needs_audit","status":"active","pair_type":null,"audit_note":"Confirm Vals 'swe-bench' runs the Verified subset before promoting to 'verified'.","origin":"seed","versions":[{"key":"swe-bench-verified/current","name":"swe-bench-verified","harness":"published","n":78},{"key":"valsswebench/current","name":"valsswebench","harness":"vals-ai","n":60}]},{"family_id":"terminal-bench-2-1","label":"Terminal-Bench 2.1","capability":"agentic-terminal","tier":"agentic","same_item_set":"verified","status":"active","pair_type":null,"audit_note":null,"origin":"seed","versions":[{"key":"terminalbench21/current","name":"terminalbench21","harness":"published","n":65},{"key":"vals-terminal-bench21/current","name":"vals-terminal-bench21","harness":"vals-ai","n":61},{"key":"aa-terminal-bench21/current","name":"aa-terminal-bench21","harness":"artificial-analysis","n":12}]},{"family_id":"terminal-bench-3","label":"Terminal-Bench 3","capability":"agentic-terminal","tier":"agentic","same_item_set":"verified","status":"candidate","pair_type":null,"audit_note":"Overlap currently 0 shared models; keep registered, exclude from aggregates.","origin":"seed","versions":[{"key":"terminal-bench-3/current","name":"terminal-bench-3","harness":"published","n":13},{"key":"terminal-bench3/current","name":"terminal-bench3","harness":"published","n":2}]},{"family_id":"terminal-bench-4-0","label":"Terminal-Bench 4.0","capability":"agentic-terminal","tier":"agentic","same_item_set":"verified","status":"active","pair_type":null,"audit_note":null,"origin":"seed","versions":[{"key":"terminal-bench-4/current","name":"terminal-bench-4","harness":"published","n":17},{"key":"aa-terminal-bench4/current","name":"aa-terminal-bench4","harness":"artificial-analysis","n":24}]},{"family_id":"terminal-bench-science-variants","label":"Terminal-Bench Science verifier variants","capability":"agentic-terminal","tier":"protocol_layer","same_item_set":"verified","status":"candidate","pair_type":"protocol_variant","audit_note":"1x vs 6x verifier count — pure scaffold knob. Currently only 1 shared model; keep for growth.","origin":"seed","versions":[{"key":"terminal-bench-science/current","name":"terminal-bench-science","harness":"published","n":5},{"key":"terminal-bench-science-6x-verifier/current","name":"terminal-bench-science-6x-verifier","harness":"published","n":1}]}],"pairs":[{"id":36,"family_id":"chartography","tier":null,"capability":"vision","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"chartography/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/chartography"},"b":{"key":"chartographywithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/chartographywithtools"},"n":3,"mean_abs_pp":35.53,"median_abs_pp":28.6,"max_abs_pp":53.4,"share_gt_5":1,"share_gt_10":1,"kendall_tau":0.333,"n_rank_flips":1,"n_positive":0,"n_negative":3,"sign_p":0.25,"directionality":1},{"id":34,"family_id":"automationbench","tier":null,"capability":"agentic-tool","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":false,"a":{"key":"aa-automation-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aaautomationbench"},"b":{"key":"automationbench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/automationbench"},"n":13,"mean_abs_pp":19.71,"median_abs_pp":26.2,"max_abs_pp":29.5,"share_gt_5":1,"share_gt_10":0.769,"kendall_tau":0.154,"n_rank_flips":33,"n_positive":13,"n_negative":0,"sign_p":0.0002,"directionality":1},{"id":42,"family_id":"riemannbench","tier":null,"capability":"math","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"riemannbench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/riemannbench"},"b":{"key":"riemannbenchwithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/riemannbenchwithtools"},"n":1,"mean_abs_pp":19,"median_abs_pp":19,"max_abs_pp":19,"share_gt_5":1,"share_gt_10":1,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_p":1,"directionality":1},{"id":1,"family_id":"terminal-bench-2-1","tier":"agentic","capability":"agentic-terminal","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"terminalbench21/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"b":{"key":"vals-terminal-bench21/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"n":29,"mean_abs_pp":12.09,"median_abs_pp":10.7,"max_abs_pp":33.2,"share_gt_5":0.862,"share_gt_10":0.517,"kendall_tau":0.516,"n_rank_flips":96,"n_positive":29,"n_negative":0,"sign_p":0,"directionality":1},{"id":16,"family_id":"hle-protocol-variants","tier":"protocol_layer","capability":"knowledge","pair_type":"protocol_variant","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"hlenotools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/hlenotools"},"b":{"key":"hlewithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/hlewithtools"},"n":11,"mean_abs_pp":9.99,"median_abs_pp":10.7,"max_abs_pp":14.2,"share_gt_5":0.909,"share_gt_10":0.636,"kendall_tau":0.807,"n_rank_flips":5,"n_positive":0,"n_negative":11,"sign_p":0.001,"directionality":1},{"id":3,"family_id":"terminal-bench-2-1","tier":"agentic","capability":"agentic-terminal","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"vals-terminal-bench21/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"b":{"key":"aa-terminal-bench21/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"n":11,"mean_abs_pp":9.75,"median_abs_pp":7.5,"max_abs_pp":21.4,"share_gt_5":0.727,"share_gt_10":0.455,"kendall_tau":0.881,"n_rank_flips":3,"n_positive":0,"n_negative":11,"sign_p":0.001,"directionality":1},{"id":15,"family_id":"hle","tier":"protocol_layer","capability":"knowledge","pair_type":"cross_harness","same_item_set":"needs_audit","status":"active","low_overlap":false,"a":{"key":"hle/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/hle"},"b":{"key":"aa-hle/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aahle"},"n":48,"mean_abs_pp":8.15,"median_abs_pp":4.85,"max_abs_pp":35.7,"share_gt_5":0.5,"share_gt_10":0.292,"kendall_tau":0.579,"n_rank_flips":236,"n_positive":28,"n_negative":18,"sign_p":0.1839,"directionality":0.609},{"id":24,"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-pro"},"b":{"key":"valslivecodebench/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/valslivecodebench"},"n":2,"mean_abs_pp":7.85,"median_abs_pp":7.85,"max_abs_pp":10.1,"share_gt_5":1,"share_gt_10":0.5,"kendall_tau":1,"n_rank_flips":0,"n_positive":0,"n_negative":2,"sign_p":0.5,"directionality":1},{"id":26,"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"valslivecodebench/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/valslivecodebench"},"b":{"key":"aa-live-code-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aalivecodebench"},"n":1,"mean_abs_pp":7.2,"median_abs_pp":7.2,"max_abs_pp":7.2,"share_gt_5":1,"share_gt_10":0,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_p":1,"directionality":1},{"id":32,"family_id":"arxivmathaugust2026","tier":null,"capability":"math","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"arxivmathaugust2026/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/arxivmathaugust2026"},"b":{"key":"arxivmathaugust2026withtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/arxivmathaugust2026withtools"},"n":2,"mean_abs_pp":7.05,"median_abs_pp":7.05,"max_abs_pp":8.4,"share_gt_5":1,"share_gt_10":0,"kendall_tau":1,"n_rank_flips":0,"n_positive":0,"n_negative":2,"sign_p":0.5,"directionality":1},{"id":37,"family_id":"charxiv","tier":null,"capability":"vision","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":false,"a":{"key":"charxiv/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/charxiv"},"b":{"key":"charxivnotools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/charxivnotools"},"n":12,"mean_abs_pp":6.52,"median_abs_pp":6.25,"max_abs_pp":11.3,"share_gt_5":0.667,"share_gt_10":0.083,"kendall_tau":0.657,"n_rank_flips":11,"n_positive":12,"n_negative":0,"sign_p":0.0005,"directionality":1},{"id":5,"family_id":"swe-bench-verified","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"needs_audit","status":"active","low_overlap":false,"a":{"key":"swe-bench-verified/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/swe-bench-verified"},"b":{"key":"valsswebench/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/valsswebench"},"n":24,"mean_abs_pp":6.29,"median_abs_pp":5.45,"max_abs_pp":17,"share_gt_5":0.542,"share_gt_10":0.208,"kendall_tau":0.676,"n_rank_flips":44,"n_positive":17,"n_negative":4,"sign_p":0.0072,"directionality":0.81},{"id":21,"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench-v6/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-v6"},"b":{"key":"livecodebench-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-pro"},"n":3,"mean_abs_pp":6.11,"median_abs_pp":5.1,"max_abs_pp":10.84,"share_gt_5":0.667,"share_gt_10":0.333,"kendall_tau":1,"n_rank_flips":0,"n_positive":3,"n_negative":0,"sign_p":0.25,"directionality":1},{"id":20,"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench"},"b":{"key":"aa-live-code-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aalivecodebench"},"n":1,"mean_abs_pp":4.5,"median_abs_pp":4.5,"max_abs_pp":4.5,"share_gt_5":0,"share_gt_10":0,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_p":1,"directionality":1},{"id":14,"family_id":"ifbench","tier":"knowledge_tool_free","capability":"instruction-following","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"ifbench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/ifbench"},"b":{"key":"aa-if-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/ifbench"},"n":11,"mean_abs_pp":4.19,"median_abs_pp":0.87,"max_abs_pp":15.42,"share_gt_5":0.273,"share_gt_10":0.273,"kendall_tau":0.771,"n_rank_flips":6,"n_positive":6,"n_negative":3,"sign_p":0.5078,"directionality":0.667},{"id":4,"family_id":"terminal-bench-4-0","tier":"agentic","capability":"agentic-terminal","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"terminal-bench-4/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminal-bench-4"},"b":{"key":"aa-terminal-bench4/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/terminal-bench-4"},"n":11,"mean_abs_pp":3.89,"median_abs_pp":3.8,"max_abs_pp":12.2,"share_gt_5":0.364,"share_gt_10":0.091,"kendall_tau":0.818,"n_rank_flips":5,"n_positive":8,"n_negative":2,"sign_p":0.1094,"directionality":0.8},{"id":19,"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench"},"b":{"key":"valslivecodebench/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/valslivecodebench"},"n":2,"mean_abs_pp":3.6,"median_abs_pp":3.6,"max_abs_pp":4.5,"share_gt_5":0,"share_gt_10":0,"kendall_tau":1,"n_rank_flips":0,"n_positive":2,"n_negative":0,"sign_p":0.5,"directionality":1},{"id":22,"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench-v6/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-v6"},"b":{"key":"valslivecodebench/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/valslivecodebench"},"n":4,"mean_abs_pp":3.3,"median_abs_pp":2.9,"max_abs_pp":6.3,"share_gt_5":0.25,"share_gt_10":0,"kendall_tau":-0.183,"n_rank_flips":3,"n_positive":4,"n_negative":0,"sign_p":0.125,"directionality":1},{"id":2,"family_id":"terminal-bench-2-1","tier":"agentic","capability":"agentic-terminal","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"terminalbench21/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"b":{"key":"aa-terminal-bench21/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"n":10,"mean_abs_pp":3.27,"median_abs_pp":2.9,"max_abs_pp":8.7,"share_gt_5":0.2,"share_gt_10":0,"kendall_tau":0.899,"n_rank_flips":2,"n_positive":7,"n_negative":1,"sign_p":0.0703,"directionality":0.875},{"id":9,"family_id":"gpqa-diamond","tier":"knowledge_tool_free","capability":"knowledge","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"vals-gpqa-diamond/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"b":{"key":"aa-gpqa-diamond/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"n":47,"mean_abs_pp":2.91,"median_abs_pp":1.2,"max_abs_pp":40,"share_gt_5":0.149,"share_gt_10":0.043,"kendall_tau":0.77,"n_rank_flips":119,"n_positive":15,"n_negative":31,"sign_p":0.0259,"directionality":0.674},{"id":11,"family_id":"mmlu-pro","tier":"knowledge_tool_free","capability":"knowledge","pair_type":"protocol_variant","same_item_set":"needs_audit","status":"active","low_overlap":true,"a":{"key":"mmlu-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/mmlu-pro"},"b":{"key":"mmlu-pro-arcee/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/mmlu-pro"},"n":3,"mean_abs_pp":2.4,"median_abs_pp":0.1,"max_abs_pp":7.1,"share_gt_5":0.333,"share_gt_10":0,"kendall_tau":-0.333,"n_rank_flips":2,"n_positive":0,"n_negative":2,"sign_p":0.5,"directionality":1},{"id":40,"family_id":"gdppdf","tier":null,"capability":"agentic-tool","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"gdppdf/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/gdppdf"},"b":{"key":"gdppdfwithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/gdppdfwithtools"},"n":1,"mean_abs_pp":2.1,"median_abs_pp":2.1,"max_abs_pp":2.1,"share_gt_5":0,"share_gt_10":0,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_p":1,"directionality":1},{"id":6,"family_id":"scicode","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"needs_audit","status":"active","low_overlap":false,"a":{"key":"scicode/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/scicode"},"b":{"key":"aa-sci-code/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aascicode"},"n":20,"mean_abs_pp":2.03,"median_abs_pp":1,"max_abs_pp":7.4,"share_gt_5":0.1,"share_gt_10":0,"kendall_tau":0.769,"n_rank_flips":21,"n_positive":6,"n_negative":11,"sign_p":0.3323,"directionality":0.647},{"id":13,"family_id":"mmmu-pro","tier":"knowledge_tool_free","capability":"vision","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"mmmu-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/mmmu-pro"},"b":{"key":"aa-mmmu-pro/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/mmmu-pro"},"n":32,"mean_abs_pp":2.02,"median_abs_pp":0.7,"max_abs_pp":26.3,"share_gt_5":0.031,"share_gt_10":0.031,"kendall_tau":0.695,"n_rank_flips":73,"n_positive":16,"n_negative":11,"sign_p":0.4421,"directionality":0.593},{"id":8,"family_id":"gpqa-diamond","tier":"knowledge_tool_free","capability":"knowledge","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"gpqa-diamond/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"b":{"key":"aa-gpqa-diamond/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"n":40,"mean_abs_pp":1.98,"median_abs_pp":0.65,"max_abs_pp":25.3,"share_gt_5":0.075,"share_gt_10":0.05,"kendall_tau":0.809,"n_rank_flips":71,"n_positive":21,"n_negative":16,"sign_p":0.5114,"directionality":0.568},{"id":10,"family_id":"mmlu-pro","tier":"knowledge_tool_free","capability":"knowledge","pair_type":"cross_harness","same_item_set":"needs_audit","status":"active","low_overlap":false,"a":{"key":"mmlu-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/mmlu-pro"},"b":{"key":"vals-mmlu-pro/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/mmlu-pro"},"n":7,"mean_abs_pp":1.76,"median_abs_pp":0.8,"max_abs_pp":8.1,"share_gt_5":0.143,"share_gt_10":0,"kendall_tau":0.524,"n_rank_flips":5,"n_positive":5,"n_negative":1,"sign_p":0.2188,"directionality":0.833},{"id":7,"family_id":"gpqa-diamond","tier":"knowledge_tool_free","capability":"knowledge","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"gpqa-diamond/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"b":{"key":"vals-gpqa-diamond/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"n":23,"mean_abs_pp":1.36,"median_abs_pp":0.9,"max_abs_pp":5.9,"share_gt_5":0.087,"share_gt_10":0,"kendall_tau":0.689,"n_rank_flips":38,"n_positive":15,"n_negative":8,"sign_p":0.21,"directionality":0.652},{"id":33,"family_id":"arxivmathjune2026","tier":null,"capability":"math","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"arxivmathjune2026/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/arxivmathjune2026"},"b":{"key":"arxivmathjune2026withtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/arxivmathjune2026withtools"},"n":1,"mean_abs_pp":0.5,"median_abs_pp":0.5,"max_abs_pp":0.5,"share_gt_5":0,"share_gt_10":0,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_p":1,"directionality":1},{"id":12,"family_id":"mmlu-pro","tier":"knowledge_tool_free","capability":"knowledge","pair_type":"cross_harness","same_item_set":"needs_audit","status":"active","low_overlap":true,"a":{"key":"vals-mmlu-pro/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/mmlu-pro"},"b":{"key":"mmlu-pro-arcee/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/mmlu-pro"},"n":1,"mean_abs_pp":0.4,"median_abs_pp":0.4,"max_abs_pp":0.4,"share_gt_5":0,"share_gt_10":0,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_p":1,"directionality":1},{"id":35,"family_id":"benchcadvision2code","tier":null,"capability":"vision","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"benchcadvision2code/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/benchcadvision2code"},"b":{"key":"benchcadvision2codewithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/benchcadvision2codewithtools"},"n":3,"mean_abs_pp":0.3,"median_abs_pp":0.23,"max_abs_pp":0.46,"share_gt_5":0,"share_gt_10":0,"kendall_tau":1,"n_rank_flips":0,"n_positive":0,"n_negative":3,"sign_p":0.25,"directionality":1},{"id":31,"family_id":"aime2025","tier":null,"capability":"math","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"aime2025/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/aime2025"},"b":{"key":"aime2025arcee/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/aime2025arcee"},"n":1,"mean_abs_pp":0.2,"median_abs_pp":0.2,"max_abs_pp":0.2,"share_gt_5":0,"share_gt_10":0,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_p":1,"directionality":1},{"id":29,"family_id":"aime2025","tier":null,"capability":"math","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"aa-aime2025/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aaaime2025"},"b":{"key":"aime2025/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/aime2025"},"n":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5":null,"share_gt_10":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_p":null,"directionality":null},{"id":30,"family_id":"aime2025","tier":null,"capability":"math","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"aa-aime2025/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aaaime2025"},"b":{"key":"aime2025arcee/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/aime2025arcee"},"n":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5":null,"share_gt_10":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_p":null,"directionality":null},{"id":38,"family_id":"gdppdf","tier":null,"capability":"agentic-tool","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"aa-gdp-pdf/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aagdppdf"},"b":{"key":"gdppdf/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/gdppdf"},"n":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5":null,"share_gt_10":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_p":null,"directionality":null},{"id":39,"family_id":"gdppdf","tier":null,"capability":"agentic-tool","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"aa-gdp-pdf/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aagdppdf"},"b":{"key":"gdppdfwithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/gdppdfwithtools"},"n":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5":null,"share_gt_10":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_p":null,"directionality":null},{"id":17,"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench"},"b":{"key":"livecodebench-v6/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-v6"},"n":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5":null,"share_gt_10":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_p":null,"directionality":null},{"id":18,"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench"},"b":{"key":"livecodebench-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-pro"},"n":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5":null,"share_gt_10":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_p":null,"directionality":null},{"id":23,"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench-v6/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-v6"},"b":{"key":"aa-live-code-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aalivecodebench"},"n":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5":null,"share_gt_10":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_p":null,"directionality":null},{"id":25,"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-pro"},"b":{"key":"aa-live-code-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aalivecodebench"},"n":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5":null,"share_gt_10":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_p":null,"directionality":null},{"id":41,"family_id":"math500","tier":null,"capability":"math","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"aa-math500/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/math-500"},"b":{"key":"math-500/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/math-500"},"n":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5":null,"share_gt_10":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_p":null,"directionality":null},{"id":28,"family_id":"terminal-bench-3","tier":"agentic","capability":"agentic-terminal","pair_type":"protocol_variant","same_item_set":"verified","status":"candidate","low_overlap":true,"a":{"key":"terminal-bench-3/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminal-bench-3"},"b":{"key":"terminal-bench3/current","harness":"published","source_url":"https://benchlm.ai/benchmarks"},"n":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5":null,"share_gt_10":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_p":null,"directionality":null},{"id":27,"family_id":"terminal-bench-science-variants","tier":"protocol_layer","capability":"agentic-terminal","pair_type":"protocol_variant","same_item_set":"verified","status":"candidate","low_overlap":true,"a":{"key":"terminal-bench-science/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminal-bench-science"},"b":{"key":"terminal-bench-science-6x-verifier/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminal-bench-science-6x-verifier"},"n":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5":null,"share_gt_10":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_p":null,"directionality":null}],"deltas":{"1":[{"model_id":24,"slug":"claude-fable","name":"Claude Fable 5","score_a":0.843,"score_b":0.805,"delta_pp":3.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":38,"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score_a":0.746,"score_b":0.719,"delta_pp":2.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":44,"slug":"claude-sonnet-5","name":"Claude Sonnet 5","score_a":0.804,"score_b":0.745,"delta_pp":5.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":174,"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score_a":0.827,"score_b":0.67,"delta_pp":15.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":175,"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score_a":0.879,"score_b":0.547,"delta_pp":33.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":173,"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","score_a":0.906,"score_b":0.745,"delta_pp":16.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":219,"slug":"glm-5-2","name":"GLM-5.2","score_a":0.81,"score_b":0.678,"delta_pp":13.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":220,"slug":"glm-5-3","name":"GLM-5.3","score_a":0.882,"score_b":0.715,"delta_pp":16.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":221,"slug":"glm-5-3-flash","name":"GLM-5.3-Flash","score_a":0.843,"score_b":0.629,"delta_pp":21.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":244,"slug":"gpt-5-6-luna","name":"GPT-5.6 Luna","score_a":0.847,"score_b":0.79,"delta_pp":5.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":246,"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score_a":0.919,"score_b":0.858,"delta_pp":6.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":247,"slug":"gpt-5-6-terra","name":"GPT-5.6 Terra","score_a":0.874,"score_b":0.775,"delta_pp":9.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":192,"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score_a":0.762,"score_b":0.742,"delta_pp":2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":194,"slug":"gemini-3-5-flash-lite","name":"Gemini 3.5 Flash-Lite","score_a":0.54,"score_b":0.502,"delta_pp":3.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":196,"slug":"gemini-3-7-flash","name":"Gemini 3.7 Flash","score_a":0.858,"score_b":0.775,"delta_pp":8.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":197,"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score_a":0.894,"score_b":0.813,"delta_pp":8.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":272,"slug":"grok-4-5","name":"Grok 4.5","score_a":0.833,"score_b":0.678,"delta_pp":15.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":291,"slug":"hy4-preview","name":"Hy4 preview","score_a":0.854,"score_b":0.551,"delta_pp":30.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":294,"slug":"inkling","name":"Inkling","score_a":0.638,"score_b":0.476,"delta_pp":16.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":295,"slug":"inkling-small","name":"Inkling-Small","score_a":0.647,"score_b":0.551,"delta_pp":9.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":323,"slug":"kimi-k3","name":"Kimi K3","score_a":0.883,"score_b":0.809,"delta_pp":7.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":342,"slug":"ling-3-0-flash","name":"Ling 3.0 Flash","score_a":0.57,"score_b":0.502,"delta_pp":6.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":379,"slug":"minimax-m3","name":"MiniMax M3","score_a":0.66,"score_b":0.536,"delta_pp":12.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":390,"slug":"muse-spark-1-1","name":"Muse Spark 1.1","score_a":0.8,"score_b":0.693,"delta_pp":10.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":391,"slug":"muse-spark-1-2","name":"Muse Spark 1.2","score_a":0.829,"score_b":0.697,"delta_pp":13.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":392,"slug":"muse-spark-1-3","name":"Muse Spark 1.3","score_a":0.888,"score_b":0.723,"delta_pp":16.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":400,"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score_a":0.564,"score_b":0.509,"delta_pp":5.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":465,"slug":"qwen3-8-max","name":"Qwen3.8 Max","score_a":0.866,"score_b":0.674,"delta_pp":19.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":462,"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score_a":0.73,"score_b":0.584,"delta_pp":14.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"2":[{"model_id":220,"slug":"glm-5-3","name":"GLM-5.3","score_a":0.882,"score_b":0.839,"delta_pp":4.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":221,"slug":"glm-5-3-flash","name":"GLM-5.3-Flash","score_a":0.843,"score_b":0.843,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":197,"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score_a":0.894,"score_b":0.876,"delta_pp":1.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":294,"slug":"inkling","name":"Inkling","score_a":0.638,"score_b":0.551,"delta_pp":8.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":323,"slug":"kimi-k3","name":"Kimi K3","score_a":0.883,"score_b":0.85,"delta_pp":3.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":379,"slug":"minimax-m3","name":"MiniMax M3","score_a":0.66,"score_b":0.652,"delta_pp":0.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":388,"slug":"muse-glimmer-30b","name":"Muse Glimmer 30B","score_a":0.517,"score_b":0.517,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":392,"slug":"muse-spark-1-3","name":"Muse Spark 1.3","score_a":0.888,"score_b":0.843,"delta_pp":4.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":400,"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score_a":0.564,"score_b":0.539,"delta_pp":2.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":462,"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score_a":0.73,"score_b":0.798,"delta_pp":-6.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"3":[{"model_id":25,"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score_a":0.85,"score_b":0.914,"delta_pp":-6.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":220,"slug":"glm-5-3","name":"GLM-5.3","score_a":0.715,"score_b":0.839,"delta_pp":-12.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":221,"slug":"glm-5-3-flash","name":"GLM-5.3-Flash","score_a":0.629,"score_b":0.843,"delta_pp":-21.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":252,"slug":"gpt-6-astra","name":"GPT-6 Astra","score_a":0.873,"score_b":0.884,"delta_pp":-1.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":197,"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score_a":0.813,"score_b":0.876,"delta_pp":-6.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":294,"slug":"inkling","name":"Inkling","score_a":0.476,"score_b":0.551,"delta_pp":-7.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":323,"slug":"kimi-k3","name":"Kimi K3","score_a":0.809,"score_b":0.85,"delta_pp":-4.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":379,"slug":"minimax-m3","name":"MiniMax M3","score_a":0.536,"score_b":0.652,"delta_pp":-11.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":392,"slug":"muse-spark-1-3","name":"Muse Spark 1.3","score_a":0.723,"score_b":0.843,"delta_pp":-12,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":400,"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score_a":0.509,"score_b":0.539,"delta_pp":-3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":462,"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score_a":0.584,"score_b":0.798,"delta_pp":-21.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"4":[{"model_id":25,"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score_a":0.558,"score_b":0.52,"delta_pp":3.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":28,"slug":"claude-haiku-5-5","name":"Claude Haiku 5.5","score_a":0.392,"score_b":0.328,"delta_pp":6.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":40,"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score_a":0.664,"score_b":0.596,"delta_pp":6.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":45,"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score_a":0.706,"score_b":0.636,"delta_pp":7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":173,"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","score_a":0.312,"score_b":0.268,"delta_pp":4.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":252,"slug":"gpt-6-astra","name":"GPT-6 Astra","score_a":0.579,"score_b":0.591,"delta_pp":-1.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":197,"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score_a":0.191,"score_b":0.197,"delta_pp":-0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":202,"slug":"gemini-4-argon","name":"Gemini 4 Argon","score_a":0.574,"score_b":0.571,"delta_pp":0.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":274,"slug":"grok-4-7","name":"Grok 4.7","score_a":0.38,"score_b":0.258,"delta_pp":12.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":371,"slug":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro","score_a":0.349,"score_b":0.348,"delta_pp":0.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":501,"slug":"step-5-preview","name":"Step 5 Preview","score_a":0.333,"score_b":0.333,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"5":[{"model_id":24,"slug":"claude-fable","name":"Claude Fable 5","score_a":0.95,"score_b":0.95,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":26,"slug":"claude-haiku-4-5","name":"Claude Haiku 4.5","score_a":0.733,"score_b":0.666,"delta_pp":6.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":38,"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score_a":0.886,"score_b":0.886,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":39,"slug":"claude-opus-5","name":"Claude Opus 5","score_a":0.96,"score_b":0.97,"delta_pp":-1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":43,"slug":"claude-sonnet-4-6","name":"Claude Sonnet 4.6","score_a":0.796,"score_b":0.774,"delta_pp":2.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":44,"slug":"claude-sonnet-5","name":"Claude Sonnet 5","score_a":0.852,"score_b":0.796,"delta_pp":5.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":174,"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score_a":0.79,"score_b":0.888,"delta_pp":-9.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":175,"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score_a":0.806,"score_b":0.964,"delta_pp":-15.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":216,"slug":"glm-4-7","name":"GLM-4.7","score_a":0.738,"score_b":0.694,"delta_pp":4.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":236,"slug":"gpt-5-3-codex","name":"GPT-5.3 Codex","score_a":0.85,"score_b":0.78,"delta_pp":7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":189,"slug":"gemini-2-5-pro","name":"Gemini 2.5 Pro","score_a":0.638,"score_b":0.544,"delta_pp":9.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":270,"slug":"grok-4-20-beta","name":"Grok 4.20","score_a":0.767,"score_b":0.722,"delta_pp":4.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":294,"slug":"inkling","name":"Inkling","score_a":0.776,"score_b":0.776,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":295,"slug":"inkling-small","name":"Inkling-Small","score_a":0.802,"score_b":0.822,"delta_pp":-2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":318,"slug":"kimi-2-6","name":"Kimi K2.6","score_a":0.802,"score_b":0.762,"delta_pp":4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":324,"slug":"laguna-m-1","name":"Laguna M.1","score_a":0.746,"score_b":0.576,"delta_pp":17,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":326,"slug":"laguna-xs-2","name":"Laguna XS.2","score_a":0.699,"score_b":0.552,"delta_pp":14.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":379,"slug":"minimax-m3","name":"MiniMax M3","score_a":0.805,"score_b":0.75,"delta_pp":5.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":385,"slug":"mistral-medium-3-5-128b","name":"Mistral Medium 3.5 128B","score_a":0.776,"score_b":0.664,"delta_pp":11.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":389,"slug":"muse-spark","name":"Muse Spark","score_a":0.774,"score_b":0.744,"delta_pp":3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":400,"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score_a":0.719,"score_b":0.69,"delta_pp":2.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":459,"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score_a":0.788,"score_b":0.734,"delta_pp":5.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":456,"slug":"qwen3-6-27b","name":"Qwen3.6-27B","score_a":0.772,"score_b":0.7,"delta_pp":7.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":460,"slug":"qwen3-7-max","name":"Qwen3.7 Max","score_a":0.804,"score_b":0.688,"delta_pp":11.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"6":[{"model_id":1,"slug":"a-x-k2","name":"A.X K2","score_a":0.41,"score_b":0.41,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":192,"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score_a":0.531,"score_b":0.539,"delta_pp":-0.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":262,"slug":"granite-4-2-30b","name":"Granite 4.2 30B","score_a":0.3876,"score_b":0.378,"delta_pp":0.96,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":263,"slug":"granite-4-2-3b","name":"Granite 4.2 3B","score_a":0.2411,"score_b":0.253,"delta_pp":-1.19,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":264,"slug":"granite-4-2-8b","name":"Granite 4.2 8B","score_a":0.3609,"score_b":0.315,"delta_pp":4.59,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":271,"slug":"grok-4-3","name":"Grok 4.3","score_a":0.473,"score_b":0.483,"delta_pp":-1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":290,"slug":"hy3-preview","name":"Hy3 Preview","score_a":0.412,"score_b":0.486,"delta_pp":-7.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":295,"slug":"inkling-small","name":"Inkling-Small","score_a":0.487,"score_b":0.497,"delta_pp":-1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":311,"slug":"k-exaone-2-0","name":"K-EXAONE 2.0","score_a":0.374,"score_b":0.42,"delta_pp":-4.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":318,"slug":"kimi-2-6","name":"Kimi K2.6","score_a":0.522,"score_b":0.515,"delta_pp":0.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":342,"slug":"ling-3-0-flash","name":"Ling 3.0 Flash","score_a":0.4124,"score_b":0.42,"delta_pp":-0.76,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":344,"slug":"ling-3-0-flash-fp8","name":"Ling 3.0 Flash FP8","score_a":0.4037,"score_b":0.42,"delta_pp":-1.63,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":365,"slug":"mercury-2-5","name":"Mercury 2.5","score_a":0.38,"score_b":0.385,"delta_pp":-0.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":376,"slug":"minicpm5-2b","name":"MiniCPM5-2B","score_a":0.263,"score_b":0.263,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":388,"slug":"muse-glimmer-30b","name":"Muse Glimmer 30B","score_a":0.436,"score_b":0.449,"delta_pp":-1.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":400,"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score_a":0.446,"score_b":0.403,"delta_pp":4.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":396,"slug":"nemotron-3-5-lightning-30b-a3b-nvfp4","name":"Nemotron 3.5 Lightning 30B A3B NVFP4","score_a":0.3138,"score_b":0.321,"delta_pp":-0.72,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":460,"slug":"qwen3-7-max","name":"Qwen3.7 Max","score_a":0.535,"score_b":0.495,"delta_pp":4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":461,"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score_a":0.513,"score_b":0.461,"delta_pp":5.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":501,"slug":"step-5-preview","name":"Step 5 Preview","score_a":0.589,"score_b":0.589,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"7":[{"model_id":38,"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score_a":0.936,"score_b":0.924,"delta_pp":1.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":174,"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score_a":0.881,"score_b":0.899,"delta_pp":-1.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":175,"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score_a":0.901,"score_b":0.924,"delta_pp":-2.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":218,"slug":"glm-5-1","name":"GLM-5.1","score_a":0.862,"score_b":0.845,"delta_pp":1.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":219,"slug":"glm-5-2","name":"GLM-5.2","score_a":0.912,"score_b":0.856,"delta_pp":5.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":241,"slug":"gpt-5-5","name":"GPT-5.5","score_a":0.936,"score_b":0.932,"delta_pp":0.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":244,"slug":"gpt-5-6-luna","name":"GPT-5.6 Luna","score_a":0.923,"score_b":0.917,"delta_pp":0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":246,"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score_a":0.946,"score_b":0.952,"delta_pp":-0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":247,"slug":"gpt-5-6-terra","name":"GPT-5.6 Terra","score_a":0.929,"score_b":0.909,"delta_pp":2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":191,"slug":"gemini-3-1-pro","name":"Gemini 3.1 Pro","score_a":0.943,"score_b":0.955,"delta_pp":-1.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":192,"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score_a":0.9268,"score_b":0.927,"delta_pp":-0.02,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":270,"slug":"grok-4-20-beta","name":"Grok 4.20","score_a":0.885,"score_b":0.886,"delta_pp":-0.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":294,"slug":"inkling","name":"Inkling","score_a":0.879,"score_b":0.871,"delta_pp":0.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":295,"slug":"inkling-small","name":"Inkling-Small","score_a":0.895,"score_b":0.836,"delta_pp":5.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":318,"slug":"kimi-2-6","name":"Kimi K2.6","score_a":0.905,"score_b":0.891,"delta_pp":1.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":323,"slug":"kimi-k3","name":"Kimi K3","score_a":0.935,"score_b":0.929,"delta_pp":0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":342,"slug":"ling-3-0-flash","name":"Ling 3.0 Flash","score_a":0.8497,"score_b":0.848,"delta_pp":0.17,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":378,"slug":"minimax-m2-7","name":"MiniMax M2.7","score_a":0.87,"score_b":0.866,"delta_pp":0.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":389,"slug":"muse-spark","name":"Muse Spark","score_a":0.895,"score_b":0.896,"delta_pp":-0.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":400,"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score_a":0.87,"score_b":0.861,"delta_pp":0.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":460,"slug":"qwen3-7-max","name":"Qwen3.7 Max","score_a":0.924,"score_b":0.902,"delta_pp":2.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":465,"slug":"qwen3-8-max","name":"Qwen3.8 Max","score_a":0.926,"score_b":0.937,"delta_pp":-1.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":462,"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score_a":0.892,"score_b":0.889,"delta_pp":0.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"8":[{"model_id":1,"slug":"a-x-k2","name":"A.X K2","score_a":0.856,"score_b":0.857,"delta_pp":-0.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":34,"slug":"claude-opus-4-6","name":"Claude Opus 4.6","score_a":0.892,"score_b":0.84,"delta_pp":5.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":37,"slug":"claude-opus-4-7-adaptive","name":"Claude Opus 4.7 (Adaptive)","score_a":0.942,"score_b":0.914,"delta_pp":2.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":38,"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score_a":0.936,"score_b":0.92,"delta_pp":1.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":174,"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score_a":0.881,"score_b":0.908,"delta_pp":-2.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":175,"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score_a":0.901,"score_b":0.928,"delta_pp":-2.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":217,"slug":"glm-5","name":"GLM-5","score_a":0.86,"score_b":0.82,"delta_pp":4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":218,"slug":"glm-5-1","name":"GLM-5.1","score_a":0.862,"score_b":0.868,"delta_pp":-0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":219,"slug":"glm-5-2","name":"GLM-5.2","score_a":0.912,"score_b":0.895,"delta_pp":1.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":237,"slug":"gpt-5-4","name":"GPT-5.4","score_a":0.928,"score_b":0.92,"delta_pp":0.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":241,"slug":"gpt-5-5","name":"GPT-5.5","score_a":0.936,"score_b":0.935,"delta_pp":0.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":244,"slug":"gpt-5-6-luna","name":"GPT-5.6 Luna","score_a":0.923,"score_b":0.911,"delta_pp":1.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":246,"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score_a":0.946,"score_b":0.941,"delta_pp":0.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":247,"slug":"gpt-5-6-terra","name":"GPT-5.6 Terra","score_a":0.929,"score_b":0.925,"delta_pp":0.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":252,"slug":"gpt-6-astra","name":"GPT-6 Astra","score_a":0.96,"score_b":0.961,"delta_pp":-0.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":191,"slug":"gemini-3-1-pro","name":"Gemini 3.1 Pro","score_a":0.943,"score_b":0.941,"delta_pp":0.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":192,"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score_a":0.9268,"score_b":0.922,"delta_pp":0.48,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":204,"slug":"gemma-4-12b","name":"Gemma 4 12B","score_a":0.788,"score_b":0.753,"delta_pp":3.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":290,"slug":"hy3-preview","name":"Hy3 Preview","score_a":0.872,"score_b":0.897,"delta_pp":-2.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":294,"slug":"inkling","name":"Inkling","score_a":0.879,"score_b":0.872,"delta_pp":0.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":295,"slug":"inkling-small","name":"Inkling-Small","score_a":0.895,"score_b":0.895,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":311,"slug":"k-exaone-2-0","name":"K-EXAONE 2.0","score_a":0.822,"score_b":0.829,"delta_pp":-0.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":320,"slug":"kimi-k2-5","name":"Kimi K2.5","score_a":0.876,"score_b":0.879,"delta_pp":-0.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":318,"slug":"kimi-2-6","name":"Kimi K2.6","score_a":0.905,"score_b":0.911,"delta_pp":-0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":323,"slug":"kimi-k3","name":"Kimi K3","score_a":0.935,"score_b":0.935,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":342,"slug":"ling-3-0-flash","name":"Ling 3.0 Flash","score_a":0.8497,"score_b":0.855,"delta_pp":-0.53,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":344,"slug":"ling-3-0-flash-fp8","name":"Ling 3.0 Flash FP8","score_a":0.84,"score_b":0.855,"delta_pp":-1.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":376,"slug":"minicpm5-2b","name":"MiniCPM5-2B","score_a":0.702,"score_b":0.702,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":378,"slug":"minimax-m2-7","name":"MiniMax M2.7","score_a":0.87,"score_b":0.874,"delta_pp":-0.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":389,"slug":"muse-spark","name":"Muse Spark","score_a":0.895,"score_b":0.884,"delta_pp":1.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":398,"slug":"nemotron-3-nano-omni-30b-a3b","name":"Nemotron 3 Nano Omni 30B A3B","score_a":0.722,"score_b":0.469,"delta_pp":25.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":400,"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score_a":0.87,"score_b":0.867,"delta_pp":0.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":396,"slug":"nemotron-3-5-lightning-30b-a3b-nvfp4","name":"Nemotron 3.5 Lightning 30B A3B NVFP4","score_a":0.7557,"score_b":0.743,"delta_pp":1.27,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":460,"slug":"qwen3-7-max","name":"Qwen3.7 Max","score_a":0.924,"score_b":0.923,"delta_pp":0.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":461,"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score_a":0.903,"score_b":0.9,"delta_pp":0.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":462,"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score_a":0.892,"score_b":0.905,"delta_pp":-1.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":464,"slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","score_a":0.917,"score_b":0.923,"delta_pp":-0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":496,"slug":"solar-pro-4","name":"Solar Pro 4","score_a":0.89,"score_b":0.891,"delta_pp":-0.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":509,"slug":"trinity-large-preview","name":"Trinity-Large-Preview","score_a":0.633,"score_b":0.752,"delta_pp":-11.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":510,"slug":"trinity-large-thinking","name":"Trinity-Large-Thinking","score_a":0.763,"score_b":0.752,"delta_pp":1.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"9":[{"model_id":24,"slug":"claude-fable","name":"Claude Fable 5","score_a":0.932,"score_b":0.926,"delta_pp":0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":25,"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score_a":0.934,"score_b":0.937,"delta_pp":-0.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":36,"slug":"claude-opus-4-7","name":"Claude Opus 4.7","score_a":0.902,"score_b":0.885,"delta_pp":1.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":38,"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score_a":0.924,"score_b":0.92,"delta_pp":0.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":39,"slug":"claude-opus-5","name":"Claude Opus 5","score_a":0.934,"score_b":0.932,"delta_pp":0.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":43,"slug":"claude-sonnet-4-6","name":"Claude Sonnet 4.6","score_a":0.856,"score_b":0.799,"delta_pp":5.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":44,"slug":"claude-sonnet-5","name":"Claude Sonnet 5","score_a":0.889,"score_b":0.911,"delta_pp":-2.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":174,"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score_a":0.899,"score_b":0.908,"delta_pp":-0.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":175,"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score_a":0.924,"score_b":0.928,"delta_pp":-0.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":215,"slug":"glm-4-6","name":"GLM-4.6","score_a":0.745,"score_b":0.632,"delta_pp":11.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":216,"slug":"glm-4-7","name":"GLM-4.7","score_a":0.801,"score_b":0.859,"delta_pp":-5.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":218,"slug":"glm-5-1","name":"GLM-5.1","score_a":0.845,"score_b":0.868,"delta_pp":-2.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":219,"slug":"glm-5-2","name":"GLM-5.2","score_a":0.856,"score_b":0.895,"delta_pp":-3.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":220,"slug":"glm-5-3","name":"GLM-5.3","score_a":0.881,"score_b":0.917,"delta_pp":-3.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":221,"slug":"glm-5-3-flash","name":"GLM-5.3-Flash","score_a":0.864,"score_b":0.912,"delta_pp":-4.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":238,"slug":"gpt-5-4-mini","name":"GPT-5.4 mini","score_a":0.831,"score_b":0.875,"delta_pp":-4.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":239,"slug":"gpt-5-4-nano","name":"GPT-5.4 nano","score_a":0.775,"score_b":0.817,"delta_pp":-4.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":241,"slug":"gpt-5-5","name":"GPT-5.5","score_a":0.932,"score_b":0.935,"delta_pp":-0.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":244,"slug":"gpt-5-6-luna","name":"GPT-5.6 Luna","score_a":0.917,"score_b":0.911,"delta_pp":0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":246,"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score_a":0.952,"score_b":0.941,"delta_pp":1.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":247,"slug":"gpt-5-6-terra","name":"GPT-5.6 Terra","score_a":0.909,"score_b":0.925,"delta_pp":-1.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":199,"slug":"gemini-3-flash","name":"Gemini 3 Flash","score_a":0.879,"score_b":0.812,"delta_pp":6.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":191,"slug":"gemini-3-1-pro","name":"Gemini 3.1 Pro","score_a":0.955,"score_b":0.941,"delta_pp":1.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":192,"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score_a":0.927,"score_b":0.922,"delta_pp":0.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":194,"slug":"gemini-3-5-flash-lite","name":"Gemini 3.5 Flash-Lite","score_a":0.838,"score_b":0.838,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":195,"slug":"gemini-3-6-flash","name":"Gemini 3.6 Flash","score_a":0.934,"score_b":0.928,"delta_pp":0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":196,"slug":"gemini-3-7-flash","name":"Gemini 3.7 Flash","score_a":0.939,"score_b":0.945,"delta_pp":-0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":197,"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score_a":0.944,"score_b":0.953,"delta_pp":-0.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":271,"slug":"grok-4-3","name":"Grok 4.3","score_a":0.914,"score_b":0.901,"delta_pp":1.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":272,"slug":"grok-4-5","name":"Grok 4.5","score_a":0.929,"score_b":0.931,"delta_pp":-0.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":273,"slug":"grok-4-6","name":"Grok 4.6","score_a":0.947,"score_b":0.949,"delta_pp":-0.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":294,"slug":"inkling","name":"Inkling","score_a":0.871,"score_b":0.872,"delta_pp":-0.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":295,"slug":"inkling-small","name":"Inkling-Small","score_a":0.836,"score_b":0.895,"delta_pp":-5.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":318,"slug":"kimi-2-6","name":"Kimi K2.6","score_a":0.891,"score_b":0.911,"delta_pp":-2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":323,"slug":"kimi-k3","name":"Kimi K3","score_a":0.929,"score_b":0.935,"delta_pp":-0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":342,"slug":"ling-3-0-flash","name":"Ling 3.0 Flash","score_a":0.848,"score_b":0.855,"delta_pp":-0.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":372,"slug":"mimo-v2-flash","name":"MiMo-V2-Flash","score_a":0.593,"score_b":0.656,"delta_pp":-6.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":369,"slug":"mimo-v2-5-pro","name":"MiMo-V2.5-Pro","score_a":0.826,"score_b":0.866,"delta_pp":-4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":378,"slug":"minimax-m2-7","name":"MiniMax M2.7","score_a":0.866,"score_b":0.874,"delta_pp":-0.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":379,"slug":"minimax-m3","name":"MiniMax M3","score_a":0.927,"score_b":0.929,"delta_pp":-0.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":385,"slug":"mistral-medium-3-5-128b","name":"Mistral Medium 3.5 128B","score_a":0.348,"score_b":0.748,"delta_pp":-40,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":389,"slug":"muse-spark","name":"Muse Spark","score_a":0.896,"score_b":0.884,"delta_pp":1.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":390,"slug":"muse-spark-1-1","name":"Muse Spark 1.1","score_a":0.912,"score_b":0.898,"delta_pp":1.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":400,"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score_a":0.861,"score_b":0.867,"delta_pp":-0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":459,"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score_a":0.874,"score_b":0.882,"delta_pp":-0.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":460,"slug":"qwen3-7-max","name":"Qwen3.7 Max","score_a":0.902,"score_b":0.923,"delta_pp":-2.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":462,"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score_a":0.889,"score_b":0.905,"delta_pp":-1.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"10":[{"model_id":43,"slug":"claude-sonnet-4-6","name":"Claude Sonnet 4.6","score_a":0.792,"score_b":0.873,"delta_pp":-8.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":174,"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score_a":0.862,"score_b":0.862,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":175,"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score_a":0.875,"score_b":0.87,"delta_pp":0.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":216,"slug":"glm-4-7","name":"GLM-4.7","score_a":0.843,"score_b":0.827,"delta_pp":1.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":400,"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score_a":0.868,"score_b":0.858,"delta_pp":1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":459,"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score_a":0.885,"score_b":0.877,"delta_pp":0.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":460,"slug":"qwen3-7-max","name":"Qwen3.7 Max","score_a":0.896,"score_b":0.893,"delta_pp":0.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"11":[{"model_id":34,"slug":"claude-opus-4-6","name":"Claude Opus 4.6","score_a":0.82,"score_b":0.891,"delta_pp":-7.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":217,"slug":"glm-5","name":"GLM-5","score_a":0.857,"score_b":0.858,"delta_pp":-0.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":320,"slug":"kimi-k2-5","name":"Kimi K2.5","score_a":0.871,"score_b":0.871,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"12":[{"model_id":378,"slug":"minimax-m2-7","name":"MiniMax M2.7","score_a":0.804,"score_b":0.808,"delta_pp":-0.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"13":[{"model_id":32,"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score_a":0.706,"score_b":0.712,"delta_pp":-0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":34,"slug":"claude-opus-4-6","name":"Claude Opus 4.6","score_a":0.773,"score_b":0.725,"delta_pp":4.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":49,"slug":"command-a-plus","name":"Command A+","score_a":0.63,"score_b":0.632,"delta_pp":-0.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":237,"slug":"gpt-5-4","name":"GPT-5.4","score_a":0.812,"score_b":0.784,"delta_pp":2.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":238,"slug":"gpt-5-4-mini","name":"GPT-5.4 mini","score_a":0.766,"score_b":0.733,"delta_pp":3.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":239,"slug":"gpt-5-4-nano","name":"GPT-5.4 nano","score_a":0.661,"score_b":0.654,"delta_pp":0.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":241,"slug":"gpt-5-5","name":"GPT-5.5","score_a":0.812,"score_b":0.799,"delta_pp":1.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":244,"slug":"gpt-5-6-luna","name":"GPT-5.6 Luna","score_a":0.784,"score_b":0.786,"delta_pp":-0.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":246,"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score_a":0.83,"score_b":0.834,"delta_pp":-0.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":247,"slug":"gpt-5-6-terra","name":"GPT-5.6 Terra","score_a":0.807,"score_b":0.807,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":200,"slug":"gemini-3-pro","name":"Gemini 3 Pro","score_a":0.81,"score_b":0.802,"delta_pp":0.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":191,"slug":"gemini-3-1-pro","name":"Gemini 3.1 Pro","score_a":0.839,"score_b":0.824,"delta_pp":1.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":192,"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score_a":0.836,"score_b":0.843,"delta_pp":-0.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":204,"slug":"gemma-4-12b","name":"Gemma 4 12B","score_a":0.691,"score_b":0.697,"delta_pp":-0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":205,"slug":"gemma-4-26b-a4b","name":"Gemma 4 26B A4B","score_a":0.738,"score_b":0.692,"delta_pp":4.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":206,"slug":"gemma-4-31b","name":"Gemma 4 31B","score_a":0.769,"score_b":0.734,"delta_pp":3.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":271,"slug":"grok-4-3","name":"Grok 4.3","score_a":0.781,"score_b":0.781,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":294,"slug":"inkling","name":"Inkling","score_a":0.735,"score_b":0.735,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":295,"slug":"inkling-small","name":"Inkling-Small","score_a":0.74,"score_b":0.74,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":320,"slug":"kimi-k2-5","name":"Kimi K2.5","score_a":0.785,"score_b":0.754,"delta_pp":3.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":321,"slug":"kimi-k2-5-reasoning","name":"Kimi K2.5 (Reasoning)","score_a":0.785,"score_b":0.754,"delta_pp":3.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":318,"slug":"kimi-2-6","name":"Kimi K2.6","score_a":0.794,"score_b":0.794,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":323,"slug":"kimi-k3","name":"Kimi K3","score_a":0.816,"score_b":0.805,"delta_pp":1.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":379,"slug":"minimax-m3","name":"MiniMax M3","score_a":0.781,"score_b":0.786,"delta_pp":-0.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":388,"slug":"muse-glimmer-30b","name":"Muse Glimmer 30B","score_a":0.74,"score_b":0.743,"delta_pp":-0.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":389,"slug":"muse-spark","name":"Muse Spark","score_a":0.804,"score_b":0.805,"delta_pp":-0.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":452,"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score_a":0.79,"score_b":0.527,"delta_pp":26.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":459,"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score_a":0.788,"score_b":0.78,"delta_pp":0.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":456,"slug":"qwen3-6-27b","name":"Qwen3.6-27B","score_a":0.758,"score_b":0.746,"delta_pp":1.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":457,"slug":"qwen3-6-35b-a3b","name":"Qwen3.6-35B-A3B","score_a":0.753,"score_b":0.75,"delta_pp":0.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":461,"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score_a":0.79,"score_b":0.805,"delta_pp":-1.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":501,"slug":"step-5-preview","name":"Step 5 Preview","score_a":0.76,"score_b":0.764,"delta_pp":-0.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"14":[{"model_id":32,"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score_a":0.58,"score_b":0.43,"delta_pp":15,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":192,"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score_a":0.763,"score_b":0.763,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":271,"slug":"grok-4-3","name":"Grok 4.3","score_a":0.813,"score_b":0.813,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":334,"slug":"lfm2-5-8b-a1b","name":"LFM2.5-8B-A1B","score_a":0.5647,"score_b":0.556,"delta_pp":0.87,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":341,"slug":"ling-2-6-flash","name":"Ling 2.6 Flash","score_a":0.57,"score_b":0.574,"delta_pp":-0.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":398,"slug":"nemotron-3-nano-omni-30b-a3b","name":"Nemotron 3 Nano Omni 30B A3B","score_a":0.742,"score_b":0.632,"delta_pp":11,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":400,"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score_a":0.817,"score_b":0.814,"delta_pp":0.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":459,"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score_a":0.758,"score_b":0.752,"delta_pp":0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":460,"slug":"qwen3-7-max","name":"Qwen3.7 Max","score_a":0.791,"score_b":0.805,"delta_pp":-1.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":461,"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score_a":0.791,"score_b":0.78,"delta_pp":1.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":495,"slug":"solar-pro-3","name":"Solar Pro 3","score_a":0.5578,"score_b":0.712,"delta_pp":-15.42,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"15":[{"model_id":1,"slug":"a-x-k2","name":"A.X K2","score_a":0.278,"score_b":0.296,"delta_pp":-1.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":4,"slug":"apodex-1-1","name":"Apodex 1.1","score_a":0.561,"score_b":0.341,"delta_pp":22,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":25,"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score_a":0.65,"score_b":0.591,"delta_pp":5.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":32,"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score_a":0.308,"score_b":0.132,"delta_pp":17.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":34,"slug":"claude-opus-4-6","name":"Claude Opus 4.6","score_a":0.53,"score_b":0.191,"delta_pp":33.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":37,"slug":"claude-opus-4-7-adaptive","name":"Claude Opus 4.7 (Adaptive)","score_a":0.547,"score_b":0.423,"delta_pp":12.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":38,"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score_a":0.579,"score_b":0.487,"delta_pp":9.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":39,"slug":"claude-opus-5","name":"Claude Opus 5","score_a":0.647,"score_b":0.549,"delta_pp":9.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":43,"slug":"claude-sonnet-4-6","name":"Claude Sonnet 4.6","score_a":0.49,"score_b":0.133,"delta_pp":35.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":44,"slug":"claude-sonnet-5","name":"Claude Sonnet 5","score_a":0.574,"score_b":0.413,"delta_pp":16.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":174,"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score_a":0.348,"score_b":0.386,"delta_pp":-3.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":175,"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score_a":0.427,"score_b":0.41,"delta_pp":1.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":173,"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","score_a":0.368,"score_b":0.392,"delta_pp":-2.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":216,"slug":"glm-4-7","name":"GLM-4.7","score_a":0.248,"score_b":0.274,"delta_pp":-2.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":217,"slug":"glm-5","name":"GLM-5","score_a":0.504,"score_b":0.293,"delta_pp":21.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":218,"slug":"glm-5-1","name":"GLM-5.1","score_a":0.523,"score_b":0.301,"delta_pp":22.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":219,"slug":"glm-5-2","name":"GLM-5.2","score_a":0.547,"score_b":0.411,"delta_pp":13.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":237,"slug":"gpt-5-4","name":"GPT-5.4","score_a":0.521,"score_b":0.437,"delta_pp":8.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":238,"slug":"gpt-5-4-mini","name":"GPT-5.4 mini","score_a":0.415,"score_b":0.281,"delta_pp":13.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":239,"slug":"gpt-5-4-nano","name":"GPT-5.4 nano","score_a":0.377,"score_b":0.283,"delta_pp":9.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":241,"slug":"gpt-5-5","name":"GPT-5.5","score_a":0.522,"score_b":0.458,"delta_pp":6.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":189,"slug":"gemini-2-5-pro","name":"Gemini 2.5 Pro","score_a":0.188,"score_b":0.225,"delta_pp":-3.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":192,"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score_a":0.402,"score_b":0.427,"delta_pp":-2.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":205,"slug":"gemma-4-26b-a4b","name":"Gemma 4 26B A4B","score_a":0.172,"score_b":0.193,"delta_pp":-2.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":206,"slug":"gemma-4-31b","name":"Gemma 4 31B","score_a":0.265,"score_b":0.236,"delta_pp":2.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":271,"slug":"grok-4-3","name":"Grok 4.3","score_a":0.35,"score_b":0.372,"delta_pp":-2.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":290,"slug":"hy3-preview","name":"Hy3 Preview","score_a":0.255,"score_b":0.335,"delta_pp":-8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":294,"slug":"inkling","name":"Inkling","score_a":0.46,"score_b":0.319,"delta_pp":14.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":295,"slug":"inkling-small","name":"Inkling-Small","score_a":0.478,"score_b":0.333,"delta_pp":14.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":311,"slug":"k-exaone-2-0","name":"K-EXAONE 2.0","score_a":0.183,"score_b":0.186,"delta_pp":-0.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":320,"slug":"kimi-k2-5","name":"Kimi K2.5","score_a":0.301,"score_b":0.307,"delta_pp":-0.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":318,"slug":"kimi-2-6","name":"Kimi K2.6","score_a":0.347,"score_b":0.375,"delta_pp":-2.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":323,"slug":"kimi-k3","name":"Kimi K3","score_a":0.56,"score_b":0.469,"delta_pp":9.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":342,"slug":"ling-3-0-flash","name":"Ling 3.0 Flash","score_a":0.227,"score_b":0.237,"delta_pp":-1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":369,"slug":"mimo-v2-5-pro","name":"MiMo-V2.5-Pro","score_a":0.48,"score_b":0.357,"delta_pp":12.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":376,"slug":"minicpm5-2b","name":"MiniCPM5-2B","score_a":0.089,"score_b":0.089,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":389,"slug":"muse-spark","name":"Muse Spark","score_a":0.504,"score_b":0.407,"delta_pp":9.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":390,"slug":"muse-spark-1-1","name":"Muse Spark 1.1","score_a":0.621,"score_b":0.462,"delta_pp":15.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":400,"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score_a":0.267,"score_b":0.284,"delta_pp":-1.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":452,"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score_a":0.287,"score_b":0.198,"delta_pp":8.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":459,"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score_a":0.288,"score_b":0.278,"delta_pp":1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":456,"slug":"qwen3-6-27b","name":"Qwen3.6-27B","score_a":0.24,"score_b":0.231,"delta_pp":0.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":457,"slug":"qwen3-6-35b-a3b","name":"Qwen3.6-35B-A3B","score_a":0.214,"score_b":0.222,"delta_pp":-0.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":460,"slug":"qwen3-7-max","name":"Qwen3.7 Max","score_a":0.414,"score_b":0.405,"delta_pp":0.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":461,"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score_a":0.347,"score_b":0.356,"delta_pp":-0.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":462,"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score_a":0.308,"score_b":0.339,"delta_pp":-3.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":464,"slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","score_a":0.359,"score_b":0.38,"delta_pp":-2.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":501,"slug":"step-5-preview","name":"Step 5 Preview","score_a":0.465,"score_b":0.465,"delta_pp":0,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"16":[{"model_id":28,"slug":"claude-haiku-5-5","name":"Claude Haiku 5.5","score_a":0.459,"score_b":0.574,"delta_pp":-11.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":39,"slug":"claude-opus-5","name":"Claude Opus 5","score_a":0.563,"score_b":0.647,"delta_pp":-8.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":40,"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score_a":0.644,"score_b":0.677,"delta_pp":-3.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":44,"slug":"claude-sonnet-5","name":"Claude Sonnet 5","score_a":0.432,"score_b":0.574,"delta_pp":-14.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":45,"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score_a":0.569,"score_b":0.645,"delta_pp":-7.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":291,"slug":"hy4-preview","name":"Hy4 preview","score_a":0.434,"score_b":0.554,"delta_pp":-12,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":400,"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score_a":0.267,"score_b":0.374,"delta_pp":-10.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":433,"slug":"ornith-1-5-35b-a3b","name":"Ornith-1.5-35B-A3B","score_a":0.256,"score_b":0.334,"delta_pp":-7.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":434,"slug":"ornith-1-5-397b","name":"Ornith-1.5-397B","score_a":0.446,"score_b":0.561,"delta_pp":-11.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":435,"slug":"ornith-1-5-9b","name":"Ornith-1.5-9B","score_a":0.202,"score_b":0.305,"delta_pp":-10.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":465,"slug":"qwen3-8-max","name":"Qwen3.8 Max","score_a":0.436,"score_b":0.562,"delta_pp":-12.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"19":[{"model_id":216,"slug":"glm-4-7","name":"GLM-4.7","score_a":0.849,"score_b":0.822,"delta_pp":2.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":460,"slug":"qwen3-7-max","name":"Qwen3.7 Max","score_a":0.916,"score_b":0.871,"delta_pp":4.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"20":[{"model_id":216,"slug":"glm-4-7","name":"GLM-4.7","score_a":0.849,"score_b":0.894,"delta_pp":-4.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"21":[{"model_id":375,"slug":"minicpm5-1b","name":"MiniCPM5-1B","score_a":0.3352,"score_b":0.2268,"delta_pp":10.84,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":481,"slug":"sakana-fugu","name":"Sakana Fugu","score_a":0.929,"score_b":0.878,"delta_pp":5.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":483,"slug":"sakana-fugu-ultra","name":"Sakana Fugu-Ultra","score_a":0.932,"score_b":0.908,"delta_pp":2.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"22":[{"model_id":318,"slug":"kimi-2-6","name":"Kimi K2.6","score_a":0.896,"score_b":0.868,"delta_pp":2.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":400,"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score_a":0.89,"score_b":0.86,"delta_pp":3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":459,"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score_a":0.871,"score_b":0.86,"delta_pp":1.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":462,"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score_a":0.903,"score_b":0.84,"delta_pp":6.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"24":[{"model_id":191,"slug":"gemini-3-1-pro","name":"Gemini 3.1 Pro","score_a":0.829,"score_b":0.885,"delta_pp":-5.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":270,"slug":"grok-4-20-beta","name":"Grok 4.20","score_a":0.742,"score_b":0.843,"delta_pp":-10.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"26":[{"model_id":216,"slug":"glm-4-7","name":"GLM-4.7","score_a":0.822,"score_b":0.894,"delta_pp":-7.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"31":[{"model_id":320,"slug":"kimi-k2-5","name":"Kimi K2.5","score_a":0.961,"score_b":0.963,"delta_pp":-0.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"32":[{"model_id":40,"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score_a":0.912,"score_b":0.969,"delta_pp":-5.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":45,"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score_a":0.868,"score_b":0.952,"delta_pp":-8.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"33":[{"model_id":39,"slug":"claude-opus-5","name":"Claude Opus 5","score_a":0.908,"score_b":0.913,"delta_pp":-0.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"34":[{"model_id":25,"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score_a":0.594,"score_b":0.314,"delta_pp":28,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":40,"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score_a":0.695,"score_b":0.4,"delta_pp":29.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":173,"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","score_a":0.689,"score_b":0.548,"delta_pp":14.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":220,"slug":"glm-5-3","name":"GLM-5.3","score_a":0.622,"score_b":0.482,"delta_pp":14,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":221,"slug":"glm-5-3-flash","name":"GLM-5.3-Flash","score_a":0.604,"score_b":0.488,"delta_pp":11.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":252,"slug":"gpt-6-astra","name":"GPT-6 Astra","score_a":0.685,"score_b":0.414,"delta_pp":27.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":256,"slug":"gpt-6-sol","name":"GPT-6 Sol","score_a":0.616,"score_b":0.332,"delta_pp":28.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":251,"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score_a":0.649,"score_b":0.361,"delta_pp":28.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":202,"slug":"gemini-4-argon","name":"Gemini 4 Argon","score_a":0.775,"score_b":0.513,"delta_pp":26.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":323,"slug":"kimi-k3","name":"Kimi K3","score_a":0.583,"score_b":0.308,"delta_pp":27.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":371,"slug":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro","score_a":0.586,"score_b":0.531,"delta_pp":5.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":392,"slug":"muse-spark-1-3","name":"Muse Spark 1.3","score_a":0.579,"score_b":0.494,"delta_pp":8.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":501,"slug":"step-5-preview","name":"Step 5 Preview","score_a":0.51,"score_b":0.44,"delta_pp":7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"35":[{"model_id":39,"slug":"claude-opus-5","name":"Claude Opus 5","score_a":0.0037,"score_b":0.0082,"delta_pp":-0.46,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":40,"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score_a":0.0073,"score_b":0.0096,"delta_pp":-0.23,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":45,"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score_a":0.0075,"score_b":0.0096,"delta_pp":-0.22,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"36":[{"model_id":39,"slug":"claude-opus-5","name":"Claude Opus 5","score_a":0.296,"score_b":0.83,"delta_pp":-53.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":40,"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score_a":0.644,"score_b":0.89,"delta_pp":-24.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":45,"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score_a":0.616,"score_b":0.902,"delta_pp":-28.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"37":[{"model_id":29,"slug":"claude-mythos-5","name":"Claude Mythos 5","score_a":0.935,"score_b":0.889,"delta_pp":4.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":37,"slug":"claude-opus-4-7-adaptive","name":"Claude Opus 4.7 (Adaptive)","score_a":0.91,"score_b":0.821,"delta_pp":8.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":38,"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score_a":0.899,"score_b":0.805,"delta_pp":9.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":44,"slug":"claude-sonnet-5","name":"Claude Sonnet 5","score_a":0.883,"score_b":0.77,"delta_pp":11.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":196,"slug":"gemini-3-7-flash","name":"Gemini 3.7 Flash","score_a":0.887,"score_b":0.845,"delta_pp":4.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":294,"slug":"inkling","name":"Inkling","score_a":0.82,"score_b":0.781,"delta_pp":3.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":295,"slug":"inkling-small","name":"Inkling-Small","score_a":0.813,"score_b":0.774,"delta_pp":3.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":323,"slug":"kimi-k3","name":"Kimi K3","score_a":0.913,"score_b":0.848,"delta_pp":6.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":465,"slug":"qwen3-8-max","name":"Qwen3.8 Max","score_a":0.935,"score_b":0.884,"delta_pp":5.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":462,"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score_a":0.902,"score_b":0.837,"delta_pp":6.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":464,"slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","score_a":0.906,"score_b":0.846,"delta_pp":6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"},{"model_id":467,"slug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","score_a":0.914,"score_b":0.835,"delta_pp":7.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"40":[{"model_id":39,"slug":"claude-opus-5","name":"Claude Opus 5","score_a":0.834,"score_b":0.855,"delta_pp":-2.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}],"42":[{"model_id":39,"slug":"claude-opus-5","name":"Claude Opus 5","score_a":0.6,"score_b":0.79,"delta_pp":-19,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09"}]},"aggregates":{"by_tier":{"agentic":{"n_pairs":6,"n_model_pairs":105,"pooled_mean_abs_pp":6.902380952380953},"knowledge_tool_free":{"n_pairs":6,"n_model_pairs":160,"pooled_mean_abs_pp":2.3172500000000005},"protocol_layer":{"n_pairs":2,"n_model_pairs":59,"pooled_mean_abs_pp":8.496610169491525}},"by_capability":{"agentic-terminal":{"n_pairs":4,"n_model_pairs":61,"pooled_mean_abs_pp":8.740983606557378},"coding":{"n_pairs":2,"n_model_pairs":44,"pooled_mean_abs_pp":4.353409090909091},"instruction-following":{"n_pairs":1,"n_model_pairs":11,"pooled_mean_abs_pp":4.189999999999999},"knowledge":{"n_pairs":6,"n_model_pairs":176,"pooled_mean_abs_pp":4.325397727272728},"vision":{"n_pairs":1,"n_model_pairs":32,"pooled_mean_abs_pp":2.0218750000000005}},"by_family":{"gpqa-diamond":{"n_pairs":3,"n_model_pairs":110,"pooled_mean_abs_pp":2.251545454545455},"hle":{"n_pairs":1,"n_model_pairs":48,"pooled_mean_abs_pp":8.154166666666667},"hle-protocol-variants":{"n_pairs":1,"n_model_pairs":11,"pooled_mean_abs_pp":9.990909090909092},"ifbench":{"n_pairs":1,"n_model_pairs":11,"pooled_mean_abs_pp":4.189999999999999},"mmlu-pro":{"n_pairs":1,"n_model_pairs":7,"pooled_mean_abs_pp":1.7571428571428573},"mmmu-pro":{"n_pairs":1,"n_model_pairs":32,"pooled_mean_abs_pp":2.0218750000000005},"scicode":{"n_pairs":1,"n_model_pairs":20,"pooled_mean_abs_pp":2.0324999999999998},"swe-bench-verified":{"n_pairs":1,"n_model_pairs":24,"pooled_mean_abs_pp":6.2875000000000005},"terminal-bench-2-1":{"n_pairs":3,"n_model_pairs":50,"pooled_mean_abs_pp":9.808},"terminal-bench-4-0":{"n_pairs":1,"n_model_pairs":11,"pooled_mean_abs_pp":3.890909090909091}},"headline":{"by_tier":{"agentic":{"n_pairs":4,"n_model_pairs":61,"pooled_mean_abs_pp":8.740983606557378},"knowledge_tool_free":{"n_pairs":5,"n_model_pairs":153,"pooled_mean_abs_pp":2.3428758169934647},"protocol_layer":{"n_pairs":1,"n_model_pairs":11,"pooled_mean_abs_pp":9.990909090909092}},"ratio":3.730877899356354}}}}