{"api_version":"v1","generated_at":"2026-10-09T11:11:45Z","data_retrieved_at":"2026-10-09","harnesses":["artificial-analysis","published","vals-ai"],"counts":{"models":402,"benchmarks":148,"measured":4919,"estimated":16584,"mappings":1192,"cross_mappings":6884,"cross_multi_mappings":119,"confidence":{"high":2526,"medium":5206,"low":8852}},"measured_only":true,"families":[{"family_id":"aime2025","label":"Aime2025","capability":"math","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"aa-aime2025/current","name":"aa-aime2025","harness":"artificial-analysis","n":1},{"key":"aime2025/current","name":"aime2025","harness":"published","n":16},{"key":"aime2025arcee/current","name":"aime2025arcee","harness":"published","n":6}],"url":"https://benchgap.net/api/v1/harness-tax/aime2025.json"},{"family_id":"arxivmathaugust2026","label":"Arxivmathaugust2026","capability":"math","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"arxivmathaugust2026/current","name":"arxivmathaugust2026","harness":"published","n":2},{"key":"arxivmathaugust2026withtools/current","name":"arxivmathaugust2026withtools","harness":"published","n":2}],"url":"https://benchgap.net/api/v1/harness-tax/arxivmathaugust2026.json"},{"family_id":"arxivmathjune2026","label":"Arxivmathjune2026","capability":"math","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"arxivmathjune2026/current","name":"arxivmathjune2026","harness":"published","n":1},{"key":"arxivmathjune2026withtools/current","name":"arxivmathjune2026withtools","harness":"published","n":1}],"url":"https://benchgap.net/api/v1/harness-tax/arxivmathjune2026.json"},{"family_id":"automationbench","label":"Automationbench","capability":"agentic-tool","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"aa-automation-bench/current","name":"aa-automation-bench","harness":"artificial-analysis","n":25},{"key":"automationbench/current","name":"automationbench","harness":"published","n":22}],"url":"https://benchgap.net/api/v1/harness-tax/automationbench.json"},{"family_id":"benchcadvision2code","label":"Benchcadvision2Code","capability":"vision","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"benchcadvision2code/current","name":"benchcadvision2code","harness":"published","n":3},{"key":"benchcadvision2codewithtools/current","name":"benchcadvision2codewithtools","harness":"published","n":4}],"url":"https://benchgap.net/api/v1/harness-tax/benchcadvision2code.json"},{"family_id":"chartography","label":"Chartography","capability":"vision","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"chartography/current","name":"chartography","harness":"published","n":5},{"key":"chartographywithtools/current","name":"chartographywithtools","harness":"published","n":5}],"url":"https://benchgap.net/api/v1/harness-tax/chartography.json"},{"family_id":"charxiv","label":"Charxiv","capability":"vision","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"charxiv/current","name":"charxiv","harness":"published","n":39},{"key":"charxivnotools/current","name":"charxivnotools","harness":"published","n":14}],"url":"https://benchgap.net/api/v1/harness-tax/charxiv.json"},{"family_id":"gdppdf","label":"Gdppdf","capability":"agentic-tool","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"aa-gdp-pdf/current","name":"aa-gdp-pdf","harness":"artificial-analysis","n":24},{"key":"gdppdf/current","name":"gdppdf","harness":"published","n":2},{"key":"gdppdfwithtools/current","name":"gdppdfwithtools","harness":"published","n":1}],"url":"https://benchgap.net/api/v1/harness-tax/gdppdf.json"},{"family_id":"gpqa-diamond","label":"GPQA Diamond","capability":"knowledge","tier":"knowledge_tool_free","same_item_set":"verified","status":"active","pair_type":null,"audit_note":null,"origin":"seed","versions":[{"key":"gpqa-diamond/current","name":"gpqa-diamond","harness":"published","n":68},{"key":"vals-gpqa-diamond/current","name":"vals-gpqa-diamond","harness":"vals-ai","n":56},{"key":"aa-gpqa-diamond/current","name":"aa-gpqa-diamond","harness":"artificial-analysis","n":181}],"url":"https://benchgap.net/api/v1/harness-tax/gpqa-diamond.json"},{"family_id":"hle","label":"Humanity's Last Exam","capability":"knowledge","tier":"protocol_layer","same_item_set":"needs_audit","status":"active","pair_type":null,"audit_note":"AA HLE protocol may include tools; this pair is the pivot case for the paper. Audit before claiming pure harness tax.","origin":"seed","versions":[{"key":"hle/current","name":"hle","harness":"published","n":59},{"key":"aa-hle/current","name":"aa-hle","harness":"artificial-analysis","n":195}],"url":"https://benchgap.net/api/v1/harness-tax/hle.json"},{"family_id":"hle-protocol-variants","label":"HLE tools on/off (same harness)","capability":"knowledge","tier":"protocol_layer","same_item_set":"verified","status":"active","pair_type":"protocol_variant","audit_note":null,"origin":"seed","versions":[{"key":"hlenotools/current","name":"hlenotools","harness":"published","n":42},{"key":"hlewithtools/current","name":"hlewithtools","harness":"published","n":22}],"url":"https://benchgap.net/api/v1/harness-tax/hle-protocol-variants.json"},{"family_id":"ifbench","label":"IFBench","capability":"instruction-following","tier":"knowledge_tool_free","same_item_set":"verified","status":"active","pair_type":null,"audit_note":null,"origin":"seed","versions":[{"key":"ifbench/current","name":"ifbench","harness":"published","n":43},{"key":"aa-if-bench/current","name":"aa-if-bench","harness":"artificial-analysis","n":130}],"url":"https://benchgap.net/api/v1/harness-tax/ifbench.json"},{"family_id":"livecodebench","label":"LiveCodeBench","capability":"coding","tier":"agentic","same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Versions differ across harnesses (v6 etc.) so cross-harness pairs conflate version and harness. Keep pairs but flag weak_item_set_alignment; do not use in headline aggregates until aligned versions exist.","origin":"seed","versions":[{"key":"livecodebench/current","name":"livecodebench","harness":"published","n":7},{"key":"livecodebench-v6/current","name":"livecodebench-v6","harness":"published","n":32},{"key":"livecodebench-pro/current","name":"livecodebench-pro","harness":"published","n":8},{"key":"valslivecodebench/current","name":"valslivecodebench","harness":"vals-ai","n":58},{"key":"aa-live-code-bench/current","name":"aa-live-code-bench","harness":"artificial-analysis","n":2}],"url":"https://benchgap.net/api/v1/harness-tax/livecodebench.json"},{"family_id":"math500","label":"Math500","capability":"math","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"aa-math500/current","name":"aa-math500","harness":"artificial-analysis","n":3},{"key":"math-500/current","name":"math-500","harness":"published","n":7}],"url":"https://benchgap.net/api/v1/harness-tax/math500.json"},{"family_id":"mmlu-pro","label":"MMLU-Pro","capability":"knowledge","tier":"knowledge_tool_free","same_item_set":"needs_audit","status":"active","pair_type":null,"audit_note":"mmlu-pro-arcee is a re-run protocol variant; confirm item set and treat separately from plain published run if it differs.","origin":"seed","versions":[{"key":"mmlu-pro/current","name":"mmlu-pro","harness":"published","n":46},{"key":"vals-mmlu-pro/current","name":"vals-mmlu-pro","harness":"vals-ai","n":56},{"key":"mmlu-pro-arcee/current","name":"mmlu-pro-arcee","harness":"published","n":6}],"url":"https://benchgap.net/api/v1/harness-tax/mmlu-pro.json"},{"family_id":"mmmu-pro","label":"MMMU-Pro","capability":"vision","tier":"knowledge_tool_free","same_item_set":"verified","status":"active","pair_type":null,"audit_note":null,"origin":"seed","versions":[{"key":"mmmu-pro/current","name":"mmmu-pro","harness":"published","n":42},{"key":"aa-mmmu-pro/current","name":"aa-mmmu-pro","harness":"artificial-analysis","n":104}],"url":"https://benchgap.net/api/v1/harness-tax/mmmu-pro.json"},{"family_id":"riemannbench","label":"Riemannbench","capability":"math","tier":null,"same_item_set":"weak_alignment","status":"candidate","pair_type":null,"audit_note":"Auto-detected by normalized base name; review the item set before promoting to the seed.","origin":"auto","versions":[{"key":"riemannbench/current","name":"riemannbench","harness":"published","n":1},{"key":"riemannbenchwithtools/current","name":"riemannbenchwithtools","harness":"published","n":1}],"url":"https://benchgap.net/api/v1/harness-tax/riemannbench.json"},{"family_id":"scicode","label":"SciCode","capability":"coding","tier":"agentic","same_item_set":"needs_audit","status":"active","pair_type":null,"audit_note":"Confirm the AA 'aa-sci-code' run uses the same item set as the published SciCode runs.","origin":"seed","versions":[{"key":"scicode/current","name":"scicode","harness":"published","n":28},{"key":"aa-sci-code/current","name":"aa-sci-code","harness":"artificial-analysis","n":105}],"url":"https://benchgap.net/api/v1/harness-tax/scicode.json"},{"family_id":"swe-bench-verified","label":"SWE-bench Verified","capability":"coding","tier":"agentic","same_item_set":"needs_audit","status":"active","pair_type":null,"audit_note":"Confirm Vals 'swe-bench' runs the Verified subset before promoting to 'verified'.","origin":"seed","versions":[{"key":"swe-bench-verified/current","name":"swe-bench-verified","harness":"published","n":78},{"key":"valsswebench/current","name":"valsswebench","harness":"vals-ai","n":60}],"url":"https://benchgap.net/api/v1/harness-tax/swe-bench-verified.json"},{"family_id":"terminal-bench-2-1","label":"Terminal-Bench 2.1","capability":"agentic-terminal","tier":"agentic","same_item_set":"verified","status":"active","pair_type":null,"audit_note":null,"origin":"seed","versions":[{"key":"terminalbench21/current","name":"terminalbench21","harness":"published","n":65},{"key":"vals-terminal-bench21/current","name":"vals-terminal-bench21","harness":"vals-ai","n":61},{"key":"aa-terminal-bench21/current","name":"aa-terminal-bench21","harness":"artificial-analysis","n":12}],"url":"https://benchgap.net/api/v1/harness-tax/terminal-bench-2-1.json"},{"family_id":"terminal-bench-3","label":"Terminal-Bench 3","capability":"agentic-terminal","tier":"agentic","same_item_set":"verified","status":"candidate","pair_type":null,"audit_note":"Overlap currently 0 shared models; keep registered, exclude from aggregates.","origin":"seed","versions":[{"key":"terminal-bench-3/current","name":"terminal-bench-3","harness":"published","n":13},{"key":"terminal-bench3/current","name":"terminal-bench3","harness":"published","n":2}],"url":"https://benchgap.net/api/v1/harness-tax/terminal-bench-3.json"},{"family_id":"terminal-bench-4-0","label":"Terminal-Bench 4.0","capability":"agentic-terminal","tier":"agentic","same_item_set":"verified","status":"active","pair_type":null,"audit_note":null,"origin":"seed","versions":[{"key":"terminal-bench-4/current","name":"terminal-bench-4","harness":"published","n":17},{"key":"aa-terminal-bench4/current","name":"aa-terminal-bench4","harness":"artificial-analysis","n":24}],"url":"https://benchgap.net/api/v1/harness-tax/terminal-bench-4-0.json"},{"family_id":"terminal-bench-science-variants","label":"Terminal-Bench Science verifier variants","capability":"agentic-terminal","tier":"protocol_layer","same_item_set":"verified","status":"candidate","pair_type":"protocol_variant","audit_note":"1x vs 6x verifier count — pure scaffold knob. Currently only 1 shared model; keep for growth.","origin":"seed","versions":[{"key":"terminal-bench-science/current","name":"terminal-bench-science","harness":"published","n":5},{"key":"terminal-bench-science-6x-verifier/current","name":"terminal-bench-science-6x-verifier","harness":"published","n":1}],"url":"https://benchgap.net/api/v1/harness-tax/terminal-bench-science-variants.json"}],"pairs":[{"family_id":"chartography","tier":null,"capability":"vision","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"chartography/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/chartography"},"b":{"key":"chartographywithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/chartographywithtools"},"n_models":3,"mean_abs_pp":35.53,"median_abs_pp":28.6,"max_abs_pp":53.4,"share_gt_5pp":1,"share_gt_10pp":1,"kendall_tau":0.333,"n_rank_flips":1,"n_positive":0,"n_negative":3,"sign_test_p":0.25,"directionality":1},{"family_id":"automationbench","tier":null,"capability":"agentic-tool","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":false,"a":{"key":"aa-automation-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aaautomationbench"},"b":{"key":"automationbench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/automationbench"},"n_models":13,"mean_abs_pp":19.71,"median_abs_pp":26.2,"max_abs_pp":29.5,"share_gt_5pp":1,"share_gt_10pp":0.769,"kendall_tau":0.154,"n_rank_flips":33,"n_positive":13,"n_negative":0,"sign_test_p":0.0002,"directionality":1},{"family_id":"riemannbench","tier":null,"capability":"math","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"riemannbench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/riemannbench"},"b":{"key":"riemannbenchwithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/riemannbenchwithtools"},"n_models":1,"mean_abs_pp":19,"median_abs_pp":19,"max_abs_pp":19,"share_gt_5pp":1,"share_gt_10pp":1,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_test_p":1,"directionality":1},{"family_id":"terminal-bench-2-1","tier":"agentic","capability":"agentic-terminal","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"terminalbench21/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"b":{"key":"vals-terminal-bench21/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"n_models":29,"mean_abs_pp":12.09,"median_abs_pp":10.7,"max_abs_pp":33.2,"share_gt_5pp":0.862,"share_gt_10pp":0.517,"kendall_tau":0.516,"n_rank_flips":96,"n_positive":29,"n_negative":0,"sign_test_p":0,"directionality":1},{"family_id":"hle-protocol-variants","tier":"protocol_layer","capability":"knowledge","pair_type":"protocol_variant","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"hlenotools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/hlenotools"},"b":{"key":"hlewithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/hlewithtools"},"n_models":11,"mean_abs_pp":9.99,"median_abs_pp":10.7,"max_abs_pp":14.2,"share_gt_5pp":0.909,"share_gt_10pp":0.636,"kendall_tau":0.807,"n_rank_flips":5,"n_positive":0,"n_negative":11,"sign_test_p":0.001,"directionality":1},{"family_id":"terminal-bench-2-1","tier":"agentic","capability":"agentic-terminal","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"vals-terminal-bench21/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"b":{"key":"aa-terminal-bench21/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"n_models":11,"mean_abs_pp":9.75,"median_abs_pp":7.5,"max_abs_pp":21.4,"share_gt_5pp":0.727,"share_gt_10pp":0.455,"kendall_tau":0.881,"n_rank_flips":3,"n_positive":0,"n_negative":11,"sign_test_p":0.001,"directionality":1},{"family_id":"hle","tier":"protocol_layer","capability":"knowledge","pair_type":"cross_harness","same_item_set":"needs_audit","status":"active","low_overlap":false,"a":{"key":"hle/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/hle"},"b":{"key":"aa-hle/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aahle"},"n_models":48,"mean_abs_pp":8.15,"median_abs_pp":4.85,"max_abs_pp":35.7,"share_gt_5pp":0.5,"share_gt_10pp":0.292,"kendall_tau":0.579,"n_rank_flips":236,"n_positive":28,"n_negative":18,"sign_test_p":0.1839,"directionality":0.609},{"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-pro"},"b":{"key":"valslivecodebench/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/valslivecodebench"},"n_models":2,"mean_abs_pp":7.85,"median_abs_pp":7.85,"max_abs_pp":10.1,"share_gt_5pp":1,"share_gt_10pp":0.5,"kendall_tau":1,"n_rank_flips":0,"n_positive":0,"n_negative":2,"sign_test_p":0.5,"directionality":1},{"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"valslivecodebench/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/valslivecodebench"},"b":{"key":"aa-live-code-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aalivecodebench"},"n_models":1,"mean_abs_pp":7.2,"median_abs_pp":7.2,"max_abs_pp":7.2,"share_gt_5pp":1,"share_gt_10pp":0,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_test_p":1,"directionality":1},{"family_id":"arxivmathaugust2026","tier":null,"capability":"math","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"arxivmathaugust2026/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/arxivmathaugust2026"},"b":{"key":"arxivmathaugust2026withtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/arxivmathaugust2026withtools"},"n_models":2,"mean_abs_pp":7.05,"median_abs_pp":7.05,"max_abs_pp":8.4,"share_gt_5pp":1,"share_gt_10pp":0,"kendall_tau":1,"n_rank_flips":0,"n_positive":0,"n_negative":2,"sign_test_p":0.5,"directionality":1},{"family_id":"charxiv","tier":null,"capability":"vision","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":false,"a":{"key":"charxiv/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/charxiv"},"b":{"key":"charxivnotools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/charxivnotools"},"n_models":12,"mean_abs_pp":6.52,"median_abs_pp":6.25,"max_abs_pp":11.3,"share_gt_5pp":0.667,"share_gt_10pp":0.083,"kendall_tau":0.657,"n_rank_flips":11,"n_positive":12,"n_negative":0,"sign_test_p":0.0005,"directionality":1},{"family_id":"swe-bench-verified","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"needs_audit","status":"active","low_overlap":false,"a":{"key":"swe-bench-verified/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/swe-bench-verified"},"b":{"key":"valsswebench/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/valsswebench"},"n_models":24,"mean_abs_pp":6.29,"median_abs_pp":5.45,"max_abs_pp":17,"share_gt_5pp":0.542,"share_gt_10pp":0.208,"kendall_tau":0.676,"n_rank_flips":44,"n_positive":17,"n_negative":4,"sign_test_p":0.0072,"directionality":0.81},{"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench-v6/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-v6"},"b":{"key":"livecodebench-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-pro"},"n_models":3,"mean_abs_pp":6.11,"median_abs_pp":5.1,"max_abs_pp":10.84,"share_gt_5pp":0.667,"share_gt_10pp":0.333,"kendall_tau":1,"n_rank_flips":0,"n_positive":3,"n_negative":0,"sign_test_p":0.25,"directionality":1},{"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench"},"b":{"key":"aa-live-code-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aalivecodebench"},"n_models":1,"mean_abs_pp":4.5,"median_abs_pp":4.5,"max_abs_pp":4.5,"share_gt_5pp":0,"share_gt_10pp":0,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_test_p":1,"directionality":1},{"family_id":"ifbench","tier":"knowledge_tool_free","capability":"instruction-following","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"ifbench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/ifbench"},"b":{"key":"aa-if-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/ifbench"},"n_models":11,"mean_abs_pp":4.19,"median_abs_pp":0.87,"max_abs_pp":15.42,"share_gt_5pp":0.273,"share_gt_10pp":0.273,"kendall_tau":0.771,"n_rank_flips":6,"n_positive":6,"n_negative":3,"sign_test_p":0.5078,"directionality":0.667},{"family_id":"terminal-bench-4-0","tier":"agentic","capability":"agentic-terminal","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"terminal-bench-4/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminal-bench-4"},"b":{"key":"aa-terminal-bench4/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/terminal-bench-4"},"n_models":11,"mean_abs_pp":3.89,"median_abs_pp":3.8,"max_abs_pp":12.2,"share_gt_5pp":0.364,"share_gt_10pp":0.091,"kendall_tau":0.818,"n_rank_flips":5,"n_positive":8,"n_negative":2,"sign_test_p":0.1094,"directionality":0.8},{"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench"},"b":{"key":"valslivecodebench/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/valslivecodebench"},"n_models":2,"mean_abs_pp":3.6,"median_abs_pp":3.6,"max_abs_pp":4.5,"share_gt_5pp":0,"share_gt_10pp":0,"kendall_tau":1,"n_rank_flips":0,"n_positive":2,"n_negative":0,"sign_test_p":0.5,"directionality":1},{"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench-v6/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-v6"},"b":{"key":"valslivecodebench/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/valslivecodebench"},"n_models":4,"mean_abs_pp":3.3,"median_abs_pp":2.9,"max_abs_pp":6.3,"share_gt_5pp":0.25,"share_gt_10pp":0,"kendall_tau":-0.183,"n_rank_flips":3,"n_positive":4,"n_negative":0,"sign_test_p":0.125,"directionality":1},{"family_id":"terminal-bench-2-1","tier":"agentic","capability":"agentic-terminal","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"terminalbench21/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"b":{"key":"aa-terminal-bench21/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"n_models":10,"mean_abs_pp":3.27,"median_abs_pp":2.9,"max_abs_pp":8.7,"share_gt_5pp":0.2,"share_gt_10pp":0,"kendall_tau":0.899,"n_rank_flips":2,"n_positive":7,"n_negative":1,"sign_test_p":0.0703,"directionality":0.875},{"family_id":"gpqa-diamond","tier":"knowledge_tool_free","capability":"knowledge","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"vals-gpqa-diamond/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"b":{"key":"aa-gpqa-diamond/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"n_models":47,"mean_abs_pp":2.91,"median_abs_pp":1.2,"max_abs_pp":40,"share_gt_5pp":0.149,"share_gt_10pp":0.043,"kendall_tau":0.77,"n_rank_flips":119,"n_positive":15,"n_negative":31,"sign_test_p":0.0259,"directionality":0.674},{"family_id":"mmlu-pro","tier":"knowledge_tool_free","capability":"knowledge","pair_type":"protocol_variant","same_item_set":"needs_audit","status":"active","low_overlap":true,"a":{"key":"mmlu-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/mmlu-pro"},"b":{"key":"mmlu-pro-arcee/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/mmlu-pro"},"n_models":3,"mean_abs_pp":2.4,"median_abs_pp":0.1,"max_abs_pp":7.1,"share_gt_5pp":0.333,"share_gt_10pp":0,"kendall_tau":-0.333,"n_rank_flips":2,"n_positive":0,"n_negative":2,"sign_test_p":0.5,"directionality":1},{"family_id":"gdppdf","tier":null,"capability":"agentic-tool","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"gdppdf/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/gdppdf"},"b":{"key":"gdppdfwithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/gdppdfwithtools"},"n_models":1,"mean_abs_pp":2.1,"median_abs_pp":2.1,"max_abs_pp":2.1,"share_gt_5pp":0,"share_gt_10pp":0,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_test_p":1,"directionality":1},{"family_id":"scicode","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"needs_audit","status":"active","low_overlap":false,"a":{"key":"scicode/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/scicode"},"b":{"key":"aa-sci-code/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aascicode"},"n_models":20,"mean_abs_pp":2.03,"median_abs_pp":1,"max_abs_pp":7.4,"share_gt_5pp":0.1,"share_gt_10pp":0,"kendall_tau":0.769,"n_rank_flips":21,"n_positive":6,"n_negative":11,"sign_test_p":0.3323,"directionality":0.647},{"family_id":"mmmu-pro","tier":"knowledge_tool_free","capability":"vision","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"mmmu-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/mmmu-pro"},"b":{"key":"aa-mmmu-pro/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/mmmu-pro"},"n_models":32,"mean_abs_pp":2.02,"median_abs_pp":0.7,"max_abs_pp":26.3,"share_gt_5pp":0.031,"share_gt_10pp":0.031,"kendall_tau":0.695,"n_rank_flips":73,"n_positive":16,"n_negative":11,"sign_test_p":0.4421,"directionality":0.593},{"family_id":"gpqa-diamond","tier":"knowledge_tool_free","capability":"knowledge","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"gpqa-diamond/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"b":{"key":"aa-gpqa-diamond/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"n_models":40,"mean_abs_pp":1.98,"median_abs_pp":0.65,"max_abs_pp":25.3,"share_gt_5pp":0.075,"share_gt_10pp":0.05,"kendall_tau":0.809,"n_rank_flips":71,"n_positive":21,"n_negative":16,"sign_test_p":0.5114,"directionality":0.568},{"family_id":"mmlu-pro","tier":"knowledge_tool_free","capability":"knowledge","pair_type":"cross_harness","same_item_set":"needs_audit","status":"active","low_overlap":false,"a":{"key":"mmlu-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/mmlu-pro"},"b":{"key":"vals-mmlu-pro/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/mmlu-pro"},"n_models":7,"mean_abs_pp":1.76,"median_abs_pp":0.8,"max_abs_pp":8.1,"share_gt_5pp":0.143,"share_gt_10pp":0,"kendall_tau":0.524,"n_rank_flips":5,"n_positive":5,"n_negative":1,"sign_test_p":0.2188,"directionality":0.833},{"family_id":"gpqa-diamond","tier":"knowledge_tool_free","capability":"knowledge","pair_type":"cross_harness","same_item_set":"verified","status":"active","low_overlap":false,"a":{"key":"gpqa-diamond/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"b":{"key":"vals-gpqa-diamond/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"n_models":23,"mean_abs_pp":1.36,"median_abs_pp":0.9,"max_abs_pp":5.9,"share_gt_5pp":0.087,"share_gt_10pp":0,"kendall_tau":0.689,"n_rank_flips":38,"n_positive":15,"n_negative":8,"sign_test_p":0.21,"directionality":0.652},{"family_id":"arxivmathjune2026","tier":null,"capability":"math","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"arxivmathjune2026/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/arxivmathjune2026"},"b":{"key":"arxivmathjune2026withtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/arxivmathjune2026withtools"},"n_models":1,"mean_abs_pp":0.5,"median_abs_pp":0.5,"max_abs_pp":0.5,"share_gt_5pp":0,"share_gt_10pp":0,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_test_p":1,"directionality":1},{"family_id":"mmlu-pro","tier":"knowledge_tool_free","capability":"knowledge","pair_type":"cross_harness","same_item_set":"needs_audit","status":"active","low_overlap":true,"a":{"key":"vals-mmlu-pro/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/mmlu-pro"},"b":{"key":"mmlu-pro-arcee/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/mmlu-pro"},"n_models":1,"mean_abs_pp":0.4,"median_abs_pp":0.4,"max_abs_pp":0.4,"share_gt_5pp":0,"share_gt_10pp":0,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_test_p":1,"directionality":1},{"family_id":"benchcadvision2code","tier":null,"capability":"vision","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"benchcadvision2code/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/benchcadvision2code"},"b":{"key":"benchcadvision2codewithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/benchcadvision2codewithtools"},"n_models":3,"mean_abs_pp":0.3,"median_abs_pp":0.23,"max_abs_pp":0.46,"share_gt_5pp":0,"share_gt_10pp":0,"kendall_tau":1,"n_rank_flips":0,"n_positive":0,"n_negative":3,"sign_test_p":0.25,"directionality":1},{"family_id":"aime2025","tier":null,"capability":"math","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"aime2025/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/aime2025"},"b":{"key":"aime2025arcee/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/aime2025arcee"},"n_models":1,"mean_abs_pp":0.2,"median_abs_pp":0.2,"max_abs_pp":0.2,"share_gt_5pp":0,"share_gt_10pp":0,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":1,"sign_test_p":1,"directionality":1},{"family_id":"aime2025","tier":null,"capability":"math","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"aa-aime2025/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aaaime2025"},"b":{"key":"aime2025/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/aime2025"},"n_models":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5pp":null,"share_gt_10pp":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_test_p":null,"directionality":null},{"family_id":"aime2025","tier":null,"capability":"math","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"aa-aime2025/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aaaime2025"},"b":{"key":"aime2025arcee/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/aime2025arcee"},"n_models":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5pp":null,"share_gt_10pp":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_test_p":null,"directionality":null},{"family_id":"gdppdf","tier":null,"capability":"agentic-tool","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"aa-gdp-pdf/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aagdppdf"},"b":{"key":"gdppdf/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/gdppdf"},"n_models":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5pp":null,"share_gt_10pp":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_test_p":null,"directionality":null},{"family_id":"gdppdf","tier":null,"capability":"agentic-tool","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"aa-gdp-pdf/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aagdppdf"},"b":{"key":"gdppdfwithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/gdppdfwithtools"},"n_models":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5pp":null,"share_gt_10pp":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_test_p":null,"directionality":null},{"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench"},"b":{"key":"livecodebench-v6/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-v6"},"n_models":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5pp":null,"share_gt_10pp":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_test_p":null,"directionality":null},{"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"protocol_variant","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench"},"b":{"key":"livecodebench-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-pro"},"n_models":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5pp":null,"share_gt_10pp":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_test_p":null,"directionality":null},{"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench-v6/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-v6"},"b":{"key":"aa-live-code-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aalivecodebench"},"n_models":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5pp":null,"share_gt_10pp":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_test_p":null,"directionality":null},{"family_id":"livecodebench","tier":"agentic","capability":"coding","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"livecodebench-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/livecodebench-pro"},"b":{"key":"aa-live-code-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aalivecodebench"},"n_models":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5pp":null,"share_gt_10pp":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_test_p":null,"directionality":null},{"family_id":"math500","tier":null,"capability":"math","pair_type":"cross_harness","same_item_set":"weak_alignment","status":"candidate","low_overlap":true,"a":{"key":"aa-math500/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/math-500"},"b":{"key":"math-500/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/math-500"},"n_models":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5pp":null,"share_gt_10pp":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_test_p":null,"directionality":null},{"family_id":"terminal-bench-3","tier":"agentic","capability":"agentic-terminal","pair_type":"protocol_variant","same_item_set":"verified","status":"candidate","low_overlap":true,"a":{"key":"terminal-bench-3/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminal-bench-3"},"b":{"key":"terminal-bench3/current","harness":"published","source_url":"https://benchlm.ai/benchmarks"},"n_models":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5pp":null,"share_gt_10pp":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_test_p":null,"directionality":null},{"family_id":"terminal-bench-science-variants","tier":"protocol_layer","capability":"agentic-terminal","pair_type":"protocol_variant","same_item_set":"verified","status":"candidate","low_overlap":true,"a":{"key":"terminal-bench-science/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminal-bench-science"},"b":{"key":"terminal-bench-science-6x-verifier/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminal-bench-science-6x-verifier"},"n_models":0,"mean_abs_pp":null,"median_abs_pp":null,"max_abs_pp":null,"share_gt_5pp":null,"share_gt_10pp":null,"kendall_tau":null,"n_rank_flips":0,"n_positive":0,"n_negative":0,"sign_test_p":null,"directionality":null}],"aggregates":{"by_tier":{"agentic":{"n_pairs":6,"n_model_pairs":105,"pooled_mean_abs_pp":6.902380952380953},"knowledge_tool_free":{"n_pairs":6,"n_model_pairs":160,"pooled_mean_abs_pp":2.3172500000000005},"protocol_layer":{"n_pairs":2,"n_model_pairs":59,"pooled_mean_abs_pp":8.496610169491525}},"by_capability":{"agentic-terminal":{"n_pairs":4,"n_model_pairs":61,"pooled_mean_abs_pp":8.740983606557378},"coding":{"n_pairs":2,"n_model_pairs":44,"pooled_mean_abs_pp":4.353409090909091},"instruction-following":{"n_pairs":1,"n_model_pairs":11,"pooled_mean_abs_pp":4.189999999999999},"knowledge":{"n_pairs":6,"n_model_pairs":176,"pooled_mean_abs_pp":4.325397727272728},"vision":{"n_pairs":1,"n_model_pairs":32,"pooled_mean_abs_pp":2.0218750000000005}},"by_family":{"gpqa-diamond":{"n_pairs":3,"n_model_pairs":110,"pooled_mean_abs_pp":2.251545454545455},"hle":{"n_pairs":1,"n_model_pairs":48,"pooled_mean_abs_pp":8.154166666666667},"hle-protocol-variants":{"n_pairs":1,"n_model_pairs":11,"pooled_mean_abs_pp":9.990909090909092},"ifbench":{"n_pairs":1,"n_model_pairs":11,"pooled_mean_abs_pp":4.189999999999999},"mmlu-pro":{"n_pairs":1,"n_model_pairs":7,"pooled_mean_abs_pp":1.7571428571428573},"mmmu-pro":{"n_pairs":1,"n_model_pairs":32,"pooled_mean_abs_pp":2.0218750000000005},"scicode":{"n_pairs":1,"n_model_pairs":20,"pooled_mean_abs_pp":2.0324999999999998},"swe-bench-verified":{"n_pairs":1,"n_model_pairs":24,"pooled_mean_abs_pp":6.2875000000000005},"terminal-bench-2-1":{"n_pairs":3,"n_model_pairs":50,"pooled_mean_abs_pp":9.808},"terminal-bench-4-0":{"n_pairs":1,"n_model_pairs":11,"pooled_mean_abs_pp":3.890909090909091}},"headline":{"by_tier":{"agentic":{"n_pairs":4,"n_model_pairs":61,"pooled_mean_abs_pp":8.740983606557378},"knowledge_tool_free":{"n_pairs":5,"n_model_pairs":153,"pooled_mean_abs_pp":2.3428758169934647},"protocol_layer":{"n_pairs":1,"n_model_pairs":11,"pooled_mean_abs_pp":9.990909090909092}},"ratio":3.730877899356354}},"audit":{"outliers":[{"model":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score_a":0.879,"score_b":0.547,"delta_pp":33.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"terminal-bench-2-1","a":{"key":"terminalbench21/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"b":{"key":"vals-terminal-bench21/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/terminalbench21"}},{"model":"glm-5-3-flash","name":"GLM-5.3-Flash","score_a":0.843,"score_b":0.629,"delta_pp":21.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"terminal-bench-2-1","a":{"key":"terminalbench21/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"b":{"key":"vals-terminal-bench21/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/terminalbench21"}},{"model":"hy4-preview","name":"Hy4 preview","score_a":0.854,"score_b":0.551,"delta_pp":30.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"terminal-bench-2-1","a":{"key":"terminalbench21/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"b":{"key":"vals-terminal-bench21/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/terminalbench21"}},{"model":"glm-5-3-flash","name":"GLM-5.3-Flash","score_a":0.629,"score_b":0.843,"delta_pp":-21.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"terminal-bench-2-1","a":{"key":"vals-terminal-bench21/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"b":{"key":"aa-terminal-bench21/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/terminalbench21"}},{"model":"qwen3-8-27b","name":"Qwen3.8-27B","score_a":0.584,"score_b":0.798,"delta_pp":-21.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"terminal-bench-2-1","a":{"key":"vals-terminal-bench21/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/terminalbench21"},"b":{"key":"aa-terminal-bench21/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/terminalbench21"}},{"model":"nemotron-3-nano-omni-30b-a3b","name":"Nemotron 3 Nano Omni 30B A3B","score_a":0.722,"score_b":0.469,"delta_pp":25.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"gpqa-diamond","a":{"key":"gpqa-diamond/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"b":{"key":"aa-gpqa-diamond/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"}},{"model":"mistral-medium-3-5-128b","name":"Mistral Medium 3.5 128B","score_a":0.348,"score_b":0.748,"delta_pp":-40,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"gpqa-diamond","a":{"key":"vals-gpqa-diamond/current","harness":"vals-ai","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"},"b":{"key":"aa-gpqa-diamond/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/gpqa-diamond"}},{"model":"qwen3-5-397b","name":"Qwen3.5 397B","score_a":0.79,"score_b":0.527,"delta_pp":26.3,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"mmmu-pro","a":{"key":"mmmu-pro/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/mmmu-pro"},"b":{"key":"aa-mmmu-pro/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/mmmu-pro"}},{"model":"apodex-1-1","name":"Apodex 1.1","score_a":0.561,"score_b":0.341,"delta_pp":22,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"hle","a":{"key":"hle/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/hle"},"b":{"key":"aa-hle/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aahle"}},{"model":"claude-opus-4-6","name":"Claude Opus 4.6","score_a":0.53,"score_b":0.191,"delta_pp":33.9,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"hle","a":{"key":"hle/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/hle"},"b":{"key":"aa-hle/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aahle"}},{"model":"claude-sonnet-4-6","name":"Claude Sonnet 4.6","score_a":0.49,"score_b":0.133,"delta_pp":35.7,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"hle","a":{"key":"hle/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/hle"},"b":{"key":"aa-hle/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aahle"}},{"model":"glm-5","name":"GLM-5","score_a":0.504,"score_b":0.293,"delta_pp":21.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"hle","a":{"key":"hle/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/hle"},"b":{"key":"aa-hle/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aahle"}},{"model":"glm-5-1","name":"GLM-5.1","score_a":0.523,"score_b":0.301,"delta_pp":22.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"hle","a":{"key":"hle/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/hle"},"b":{"key":"aa-hle/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aahle"}},{"model":"claude-fable-5-1","name":"Claude Fable 5.1","score_a":0.594,"score_b":0.314,"delta_pp":28,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"automationbench","a":{"key":"aa-automation-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aaautomationbench"},"b":{"key":"automationbench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/automationbench"}},{"model":"claude-opus-5-5","name":"Claude Opus 5.5","score_a":0.695,"score_b":0.4,"delta_pp":29.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"automationbench","a":{"key":"aa-automation-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aaautomationbench"},"b":{"key":"automationbench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/automationbench"}},{"model":"gpt-6-astra","name":"GPT-6 Astra","score_a":0.685,"score_b":0.414,"delta_pp":27.1,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"automationbench","a":{"key":"aa-automation-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aaautomationbench"},"b":{"key":"automationbench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/automationbench"}},{"model":"gpt-6-sol","name":"GPT-6 Sol","score_a":0.616,"score_b":0.332,"delta_pp":28.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"automationbench","a":{"key":"aa-automation-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aaautomationbench"},"b":{"key":"automationbench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/automationbench"}},{"model":"gpt-6-1-sol","name":"GPT-6.1 Sol","score_a":0.649,"score_b":0.361,"delta_pp":28.8,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"automationbench","a":{"key":"aa-automation-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aaautomationbench"},"b":{"key":"automationbench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/automationbench"}},{"model":"gemini-4-argon","name":"Gemini 4 Argon","score_a":0.775,"score_b":0.513,"delta_pp":26.2,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"automationbench","a":{"key":"aa-automation-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aaautomationbench"},"b":{"key":"automationbench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/automationbench"}},{"model":"kimi-k3","name":"Kimi K3","score_a":0.583,"score_b":0.308,"delta_pp":27.5,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"automationbench","a":{"key":"aa-automation-bench/current","harness":"artificial-analysis","source_url":"https://benchlm.ai/benchmarks/aaautomationbench"},"b":{"key":"automationbench/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/automationbench"}},{"model":"claude-opus-5","name":"Claude Opus 5","score_a":0.296,"score_b":0.83,"delta_pp":-53.4,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"chartography","a":{"key":"chartography/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/chartography"},"b":{"key":"chartographywithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/chartographywithtools"}},{"model":"claude-opus-5-5","name":"Claude Opus 5.5","score_a":0.644,"score_b":0.89,"delta_pp":-24.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"chartography","a":{"key":"chartography/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/chartography"},"b":{"key":"chartographywithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/chartographywithtools"}},{"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score_a":0.616,"score_b":0.902,"delta_pp":-28.6,"retrieved_a":"2026-10-09","retrieved_b":"2026-10-09","family_id":"chartography","a":{"key":"chartography/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/chartography"},"b":{"key":"chartographywithtools/current","harness":"published","source_url":"https://benchlm.ai/benchmarks/chartographywithtools"}}]},"url":"https://benchgap.net/api/v1/harness-tax.json"}